1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Framework for buffer objects that can be shared across devices/subsystems. 4 * 5 * Copyright(C) 2011 Linaro Limited. All rights reserved. 6 * Author: Sumit Semwal <sumit.semwal@ti.com> 7 * 8 * Many thanks to linaro-mm-sig list, and specially 9 * Arnd Bergmann <arnd@arndb.de>, Rob Clark <rob@ti.com> and 10 * Daniel Vetter <daniel@ffwll.ch> for their support in creation and 11 * refining of this idea. 12 */ 13 14 #include <linux/fs.h> 15 #include <linux/slab.h> 16 #include <linux/dma-buf.h> 17 #include <linux/dma-fence.h> 18 #include <linux/anon_inodes.h> 19 #include <linux/export.h> 20 #include <linux/debugfs.h> 21 #include <linux/module.h> 22 #include <linux/seq_file.h> 23 #include <linux/poll.h> 24 #include <linux/dma-resv.h> 25 #include <linux/mm.h> 26 #include <linux/mount.h> 27 #include <linux/pseudo_fs.h> 28 29 #include <uapi/linux/dma-buf.h> 30 #include <uapi/linux/magic.h> 31 32 #include "dma-buf-sysfs-stats.h" 33 34 static inline int is_dma_buf_file(struct file *); 35 36 struct dma_buf_list { 37 struct list_head head; 38 struct mutex lock; 39 }; 40 41 static struct dma_buf_list db_list; 42 43 static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen) 44 { 45 struct dma_buf *dmabuf; 46 char name[DMA_BUF_NAME_LEN]; 47 size_t ret = 0; 48 49 dmabuf = dentry->d_fsdata; 50 spin_lock(&dmabuf->name_lock); 51 if (dmabuf->name) 52 ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN); 53 spin_unlock(&dmabuf->name_lock); 54 55 return dynamic_dname(dentry, buffer, buflen, "/%s:%s", 56 dentry->d_name.name, ret > 0 ? name : ""); 57 } 58 59 static void dma_buf_release(struct dentry *dentry) 60 { 61 struct dma_buf *dmabuf; 62 63 dmabuf = dentry->d_fsdata; 64 if (unlikely(!dmabuf)) 65 return; 66 67 BUG_ON(dmabuf->vmapping_counter); 68 69 /* 70 * Any fences that a dma-buf poll can wait on should be signaled 71 * before releasing dma-buf. This is the responsibility of each 72 * driver that uses the reservation objects. 73 * 74 * If you hit this BUG() it means someone dropped their ref to the 75 * dma-buf while still having pending operation to the buffer. 76 */ 77 BUG_ON(dmabuf->cb_shared.active || dmabuf->cb_excl.active); 78 79 dmabuf->ops->release(dmabuf); 80 81 if (dmabuf->resv == (struct dma_resv *)&dmabuf[1]) 82 dma_resv_fini(dmabuf->resv); 83 84 dma_buf_stats_teardown(dmabuf); 85 module_put(dmabuf->owner); 86 kfree(dmabuf->name); 87 kfree(dmabuf); 88 } 89 90 static int dma_buf_file_release(struct inode *inode, struct file *file) 91 { 92 struct dma_buf *dmabuf; 93 94 if (!is_dma_buf_file(file)) 95 return -EINVAL; 96 97 dmabuf = file->private_data; 98 99 mutex_lock(&db_list.lock); 100 list_del(&dmabuf->list_node); 101 mutex_unlock(&db_list.lock); 102 103 return 0; 104 } 105 106 static const struct dentry_operations dma_buf_dentry_ops = { 107 .d_dname = dmabuffs_dname, 108 .d_release = dma_buf_release, 109 }; 110 111 static struct vfsmount *dma_buf_mnt; 112 113 static int dma_buf_fs_init_context(struct fs_context *fc) 114 { 115 struct pseudo_fs_context *ctx; 116 117 ctx = init_pseudo(fc, DMA_BUF_MAGIC); 118 if (!ctx) 119 return -ENOMEM; 120 ctx->dops = &dma_buf_dentry_ops; 121 return 0; 122 } 123 124 static struct file_system_type dma_buf_fs_type = { 125 .name = "dmabuf", 126 .init_fs_context = dma_buf_fs_init_context, 127 .kill_sb = kill_anon_super, 128 }; 129 130 static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) 131 { 132 struct dma_buf *dmabuf; 133 134 if (!is_dma_buf_file(file)) 135 return -EINVAL; 136 137 dmabuf = file->private_data; 138 139 /* check if buffer supports mmap */ 140 if (!dmabuf->ops->mmap) 141 return -EINVAL; 142 143 /* check for overflowing the buffer's size */ 144 if (vma->vm_pgoff + vma_pages(vma) > 145 dmabuf->size >> PAGE_SHIFT) 146 return -EINVAL; 147 148 return dmabuf->ops->mmap(dmabuf, vma); 149 } 150 151 static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence) 152 { 153 struct dma_buf *dmabuf; 154 loff_t base; 155 156 if (!is_dma_buf_file(file)) 157 return -EBADF; 158 159 dmabuf = file->private_data; 160 161 /* only support discovering the end of the buffer, 162 but also allow SEEK_SET to maintain the idiomatic 163 SEEK_END(0), SEEK_CUR(0) pattern */ 164 if (whence == SEEK_END) 165 base = dmabuf->size; 166 else if (whence == SEEK_SET) 167 base = 0; 168 else 169 return -EINVAL; 170 171 if (offset != 0) 172 return -EINVAL; 173 174 return base + offset; 175 } 176 177 /** 178 * DOC: implicit fence polling 179 * 180 * To support cross-device and cross-driver synchronization of buffer access 181 * implicit fences (represented internally in the kernel with &struct dma_fence) 182 * can be attached to a &dma_buf. The glue for that and a few related things are 183 * provided in the &dma_resv structure. 184 * 185 * Userspace can query the state of these implicitly tracked fences using poll() 186 * and related system calls: 187 * 188 * - Checking for EPOLLIN, i.e. read access, can be use to query the state of the 189 * most recent write or exclusive fence. 190 * 191 * - Checking for EPOLLOUT, i.e. write access, can be used to query the state of 192 * all attached fences, shared and exclusive ones. 193 * 194 * Note that this only signals the completion of the respective fences, i.e. the 195 * DMA transfers are complete. Cache flushing and any other necessary 196 * preparations before CPU access can begin still need to happen. 197 */ 198 199 static void dma_buf_poll_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 200 { 201 struct dma_buf_poll_cb_t *dcb = (struct dma_buf_poll_cb_t *)cb; 202 unsigned long flags; 203 204 spin_lock_irqsave(&dcb->poll->lock, flags); 205 wake_up_locked_poll(dcb->poll, dcb->active); 206 dcb->active = 0; 207 spin_unlock_irqrestore(&dcb->poll->lock, flags); 208 } 209 210 static __poll_t dma_buf_poll(struct file *file, poll_table *poll) 211 { 212 struct dma_buf *dmabuf; 213 struct dma_resv *resv; 214 struct dma_resv_list *fobj; 215 struct dma_fence *fence_excl; 216 __poll_t events; 217 unsigned shared_count, seq; 218 219 dmabuf = file->private_data; 220 if (!dmabuf || !dmabuf->resv) 221 return EPOLLERR; 222 223 resv = dmabuf->resv; 224 225 poll_wait(file, &dmabuf->poll, poll); 226 227 events = poll_requested_events(poll) & (EPOLLIN | EPOLLOUT); 228 if (!events) 229 return 0; 230 231 retry: 232 seq = read_seqcount_begin(&resv->seq); 233 rcu_read_lock(); 234 235 fobj = rcu_dereference(resv->fence); 236 if (fobj) 237 shared_count = fobj->shared_count; 238 else 239 shared_count = 0; 240 fence_excl = dma_resv_excl_fence(resv); 241 if (read_seqcount_retry(&resv->seq, seq)) { 242 rcu_read_unlock(); 243 goto retry; 244 } 245 246 if (fence_excl && (!(events & EPOLLOUT) || shared_count == 0)) { 247 struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_excl; 248 __poll_t pevents = EPOLLIN; 249 250 if (shared_count == 0) 251 pevents |= EPOLLOUT; 252 253 spin_lock_irq(&dmabuf->poll.lock); 254 if (dcb->active) { 255 dcb->active |= pevents; 256 events &= ~pevents; 257 } else 258 dcb->active = pevents; 259 spin_unlock_irq(&dmabuf->poll.lock); 260 261 if (events & pevents) { 262 if (!dma_fence_get_rcu(fence_excl)) { 263 /* force a recheck */ 264 events &= ~pevents; 265 dma_buf_poll_cb(NULL, &dcb->cb); 266 } else if (!dma_fence_add_callback(fence_excl, &dcb->cb, 267 dma_buf_poll_cb)) { 268 events &= ~pevents; 269 dma_fence_put(fence_excl); 270 } else { 271 /* 272 * No callback queued, wake up any additional 273 * waiters. 274 */ 275 dma_fence_put(fence_excl); 276 dma_buf_poll_cb(NULL, &dcb->cb); 277 } 278 } 279 } 280 281 if ((events & EPOLLOUT) && shared_count > 0) { 282 struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_shared; 283 int i; 284 285 /* Only queue a new callback if no event has fired yet */ 286 spin_lock_irq(&dmabuf->poll.lock); 287 if (dcb->active) 288 events &= ~EPOLLOUT; 289 else 290 dcb->active = EPOLLOUT; 291 spin_unlock_irq(&dmabuf->poll.lock); 292 293 if (!(events & EPOLLOUT)) 294 goto out; 295 296 for (i = 0; i < shared_count; ++i) { 297 struct dma_fence *fence = rcu_dereference(fobj->shared[i]); 298 299 if (!dma_fence_get_rcu(fence)) { 300 /* 301 * fence refcount dropped to zero, this means 302 * that fobj has been freed 303 * 304 * call dma_buf_poll_cb and force a recheck! 305 */ 306 events &= ~EPOLLOUT; 307 dma_buf_poll_cb(NULL, &dcb->cb); 308 break; 309 } 310 if (!dma_fence_add_callback(fence, &dcb->cb, 311 dma_buf_poll_cb)) { 312 dma_fence_put(fence); 313 events &= ~EPOLLOUT; 314 break; 315 } 316 dma_fence_put(fence); 317 } 318 319 /* No callback queued, wake up any additional waiters. */ 320 if (i == shared_count) 321 dma_buf_poll_cb(NULL, &dcb->cb); 322 } 323 324 out: 325 rcu_read_unlock(); 326 return events; 327 } 328 329 /** 330 * dma_buf_set_name - Set a name to a specific dma_buf to track the usage. 331 * The name of the dma-buf buffer can only be set when the dma-buf is not 332 * attached to any devices. It could theoritically support changing the 333 * name of the dma-buf if the same piece of memory is used for multiple 334 * purpose between different devices. 335 * 336 * @dmabuf: [in] dmabuf buffer that will be renamed. 337 * @buf: [in] A piece of userspace memory that contains the name of 338 * the dma-buf. 339 * 340 * Returns 0 on success. If the dma-buf buffer is already attached to 341 * devices, return -EBUSY. 342 * 343 */ 344 static long dma_buf_set_name(struct dma_buf *dmabuf, const char __user *buf) 345 { 346 char *name = strndup_user(buf, DMA_BUF_NAME_LEN); 347 long ret = 0; 348 349 if (IS_ERR(name)) 350 return PTR_ERR(name); 351 352 dma_resv_lock(dmabuf->resv, NULL); 353 if (!list_empty(&dmabuf->attachments)) { 354 ret = -EBUSY; 355 kfree(name); 356 goto out_unlock; 357 } 358 spin_lock(&dmabuf->name_lock); 359 kfree(dmabuf->name); 360 dmabuf->name = name; 361 spin_unlock(&dmabuf->name_lock); 362 363 out_unlock: 364 dma_resv_unlock(dmabuf->resv); 365 return ret; 366 } 367 368 static long dma_buf_ioctl(struct file *file, 369 unsigned int cmd, unsigned long arg) 370 { 371 struct dma_buf *dmabuf; 372 struct dma_buf_sync sync; 373 enum dma_data_direction direction; 374 int ret; 375 376 dmabuf = file->private_data; 377 378 switch (cmd) { 379 case DMA_BUF_IOCTL_SYNC: 380 if (copy_from_user(&sync, (void __user *) arg, sizeof(sync))) 381 return -EFAULT; 382 383 if (sync.flags & ~DMA_BUF_SYNC_VALID_FLAGS_MASK) 384 return -EINVAL; 385 386 switch (sync.flags & DMA_BUF_SYNC_RW) { 387 case DMA_BUF_SYNC_READ: 388 direction = DMA_FROM_DEVICE; 389 break; 390 case DMA_BUF_SYNC_WRITE: 391 direction = DMA_TO_DEVICE; 392 break; 393 case DMA_BUF_SYNC_RW: 394 direction = DMA_BIDIRECTIONAL; 395 break; 396 default: 397 return -EINVAL; 398 } 399 400 if (sync.flags & DMA_BUF_SYNC_END) 401 ret = dma_buf_end_cpu_access(dmabuf, direction); 402 else 403 ret = dma_buf_begin_cpu_access(dmabuf, direction); 404 405 return ret; 406 407 case DMA_BUF_SET_NAME_A: 408 case DMA_BUF_SET_NAME_B: 409 return dma_buf_set_name(dmabuf, (const char __user *)arg); 410 411 default: 412 return -ENOTTY; 413 } 414 } 415 416 static void dma_buf_show_fdinfo(struct seq_file *m, struct file *file) 417 { 418 struct dma_buf *dmabuf = file->private_data; 419 420 seq_printf(m, "size:\t%zu\n", dmabuf->size); 421 /* Don't count the temporary reference taken inside procfs seq_show */ 422 seq_printf(m, "count:\t%ld\n", file_count(dmabuf->file) - 1); 423 seq_printf(m, "exp_name:\t%s\n", dmabuf->exp_name); 424 spin_lock(&dmabuf->name_lock); 425 if (dmabuf->name) 426 seq_printf(m, "name:\t%s\n", dmabuf->name); 427 spin_unlock(&dmabuf->name_lock); 428 } 429 430 static const struct file_operations dma_buf_fops = { 431 .release = dma_buf_file_release, 432 .mmap = dma_buf_mmap_internal, 433 .llseek = dma_buf_llseek, 434 .poll = dma_buf_poll, 435 .unlocked_ioctl = dma_buf_ioctl, 436 .compat_ioctl = compat_ptr_ioctl, 437 .show_fdinfo = dma_buf_show_fdinfo, 438 }; 439 440 /* 441 * is_dma_buf_file - Check if struct file* is associated with dma_buf 442 */ 443 static inline int is_dma_buf_file(struct file *file) 444 { 445 return file->f_op == &dma_buf_fops; 446 } 447 448 static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags) 449 { 450 struct file *file; 451 struct inode *inode = alloc_anon_inode(dma_buf_mnt->mnt_sb); 452 453 if (IS_ERR(inode)) 454 return ERR_CAST(inode); 455 456 inode->i_size = dmabuf->size; 457 inode_set_bytes(inode, dmabuf->size); 458 459 file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf", 460 flags, &dma_buf_fops); 461 if (IS_ERR(file)) 462 goto err_alloc_file; 463 file->f_flags = flags & (O_ACCMODE | O_NONBLOCK); 464 file->private_data = dmabuf; 465 file->f_path.dentry->d_fsdata = dmabuf; 466 467 return file; 468 469 err_alloc_file: 470 iput(inode); 471 return file; 472 } 473 474 /** 475 * DOC: dma buf device access 476 * 477 * For device DMA access to a shared DMA buffer the usual sequence of operations 478 * is fairly simple: 479 * 480 * 1. The exporter defines his exporter instance using 481 * DEFINE_DMA_BUF_EXPORT_INFO() and calls dma_buf_export() to wrap a private 482 * buffer object into a &dma_buf. It then exports that &dma_buf to userspace 483 * as a file descriptor by calling dma_buf_fd(). 484 * 485 * 2. Userspace passes this file-descriptors to all drivers it wants this buffer 486 * to share with: First the filedescriptor is converted to a &dma_buf using 487 * dma_buf_get(). Then the buffer is attached to the device using 488 * dma_buf_attach(). 489 * 490 * Up to this stage the exporter is still free to migrate or reallocate the 491 * backing storage. 492 * 493 * 3. Once the buffer is attached to all devices userspace can initiate DMA 494 * access to the shared buffer. In the kernel this is done by calling 495 * dma_buf_map_attachment() and dma_buf_unmap_attachment(). 496 * 497 * 4. Once a driver is done with a shared buffer it needs to call 498 * dma_buf_detach() (after cleaning up any mappings) and then release the 499 * reference acquired with dma_buf_get() by calling dma_buf_put(). 500 * 501 * For the detailed semantics exporters are expected to implement see 502 * &dma_buf_ops. 503 */ 504 505 /** 506 * dma_buf_export - Creates a new dma_buf, and associates an anon file 507 * with this buffer, so it can be exported. 508 * Also connect the allocator specific data and ops to the buffer. 509 * Additionally, provide a name string for exporter; useful in debugging. 510 * 511 * @exp_info: [in] holds all the export related information provided 512 * by the exporter. see &struct dma_buf_export_info 513 * for further details. 514 * 515 * Returns, on success, a newly created struct dma_buf object, which wraps the 516 * supplied private data and operations for struct dma_buf_ops. On either 517 * missing ops, or error in allocating struct dma_buf, will return negative 518 * error. 519 * 520 * For most cases the easiest way to create @exp_info is through the 521 * %DEFINE_DMA_BUF_EXPORT_INFO macro. 522 */ 523 struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) 524 { 525 struct dma_buf *dmabuf; 526 struct dma_resv *resv = exp_info->resv; 527 struct file *file; 528 size_t alloc_size = sizeof(struct dma_buf); 529 int ret; 530 531 if (!exp_info->resv) 532 alloc_size += sizeof(struct dma_resv); 533 else 534 /* prevent &dma_buf[1] == dma_buf->resv */ 535 alloc_size += 1; 536 537 if (WARN_ON(!exp_info->priv 538 || !exp_info->ops 539 || !exp_info->ops->map_dma_buf 540 || !exp_info->ops->unmap_dma_buf 541 || !exp_info->ops->release)) { 542 return ERR_PTR(-EINVAL); 543 } 544 545 if (WARN_ON(exp_info->ops->cache_sgt_mapping && 546 (exp_info->ops->pin || exp_info->ops->unpin))) 547 return ERR_PTR(-EINVAL); 548 549 if (WARN_ON(!exp_info->ops->pin != !exp_info->ops->unpin)) 550 return ERR_PTR(-EINVAL); 551 552 if (!try_module_get(exp_info->owner)) 553 return ERR_PTR(-ENOENT); 554 555 dmabuf = kzalloc(alloc_size, GFP_KERNEL); 556 if (!dmabuf) { 557 ret = -ENOMEM; 558 goto err_module; 559 } 560 561 dmabuf->priv = exp_info->priv; 562 dmabuf->ops = exp_info->ops; 563 dmabuf->size = exp_info->size; 564 dmabuf->exp_name = exp_info->exp_name; 565 dmabuf->owner = exp_info->owner; 566 spin_lock_init(&dmabuf->name_lock); 567 init_waitqueue_head(&dmabuf->poll); 568 dmabuf->cb_excl.poll = dmabuf->cb_shared.poll = &dmabuf->poll; 569 dmabuf->cb_excl.active = dmabuf->cb_shared.active = 0; 570 571 if (!resv) { 572 resv = (struct dma_resv *)&dmabuf[1]; 573 dma_resv_init(resv); 574 } 575 dmabuf->resv = resv; 576 577 file = dma_buf_getfile(dmabuf, exp_info->flags); 578 if (IS_ERR(file)) { 579 ret = PTR_ERR(file); 580 goto err_dmabuf; 581 } 582 583 file->f_mode |= FMODE_LSEEK; 584 dmabuf->file = file; 585 586 ret = dma_buf_stats_setup(dmabuf); 587 if (ret) 588 goto err_sysfs; 589 590 mutex_init(&dmabuf->lock); 591 INIT_LIST_HEAD(&dmabuf->attachments); 592 593 mutex_lock(&db_list.lock); 594 list_add(&dmabuf->list_node, &db_list.head); 595 mutex_unlock(&db_list.lock); 596 597 return dmabuf; 598 599 err_sysfs: 600 /* 601 * Set file->f_path.dentry->d_fsdata to NULL so that when 602 * dma_buf_release() gets invoked by dentry_ops, it exits 603 * early before calling the release() dma_buf op. 604 */ 605 file->f_path.dentry->d_fsdata = NULL; 606 fput(file); 607 err_dmabuf: 608 kfree(dmabuf); 609 err_module: 610 module_put(exp_info->owner); 611 return ERR_PTR(ret); 612 } 613 EXPORT_SYMBOL_GPL(dma_buf_export); 614 615 /** 616 * dma_buf_fd - returns a file descriptor for the given struct dma_buf 617 * @dmabuf: [in] pointer to dma_buf for which fd is required. 618 * @flags: [in] flags to give to fd 619 * 620 * On success, returns an associated 'fd'. Else, returns error. 621 */ 622 int dma_buf_fd(struct dma_buf *dmabuf, int flags) 623 { 624 int fd; 625 626 if (!dmabuf || !dmabuf->file) 627 return -EINVAL; 628 629 fd = get_unused_fd_flags(flags); 630 if (fd < 0) 631 return fd; 632 633 fd_install(fd, dmabuf->file); 634 635 return fd; 636 } 637 EXPORT_SYMBOL_GPL(dma_buf_fd); 638 639 /** 640 * dma_buf_get - returns the struct dma_buf related to an fd 641 * @fd: [in] fd associated with the struct dma_buf to be returned 642 * 643 * On success, returns the struct dma_buf associated with an fd; uses 644 * file's refcounting done by fget to increase refcount. returns ERR_PTR 645 * otherwise. 646 */ 647 struct dma_buf *dma_buf_get(int fd) 648 { 649 struct file *file; 650 651 file = fget(fd); 652 653 if (!file) 654 return ERR_PTR(-EBADF); 655 656 if (!is_dma_buf_file(file)) { 657 fput(file); 658 return ERR_PTR(-EINVAL); 659 } 660 661 return file->private_data; 662 } 663 EXPORT_SYMBOL_GPL(dma_buf_get); 664 665 /** 666 * dma_buf_put - decreases refcount of the buffer 667 * @dmabuf: [in] buffer to reduce refcount of 668 * 669 * Uses file's refcounting done implicitly by fput(). 670 * 671 * If, as a result of this call, the refcount becomes 0, the 'release' file 672 * operation related to this fd is called. It calls &dma_buf_ops.release vfunc 673 * in turn, and frees the memory allocated for dmabuf when exported. 674 */ 675 void dma_buf_put(struct dma_buf *dmabuf) 676 { 677 if (WARN_ON(!dmabuf || !dmabuf->file)) 678 return; 679 680 fput(dmabuf->file); 681 } 682 EXPORT_SYMBOL_GPL(dma_buf_put); 683 684 static void mangle_sg_table(struct sg_table *sg_table) 685 { 686 #ifdef CONFIG_DMABUF_DEBUG 687 int i; 688 struct scatterlist *sg; 689 690 /* To catch abuse of the underlying struct page by importers mix 691 * up the bits, but take care to preserve the low SG_ bits to 692 * not corrupt the sgt. The mixing is undone in __unmap_dma_buf 693 * before passing the sgt back to the exporter. */ 694 for_each_sgtable_sg(sg_table, sg, i) 695 sg->page_link ^= ~0xffUL; 696 #endif 697 698 } 699 static struct sg_table * __map_dma_buf(struct dma_buf_attachment *attach, 700 enum dma_data_direction direction) 701 { 702 struct sg_table *sg_table; 703 704 sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction); 705 706 if (!IS_ERR_OR_NULL(sg_table)) 707 mangle_sg_table(sg_table); 708 709 return sg_table; 710 } 711 712 /** 713 * dma_buf_dynamic_attach - Add the device to dma_buf's attachments list 714 * @dmabuf: [in] buffer to attach device to. 715 * @dev: [in] device to be attached. 716 * @importer_ops: [in] importer operations for the attachment 717 * @importer_priv: [in] importer private pointer for the attachment 718 * 719 * Returns struct dma_buf_attachment pointer for this attachment. Attachments 720 * must be cleaned up by calling dma_buf_detach(). 721 * 722 * Optionally this calls &dma_buf_ops.attach to allow device-specific attach 723 * functionality. 724 * 725 * Returns: 726 * 727 * A pointer to newly created &dma_buf_attachment on success, or a negative 728 * error code wrapped into a pointer on failure. 729 * 730 * Note that this can fail if the backing storage of @dmabuf is in a place not 731 * accessible to @dev, and cannot be moved to a more suitable place. This is 732 * indicated with the error code -EBUSY. 733 */ 734 struct dma_buf_attachment * 735 dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, 736 const struct dma_buf_attach_ops *importer_ops, 737 void *importer_priv) 738 { 739 struct dma_buf_attachment *attach; 740 int ret; 741 unsigned int attach_uid; 742 743 if (WARN_ON(!dmabuf || !dev)) 744 return ERR_PTR(-EINVAL); 745 746 if (WARN_ON(importer_ops && !importer_ops->move_notify)) 747 return ERR_PTR(-EINVAL); 748 749 attach = kzalloc(sizeof(*attach), GFP_KERNEL); 750 if (!attach) 751 return ERR_PTR(-ENOMEM); 752 753 attach->dev = dev; 754 attach->dmabuf = dmabuf; 755 if (importer_ops) 756 attach->peer2peer = importer_ops->allow_peer2peer; 757 attach->importer_ops = importer_ops; 758 attach->importer_priv = importer_priv; 759 760 if (dmabuf->ops->attach) { 761 ret = dmabuf->ops->attach(dmabuf, attach); 762 if (ret) 763 goto err_attach; 764 } 765 dma_resv_lock(dmabuf->resv, NULL); 766 list_add(&attach->node, &dmabuf->attachments); 767 attach_uid = dma_buf_update_attach_uid(dmabuf); 768 dma_resv_unlock(dmabuf->resv); 769 770 ret = dma_buf_attach_stats_setup(attach, attach_uid); 771 if (ret) 772 goto err_sysfs; 773 774 /* When either the importer or the exporter can't handle dynamic 775 * mappings we cache the mapping here to avoid issues with the 776 * reservation object lock. 777 */ 778 if (dma_buf_attachment_is_dynamic(attach) != 779 dma_buf_is_dynamic(dmabuf)) { 780 struct sg_table *sgt; 781 782 if (dma_buf_is_dynamic(attach->dmabuf)) { 783 dma_resv_lock(attach->dmabuf->resv, NULL); 784 ret = dmabuf->ops->pin(attach); 785 if (ret) 786 goto err_unlock; 787 } 788 789 sgt = __map_dma_buf(attach, DMA_BIDIRECTIONAL); 790 if (!sgt) 791 sgt = ERR_PTR(-ENOMEM); 792 if (IS_ERR(sgt)) { 793 ret = PTR_ERR(sgt); 794 goto err_unpin; 795 } 796 if (dma_buf_is_dynamic(attach->dmabuf)) 797 dma_resv_unlock(attach->dmabuf->resv); 798 attach->sgt = sgt; 799 attach->dir = DMA_BIDIRECTIONAL; 800 dma_buf_update_attachment_map_count(attach, 1 /* delta */); 801 } 802 803 return attach; 804 805 err_attach: 806 kfree(attach); 807 return ERR_PTR(ret); 808 809 err_unpin: 810 if (dma_buf_is_dynamic(attach->dmabuf)) 811 dmabuf->ops->unpin(attach); 812 813 err_unlock: 814 if (dma_buf_is_dynamic(attach->dmabuf)) 815 dma_resv_unlock(attach->dmabuf->resv); 816 817 err_sysfs: 818 dma_buf_detach(dmabuf, attach); 819 return ERR_PTR(ret); 820 } 821 EXPORT_SYMBOL_GPL(dma_buf_dynamic_attach); 822 823 /** 824 * dma_buf_attach - Wrapper for dma_buf_dynamic_attach 825 * @dmabuf: [in] buffer to attach device to. 826 * @dev: [in] device to be attached. 827 * 828 * Wrapper to call dma_buf_dynamic_attach() for drivers which still use a static 829 * mapping. 830 */ 831 struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf, 832 struct device *dev) 833 { 834 return dma_buf_dynamic_attach(dmabuf, dev, NULL, NULL); 835 } 836 EXPORT_SYMBOL_GPL(dma_buf_attach); 837 838 static void __unmap_dma_buf(struct dma_buf_attachment *attach, 839 struct sg_table *sg_table, 840 enum dma_data_direction direction) 841 { 842 /* uses XOR, hence this unmangles */ 843 mangle_sg_table(sg_table); 844 845 attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction); 846 } 847 848 /** 849 * dma_buf_detach - Remove the given attachment from dmabuf's attachments list 850 * @dmabuf: [in] buffer to detach from. 851 * @attach: [in] attachment to be detached; is free'd after this call. 852 * 853 * Clean up a device attachment obtained by calling dma_buf_attach(). 854 * 855 * Optionally this calls &dma_buf_ops.detach for device-specific detach. 856 */ 857 void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) 858 { 859 if (WARN_ON(!dmabuf || !attach)) 860 return; 861 862 if (attach->sgt) { 863 if (dma_buf_is_dynamic(attach->dmabuf)) 864 dma_resv_lock(attach->dmabuf->resv, NULL); 865 866 __unmap_dma_buf(attach, attach->sgt, attach->dir); 867 dma_buf_update_attachment_map_count(attach, -1 /* delta */); 868 869 if (dma_buf_is_dynamic(attach->dmabuf)) { 870 dmabuf->ops->unpin(attach); 871 dma_resv_unlock(attach->dmabuf->resv); 872 } 873 } 874 875 dma_resv_lock(dmabuf->resv, NULL); 876 list_del(&attach->node); 877 dma_resv_unlock(dmabuf->resv); 878 if (dmabuf->ops->detach) 879 dmabuf->ops->detach(dmabuf, attach); 880 881 dma_buf_attach_stats_teardown(attach); 882 kfree(attach); 883 } 884 EXPORT_SYMBOL_GPL(dma_buf_detach); 885 886 /** 887 * dma_buf_pin - Lock down the DMA-buf 888 * @attach: [in] attachment which should be pinned 889 * 890 * Only dynamic importers (who set up @attach with dma_buf_dynamic_attach()) may 891 * call this, and only for limited use cases like scanout and not for temporary 892 * pin operations. It is not permitted to allow userspace to pin arbitrary 893 * amounts of buffers through this interface. 894 * 895 * Buffers must be unpinned by calling dma_buf_unpin(). 896 * 897 * Returns: 898 * 0 on success, negative error code on failure. 899 */ 900 int dma_buf_pin(struct dma_buf_attachment *attach) 901 { 902 struct dma_buf *dmabuf = attach->dmabuf; 903 int ret = 0; 904 905 WARN_ON(!dma_buf_attachment_is_dynamic(attach)); 906 907 dma_resv_assert_held(dmabuf->resv); 908 909 if (dmabuf->ops->pin) 910 ret = dmabuf->ops->pin(attach); 911 912 return ret; 913 } 914 EXPORT_SYMBOL_GPL(dma_buf_pin); 915 916 /** 917 * dma_buf_unpin - Unpin a DMA-buf 918 * @attach: [in] attachment which should be unpinned 919 * 920 * This unpins a buffer pinned by dma_buf_pin() and allows the exporter to move 921 * any mapping of @attach again and inform the importer through 922 * &dma_buf_attach_ops.move_notify. 923 */ 924 void dma_buf_unpin(struct dma_buf_attachment *attach) 925 { 926 struct dma_buf *dmabuf = attach->dmabuf; 927 928 WARN_ON(!dma_buf_attachment_is_dynamic(attach)); 929 930 dma_resv_assert_held(dmabuf->resv); 931 932 if (dmabuf->ops->unpin) 933 dmabuf->ops->unpin(attach); 934 } 935 EXPORT_SYMBOL_GPL(dma_buf_unpin); 936 937 /** 938 * dma_buf_map_attachment - Returns the scatterlist table of the attachment; 939 * mapped into _device_ address space. Is a wrapper for map_dma_buf() of the 940 * dma_buf_ops. 941 * @attach: [in] attachment whose scatterlist is to be returned 942 * @direction: [in] direction of DMA transfer 943 * 944 * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR 945 * on error. May return -EINTR if it is interrupted by a signal. 946 * 947 * On success, the DMA addresses and lengths in the returned scatterlist are 948 * PAGE_SIZE aligned. 949 * 950 * A mapping must be unmapped by using dma_buf_unmap_attachment(). Note that 951 * the underlying backing storage is pinned for as long as a mapping exists, 952 * therefore users/importers should not hold onto a mapping for undue amounts of 953 * time. 954 * 955 * Important: Dynamic importers must wait for the exclusive fence of the struct 956 * dma_resv attached to the DMA-BUF first. 957 */ 958 struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach, 959 enum dma_data_direction direction) 960 { 961 struct sg_table *sg_table; 962 int r; 963 964 might_sleep(); 965 966 if (WARN_ON(!attach || !attach->dmabuf)) 967 return ERR_PTR(-EINVAL); 968 969 if (dma_buf_attachment_is_dynamic(attach)) 970 dma_resv_assert_held(attach->dmabuf->resv); 971 972 if (attach->sgt) { 973 /* 974 * Two mappings with different directions for the same 975 * attachment are not allowed. 976 */ 977 if (attach->dir != direction && 978 attach->dir != DMA_BIDIRECTIONAL) 979 return ERR_PTR(-EBUSY); 980 981 return attach->sgt; 982 } 983 984 if (dma_buf_is_dynamic(attach->dmabuf)) { 985 dma_resv_assert_held(attach->dmabuf->resv); 986 if (!IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) { 987 r = attach->dmabuf->ops->pin(attach); 988 if (r) 989 return ERR_PTR(r); 990 } 991 } 992 993 sg_table = __map_dma_buf(attach, direction); 994 if (!sg_table) 995 sg_table = ERR_PTR(-ENOMEM); 996 997 if (IS_ERR(sg_table) && dma_buf_is_dynamic(attach->dmabuf) && 998 !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) 999 attach->dmabuf->ops->unpin(attach); 1000 1001 if (!IS_ERR(sg_table) && attach->dmabuf->ops->cache_sgt_mapping) { 1002 attach->sgt = sg_table; 1003 attach->dir = direction; 1004 } 1005 1006 #ifdef CONFIG_DMA_API_DEBUG 1007 if (!IS_ERR(sg_table)) { 1008 struct scatterlist *sg; 1009 u64 addr; 1010 int len; 1011 int i; 1012 1013 for_each_sgtable_dma_sg(sg_table, sg, i) { 1014 addr = sg_dma_address(sg); 1015 len = sg_dma_len(sg); 1016 if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(len)) { 1017 pr_debug("%s: addr %llx or len %x is not page aligned!\n", 1018 __func__, addr, len); 1019 } 1020 } 1021 } 1022 #endif /* CONFIG_DMA_API_DEBUG */ 1023 1024 if (!IS_ERR(sg_table)) 1025 dma_buf_update_attachment_map_count(attach, 1 /* delta */); 1026 1027 return sg_table; 1028 } 1029 EXPORT_SYMBOL_GPL(dma_buf_map_attachment); 1030 1031 /** 1032 * dma_buf_unmap_attachment - unmaps and decreases usecount of the buffer;might 1033 * deallocate the scatterlist associated. Is a wrapper for unmap_dma_buf() of 1034 * dma_buf_ops. 1035 * @attach: [in] attachment to unmap buffer from 1036 * @sg_table: [in] scatterlist info of the buffer to unmap 1037 * @direction: [in] direction of DMA transfer 1038 * 1039 * This unmaps a DMA mapping for @attached obtained by dma_buf_map_attachment(). 1040 */ 1041 void dma_buf_unmap_attachment(struct dma_buf_attachment *attach, 1042 struct sg_table *sg_table, 1043 enum dma_data_direction direction) 1044 { 1045 might_sleep(); 1046 1047 if (WARN_ON(!attach || !attach->dmabuf || !sg_table)) 1048 return; 1049 1050 if (dma_buf_attachment_is_dynamic(attach)) 1051 dma_resv_assert_held(attach->dmabuf->resv); 1052 1053 if (attach->sgt == sg_table) 1054 return; 1055 1056 if (dma_buf_is_dynamic(attach->dmabuf)) 1057 dma_resv_assert_held(attach->dmabuf->resv); 1058 1059 __unmap_dma_buf(attach, sg_table, direction); 1060 1061 if (dma_buf_is_dynamic(attach->dmabuf) && 1062 !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) 1063 dma_buf_unpin(attach); 1064 1065 dma_buf_update_attachment_map_count(attach, -1 /* delta */); 1066 } 1067 EXPORT_SYMBOL_GPL(dma_buf_unmap_attachment); 1068 1069 /** 1070 * dma_buf_move_notify - notify attachments that DMA-buf is moving 1071 * 1072 * @dmabuf: [in] buffer which is moving 1073 * 1074 * Informs all attachmenst that they need to destroy and recreated all their 1075 * mappings. 1076 */ 1077 void dma_buf_move_notify(struct dma_buf *dmabuf) 1078 { 1079 struct dma_buf_attachment *attach; 1080 1081 dma_resv_assert_held(dmabuf->resv); 1082 1083 list_for_each_entry(attach, &dmabuf->attachments, node) 1084 if (attach->importer_ops) 1085 attach->importer_ops->move_notify(attach); 1086 } 1087 EXPORT_SYMBOL_GPL(dma_buf_move_notify); 1088 1089 /** 1090 * DOC: cpu access 1091 * 1092 * There are mutliple reasons for supporting CPU access to a dma buffer object: 1093 * 1094 * - Fallback operations in the kernel, for example when a device is connected 1095 * over USB and the kernel needs to shuffle the data around first before 1096 * sending it away. Cache coherency is handled by braketing any transactions 1097 * with calls to dma_buf_begin_cpu_access() and dma_buf_end_cpu_access() 1098 * access. 1099 * 1100 * Since for most kernel internal dma-buf accesses need the entire buffer, a 1101 * vmap interface is introduced. Note that on very old 32-bit architectures 1102 * vmalloc space might be limited and result in vmap calls failing. 1103 * 1104 * Interfaces:: 1105 * 1106 * void \*dma_buf_vmap(struct dma_buf \*dmabuf) 1107 * void dma_buf_vunmap(struct dma_buf \*dmabuf, void \*vaddr) 1108 * 1109 * The vmap call can fail if there is no vmap support in the exporter, or if 1110 * it runs out of vmalloc space. Note that the dma-buf layer keeps a reference 1111 * count for all vmap access and calls down into the exporter's vmap function 1112 * only when no vmapping exists, and only unmaps it once. Protection against 1113 * concurrent vmap/vunmap calls is provided by taking the &dma_buf.lock mutex. 1114 * 1115 * - For full compatibility on the importer side with existing userspace 1116 * interfaces, which might already support mmap'ing buffers. This is needed in 1117 * many processing pipelines (e.g. feeding a software rendered image into a 1118 * hardware pipeline, thumbnail creation, snapshots, ...). Also, Android's ION 1119 * framework already supported this and for DMA buffer file descriptors to 1120 * replace ION buffers mmap support was needed. 1121 * 1122 * There is no special interfaces, userspace simply calls mmap on the dma-buf 1123 * fd. But like for CPU access there's a need to braket the actual access, 1124 * which is handled by the ioctl (DMA_BUF_IOCTL_SYNC). Note that 1125 * DMA_BUF_IOCTL_SYNC can fail with -EAGAIN or -EINTR, in which case it must 1126 * be restarted. 1127 * 1128 * Some systems might need some sort of cache coherency management e.g. when 1129 * CPU and GPU domains are being accessed through dma-buf at the same time. 1130 * To circumvent this problem there are begin/end coherency markers, that 1131 * forward directly to existing dma-buf device drivers vfunc hooks. Userspace 1132 * can make use of those markers through the DMA_BUF_IOCTL_SYNC ioctl. The 1133 * sequence would be used like following: 1134 * 1135 * - mmap dma-buf fd 1136 * - for each drawing/upload cycle in CPU 1. SYNC_START ioctl, 2. read/write 1137 * to mmap area 3. SYNC_END ioctl. This can be repeated as often as you 1138 * want (with the new data being consumed by say the GPU or the scanout 1139 * device) 1140 * - munmap once you don't need the buffer any more 1141 * 1142 * For correctness and optimal performance, it is always required to use 1143 * SYNC_START and SYNC_END before and after, respectively, when accessing the 1144 * mapped address. Userspace cannot rely on coherent access, even when there 1145 * are systems where it just works without calling these ioctls. 1146 * 1147 * - And as a CPU fallback in userspace processing pipelines. 1148 * 1149 * Similar to the motivation for kernel cpu access it is again important that 1150 * the userspace code of a given importing subsystem can use the same 1151 * interfaces with a imported dma-buf buffer object as with a native buffer 1152 * object. This is especially important for drm where the userspace part of 1153 * contemporary OpenGL, X, and other drivers is huge, and reworking them to 1154 * use a different way to mmap a buffer rather invasive. 1155 * 1156 * The assumption in the current dma-buf interfaces is that redirecting the 1157 * initial mmap is all that's needed. A survey of some of the existing 1158 * subsystems shows that no driver seems to do any nefarious thing like 1159 * syncing up with outstanding asynchronous processing on the device or 1160 * allocating special resources at fault time. So hopefully this is good 1161 * enough, since adding interfaces to intercept pagefaults and allow pte 1162 * shootdowns would increase the complexity quite a bit. 1163 * 1164 * Interface:: 1165 * 1166 * int dma_buf_mmap(struct dma_buf \*, struct vm_area_struct \*, 1167 * unsigned long); 1168 * 1169 * If the importing subsystem simply provides a special-purpose mmap call to 1170 * set up a mapping in userspace, calling do_mmap with &dma_buf.file will 1171 * equally achieve that for a dma-buf object. 1172 */ 1173 1174 static int __dma_buf_begin_cpu_access(struct dma_buf *dmabuf, 1175 enum dma_data_direction direction) 1176 { 1177 bool write = (direction == DMA_BIDIRECTIONAL || 1178 direction == DMA_TO_DEVICE); 1179 struct dma_resv *resv = dmabuf->resv; 1180 long ret; 1181 1182 /* Wait on any implicit rendering fences */ 1183 ret = dma_resv_wait_timeout(resv, write, true, MAX_SCHEDULE_TIMEOUT); 1184 if (ret < 0) 1185 return ret; 1186 1187 return 0; 1188 } 1189 1190 /** 1191 * dma_buf_begin_cpu_access - Must be called before accessing a dma_buf from the 1192 * cpu in the kernel context. Calls begin_cpu_access to allow exporter-specific 1193 * preparations. Coherency is only guaranteed in the specified range for the 1194 * specified access direction. 1195 * @dmabuf: [in] buffer to prepare cpu access for. 1196 * @direction: [in] length of range for cpu access. 1197 * 1198 * After the cpu access is complete the caller should call 1199 * dma_buf_end_cpu_access(). Only when cpu access is braketed by both calls is 1200 * it guaranteed to be coherent with other DMA access. 1201 * 1202 * This function will also wait for any DMA transactions tracked through 1203 * implicit synchronization in &dma_buf.resv. For DMA transactions with explicit 1204 * synchronization this function will only ensure cache coherency, callers must 1205 * ensure synchronization with such DMA transactions on their own. 1206 * 1207 * Can return negative error values, returns 0 on success. 1208 */ 1209 int dma_buf_begin_cpu_access(struct dma_buf *dmabuf, 1210 enum dma_data_direction direction) 1211 { 1212 int ret = 0; 1213 1214 if (WARN_ON(!dmabuf)) 1215 return -EINVAL; 1216 1217 might_lock(&dmabuf->resv->lock.base); 1218 1219 if (dmabuf->ops->begin_cpu_access) 1220 ret = dmabuf->ops->begin_cpu_access(dmabuf, direction); 1221 1222 /* Ensure that all fences are waited upon - but we first allow 1223 * the native handler the chance to do so more efficiently if it 1224 * chooses. A double invocation here will be reasonably cheap no-op. 1225 */ 1226 if (ret == 0) 1227 ret = __dma_buf_begin_cpu_access(dmabuf, direction); 1228 1229 return ret; 1230 } 1231 EXPORT_SYMBOL_GPL(dma_buf_begin_cpu_access); 1232 1233 /** 1234 * dma_buf_end_cpu_access - Must be called after accessing a dma_buf from the 1235 * cpu in the kernel context. Calls end_cpu_access to allow exporter-specific 1236 * actions. Coherency is only guaranteed in the specified range for the 1237 * specified access direction. 1238 * @dmabuf: [in] buffer to complete cpu access for. 1239 * @direction: [in] length of range for cpu access. 1240 * 1241 * This terminates CPU access started with dma_buf_begin_cpu_access(). 1242 * 1243 * Can return negative error values, returns 0 on success. 1244 */ 1245 int dma_buf_end_cpu_access(struct dma_buf *dmabuf, 1246 enum dma_data_direction direction) 1247 { 1248 int ret = 0; 1249 1250 WARN_ON(!dmabuf); 1251 1252 might_lock(&dmabuf->resv->lock.base); 1253 1254 if (dmabuf->ops->end_cpu_access) 1255 ret = dmabuf->ops->end_cpu_access(dmabuf, direction); 1256 1257 return ret; 1258 } 1259 EXPORT_SYMBOL_GPL(dma_buf_end_cpu_access); 1260 1261 1262 /** 1263 * dma_buf_mmap - Setup up a userspace mmap with the given vma 1264 * @dmabuf: [in] buffer that should back the vma 1265 * @vma: [in] vma for the mmap 1266 * @pgoff: [in] offset in pages where this mmap should start within the 1267 * dma-buf buffer. 1268 * 1269 * This function adjusts the passed in vma so that it points at the file of the 1270 * dma_buf operation. It also adjusts the starting pgoff and does bounds 1271 * checking on the size of the vma. Then it calls the exporters mmap function to 1272 * set up the mapping. 1273 * 1274 * Can return negative error values, returns 0 on success. 1275 */ 1276 int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, 1277 unsigned long pgoff) 1278 { 1279 if (WARN_ON(!dmabuf || !vma)) 1280 return -EINVAL; 1281 1282 /* check if buffer supports mmap */ 1283 if (!dmabuf->ops->mmap) 1284 return -EINVAL; 1285 1286 /* check for offset overflow */ 1287 if (pgoff + vma_pages(vma) < pgoff) 1288 return -EOVERFLOW; 1289 1290 /* check for overflowing the buffer's size */ 1291 if (pgoff + vma_pages(vma) > 1292 dmabuf->size >> PAGE_SHIFT) 1293 return -EINVAL; 1294 1295 /* readjust the vma */ 1296 vma_set_file(vma, dmabuf->file); 1297 vma->vm_pgoff = pgoff; 1298 1299 return dmabuf->ops->mmap(dmabuf, vma); 1300 } 1301 EXPORT_SYMBOL_GPL(dma_buf_mmap); 1302 1303 /** 1304 * dma_buf_vmap - Create virtual mapping for the buffer object into kernel 1305 * address space. Same restrictions as for vmap and friends apply. 1306 * @dmabuf: [in] buffer to vmap 1307 * @map: [out] returns the vmap pointer 1308 * 1309 * This call may fail due to lack of virtual mapping address space. 1310 * These calls are optional in drivers. The intended use for them 1311 * is for mapping objects linear in kernel space for high use objects. 1312 * 1313 * To ensure coherency users must call dma_buf_begin_cpu_access() and 1314 * dma_buf_end_cpu_access() around any cpu access performed through this 1315 * mapping. 1316 * 1317 * Returns 0 on success, or a negative errno code otherwise. 1318 */ 1319 int dma_buf_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map) 1320 { 1321 struct dma_buf_map ptr; 1322 int ret = 0; 1323 1324 dma_buf_map_clear(map); 1325 1326 if (WARN_ON(!dmabuf)) 1327 return -EINVAL; 1328 1329 if (!dmabuf->ops->vmap) 1330 return -EINVAL; 1331 1332 mutex_lock(&dmabuf->lock); 1333 if (dmabuf->vmapping_counter) { 1334 dmabuf->vmapping_counter++; 1335 BUG_ON(dma_buf_map_is_null(&dmabuf->vmap_ptr)); 1336 *map = dmabuf->vmap_ptr; 1337 goto out_unlock; 1338 } 1339 1340 BUG_ON(dma_buf_map_is_set(&dmabuf->vmap_ptr)); 1341 1342 ret = dmabuf->ops->vmap(dmabuf, &ptr); 1343 if (WARN_ON_ONCE(ret)) 1344 goto out_unlock; 1345 1346 dmabuf->vmap_ptr = ptr; 1347 dmabuf->vmapping_counter = 1; 1348 1349 *map = dmabuf->vmap_ptr; 1350 1351 out_unlock: 1352 mutex_unlock(&dmabuf->lock); 1353 return ret; 1354 } 1355 EXPORT_SYMBOL_GPL(dma_buf_vmap); 1356 1357 /** 1358 * dma_buf_vunmap - Unmap a vmap obtained by dma_buf_vmap. 1359 * @dmabuf: [in] buffer to vunmap 1360 * @map: [in] vmap pointer to vunmap 1361 */ 1362 void dma_buf_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map) 1363 { 1364 if (WARN_ON(!dmabuf)) 1365 return; 1366 1367 BUG_ON(dma_buf_map_is_null(&dmabuf->vmap_ptr)); 1368 BUG_ON(dmabuf->vmapping_counter == 0); 1369 BUG_ON(!dma_buf_map_is_equal(&dmabuf->vmap_ptr, map)); 1370 1371 mutex_lock(&dmabuf->lock); 1372 if (--dmabuf->vmapping_counter == 0) { 1373 if (dmabuf->ops->vunmap) 1374 dmabuf->ops->vunmap(dmabuf, map); 1375 dma_buf_map_clear(&dmabuf->vmap_ptr); 1376 } 1377 mutex_unlock(&dmabuf->lock); 1378 } 1379 EXPORT_SYMBOL_GPL(dma_buf_vunmap); 1380 1381 #ifdef CONFIG_DEBUG_FS 1382 static int dma_buf_debug_show(struct seq_file *s, void *unused) 1383 { 1384 struct dma_buf *buf_obj; 1385 struct dma_buf_attachment *attach_obj; 1386 struct dma_resv *robj; 1387 struct dma_resv_list *fobj; 1388 struct dma_fence *fence; 1389 int count = 0, attach_count, shared_count, i; 1390 size_t size = 0; 1391 int ret; 1392 1393 ret = mutex_lock_interruptible(&db_list.lock); 1394 1395 if (ret) 1396 return ret; 1397 1398 seq_puts(s, "\nDma-buf Objects:\n"); 1399 seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\t%-8s\n", 1400 "size", "flags", "mode", "count", "ino"); 1401 1402 list_for_each_entry(buf_obj, &db_list.head, list_node) { 1403 1404 ret = dma_resv_lock_interruptible(buf_obj->resv, NULL); 1405 if (ret) 1406 goto error_unlock; 1407 1408 seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\t%08lu\t%s\n", 1409 buf_obj->size, 1410 buf_obj->file->f_flags, buf_obj->file->f_mode, 1411 file_count(buf_obj->file), 1412 buf_obj->exp_name, 1413 file_inode(buf_obj->file)->i_ino, 1414 buf_obj->name ?: ""); 1415 1416 robj = buf_obj->resv; 1417 fence = dma_resv_excl_fence(robj); 1418 if (fence) 1419 seq_printf(s, "\tExclusive fence: %s %s %ssignalled\n", 1420 fence->ops->get_driver_name(fence), 1421 fence->ops->get_timeline_name(fence), 1422 dma_fence_is_signaled(fence) ? "" : "un"); 1423 1424 fobj = rcu_dereference_protected(robj->fence, 1425 dma_resv_held(robj)); 1426 shared_count = fobj ? fobj->shared_count : 0; 1427 for (i = 0; i < shared_count; i++) { 1428 fence = rcu_dereference_protected(fobj->shared[i], 1429 dma_resv_held(robj)); 1430 seq_printf(s, "\tShared fence: %s %s %ssignalled\n", 1431 fence->ops->get_driver_name(fence), 1432 fence->ops->get_timeline_name(fence), 1433 dma_fence_is_signaled(fence) ? "" : "un"); 1434 } 1435 1436 seq_puts(s, "\tAttached Devices:\n"); 1437 attach_count = 0; 1438 1439 list_for_each_entry(attach_obj, &buf_obj->attachments, node) { 1440 seq_printf(s, "\t%s\n", dev_name(attach_obj->dev)); 1441 attach_count++; 1442 } 1443 dma_resv_unlock(buf_obj->resv); 1444 1445 seq_printf(s, "Total %d devices attached\n\n", 1446 attach_count); 1447 1448 count++; 1449 size += buf_obj->size; 1450 } 1451 1452 seq_printf(s, "\nTotal %d objects, %zu bytes\n", count, size); 1453 1454 mutex_unlock(&db_list.lock); 1455 return 0; 1456 1457 error_unlock: 1458 mutex_unlock(&db_list.lock); 1459 return ret; 1460 } 1461 1462 DEFINE_SHOW_ATTRIBUTE(dma_buf_debug); 1463 1464 static struct dentry *dma_buf_debugfs_dir; 1465 1466 static int dma_buf_init_debugfs(void) 1467 { 1468 struct dentry *d; 1469 int err = 0; 1470 1471 d = debugfs_create_dir("dma_buf", NULL); 1472 if (IS_ERR(d)) 1473 return PTR_ERR(d); 1474 1475 dma_buf_debugfs_dir = d; 1476 1477 d = debugfs_create_file("bufinfo", S_IRUGO, dma_buf_debugfs_dir, 1478 NULL, &dma_buf_debug_fops); 1479 if (IS_ERR(d)) { 1480 pr_debug("dma_buf: debugfs: failed to create node bufinfo\n"); 1481 debugfs_remove_recursive(dma_buf_debugfs_dir); 1482 dma_buf_debugfs_dir = NULL; 1483 err = PTR_ERR(d); 1484 } 1485 1486 return err; 1487 } 1488 1489 static void dma_buf_uninit_debugfs(void) 1490 { 1491 debugfs_remove_recursive(dma_buf_debugfs_dir); 1492 } 1493 #else 1494 static inline int dma_buf_init_debugfs(void) 1495 { 1496 return 0; 1497 } 1498 static inline void dma_buf_uninit_debugfs(void) 1499 { 1500 } 1501 #endif 1502 1503 static int __init dma_buf_init(void) 1504 { 1505 int ret; 1506 1507 ret = dma_buf_init_sysfs_statistics(); 1508 if (ret) 1509 return ret; 1510 1511 dma_buf_mnt = kern_mount(&dma_buf_fs_type); 1512 if (IS_ERR(dma_buf_mnt)) 1513 return PTR_ERR(dma_buf_mnt); 1514 1515 mutex_init(&db_list.lock); 1516 INIT_LIST_HEAD(&db_list.head); 1517 dma_buf_init_debugfs(); 1518 return 0; 1519 } 1520 subsys_initcall(dma_buf_init); 1521 1522 static void __exit dma_buf_deinit(void) 1523 { 1524 dma_buf_uninit_debugfs(); 1525 kern_unmount(dma_buf_mnt); 1526 dma_buf_uninit_sysfs_statistics(); 1527 } 1528 __exitcall(dma_buf_deinit); 1529