1 /* 2 * Copyright © 2012 Red Hat 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Dave Airlie <airlied@redhat.com> 25 * Rob Clark <rob.clark@linaro.org> 26 * 27 */ 28 29 #include <linux/export.h> 30 #include <linux/dma-buf.h> 31 #include <linux/rbtree.h> 32 33 #include <drm/drm.h> 34 #include <drm/drm_drv.h> 35 #include <drm/drm_file.h> 36 #include <drm/drm_framebuffer.h> 37 #include <drm/drm_gem.h> 38 #include <drm/drm_prime.h> 39 40 #include "drm_internal.h" 41 42 /** 43 * DOC: overview and lifetime rules 44 * 45 * Similar to GEM global names, PRIME file descriptors are also used to share 46 * buffer objects across processes. They offer additional security: as file 47 * descriptors must be explicitly sent over UNIX domain sockets to be shared 48 * between applications, they can't be guessed like the globally unique GEM 49 * names. 50 * 51 * Drivers that support the PRIME API implement the 52 * &drm_driver.prime_handle_to_fd and &drm_driver.prime_fd_to_handle operations. 53 * GEM based drivers must use drm_gem_prime_handle_to_fd() and 54 * drm_gem_prime_fd_to_handle() to implement these. For GEM based drivers the 55 * actual driver interfaces is provided through the &drm_gem_object_funcs.export 56 * and &drm_driver.gem_prime_import hooks. 57 * 58 * &dma_buf_ops implementations for GEM drivers are all individually exported 59 * for drivers which need to overwrite or reimplement some of them. 60 * 61 * Reference Counting for GEM Drivers 62 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 63 * 64 * On the export the &dma_buf holds a reference to the exported buffer object, 65 * usually a &drm_gem_object. It takes this reference in the PRIME_HANDLE_TO_FD 66 * IOCTL, when it first calls &drm_gem_object_funcs.export 67 * and stores the exporting GEM object in the &dma_buf.priv field. This 68 * reference needs to be released when the final reference to the &dma_buf 69 * itself is dropped and its &dma_buf_ops.release function is called. For 70 * GEM-based drivers, the &dma_buf should be exported using 71 * drm_gem_dmabuf_export() and then released by drm_gem_dmabuf_release(). 72 * 73 * Thus the chain of references always flows in one direction, avoiding loops: 74 * importing GEM object -> dma-buf -> exported GEM bo. A further complication 75 * are the lookup caches for import and export. These are required to guarantee 76 * that any given object will always have only one uniqe userspace handle. This 77 * is required to allow userspace to detect duplicated imports, since some GEM 78 * drivers do fail command submissions if a given buffer object is listed more 79 * than once. These import and export caches in &drm_prime_file_private only 80 * retain a weak reference, which is cleaned up when the corresponding object is 81 * released. 82 * 83 * Self-importing: If userspace is using PRIME as a replacement for flink then 84 * it will get a fd->handle request for a GEM object that it created. Drivers 85 * should detect this situation and return back the underlying object from the 86 * dma-buf private. For GEM based drivers this is handled in 87 * drm_gem_prime_import() already. 88 */ 89 90 struct drm_prime_member { 91 struct dma_buf *dma_buf; 92 uint32_t handle; 93 94 struct rb_node dmabuf_rb; 95 struct rb_node handle_rb; 96 }; 97 98 static int drm_prime_add_buf_handle(struct drm_prime_file_private *prime_fpriv, 99 struct dma_buf *dma_buf, uint32_t handle) 100 { 101 struct drm_prime_member *member; 102 struct rb_node **p, *rb; 103 104 member = kmalloc(sizeof(*member), GFP_KERNEL); 105 if (!member) 106 return -ENOMEM; 107 108 get_dma_buf(dma_buf); 109 member->dma_buf = dma_buf; 110 member->handle = handle; 111 112 rb = NULL; 113 p = &prime_fpriv->dmabufs.rb_node; 114 while (*p) { 115 struct drm_prime_member *pos; 116 117 rb = *p; 118 pos = rb_entry(rb, struct drm_prime_member, dmabuf_rb); 119 if (dma_buf > pos->dma_buf) 120 p = &rb->rb_right; 121 else 122 p = &rb->rb_left; 123 } 124 rb_link_node(&member->dmabuf_rb, rb, p); 125 rb_insert_color(&member->dmabuf_rb, &prime_fpriv->dmabufs); 126 127 rb = NULL; 128 p = &prime_fpriv->handles.rb_node; 129 while (*p) { 130 struct drm_prime_member *pos; 131 132 rb = *p; 133 pos = rb_entry(rb, struct drm_prime_member, handle_rb); 134 if (handle > pos->handle) 135 p = &rb->rb_right; 136 else 137 p = &rb->rb_left; 138 } 139 rb_link_node(&member->handle_rb, rb, p); 140 rb_insert_color(&member->handle_rb, &prime_fpriv->handles); 141 142 return 0; 143 } 144 145 static struct dma_buf *drm_prime_lookup_buf_by_handle(struct drm_prime_file_private *prime_fpriv, 146 uint32_t handle) 147 { 148 struct rb_node *rb; 149 150 rb = prime_fpriv->handles.rb_node; 151 while (rb) { 152 struct drm_prime_member *member; 153 154 member = rb_entry(rb, struct drm_prime_member, handle_rb); 155 if (member->handle == handle) 156 return member->dma_buf; 157 else if (member->handle < handle) 158 rb = rb->rb_right; 159 else 160 rb = rb->rb_left; 161 } 162 163 return NULL; 164 } 165 166 static int drm_prime_lookup_buf_handle(struct drm_prime_file_private *prime_fpriv, 167 struct dma_buf *dma_buf, 168 uint32_t *handle) 169 { 170 struct rb_node *rb; 171 172 rb = prime_fpriv->dmabufs.rb_node; 173 while (rb) { 174 struct drm_prime_member *member; 175 176 member = rb_entry(rb, struct drm_prime_member, dmabuf_rb); 177 if (member->dma_buf == dma_buf) { 178 *handle = member->handle; 179 return 0; 180 } else if (member->dma_buf < dma_buf) { 181 rb = rb->rb_right; 182 } else { 183 rb = rb->rb_left; 184 } 185 } 186 187 return -ENOENT; 188 } 189 190 void drm_prime_remove_buf_handle_locked(struct drm_prime_file_private *prime_fpriv, 191 struct dma_buf *dma_buf) 192 { 193 struct rb_node *rb; 194 195 rb = prime_fpriv->dmabufs.rb_node; 196 while (rb) { 197 struct drm_prime_member *member; 198 199 member = rb_entry(rb, struct drm_prime_member, dmabuf_rb); 200 if (member->dma_buf == dma_buf) { 201 rb_erase(&member->handle_rb, &prime_fpriv->handles); 202 rb_erase(&member->dmabuf_rb, &prime_fpriv->dmabufs); 203 204 dma_buf_put(dma_buf); 205 kfree(member); 206 return; 207 } else if (member->dma_buf < dma_buf) { 208 rb = rb->rb_right; 209 } else { 210 rb = rb->rb_left; 211 } 212 } 213 } 214 215 void drm_prime_init_file_private(struct drm_prime_file_private *prime_fpriv) 216 { 217 mutex_init(&prime_fpriv->lock); 218 prime_fpriv->dmabufs = RB_ROOT; 219 prime_fpriv->handles = RB_ROOT; 220 } 221 222 void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv) 223 { 224 /* by now drm_gem_release should've made sure the list is empty */ 225 WARN_ON(!RB_EMPTY_ROOT(&prime_fpriv->dmabufs)); 226 } 227 228 /** 229 * drm_gem_dmabuf_export - &dma_buf export implementation for GEM 230 * @dev: parent device for the exported dmabuf 231 * @exp_info: the export information used by dma_buf_export() 232 * 233 * This wraps dma_buf_export() for use by generic GEM drivers that are using 234 * drm_gem_dmabuf_release(). In addition to calling dma_buf_export(), we take 235 * a reference to the &drm_device and the exported &drm_gem_object (stored in 236 * &dma_buf_export_info.priv) which is released by drm_gem_dmabuf_release(). 237 * 238 * Returns the new dmabuf. 239 */ 240 struct dma_buf *drm_gem_dmabuf_export(struct drm_device *dev, 241 struct dma_buf_export_info *exp_info) 242 { 243 struct dma_buf *dma_buf; 244 245 dma_buf = dma_buf_export(exp_info); 246 if (IS_ERR(dma_buf)) 247 return dma_buf; 248 249 drm_dev_get(dev); 250 drm_gem_object_get(exp_info->priv); 251 252 return dma_buf; 253 } 254 EXPORT_SYMBOL(drm_gem_dmabuf_export); 255 256 /** 257 * drm_gem_dmabuf_release - &dma_buf release implementation for GEM 258 * @dma_buf: buffer to be released 259 * 260 * Generic release function for dma_bufs exported as PRIME buffers. GEM drivers 261 * must use this in their &dma_buf_ops structure as the release callback. 262 * drm_gem_dmabuf_release() should be used in conjunction with 263 * drm_gem_dmabuf_export(). 264 */ 265 void drm_gem_dmabuf_release(struct dma_buf *dma_buf) 266 { 267 struct drm_gem_object *obj = dma_buf->priv; 268 struct drm_device *dev = obj->dev; 269 270 /* drop the reference on the export fd holds */ 271 drm_gem_object_put_unlocked(obj); 272 273 drm_dev_put(dev); 274 } 275 EXPORT_SYMBOL(drm_gem_dmabuf_release); 276 277 /** 278 * drm_gem_prime_fd_to_handle - PRIME import function for GEM drivers 279 * @dev: dev to export the buffer from 280 * @file_priv: drm file-private structure 281 * @prime_fd: fd id of the dma-buf which should be imported 282 * @handle: pointer to storage for the handle of the imported buffer object 283 * 284 * This is the PRIME import function which must be used mandatorily by GEM 285 * drivers to ensure correct lifetime management of the underlying GEM object. 286 * The actual importing of GEM object from the dma-buf is done through the 287 * &drm_driver.gem_prime_import driver callback. 288 * 289 * Returns 0 on success or a negative error code on failure. 290 */ 291 int drm_gem_prime_fd_to_handle(struct drm_device *dev, 292 struct drm_file *file_priv, int prime_fd, 293 uint32_t *handle) 294 { 295 struct dma_buf *dma_buf; 296 struct drm_gem_object *obj; 297 int ret; 298 299 dma_buf = dma_buf_get(prime_fd); 300 if (IS_ERR(dma_buf)) 301 return PTR_ERR(dma_buf); 302 303 mutex_lock(&file_priv->prime.lock); 304 305 ret = drm_prime_lookup_buf_handle(&file_priv->prime, 306 dma_buf, handle); 307 if (ret == 0) 308 goto out_put; 309 310 /* never seen this one, need to import */ 311 mutex_lock(&dev->object_name_lock); 312 if (dev->driver->gem_prime_import) 313 obj = dev->driver->gem_prime_import(dev, dma_buf); 314 else 315 obj = drm_gem_prime_import(dev, dma_buf); 316 if (IS_ERR(obj)) { 317 ret = PTR_ERR(obj); 318 goto out_unlock; 319 } 320 321 if (obj->dma_buf) { 322 WARN_ON(obj->dma_buf != dma_buf); 323 } else { 324 obj->dma_buf = dma_buf; 325 get_dma_buf(dma_buf); 326 } 327 328 /* _handle_create_tail unconditionally unlocks dev->object_name_lock. */ 329 ret = drm_gem_handle_create_tail(file_priv, obj, handle); 330 drm_gem_object_put_unlocked(obj); 331 if (ret) 332 goto out_put; 333 334 ret = drm_prime_add_buf_handle(&file_priv->prime, 335 dma_buf, *handle); 336 mutex_unlock(&file_priv->prime.lock); 337 if (ret) 338 goto fail; 339 340 dma_buf_put(dma_buf); 341 342 return 0; 343 344 fail: 345 /* hmm, if driver attached, we are relying on the free-object path 346 * to detach.. which seems ok.. 347 */ 348 drm_gem_handle_delete(file_priv, *handle); 349 dma_buf_put(dma_buf); 350 return ret; 351 352 out_unlock: 353 mutex_unlock(&dev->object_name_lock); 354 out_put: 355 mutex_unlock(&file_priv->prime.lock); 356 dma_buf_put(dma_buf); 357 return ret; 358 } 359 EXPORT_SYMBOL(drm_gem_prime_fd_to_handle); 360 361 int drm_prime_fd_to_handle_ioctl(struct drm_device *dev, void *data, 362 struct drm_file *file_priv) 363 { 364 struct drm_prime_handle *args = data; 365 366 if (!dev->driver->prime_fd_to_handle) 367 return -ENOSYS; 368 369 return dev->driver->prime_fd_to_handle(dev, file_priv, 370 args->fd, &args->handle); 371 } 372 373 static struct dma_buf *export_and_register_object(struct drm_device *dev, 374 struct drm_gem_object *obj, 375 uint32_t flags) 376 { 377 struct dma_buf *dmabuf; 378 379 /* prevent races with concurrent gem_close. */ 380 if (obj->handle_count == 0) { 381 dmabuf = ERR_PTR(-ENOENT); 382 return dmabuf; 383 } 384 385 if (obj->funcs && obj->funcs->export) 386 dmabuf = obj->funcs->export(obj, flags); 387 else if (dev->driver->gem_prime_export) 388 dmabuf = dev->driver->gem_prime_export(obj, flags); 389 else 390 dmabuf = drm_gem_prime_export(obj, flags); 391 if (IS_ERR(dmabuf)) { 392 /* normally the created dma-buf takes ownership of the ref, 393 * but if that fails then drop the ref 394 */ 395 return dmabuf; 396 } 397 398 /* 399 * Note that callers do not need to clean up the export cache 400 * since the check for obj->handle_count guarantees that someone 401 * will clean it up. 402 */ 403 obj->dma_buf = dmabuf; 404 get_dma_buf(obj->dma_buf); 405 406 return dmabuf; 407 } 408 409 /** 410 * drm_gem_prime_handle_to_fd - PRIME export function for GEM drivers 411 * @dev: dev to export the buffer from 412 * @file_priv: drm file-private structure 413 * @handle: buffer handle to export 414 * @flags: flags like DRM_CLOEXEC 415 * @prime_fd: pointer to storage for the fd id of the create dma-buf 416 * 417 * This is the PRIME export function which must be used mandatorily by GEM 418 * drivers to ensure correct lifetime management of the underlying GEM object. 419 * The actual exporting from GEM object to a dma-buf is done through the 420 * &drm_driver.gem_prime_export driver callback. 421 */ 422 int drm_gem_prime_handle_to_fd(struct drm_device *dev, 423 struct drm_file *file_priv, uint32_t handle, 424 uint32_t flags, 425 int *prime_fd) 426 { 427 struct drm_gem_object *obj; 428 int ret = 0; 429 struct dma_buf *dmabuf; 430 431 mutex_lock(&file_priv->prime.lock); 432 obj = drm_gem_object_lookup(file_priv, handle); 433 if (!obj) { 434 ret = -ENOENT; 435 goto out_unlock; 436 } 437 438 dmabuf = drm_prime_lookup_buf_by_handle(&file_priv->prime, handle); 439 if (dmabuf) { 440 get_dma_buf(dmabuf); 441 goto out_have_handle; 442 } 443 444 mutex_lock(&dev->object_name_lock); 445 /* re-export the original imported object */ 446 if (obj->import_attach) { 447 dmabuf = obj->import_attach->dmabuf; 448 get_dma_buf(dmabuf); 449 goto out_have_obj; 450 } 451 452 if (obj->dma_buf) { 453 get_dma_buf(obj->dma_buf); 454 dmabuf = obj->dma_buf; 455 goto out_have_obj; 456 } 457 458 dmabuf = export_and_register_object(dev, obj, flags); 459 if (IS_ERR(dmabuf)) { 460 /* normally the created dma-buf takes ownership of the ref, 461 * but if that fails then drop the ref 462 */ 463 ret = PTR_ERR(dmabuf); 464 mutex_unlock(&dev->object_name_lock); 465 goto out; 466 } 467 468 out_have_obj: 469 /* 470 * If we've exported this buffer then cheat and add it to the import list 471 * so we get the correct handle back. We must do this under the 472 * protection of dev->object_name_lock to ensure that a racing gem close 473 * ioctl doesn't miss to remove this buffer handle from the cache. 474 */ 475 ret = drm_prime_add_buf_handle(&file_priv->prime, 476 dmabuf, handle); 477 mutex_unlock(&dev->object_name_lock); 478 if (ret) 479 goto fail_put_dmabuf; 480 481 out_have_handle: 482 ret = dma_buf_fd(dmabuf, flags); 483 /* 484 * We must _not_ remove the buffer from the handle cache since the newly 485 * created dma buf is already linked in the global obj->dma_buf pointer, 486 * and that is invariant as long as a userspace gem handle exists. 487 * Closing the handle will clean out the cache anyway, so we don't leak. 488 */ 489 if (ret < 0) { 490 goto fail_put_dmabuf; 491 } else { 492 *prime_fd = ret; 493 ret = 0; 494 } 495 496 goto out; 497 498 fail_put_dmabuf: 499 dma_buf_put(dmabuf); 500 out: 501 drm_gem_object_put_unlocked(obj); 502 out_unlock: 503 mutex_unlock(&file_priv->prime.lock); 504 505 return ret; 506 } 507 EXPORT_SYMBOL(drm_gem_prime_handle_to_fd); 508 509 int drm_prime_handle_to_fd_ioctl(struct drm_device *dev, void *data, 510 struct drm_file *file_priv) 511 { 512 struct drm_prime_handle *args = data; 513 514 if (!dev->driver->prime_handle_to_fd) 515 return -ENOSYS; 516 517 /* check flags are valid */ 518 if (args->flags & ~(DRM_CLOEXEC | DRM_RDWR)) 519 return -EINVAL; 520 521 return dev->driver->prime_handle_to_fd(dev, file_priv, 522 args->handle, args->flags, &args->fd); 523 } 524 525 /** 526 * DOC: PRIME Helpers 527 * 528 * Drivers can implement &drm_gem_object_funcs.export and 529 * &drm_driver.gem_prime_import in terms of simpler APIs by using the helper 530 * functions drm_gem_prime_export() and drm_gem_prime_import(). These functions 531 * implement dma-buf support in terms of some lower-level helpers, which are 532 * again exported for drivers to use individually: 533 * 534 * Exporting buffers 535 * ~~~~~~~~~~~~~~~~~ 536 * 537 * Optional pinning of buffers is handled at dma-buf attach and detach time in 538 * drm_gem_map_attach() and drm_gem_map_detach(). Backing storage itself is 539 * handled by drm_gem_map_dma_buf() and drm_gem_unmap_dma_buf(), which relies on 540 * &drm_gem_object_funcs.get_sg_table. 541 * 542 * For kernel-internal access there's drm_gem_dmabuf_vmap() and 543 * drm_gem_dmabuf_vunmap(). Userspace mmap support is provided by 544 * drm_gem_dmabuf_mmap(). 545 * 546 * Note that these export helpers can only be used if the underlying backing 547 * storage is fully coherent and either permanently pinned, or it is safe to pin 548 * it indefinitely. 549 * 550 * FIXME: The underlying helper functions are named rather inconsistently. 551 * 552 * Exporting buffers 553 * ~~~~~~~~~~~~~~~~~ 554 * 555 * Importing dma-bufs using drm_gem_prime_import() relies on 556 * &drm_driver.gem_prime_import_sg_table. 557 * 558 * Note that similarly to the export helpers this permanently pins the 559 * underlying backing storage. Which is ok for scanout, but is not the best 560 * option for sharing lots of buffers for rendering. 561 */ 562 563 /** 564 * drm_gem_map_attach - dma_buf attach implementation for GEM 565 * @dma_buf: buffer to attach device to 566 * @attach: buffer attachment data 567 * 568 * Calls &drm_gem_object_funcs.pin for device specific handling. This can be 569 * used as the &dma_buf_ops.attach callback. Must be used together with 570 * drm_gem_map_detach(). 571 * 572 * Returns 0 on success, negative error code on failure. 573 */ 574 int drm_gem_map_attach(struct dma_buf *dma_buf, 575 struct dma_buf_attachment *attach) 576 { 577 struct drm_gem_object *obj = dma_buf->priv; 578 579 return drm_gem_pin(obj); 580 } 581 EXPORT_SYMBOL(drm_gem_map_attach); 582 583 /** 584 * drm_gem_map_detach - dma_buf detach implementation for GEM 585 * @dma_buf: buffer to detach from 586 * @attach: attachment to be detached 587 * 588 * Calls &drm_gem_object_funcs.pin for device specific handling. Cleans up 589 * &dma_buf_attachment from drm_gem_map_attach(). This can be used as the 590 * &dma_buf_ops.detach callback. 591 */ 592 void drm_gem_map_detach(struct dma_buf *dma_buf, 593 struct dma_buf_attachment *attach) 594 { 595 struct drm_gem_object *obj = dma_buf->priv; 596 597 drm_gem_unpin(obj); 598 } 599 EXPORT_SYMBOL(drm_gem_map_detach); 600 601 /** 602 * drm_gem_map_dma_buf - map_dma_buf implementation for GEM 603 * @attach: attachment whose scatterlist is to be returned 604 * @dir: direction of DMA transfer 605 * 606 * Calls &drm_gem_object_funcs.get_sg_table and then maps the scatterlist. This 607 * can be used as the &dma_buf_ops.map_dma_buf callback. Should be used together 608 * with drm_gem_unmap_dma_buf(). 609 * 610 * Returns:sg_table containing the scatterlist to be returned; returns ERR_PTR 611 * on error. May return -EINTR if it is interrupted by a signal. 612 */ 613 struct sg_table *drm_gem_map_dma_buf(struct dma_buf_attachment *attach, 614 enum dma_data_direction dir) 615 { 616 struct drm_gem_object *obj = attach->dmabuf->priv; 617 struct sg_table *sgt; 618 619 if (WARN_ON(dir == DMA_NONE)) 620 return ERR_PTR(-EINVAL); 621 622 if (obj->funcs) 623 sgt = obj->funcs->get_sg_table(obj); 624 else 625 sgt = obj->dev->driver->gem_prime_get_sg_table(obj); 626 627 if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir, 628 DMA_ATTR_SKIP_CPU_SYNC)) { 629 sg_free_table(sgt); 630 kfree(sgt); 631 sgt = ERR_PTR(-ENOMEM); 632 } 633 634 return sgt; 635 } 636 EXPORT_SYMBOL(drm_gem_map_dma_buf); 637 638 /** 639 * drm_gem_unmap_dma_buf - unmap_dma_buf implementation for GEM 640 * @attach: attachment to unmap buffer from 641 * @sgt: scatterlist info of the buffer to unmap 642 * @dir: direction of DMA transfer 643 * 644 * This can be used as the &dma_buf_ops.unmap_dma_buf callback. 645 */ 646 void drm_gem_unmap_dma_buf(struct dma_buf_attachment *attach, 647 struct sg_table *sgt, 648 enum dma_data_direction dir) 649 { 650 if (!sgt) 651 return; 652 653 dma_unmap_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir, 654 DMA_ATTR_SKIP_CPU_SYNC); 655 sg_free_table(sgt); 656 kfree(sgt); 657 } 658 EXPORT_SYMBOL(drm_gem_unmap_dma_buf); 659 660 /** 661 * drm_gem_dmabuf_vmap - dma_buf vmap implementation for GEM 662 * @dma_buf: buffer to be mapped 663 * 664 * Sets up a kernel virtual mapping. This can be used as the &dma_buf_ops.vmap 665 * callback. Calls into &drm_gem_object_funcs.vmap for device specific handling. 666 * 667 * Returns the kernel virtual address or NULL on failure. 668 */ 669 void *drm_gem_dmabuf_vmap(struct dma_buf *dma_buf) 670 { 671 struct drm_gem_object *obj = dma_buf->priv; 672 void *vaddr; 673 674 vaddr = drm_gem_vmap(obj); 675 if (IS_ERR(vaddr)) 676 vaddr = NULL; 677 678 return vaddr; 679 } 680 EXPORT_SYMBOL(drm_gem_dmabuf_vmap); 681 682 /** 683 * drm_gem_dmabuf_vunmap - dma_buf vunmap implementation for GEM 684 * @dma_buf: buffer to be unmapped 685 * @vaddr: the virtual address of the buffer 686 * 687 * Releases a kernel virtual mapping. This can be used as the 688 * &dma_buf_ops.vunmap callback. Calls into &drm_gem_object_funcs.vunmap for device specific handling. 689 */ 690 void drm_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr) 691 { 692 struct drm_gem_object *obj = dma_buf->priv; 693 694 drm_gem_vunmap(obj, vaddr); 695 } 696 EXPORT_SYMBOL(drm_gem_dmabuf_vunmap); 697 698 /** 699 * drm_gem_prime_mmap - PRIME mmap function for GEM drivers 700 * @obj: GEM object 701 * @vma: Virtual address range 702 * 703 * This function sets up a userspace mapping for PRIME exported buffers using 704 * the same codepath that is used for regular GEM buffer mapping on the DRM fd. 705 * The fake GEM offset is added to vma->vm_pgoff and &drm_driver->fops->mmap is 706 * called to set up the mapping. 707 * 708 * Drivers can use this as their &drm_driver.gem_prime_mmap callback. 709 */ 710 int drm_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 711 { 712 struct drm_file *priv; 713 struct file *fil; 714 int ret; 715 716 if (obj->funcs && obj->funcs->mmap) { 717 ret = obj->funcs->mmap(obj, vma); 718 if (ret) 719 return ret; 720 vma->vm_private_data = obj; 721 drm_gem_object_get(obj); 722 return 0; 723 } 724 725 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 726 fil = kzalloc(sizeof(*fil), GFP_KERNEL); 727 if (!priv || !fil) { 728 ret = -ENOMEM; 729 goto out; 730 } 731 732 /* Used by drm_gem_mmap() to lookup the GEM object */ 733 priv->minor = obj->dev->primary; 734 fil->private_data = priv; 735 736 ret = drm_vma_node_allow(&obj->vma_node, priv); 737 if (ret) 738 goto out; 739 740 vma->vm_pgoff += drm_vma_node_start(&obj->vma_node); 741 742 ret = obj->dev->driver->fops->mmap(fil, vma); 743 744 drm_vma_node_revoke(&obj->vma_node, priv); 745 out: 746 kfree(priv); 747 kfree(fil); 748 749 return ret; 750 } 751 EXPORT_SYMBOL(drm_gem_prime_mmap); 752 753 /** 754 * drm_gem_dmabuf_mmap - dma_buf mmap implementation for GEM 755 * @dma_buf: buffer to be mapped 756 * @vma: virtual address range 757 * 758 * Provides memory mapping for the buffer. This can be used as the 759 * &dma_buf_ops.mmap callback. It just forwards to &drm_driver.gem_prime_mmap, 760 * which should be set to drm_gem_prime_mmap(). 761 * 762 * FIXME: There's really no point to this wrapper, drivers which need anything 763 * else but drm_gem_prime_mmap can roll their own &dma_buf_ops.mmap callback. 764 * 765 * Returns 0 on success or a negative error code on failure. 766 */ 767 int drm_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) 768 { 769 struct drm_gem_object *obj = dma_buf->priv; 770 struct drm_device *dev = obj->dev; 771 772 if (!dev->driver->gem_prime_mmap) 773 return -ENOSYS; 774 775 return dev->driver->gem_prime_mmap(obj, vma); 776 } 777 EXPORT_SYMBOL(drm_gem_dmabuf_mmap); 778 779 static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { 780 .cache_sgt_mapping = true, 781 .attach = drm_gem_map_attach, 782 .detach = drm_gem_map_detach, 783 .map_dma_buf = drm_gem_map_dma_buf, 784 .unmap_dma_buf = drm_gem_unmap_dma_buf, 785 .release = drm_gem_dmabuf_release, 786 .mmap = drm_gem_dmabuf_mmap, 787 .vmap = drm_gem_dmabuf_vmap, 788 .vunmap = drm_gem_dmabuf_vunmap, 789 }; 790 791 /** 792 * drm_prime_pages_to_sg - converts a page array into an sg list 793 * @pages: pointer to the array of page pointers to convert 794 * @nr_pages: length of the page vector 795 * 796 * This helper creates an sg table object from a set of pages 797 * the driver is responsible for mapping the pages into the 798 * importers address space for use with dma_buf itself. 799 * 800 * This is useful for implementing &drm_gem_object_funcs.get_sg_table. 801 */ 802 struct sg_table *drm_prime_pages_to_sg(struct page **pages, unsigned int nr_pages) 803 { 804 struct sg_table *sg = NULL; 805 int ret; 806 807 sg = kmalloc(sizeof(struct sg_table), GFP_KERNEL); 808 if (!sg) { 809 ret = -ENOMEM; 810 goto out; 811 } 812 813 ret = sg_alloc_table_from_pages(sg, pages, nr_pages, 0, 814 nr_pages << PAGE_SHIFT, GFP_KERNEL); 815 if (ret) 816 goto out; 817 818 return sg; 819 out: 820 kfree(sg); 821 return ERR_PTR(ret); 822 } 823 EXPORT_SYMBOL(drm_prime_pages_to_sg); 824 825 /** 826 * drm_gem_prime_export - helper library implementation of the export callback 827 * @obj: GEM object to export 828 * @flags: flags like DRM_CLOEXEC and DRM_RDWR 829 * 830 * This is the implementation of the &drm_gem_object_funcs.export functions for GEM drivers 831 * using the PRIME helpers. It is used as the default in 832 * drm_gem_prime_handle_to_fd(). 833 */ 834 struct dma_buf *drm_gem_prime_export(struct drm_gem_object *obj, 835 int flags) 836 { 837 struct drm_device *dev = obj->dev; 838 struct dma_buf_export_info exp_info = { 839 .exp_name = KBUILD_MODNAME, /* white lie for debug */ 840 .owner = dev->driver->fops->owner, 841 .ops = &drm_gem_prime_dmabuf_ops, 842 .size = obj->size, 843 .flags = flags, 844 .priv = obj, 845 .resv = obj->resv, 846 }; 847 848 return drm_gem_dmabuf_export(dev, &exp_info); 849 } 850 EXPORT_SYMBOL(drm_gem_prime_export); 851 852 /** 853 * drm_gem_prime_import_dev - core implementation of the import callback 854 * @dev: drm_device to import into 855 * @dma_buf: dma-buf object to import 856 * @attach_dev: struct device to dma_buf attach 857 * 858 * This is the core of drm_gem_prime_import(). It's designed to be called by 859 * drivers who want to use a different device structure than &drm_device.dev for 860 * attaching via dma_buf. This function calls 861 * &drm_driver.gem_prime_import_sg_table internally. 862 * 863 * Drivers must arrange to call drm_prime_gem_destroy() from their 864 * &drm_gem_object_funcs.free hook when using this function. 865 */ 866 struct drm_gem_object *drm_gem_prime_import_dev(struct drm_device *dev, 867 struct dma_buf *dma_buf, 868 struct device *attach_dev) 869 { 870 struct dma_buf_attachment *attach; 871 struct sg_table *sgt; 872 struct drm_gem_object *obj; 873 int ret; 874 875 if (dma_buf->ops == &drm_gem_prime_dmabuf_ops) { 876 obj = dma_buf->priv; 877 if (obj->dev == dev) { 878 /* 879 * Importing dmabuf exported from out own gem increases 880 * refcount on gem itself instead of f_count of dmabuf. 881 */ 882 drm_gem_object_get(obj); 883 return obj; 884 } 885 } 886 887 if (!dev->driver->gem_prime_import_sg_table) 888 return ERR_PTR(-EINVAL); 889 890 attach = dma_buf_attach(dma_buf, attach_dev); 891 if (IS_ERR(attach)) 892 return ERR_CAST(attach); 893 894 get_dma_buf(dma_buf); 895 896 sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); 897 if (IS_ERR(sgt)) { 898 ret = PTR_ERR(sgt); 899 goto fail_detach; 900 } 901 902 obj = dev->driver->gem_prime_import_sg_table(dev, attach, sgt); 903 if (IS_ERR(obj)) { 904 ret = PTR_ERR(obj); 905 goto fail_unmap; 906 } 907 908 obj->import_attach = attach; 909 obj->resv = dma_buf->resv; 910 911 return obj; 912 913 fail_unmap: 914 dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL); 915 fail_detach: 916 dma_buf_detach(dma_buf, attach); 917 dma_buf_put(dma_buf); 918 919 return ERR_PTR(ret); 920 } 921 EXPORT_SYMBOL(drm_gem_prime_import_dev); 922 923 /** 924 * drm_gem_prime_import - helper library implementation of the import callback 925 * @dev: drm_device to import into 926 * @dma_buf: dma-buf object to import 927 * 928 * This is the implementation of the gem_prime_import functions for GEM drivers 929 * using the PRIME helpers. Drivers can use this as their 930 * &drm_driver.gem_prime_import implementation. It is used as the default 931 * implementation in drm_gem_prime_fd_to_handle(). 932 * 933 * Drivers must arrange to call drm_prime_gem_destroy() from their 934 * &drm_gem_object_funcs.free hook when using this function. 935 */ 936 struct drm_gem_object *drm_gem_prime_import(struct drm_device *dev, 937 struct dma_buf *dma_buf) 938 { 939 return drm_gem_prime_import_dev(dev, dma_buf, dev->dev); 940 } 941 EXPORT_SYMBOL(drm_gem_prime_import); 942 943 /** 944 * drm_prime_sg_to_page_addr_arrays - convert an sg table into a page array 945 * @sgt: scatter-gather table to convert 946 * @pages: optional array of page pointers to store the page array in 947 * @addrs: optional array to store the dma bus address of each page 948 * @max_entries: size of both the passed-in arrays 949 * 950 * Exports an sg table into an array of pages and addresses. This is currently 951 * required by the TTM driver in order to do correct fault handling. 952 * 953 * Drivers can use this in their &drm_driver.gem_prime_import_sg_table 954 * implementation. 955 */ 956 int drm_prime_sg_to_page_addr_arrays(struct sg_table *sgt, struct page **pages, 957 dma_addr_t *addrs, int max_entries) 958 { 959 unsigned count; 960 struct scatterlist *sg; 961 struct page *page; 962 u32 len, index; 963 dma_addr_t addr; 964 965 index = 0; 966 for_each_sg(sgt->sgl, sg, sgt->nents, count) { 967 len = sg->length; 968 page = sg_page(sg); 969 addr = sg_dma_address(sg); 970 971 while (len > 0) { 972 if (WARN_ON(index >= max_entries)) 973 return -1; 974 if (pages) 975 pages[index] = page; 976 if (addrs) 977 addrs[index] = addr; 978 979 page++; 980 addr += PAGE_SIZE; 981 len -= PAGE_SIZE; 982 index++; 983 } 984 } 985 return 0; 986 } 987 EXPORT_SYMBOL(drm_prime_sg_to_page_addr_arrays); 988 989 /** 990 * drm_prime_gem_destroy - helper to clean up a PRIME-imported GEM object 991 * @obj: GEM object which was created from a dma-buf 992 * @sg: the sg-table which was pinned at import time 993 * 994 * This is the cleanup functions which GEM drivers need to call when they use 995 * drm_gem_prime_import() or drm_gem_prime_import_dev() to import dma-bufs. 996 */ 997 void drm_prime_gem_destroy(struct drm_gem_object *obj, struct sg_table *sg) 998 { 999 struct dma_buf_attachment *attach; 1000 struct dma_buf *dma_buf; 1001 attach = obj->import_attach; 1002 if (sg) 1003 dma_buf_unmap_attachment(attach, sg, DMA_BIDIRECTIONAL); 1004 dma_buf = attach->dmabuf; 1005 dma_buf_detach(attach->dmabuf, attach); 1006 /* remove the reference */ 1007 dma_buf_put(dma_buf); 1008 } 1009 EXPORT_SYMBOL(drm_prime_gem_destroy); 1010