1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2018 Noralf Trønnes 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/export.h> 8 #include <linux/mutex.h> 9 #include <linux/shmem_fs.h> 10 #include <linux/slab.h> 11 #include <linux/vmalloc.h> 12 13 #include <drm/drm.h> 14 #include <drm/drm_device.h> 15 #include <drm/drm_drv.h> 16 #include <drm/drm_gem_shmem_helper.h> 17 #include <drm/drm_prime.h> 18 #include <drm/drm_print.h> 19 20 /** 21 * DOC: overview 22 * 23 * This library provides helpers for GEM objects backed by shmem buffers 24 * allocated using anonymous pageable memory. 25 */ 26 27 static const struct drm_gem_object_funcs drm_gem_shmem_funcs = { 28 .free = drm_gem_shmem_free_object, 29 .print_info = drm_gem_shmem_print_info, 30 .pin = drm_gem_shmem_pin, 31 .unpin = drm_gem_shmem_unpin, 32 .get_sg_table = drm_gem_shmem_get_sg_table, 33 .vmap = drm_gem_shmem_vmap, 34 .vunmap = drm_gem_shmem_vunmap, 35 .vm_ops = &drm_gem_shmem_vm_ops, 36 }; 37 38 /** 39 * drm_gem_shmem_create - Allocate an object with the given size 40 * @dev: DRM device 41 * @size: Size of the object to allocate 42 * 43 * This function creates a shmem GEM object. 44 * 45 * Returns: 46 * A struct drm_gem_shmem_object * on success or an ERR_PTR()-encoded negative 47 * error code on failure. 48 */ 49 struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size) 50 { 51 struct drm_gem_shmem_object *shmem; 52 struct drm_gem_object *obj; 53 int ret; 54 55 size = PAGE_ALIGN(size); 56 57 if (dev->driver->gem_create_object) 58 obj = dev->driver->gem_create_object(dev, size); 59 else 60 obj = kzalloc(sizeof(*shmem), GFP_KERNEL); 61 if (!obj) 62 return ERR_PTR(-ENOMEM); 63 64 if (!obj->funcs) 65 obj->funcs = &drm_gem_shmem_funcs; 66 67 ret = drm_gem_object_init(dev, obj, size); 68 if (ret) 69 goto err_free; 70 71 ret = drm_gem_create_mmap_offset(obj); 72 if (ret) 73 goto err_release; 74 75 shmem = to_drm_gem_shmem_obj(obj); 76 mutex_init(&shmem->pages_lock); 77 mutex_init(&shmem->vmap_lock); 78 INIT_LIST_HEAD(&shmem->madv_list); 79 80 /* 81 * Our buffers are kept pinned, so allocating them 82 * from the MOVABLE zone is a really bad idea, and 83 * conflicts with CMA. See comments above new_inode() 84 * why this is required _and_ expected if you're 85 * going to pin these pages. 86 */ 87 mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER | 88 __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 89 90 return shmem; 91 92 err_release: 93 drm_gem_object_release(obj); 94 err_free: 95 kfree(obj); 96 97 return ERR_PTR(ret); 98 } 99 EXPORT_SYMBOL_GPL(drm_gem_shmem_create); 100 101 /** 102 * drm_gem_shmem_free_object - Free resources associated with a shmem GEM object 103 * @obj: GEM object to free 104 * 105 * This function cleans up the GEM object state and frees the memory used to 106 * store the object itself. 107 */ 108 void drm_gem_shmem_free_object(struct drm_gem_object *obj) 109 { 110 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 111 112 WARN_ON(shmem->vmap_use_count); 113 114 if (obj->import_attach) { 115 shmem->pages_use_count--; 116 drm_prime_gem_destroy(obj, shmem->sgt); 117 kvfree(shmem->pages); 118 } else { 119 if (shmem->sgt) { 120 dma_unmap_sg(obj->dev->dev, shmem->sgt->sgl, 121 shmem->sgt->nents, DMA_BIDIRECTIONAL); 122 sg_free_table(shmem->sgt); 123 kfree(shmem->sgt); 124 } 125 if (shmem->pages) 126 drm_gem_shmem_put_pages(shmem); 127 } 128 129 WARN_ON(shmem->pages_use_count); 130 131 drm_gem_object_release(obj); 132 mutex_destroy(&shmem->pages_lock); 133 mutex_destroy(&shmem->vmap_lock); 134 kfree(shmem); 135 } 136 EXPORT_SYMBOL_GPL(drm_gem_shmem_free_object); 137 138 static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem) 139 { 140 struct drm_gem_object *obj = &shmem->base; 141 struct page **pages; 142 143 if (shmem->pages_use_count++ > 0) 144 return 0; 145 146 pages = drm_gem_get_pages(obj); 147 if (IS_ERR(pages)) { 148 DRM_DEBUG_KMS("Failed to get pages (%ld)\n", PTR_ERR(pages)); 149 shmem->pages_use_count = 0; 150 return PTR_ERR(pages); 151 } 152 153 shmem->pages = pages; 154 155 return 0; 156 } 157 158 /* 159 * drm_gem_shmem_get_pages - Allocate backing pages for a shmem GEM object 160 * @shmem: shmem GEM object 161 * 162 * This function makes sure that backing pages exists for the shmem GEM object 163 * and increases the use count. 164 * 165 * Returns: 166 * 0 on success or a negative error code on failure. 167 */ 168 int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem) 169 { 170 int ret; 171 172 ret = mutex_lock_interruptible(&shmem->pages_lock); 173 if (ret) 174 return ret; 175 ret = drm_gem_shmem_get_pages_locked(shmem); 176 mutex_unlock(&shmem->pages_lock); 177 178 return ret; 179 } 180 EXPORT_SYMBOL(drm_gem_shmem_get_pages); 181 182 static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem) 183 { 184 struct drm_gem_object *obj = &shmem->base; 185 186 if (WARN_ON_ONCE(!shmem->pages_use_count)) 187 return; 188 189 if (--shmem->pages_use_count > 0) 190 return; 191 192 drm_gem_put_pages(obj, shmem->pages, 193 shmem->pages_mark_dirty_on_put, 194 shmem->pages_mark_accessed_on_put); 195 shmem->pages = NULL; 196 } 197 198 /* 199 * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object 200 * @shmem: shmem GEM object 201 * 202 * This function decreases the use count and puts the backing pages when use drops to zero. 203 */ 204 void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem) 205 { 206 mutex_lock(&shmem->pages_lock); 207 drm_gem_shmem_put_pages_locked(shmem); 208 mutex_unlock(&shmem->pages_lock); 209 } 210 EXPORT_SYMBOL(drm_gem_shmem_put_pages); 211 212 /** 213 * drm_gem_shmem_pin - Pin backing pages for a shmem GEM object 214 * @obj: GEM object 215 * 216 * This function makes sure the backing pages are pinned in memory while the 217 * buffer is exported. 218 * 219 * Returns: 220 * 0 on success or a negative error code on failure. 221 */ 222 int drm_gem_shmem_pin(struct drm_gem_object *obj) 223 { 224 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 225 226 return drm_gem_shmem_get_pages(shmem); 227 } 228 EXPORT_SYMBOL(drm_gem_shmem_pin); 229 230 /** 231 * drm_gem_shmem_unpin - Unpin backing pages for a shmem GEM object 232 * @obj: GEM object 233 * 234 * This function removes the requirement that the backing pages are pinned in 235 * memory. 236 */ 237 void drm_gem_shmem_unpin(struct drm_gem_object *obj) 238 { 239 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 240 241 drm_gem_shmem_put_pages(shmem); 242 } 243 EXPORT_SYMBOL(drm_gem_shmem_unpin); 244 245 static void *drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem) 246 { 247 struct drm_gem_object *obj = &shmem->base; 248 int ret; 249 250 if (shmem->vmap_use_count++ > 0) 251 return shmem->vaddr; 252 253 ret = drm_gem_shmem_get_pages(shmem); 254 if (ret) 255 goto err_zero_use; 256 257 if (obj->import_attach) 258 shmem->vaddr = dma_buf_vmap(obj->import_attach->dmabuf); 259 else 260 shmem->vaddr = vmap(shmem->pages, obj->size >> PAGE_SHIFT, 261 VM_MAP, pgprot_writecombine(PAGE_KERNEL)); 262 263 if (!shmem->vaddr) { 264 DRM_DEBUG_KMS("Failed to vmap pages\n"); 265 ret = -ENOMEM; 266 goto err_put_pages; 267 } 268 269 return shmem->vaddr; 270 271 err_put_pages: 272 drm_gem_shmem_put_pages(shmem); 273 err_zero_use: 274 shmem->vmap_use_count = 0; 275 276 return ERR_PTR(ret); 277 } 278 279 /* 280 * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object 281 * @shmem: shmem GEM object 282 * 283 * This function makes sure that a virtual address exists for the buffer backing 284 * the shmem GEM object. 285 * 286 * Returns: 287 * 0 on success or a negative error code on failure. 288 */ 289 void *drm_gem_shmem_vmap(struct drm_gem_object *obj) 290 { 291 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 292 void *vaddr; 293 int ret; 294 295 ret = mutex_lock_interruptible(&shmem->vmap_lock); 296 if (ret) 297 return ERR_PTR(ret); 298 vaddr = drm_gem_shmem_vmap_locked(shmem); 299 mutex_unlock(&shmem->vmap_lock); 300 301 return vaddr; 302 } 303 EXPORT_SYMBOL(drm_gem_shmem_vmap); 304 305 static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem) 306 { 307 struct drm_gem_object *obj = &shmem->base; 308 309 if (WARN_ON_ONCE(!shmem->vmap_use_count)) 310 return; 311 312 if (--shmem->vmap_use_count > 0) 313 return; 314 315 if (obj->import_attach) 316 dma_buf_vunmap(obj->import_attach->dmabuf, shmem->vaddr); 317 else 318 vunmap(shmem->vaddr); 319 320 shmem->vaddr = NULL; 321 drm_gem_shmem_put_pages(shmem); 322 } 323 324 /* 325 * drm_gem_shmem_vunmap - Unmap a virtual mapping fo a shmem GEM object 326 * @shmem: shmem GEM object 327 * 328 * This function removes the virtual address when use count drops to zero. 329 */ 330 void drm_gem_shmem_vunmap(struct drm_gem_object *obj, void *vaddr) 331 { 332 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 333 334 mutex_lock(&shmem->vmap_lock); 335 drm_gem_shmem_vunmap_locked(shmem); 336 mutex_unlock(&shmem->vmap_lock); 337 } 338 EXPORT_SYMBOL(drm_gem_shmem_vunmap); 339 340 struct drm_gem_shmem_object * 341 drm_gem_shmem_create_with_handle(struct drm_file *file_priv, 342 struct drm_device *dev, size_t size, 343 uint32_t *handle) 344 { 345 struct drm_gem_shmem_object *shmem; 346 int ret; 347 348 shmem = drm_gem_shmem_create(dev, size); 349 if (IS_ERR(shmem)) 350 return shmem; 351 352 /* 353 * Allocate an id of idr table where the obj is registered 354 * and handle has the id what user can see. 355 */ 356 ret = drm_gem_handle_create(file_priv, &shmem->base, handle); 357 /* drop reference from allocate - handle holds it now. */ 358 drm_gem_object_put_unlocked(&shmem->base); 359 if (ret) 360 return ERR_PTR(ret); 361 362 return shmem; 363 } 364 EXPORT_SYMBOL(drm_gem_shmem_create_with_handle); 365 366 /* Update madvise status, returns true if not purged, else 367 * false or -errno. 368 */ 369 int drm_gem_shmem_madvise(struct drm_gem_object *obj, int madv) 370 { 371 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 372 373 mutex_lock(&shmem->pages_lock); 374 375 if (shmem->madv >= 0) 376 shmem->madv = madv; 377 378 madv = shmem->madv; 379 380 mutex_unlock(&shmem->pages_lock); 381 382 return (madv >= 0); 383 } 384 EXPORT_SYMBOL(drm_gem_shmem_madvise); 385 386 void drm_gem_shmem_purge_locked(struct drm_gem_object *obj) 387 { 388 struct drm_device *dev = obj->dev; 389 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 390 391 WARN_ON(!drm_gem_shmem_is_purgeable(shmem)); 392 393 drm_gem_shmem_put_pages_locked(shmem); 394 395 shmem->madv = -1; 396 397 drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); 398 drm_gem_free_mmap_offset(obj); 399 400 /* Our goal here is to return as much of the memory as 401 * is possible back to the system as we are called from OOM. 402 * To do this we must instruct the shmfs to drop all of its 403 * backing pages, *now*. 404 */ 405 shmem_truncate_range(file_inode(obj->filp), 0, (loff_t)-1); 406 407 invalidate_mapping_pages(file_inode(obj->filp)->i_mapping, 408 0, (loff_t)-1); 409 } 410 EXPORT_SYMBOL(drm_gem_shmem_purge_locked); 411 412 void drm_gem_shmem_purge(struct drm_gem_object *obj) 413 { 414 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 415 416 mutex_lock(&shmem->pages_lock); 417 drm_gem_shmem_purge_locked(obj); 418 mutex_unlock(&shmem->pages_lock); 419 } 420 EXPORT_SYMBOL(drm_gem_shmem_purge); 421 422 /** 423 * drm_gem_shmem_dumb_create - Create a dumb shmem buffer object 424 * @file: DRM file structure to create the dumb buffer for 425 * @dev: DRM device 426 * @args: IOCTL data 427 * 428 * This function computes the pitch of the dumb buffer and rounds it up to an 429 * integer number of bytes per pixel. Drivers for hardware that doesn't have 430 * any additional restrictions on the pitch can directly use this function as 431 * their &drm_driver.dumb_create callback. 432 * 433 * For hardware with additional restrictions, drivers can adjust the fields 434 * set up by userspace before calling into this function. 435 * 436 * Returns: 437 * 0 on success or a negative error code on failure. 438 */ 439 int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, 440 struct drm_mode_create_dumb *args) 441 { 442 u32 min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8); 443 struct drm_gem_shmem_object *shmem; 444 445 if (!args->pitch || !args->size) { 446 args->pitch = min_pitch; 447 args->size = args->pitch * args->height; 448 } else { 449 /* ensure sane minimum values */ 450 if (args->pitch < min_pitch) 451 args->pitch = min_pitch; 452 if (args->size < args->pitch * args->height) 453 args->size = args->pitch * args->height; 454 } 455 456 shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, &args->handle); 457 458 return PTR_ERR_OR_ZERO(shmem); 459 } 460 EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); 461 462 static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) 463 { 464 struct vm_area_struct *vma = vmf->vma; 465 struct drm_gem_object *obj = vma->vm_private_data; 466 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 467 loff_t num_pages = obj->size >> PAGE_SHIFT; 468 struct page *page; 469 470 if (vmf->pgoff >= num_pages || WARN_ON_ONCE(!shmem->pages)) 471 return VM_FAULT_SIGBUS; 472 473 page = shmem->pages[vmf->pgoff]; 474 475 return vmf_insert_page(vma, vmf->address, page); 476 } 477 478 static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) 479 { 480 struct drm_gem_object *obj = vma->vm_private_data; 481 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 482 int ret; 483 484 ret = drm_gem_shmem_get_pages(shmem); 485 WARN_ON_ONCE(ret != 0); 486 487 drm_gem_vm_open(vma); 488 } 489 490 static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) 491 { 492 struct drm_gem_object *obj = vma->vm_private_data; 493 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 494 495 drm_gem_shmem_put_pages(shmem); 496 drm_gem_vm_close(vma); 497 } 498 499 const struct vm_operations_struct drm_gem_shmem_vm_ops = { 500 .fault = drm_gem_shmem_fault, 501 .open = drm_gem_shmem_vm_open, 502 .close = drm_gem_shmem_vm_close, 503 }; 504 EXPORT_SYMBOL_GPL(drm_gem_shmem_vm_ops); 505 506 /** 507 * drm_gem_shmem_mmap - Memory-map a shmem GEM object 508 * @filp: File object 509 * @vma: VMA for the area to be mapped 510 * 511 * This function implements an augmented version of the GEM DRM file mmap 512 * operation for shmem objects. Drivers which employ the shmem helpers should 513 * use this function as their &file_operations.mmap handler in the DRM device file's 514 * file_operations structure. 515 * 516 * Instead of directly referencing this function, drivers should use the 517 * DEFINE_DRM_GEM_SHMEM_FOPS() macro. 518 * 519 * Returns: 520 * 0 on success or a negative error code on failure. 521 */ 522 int drm_gem_shmem_mmap(struct file *filp, struct vm_area_struct *vma) 523 { 524 struct drm_gem_shmem_object *shmem; 525 int ret; 526 527 ret = drm_gem_mmap(filp, vma); 528 if (ret) 529 return ret; 530 531 shmem = to_drm_gem_shmem_obj(vma->vm_private_data); 532 533 ret = drm_gem_shmem_get_pages(shmem); 534 if (ret) { 535 drm_gem_vm_close(vma); 536 return ret; 537 } 538 539 /* VM_PFNMAP was set by drm_gem_mmap() */ 540 vma->vm_flags &= ~VM_PFNMAP; 541 vma->vm_flags |= VM_MIXEDMAP; 542 543 /* Remove the fake offset */ 544 vma->vm_pgoff -= drm_vma_node_start(&shmem->base.vma_node); 545 546 return 0; 547 } 548 EXPORT_SYMBOL_GPL(drm_gem_shmem_mmap); 549 550 /** 551 * drm_gem_shmem_print_info() - Print &drm_gem_shmem_object info for debugfs 552 * @p: DRM printer 553 * @indent: Tab indentation level 554 * @obj: GEM object 555 */ 556 void drm_gem_shmem_print_info(struct drm_printer *p, unsigned int indent, 557 const struct drm_gem_object *obj) 558 { 559 const struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 560 561 drm_printf_indent(p, indent, "pages_use_count=%u\n", shmem->pages_use_count); 562 drm_printf_indent(p, indent, "vmap_use_count=%u\n", shmem->vmap_use_count); 563 drm_printf_indent(p, indent, "vaddr=%p\n", shmem->vaddr); 564 } 565 EXPORT_SYMBOL(drm_gem_shmem_print_info); 566 567 /** 568 * drm_gem_shmem_get_sg_table - Provide a scatter/gather table of pinned 569 * pages for a shmem GEM object 570 * @obj: GEM object 571 * 572 * This function exports a scatter/gather table suitable for PRIME usage by 573 * calling the standard DMA mapping API. 574 * 575 * Returns: 576 * A pointer to the scatter/gather table of pinned pages or NULL on failure. 577 */ 578 struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_object *obj) 579 { 580 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 581 582 return drm_prime_pages_to_sg(shmem->pages, obj->size >> PAGE_SHIFT); 583 } 584 EXPORT_SYMBOL_GPL(drm_gem_shmem_get_sg_table); 585 586 /** 587 * drm_gem_shmem_get_pages_sgt - Pin pages, dma map them, and return a 588 * scatter/gather table for a shmem GEM object. 589 * @obj: GEM object 590 * 591 * This function returns a scatter/gather table suitable for driver usage. If 592 * the sg table doesn't exist, the pages are pinned, dma-mapped, and a sg 593 * table created. 594 * 595 * Returns: 596 * A pointer to the scatter/gather table of pinned pages or errno on failure. 597 */ 598 struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_object *obj) 599 { 600 int ret; 601 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); 602 struct sg_table *sgt; 603 604 if (shmem->sgt) 605 return shmem->sgt; 606 607 WARN_ON(obj->import_attach); 608 609 ret = drm_gem_shmem_get_pages(shmem); 610 if (ret) 611 return ERR_PTR(ret); 612 613 sgt = drm_gem_shmem_get_sg_table(&shmem->base); 614 if (IS_ERR(sgt)) { 615 ret = PTR_ERR(sgt); 616 goto err_put_pages; 617 } 618 /* Map the pages for use by the h/w. */ 619 dma_map_sg(obj->dev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL); 620 621 shmem->sgt = sgt; 622 623 return sgt; 624 625 err_put_pages: 626 drm_gem_shmem_put_pages(shmem); 627 return ERR_PTR(ret); 628 } 629 EXPORT_SYMBOL_GPL(drm_gem_shmem_get_pages_sgt); 630 631 /** 632 * drm_gem_shmem_prime_import_sg_table - Produce a shmem GEM object from 633 * another driver's scatter/gather table of pinned pages 634 * @dev: Device to import into 635 * @attach: DMA-BUF attachment 636 * @sgt: Scatter/gather table of pinned pages 637 * 638 * This function imports a scatter/gather table exported via DMA-BUF by 639 * another driver. Drivers that use the shmem helpers should set this as their 640 * &drm_driver.gem_prime_import_sg_table callback. 641 * 642 * Returns: 643 * A pointer to a newly created GEM object or an ERR_PTR-encoded negative 644 * error code on failure. 645 */ 646 struct drm_gem_object * 647 drm_gem_shmem_prime_import_sg_table(struct drm_device *dev, 648 struct dma_buf_attachment *attach, 649 struct sg_table *sgt) 650 { 651 size_t size = PAGE_ALIGN(attach->dmabuf->size); 652 size_t npages = size >> PAGE_SHIFT; 653 struct drm_gem_shmem_object *shmem; 654 int ret; 655 656 shmem = drm_gem_shmem_create(dev, size); 657 if (IS_ERR(shmem)) 658 return ERR_CAST(shmem); 659 660 shmem->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL); 661 if (!shmem->pages) { 662 ret = -ENOMEM; 663 goto err_free_gem; 664 } 665 666 ret = drm_prime_sg_to_page_addr_arrays(sgt, shmem->pages, NULL, npages); 667 if (ret < 0) 668 goto err_free_array; 669 670 shmem->sgt = sgt; 671 shmem->pages_use_count = 1; /* Permanently pinned from our point of view */ 672 673 DRM_DEBUG_PRIME("size = %zu\n", size); 674 675 return &shmem->base; 676 677 err_free_array: 678 kvfree(shmem->pages); 679 err_free_gem: 680 drm_gem_object_put_unlocked(&shmem->base); 681 682 return ERR_PTR(ret); 683 } 684 EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_import_sg_table); 685