1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2018 Etnaviv Project 4 */ 5 6 #include <drm/drm_prime.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/shmem_fs.h> 9 #include <linux/spinlock.h> 10 #include <linux/vmalloc.h> 11 12 #include "etnaviv_drv.h" 13 #include "etnaviv_gem.h" 14 #include "etnaviv_gpu.h" 15 #include "etnaviv_mmu.h" 16 17 static struct lock_class_key etnaviv_shm_lock_class; 18 static struct lock_class_key etnaviv_userptr_lock_class; 19 20 static void etnaviv_gem_scatter_map(struct etnaviv_gem_object *etnaviv_obj) 21 { 22 struct drm_device *dev = etnaviv_obj->base.dev; 23 struct sg_table *sgt = etnaviv_obj->sgt; 24 25 /* 26 * For non-cached buffers, ensure the new pages are clean 27 * because display controller, GPU, etc. are not coherent. 28 */ 29 if (etnaviv_obj->flags & ETNA_BO_CACHE_MASK) 30 dma_map_sgtable(dev->dev, sgt, DMA_BIDIRECTIONAL, 0); 31 } 32 33 static void etnaviv_gem_scatterlist_unmap(struct etnaviv_gem_object *etnaviv_obj) 34 { 35 struct drm_device *dev = etnaviv_obj->base.dev; 36 struct sg_table *sgt = etnaviv_obj->sgt; 37 38 /* 39 * For non-cached buffers, ensure the new pages are clean 40 * because display controller, GPU, etc. are not coherent: 41 * 42 * WARNING: The DMA API does not support concurrent CPU 43 * and device access to the memory area. With BIDIRECTIONAL, 44 * we will clean the cache lines which overlap the region, 45 * and invalidate all cache lines (partially) contained in 46 * the region. 47 * 48 * If you have dirty data in the overlapping cache lines, 49 * that will corrupt the GPU-written data. If you have 50 * written into the remainder of the region, this can 51 * discard those writes. 52 */ 53 if (etnaviv_obj->flags & ETNA_BO_CACHE_MASK) 54 dma_unmap_sgtable(dev->dev, sgt, DMA_BIDIRECTIONAL, 0); 55 } 56 57 /* called with etnaviv_obj->lock held */ 58 static int etnaviv_gem_shmem_get_pages(struct etnaviv_gem_object *etnaviv_obj) 59 { 60 struct drm_device *dev = etnaviv_obj->base.dev; 61 struct page **p = drm_gem_get_pages(&etnaviv_obj->base); 62 63 if (IS_ERR(p)) { 64 dev_dbg(dev->dev, "could not get pages: %ld\n", PTR_ERR(p)); 65 return PTR_ERR(p); 66 } 67 68 etnaviv_obj->pages = p; 69 70 return 0; 71 } 72 73 static void put_pages(struct etnaviv_gem_object *etnaviv_obj) 74 { 75 if (etnaviv_obj->sgt) { 76 etnaviv_gem_scatterlist_unmap(etnaviv_obj); 77 sg_free_table(etnaviv_obj->sgt); 78 kfree(etnaviv_obj->sgt); 79 etnaviv_obj->sgt = NULL; 80 } 81 if (etnaviv_obj->pages) { 82 drm_gem_put_pages(&etnaviv_obj->base, etnaviv_obj->pages, 83 true, false); 84 85 etnaviv_obj->pages = NULL; 86 } 87 } 88 89 struct page **etnaviv_gem_get_pages(struct etnaviv_gem_object *etnaviv_obj) 90 { 91 int ret; 92 93 lockdep_assert_held(&etnaviv_obj->lock); 94 95 if (!etnaviv_obj->pages) { 96 ret = etnaviv_obj->ops->get_pages(etnaviv_obj); 97 if (ret < 0) 98 return ERR_PTR(ret); 99 } 100 101 if (!etnaviv_obj->sgt) { 102 struct drm_device *dev = etnaviv_obj->base.dev; 103 int npages = etnaviv_obj->base.size >> PAGE_SHIFT; 104 struct sg_table *sgt; 105 106 sgt = drm_prime_pages_to_sg(etnaviv_obj->base.dev, 107 etnaviv_obj->pages, npages); 108 if (IS_ERR(sgt)) { 109 dev_err(dev->dev, "failed to allocate sgt: %ld\n", 110 PTR_ERR(sgt)); 111 return ERR_CAST(sgt); 112 } 113 114 etnaviv_obj->sgt = sgt; 115 116 etnaviv_gem_scatter_map(etnaviv_obj); 117 } 118 119 return etnaviv_obj->pages; 120 } 121 122 void etnaviv_gem_put_pages(struct etnaviv_gem_object *etnaviv_obj) 123 { 124 lockdep_assert_held(&etnaviv_obj->lock); 125 /* when we start tracking the pin count, then do something here */ 126 } 127 128 static int etnaviv_gem_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, 129 struct vm_area_struct *vma) 130 { 131 pgprot_t vm_page_prot; 132 133 vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP; 134 135 vm_page_prot = vm_get_page_prot(vma->vm_flags); 136 137 if (etnaviv_obj->flags & ETNA_BO_WC) { 138 vma->vm_page_prot = pgprot_writecombine(vm_page_prot); 139 } else if (etnaviv_obj->flags & ETNA_BO_UNCACHED) { 140 vma->vm_page_prot = pgprot_noncached(vm_page_prot); 141 } else { 142 /* 143 * Shunt off cached objs to shmem file so they have their own 144 * address_space (so unmap_mapping_range does what we want, 145 * in particular in the case of mmap'd dmabufs) 146 */ 147 vma->vm_pgoff = 0; 148 vma_set_file(vma, etnaviv_obj->base.filp); 149 150 vma->vm_page_prot = vm_page_prot; 151 } 152 153 return 0; 154 } 155 156 static int etnaviv_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 157 { 158 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 159 160 return etnaviv_obj->ops->mmap(etnaviv_obj, vma); 161 } 162 163 static vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf) 164 { 165 struct vm_area_struct *vma = vmf->vma; 166 struct drm_gem_object *obj = vma->vm_private_data; 167 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 168 struct page **pages, *page; 169 pgoff_t pgoff; 170 int err; 171 172 /* 173 * Make sure we don't parallel update on a fault, nor move or remove 174 * something from beneath our feet. Note that vmf_insert_page() is 175 * specifically coded to take care of this, so we don't have to. 176 */ 177 err = mutex_lock_interruptible(&etnaviv_obj->lock); 178 if (err) 179 return VM_FAULT_NOPAGE; 180 /* make sure we have pages attached now */ 181 pages = etnaviv_gem_get_pages(etnaviv_obj); 182 mutex_unlock(&etnaviv_obj->lock); 183 184 if (IS_ERR(pages)) { 185 err = PTR_ERR(pages); 186 return vmf_error(err); 187 } 188 189 /* We don't use vmf->pgoff since that has the fake offset: */ 190 pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; 191 192 page = pages[pgoff]; 193 194 VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address, 195 page_to_pfn(page), page_to_pfn(page) << PAGE_SHIFT); 196 197 return vmf_insert_page(vma, vmf->address, page); 198 } 199 200 int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset) 201 { 202 int ret; 203 204 /* Make it mmapable */ 205 ret = drm_gem_create_mmap_offset(obj); 206 if (ret) 207 dev_err(obj->dev->dev, "could not allocate mmap offset\n"); 208 else 209 *offset = drm_vma_node_offset_addr(&obj->vma_node); 210 211 return ret; 212 } 213 214 static struct etnaviv_vram_mapping * 215 etnaviv_gem_get_vram_mapping(struct etnaviv_gem_object *obj, 216 struct etnaviv_iommu_context *context) 217 { 218 struct etnaviv_vram_mapping *mapping; 219 220 list_for_each_entry(mapping, &obj->vram_list, obj_node) { 221 if (mapping->context == context) 222 return mapping; 223 } 224 225 return NULL; 226 } 227 228 void etnaviv_gem_mapping_unreference(struct etnaviv_vram_mapping *mapping) 229 { 230 struct etnaviv_gem_object *etnaviv_obj = mapping->object; 231 232 mutex_lock(&etnaviv_obj->lock); 233 WARN_ON(mapping->use == 0); 234 mapping->use -= 1; 235 mutex_unlock(&etnaviv_obj->lock); 236 237 drm_gem_object_put(&etnaviv_obj->base); 238 } 239 240 struct etnaviv_vram_mapping *etnaviv_gem_mapping_get( 241 struct drm_gem_object *obj, struct etnaviv_iommu_context *mmu_context, 242 u64 va) 243 { 244 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 245 struct etnaviv_vram_mapping *mapping; 246 struct page **pages; 247 int ret = 0; 248 249 mutex_lock(&etnaviv_obj->lock); 250 mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, mmu_context); 251 if (mapping) { 252 /* 253 * Holding the object lock prevents the use count changing 254 * beneath us. If the use count is zero, the MMU might be 255 * reaping this object, so take the lock and re-check that 256 * the MMU owns this mapping to close this race. 257 */ 258 if (mapping->use == 0) { 259 mutex_lock(&mmu_context->lock); 260 if (mapping->context == mmu_context) 261 mapping->use += 1; 262 else 263 mapping = NULL; 264 mutex_unlock(&mmu_context->lock); 265 if (mapping) 266 goto out; 267 } else { 268 mapping->use += 1; 269 goto out; 270 } 271 } 272 273 pages = etnaviv_gem_get_pages(etnaviv_obj); 274 if (IS_ERR(pages)) { 275 ret = PTR_ERR(pages); 276 goto out; 277 } 278 279 /* 280 * See if we have a reaped vram mapping we can re-use before 281 * allocating a fresh mapping. 282 */ 283 mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, NULL); 284 if (!mapping) { 285 mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); 286 if (!mapping) { 287 ret = -ENOMEM; 288 goto out; 289 } 290 291 INIT_LIST_HEAD(&mapping->scan_node); 292 mapping->object = etnaviv_obj; 293 } else { 294 list_del(&mapping->obj_node); 295 } 296 297 etnaviv_iommu_context_get(mmu_context); 298 mapping->context = mmu_context; 299 mapping->use = 1; 300 301 ret = etnaviv_iommu_map_gem(mmu_context, etnaviv_obj, 302 mmu_context->global->memory_base, 303 mapping, va); 304 if (ret < 0) { 305 etnaviv_iommu_context_put(mmu_context); 306 kfree(mapping); 307 } else { 308 list_add_tail(&mapping->obj_node, &etnaviv_obj->vram_list); 309 } 310 311 out: 312 mutex_unlock(&etnaviv_obj->lock); 313 314 if (ret) 315 return ERR_PTR(ret); 316 317 /* Take a reference on the object */ 318 drm_gem_object_get(obj); 319 return mapping; 320 } 321 322 void *etnaviv_gem_vmap(struct drm_gem_object *obj) 323 { 324 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 325 326 if (etnaviv_obj->vaddr) 327 return etnaviv_obj->vaddr; 328 329 mutex_lock(&etnaviv_obj->lock); 330 /* 331 * Need to check again, as we might have raced with another thread 332 * while waiting for the mutex. 333 */ 334 if (!etnaviv_obj->vaddr) 335 etnaviv_obj->vaddr = etnaviv_obj->ops->vmap(etnaviv_obj); 336 mutex_unlock(&etnaviv_obj->lock); 337 338 return etnaviv_obj->vaddr; 339 } 340 341 static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj) 342 { 343 struct page **pages; 344 345 lockdep_assert_held(&obj->lock); 346 347 pages = etnaviv_gem_get_pages(obj); 348 if (IS_ERR(pages)) 349 return NULL; 350 351 return vmap(pages, obj->base.size >> PAGE_SHIFT, 352 VM_MAP, pgprot_writecombine(PAGE_KERNEL)); 353 } 354 355 static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op) 356 { 357 if (op & ETNA_PREP_READ) 358 return DMA_FROM_DEVICE; 359 else if (op & ETNA_PREP_WRITE) 360 return DMA_TO_DEVICE; 361 else 362 return DMA_BIDIRECTIONAL; 363 } 364 365 int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op, 366 struct drm_etnaviv_timespec *timeout) 367 { 368 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 369 struct drm_device *dev = obj->dev; 370 bool write = !!(op & ETNA_PREP_WRITE); 371 int ret; 372 373 if (!etnaviv_obj->sgt) { 374 void *ret; 375 376 mutex_lock(&etnaviv_obj->lock); 377 ret = etnaviv_gem_get_pages(etnaviv_obj); 378 mutex_unlock(&etnaviv_obj->lock); 379 if (IS_ERR(ret)) 380 return PTR_ERR(ret); 381 } 382 383 if (op & ETNA_PREP_NOSYNC) { 384 if (!dma_resv_test_signaled(obj->resv, write)) 385 return -EBUSY; 386 } else { 387 unsigned long remain = etnaviv_timeout_to_jiffies(timeout); 388 389 ret = dma_resv_wait_timeout(obj->resv, write, true, remain); 390 if (ret <= 0) 391 return ret == 0 ? -ETIMEDOUT : ret; 392 } 393 394 if (etnaviv_obj->flags & ETNA_BO_CACHED) { 395 dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt, 396 etnaviv_op_to_dma_dir(op)); 397 etnaviv_obj->last_cpu_prep_op = op; 398 } 399 400 return 0; 401 } 402 403 int etnaviv_gem_cpu_fini(struct drm_gem_object *obj) 404 { 405 struct drm_device *dev = obj->dev; 406 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 407 408 if (etnaviv_obj->flags & ETNA_BO_CACHED) { 409 /* fini without a prep is almost certainly a userspace error */ 410 WARN_ON(etnaviv_obj->last_cpu_prep_op == 0); 411 dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt, 412 etnaviv_op_to_dma_dir(etnaviv_obj->last_cpu_prep_op)); 413 etnaviv_obj->last_cpu_prep_op = 0; 414 } 415 416 return 0; 417 } 418 419 int etnaviv_gem_wait_bo(struct etnaviv_gpu *gpu, struct drm_gem_object *obj, 420 struct drm_etnaviv_timespec *timeout) 421 { 422 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 423 424 return etnaviv_gpu_wait_obj_inactive(gpu, etnaviv_obj, timeout); 425 } 426 427 #ifdef CONFIG_DEBUG_FS 428 static void etnaviv_gem_describe_fence(struct dma_fence *fence, 429 const char *type, struct seq_file *m) 430 { 431 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 432 seq_printf(m, "\t%9s: %s %s seq %llu\n", 433 type, 434 fence->ops->get_driver_name(fence), 435 fence->ops->get_timeline_name(fence), 436 fence->seqno); 437 } 438 439 static void etnaviv_gem_describe(struct drm_gem_object *obj, struct seq_file *m) 440 { 441 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 442 struct dma_resv *robj = obj->resv; 443 struct dma_resv_list *fobj; 444 struct dma_fence *fence; 445 unsigned long off = drm_vma_node_start(&obj->vma_node); 446 447 seq_printf(m, "%08x: %c %2d (%2d) %08lx %p %zd\n", 448 etnaviv_obj->flags, is_active(etnaviv_obj) ? 'A' : 'I', 449 obj->name, kref_read(&obj->refcount), 450 off, etnaviv_obj->vaddr, obj->size); 451 452 rcu_read_lock(); 453 fobj = dma_resv_shared_list(robj); 454 if (fobj) { 455 unsigned int i, shared_count = fobj->shared_count; 456 457 for (i = 0; i < shared_count; i++) { 458 fence = rcu_dereference(fobj->shared[i]); 459 etnaviv_gem_describe_fence(fence, "Shared", m); 460 } 461 } 462 463 fence = dma_resv_excl_fence(robj); 464 if (fence) 465 etnaviv_gem_describe_fence(fence, "Exclusive", m); 466 rcu_read_unlock(); 467 } 468 469 void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv, 470 struct seq_file *m) 471 { 472 struct etnaviv_gem_object *etnaviv_obj; 473 int count = 0; 474 size_t size = 0; 475 476 mutex_lock(&priv->gem_lock); 477 list_for_each_entry(etnaviv_obj, &priv->gem_list, gem_node) { 478 struct drm_gem_object *obj = &etnaviv_obj->base; 479 480 seq_puts(m, " "); 481 etnaviv_gem_describe(obj, m); 482 count++; 483 size += obj->size; 484 } 485 mutex_unlock(&priv->gem_lock); 486 487 seq_printf(m, "Total %d objects, %zu bytes\n", count, size); 488 } 489 #endif 490 491 static void etnaviv_gem_shmem_release(struct etnaviv_gem_object *etnaviv_obj) 492 { 493 vunmap(etnaviv_obj->vaddr); 494 put_pages(etnaviv_obj); 495 } 496 497 static const struct etnaviv_gem_ops etnaviv_gem_shmem_ops = { 498 .get_pages = etnaviv_gem_shmem_get_pages, 499 .release = etnaviv_gem_shmem_release, 500 .vmap = etnaviv_gem_vmap_impl, 501 .mmap = etnaviv_gem_mmap_obj, 502 }; 503 504 void etnaviv_gem_free_object(struct drm_gem_object *obj) 505 { 506 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 507 struct etnaviv_drm_private *priv = obj->dev->dev_private; 508 struct etnaviv_vram_mapping *mapping, *tmp; 509 510 /* object should not be active */ 511 WARN_ON(is_active(etnaviv_obj)); 512 513 mutex_lock(&priv->gem_lock); 514 list_del(&etnaviv_obj->gem_node); 515 mutex_unlock(&priv->gem_lock); 516 517 list_for_each_entry_safe(mapping, tmp, &etnaviv_obj->vram_list, 518 obj_node) { 519 struct etnaviv_iommu_context *context = mapping->context; 520 521 WARN_ON(mapping->use); 522 523 if (context) { 524 etnaviv_iommu_unmap_gem(context, mapping); 525 etnaviv_iommu_context_put(context); 526 } 527 528 list_del(&mapping->obj_node); 529 kfree(mapping); 530 } 531 532 drm_gem_free_mmap_offset(obj); 533 etnaviv_obj->ops->release(etnaviv_obj); 534 drm_gem_object_release(obj); 535 536 kfree(etnaviv_obj); 537 } 538 539 void etnaviv_gem_obj_add(struct drm_device *dev, struct drm_gem_object *obj) 540 { 541 struct etnaviv_drm_private *priv = dev->dev_private; 542 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 543 544 mutex_lock(&priv->gem_lock); 545 list_add_tail(&etnaviv_obj->gem_node, &priv->gem_list); 546 mutex_unlock(&priv->gem_lock); 547 } 548 549 static const struct vm_operations_struct vm_ops = { 550 .fault = etnaviv_gem_fault, 551 .open = drm_gem_vm_open, 552 .close = drm_gem_vm_close, 553 }; 554 555 static const struct drm_gem_object_funcs etnaviv_gem_object_funcs = { 556 .free = etnaviv_gem_free_object, 557 .pin = etnaviv_gem_prime_pin, 558 .unpin = etnaviv_gem_prime_unpin, 559 .get_sg_table = etnaviv_gem_prime_get_sg_table, 560 .vmap = etnaviv_gem_prime_vmap, 561 .mmap = etnaviv_gem_mmap, 562 .vm_ops = &vm_ops, 563 }; 564 565 static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags, 566 const struct etnaviv_gem_ops *ops, struct drm_gem_object **obj) 567 { 568 struct etnaviv_gem_object *etnaviv_obj; 569 unsigned sz = sizeof(*etnaviv_obj); 570 bool valid = true; 571 572 /* validate flags */ 573 switch (flags & ETNA_BO_CACHE_MASK) { 574 case ETNA_BO_UNCACHED: 575 case ETNA_BO_CACHED: 576 case ETNA_BO_WC: 577 break; 578 default: 579 valid = false; 580 } 581 582 if (!valid) { 583 dev_err(dev->dev, "invalid cache flag: %x\n", 584 (flags & ETNA_BO_CACHE_MASK)); 585 return -EINVAL; 586 } 587 588 etnaviv_obj = kzalloc(sz, GFP_KERNEL); 589 if (!etnaviv_obj) 590 return -ENOMEM; 591 592 etnaviv_obj->flags = flags; 593 etnaviv_obj->ops = ops; 594 595 mutex_init(&etnaviv_obj->lock); 596 INIT_LIST_HEAD(&etnaviv_obj->vram_list); 597 598 *obj = &etnaviv_obj->base; 599 (*obj)->funcs = &etnaviv_gem_object_funcs; 600 601 return 0; 602 } 603 604 /* convenience method to construct a GEM buffer object, and userspace handle */ 605 int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file, 606 u32 size, u32 flags, u32 *handle) 607 { 608 struct etnaviv_drm_private *priv = dev->dev_private; 609 struct drm_gem_object *obj = NULL; 610 int ret; 611 612 size = PAGE_ALIGN(size); 613 614 ret = etnaviv_gem_new_impl(dev, size, flags, 615 &etnaviv_gem_shmem_ops, &obj); 616 if (ret) 617 goto fail; 618 619 lockdep_set_class(&to_etnaviv_bo(obj)->lock, &etnaviv_shm_lock_class); 620 621 ret = drm_gem_object_init(dev, obj, size); 622 if (ret) 623 goto fail; 624 625 /* 626 * Our buffers are kept pinned, so allocating them from the MOVABLE 627 * zone is a really bad idea, and conflicts with CMA. See comments 628 * above new_inode() why this is required _and_ expected if you're 629 * going to pin these pages. 630 */ 631 mapping_set_gfp_mask(obj->filp->f_mapping, priv->shm_gfp_mask); 632 633 etnaviv_gem_obj_add(dev, obj); 634 635 ret = drm_gem_handle_create(file, obj, handle); 636 637 /* drop reference from allocate - handle holds it now */ 638 fail: 639 drm_gem_object_put(obj); 640 641 return ret; 642 } 643 644 int etnaviv_gem_new_private(struct drm_device *dev, size_t size, u32 flags, 645 const struct etnaviv_gem_ops *ops, struct etnaviv_gem_object **res) 646 { 647 struct drm_gem_object *obj; 648 int ret; 649 650 ret = etnaviv_gem_new_impl(dev, size, flags, ops, &obj); 651 if (ret) 652 return ret; 653 654 drm_gem_private_object_init(dev, obj, size); 655 656 *res = to_etnaviv_bo(obj); 657 658 return 0; 659 } 660 661 static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj) 662 { 663 struct page **pvec = NULL; 664 struct etnaviv_gem_userptr *userptr = &etnaviv_obj->userptr; 665 int ret, pinned = 0, npages = etnaviv_obj->base.size >> PAGE_SHIFT; 666 667 might_lock_read(¤t->mm->mmap_lock); 668 669 if (userptr->mm != current->mm) 670 return -EPERM; 671 672 pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL); 673 if (!pvec) 674 return -ENOMEM; 675 676 do { 677 unsigned num_pages = npages - pinned; 678 uint64_t ptr = userptr->ptr + pinned * PAGE_SIZE; 679 struct page **pages = pvec + pinned; 680 681 ret = pin_user_pages_fast(ptr, num_pages, 682 FOLL_WRITE | FOLL_FORCE | FOLL_LONGTERM, 683 pages); 684 if (ret < 0) { 685 unpin_user_pages(pvec, pinned); 686 kvfree(pvec); 687 return ret; 688 } 689 690 pinned += ret; 691 692 } while (pinned < npages); 693 694 etnaviv_obj->pages = pvec; 695 696 return 0; 697 } 698 699 static void etnaviv_gem_userptr_release(struct etnaviv_gem_object *etnaviv_obj) 700 { 701 if (etnaviv_obj->sgt) { 702 etnaviv_gem_scatterlist_unmap(etnaviv_obj); 703 sg_free_table(etnaviv_obj->sgt); 704 kfree(etnaviv_obj->sgt); 705 } 706 if (etnaviv_obj->pages) { 707 int npages = etnaviv_obj->base.size >> PAGE_SHIFT; 708 709 unpin_user_pages(etnaviv_obj->pages, npages); 710 kvfree(etnaviv_obj->pages); 711 } 712 } 713 714 static int etnaviv_gem_userptr_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, 715 struct vm_area_struct *vma) 716 { 717 return -EINVAL; 718 } 719 720 static const struct etnaviv_gem_ops etnaviv_gem_userptr_ops = { 721 .get_pages = etnaviv_gem_userptr_get_pages, 722 .release = etnaviv_gem_userptr_release, 723 .vmap = etnaviv_gem_vmap_impl, 724 .mmap = etnaviv_gem_userptr_mmap_obj, 725 }; 726 727 int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file, 728 uintptr_t ptr, u32 size, u32 flags, u32 *handle) 729 { 730 struct etnaviv_gem_object *etnaviv_obj; 731 int ret; 732 733 ret = etnaviv_gem_new_private(dev, size, ETNA_BO_CACHED, 734 &etnaviv_gem_userptr_ops, &etnaviv_obj); 735 if (ret) 736 return ret; 737 738 lockdep_set_class(&etnaviv_obj->lock, &etnaviv_userptr_lock_class); 739 740 etnaviv_obj->userptr.ptr = ptr; 741 etnaviv_obj->userptr.mm = current->mm; 742 etnaviv_obj->userptr.ro = !(flags & ETNA_USERPTR_WRITE); 743 744 etnaviv_gem_obj_add(dev, &etnaviv_obj->base); 745 746 ret = drm_gem_handle_create(file, &etnaviv_obj->base, handle); 747 748 /* drop reference from allocate - handle holds it now */ 749 drm_gem_object_put(&etnaviv_obj->base); 750 return ret; 751 } 752