1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2018 Etnaviv Project 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/shmem_fs.h> 8 #include <linux/sched/mm.h> 9 #include <linux/sched/task.h> 10 11 #include "etnaviv_drv.h" 12 #include "etnaviv_gem.h" 13 #include "etnaviv_gpu.h" 14 #include "etnaviv_mmu.h" 15 16 static struct lock_class_key etnaviv_shm_lock_class; 17 static struct lock_class_key etnaviv_userptr_lock_class; 18 19 static void etnaviv_gem_scatter_map(struct etnaviv_gem_object *etnaviv_obj) 20 { 21 struct drm_device *dev = etnaviv_obj->base.dev; 22 struct sg_table *sgt = etnaviv_obj->sgt; 23 24 /* 25 * For non-cached buffers, ensure the new pages are clean 26 * because display controller, GPU, etc. are not coherent. 27 */ 28 if (etnaviv_obj->flags & ETNA_BO_CACHE_MASK) 29 dma_map_sg(dev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL); 30 } 31 32 static void etnaviv_gem_scatterlist_unmap(struct etnaviv_gem_object *etnaviv_obj) 33 { 34 struct drm_device *dev = etnaviv_obj->base.dev; 35 struct sg_table *sgt = etnaviv_obj->sgt; 36 37 /* 38 * For non-cached buffers, ensure the new pages are clean 39 * because display controller, GPU, etc. are not coherent: 40 * 41 * WARNING: The DMA API does not support concurrent CPU 42 * and device access to the memory area. With BIDIRECTIONAL, 43 * we will clean the cache lines which overlap the region, 44 * and invalidate all cache lines (partially) contained in 45 * the region. 46 * 47 * If you have dirty data in the overlapping cache lines, 48 * that will corrupt the GPU-written data. If you have 49 * written into the remainder of the region, this can 50 * discard those writes. 51 */ 52 if (etnaviv_obj->flags & ETNA_BO_CACHE_MASK) 53 dma_unmap_sg(dev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL); 54 } 55 56 /* called with etnaviv_obj->lock held */ 57 static int etnaviv_gem_shmem_get_pages(struct etnaviv_gem_object *etnaviv_obj) 58 { 59 struct drm_device *dev = etnaviv_obj->base.dev; 60 struct page **p = drm_gem_get_pages(&etnaviv_obj->base); 61 62 if (IS_ERR(p)) { 63 dev_dbg(dev->dev, "could not get pages: %ld\n", PTR_ERR(p)); 64 return PTR_ERR(p); 65 } 66 67 etnaviv_obj->pages = p; 68 69 return 0; 70 } 71 72 static void put_pages(struct etnaviv_gem_object *etnaviv_obj) 73 { 74 if (etnaviv_obj->sgt) { 75 etnaviv_gem_scatterlist_unmap(etnaviv_obj); 76 sg_free_table(etnaviv_obj->sgt); 77 kfree(etnaviv_obj->sgt); 78 etnaviv_obj->sgt = NULL; 79 } 80 if (etnaviv_obj->pages) { 81 drm_gem_put_pages(&etnaviv_obj->base, etnaviv_obj->pages, 82 true, false); 83 84 etnaviv_obj->pages = NULL; 85 } 86 } 87 88 struct page **etnaviv_gem_get_pages(struct etnaviv_gem_object *etnaviv_obj) 89 { 90 int ret; 91 92 lockdep_assert_held(&etnaviv_obj->lock); 93 94 if (!etnaviv_obj->pages) { 95 ret = etnaviv_obj->ops->get_pages(etnaviv_obj); 96 if (ret < 0) 97 return ERR_PTR(ret); 98 } 99 100 if (!etnaviv_obj->sgt) { 101 struct drm_device *dev = etnaviv_obj->base.dev; 102 int npages = etnaviv_obj->base.size >> PAGE_SHIFT; 103 struct sg_table *sgt; 104 105 sgt = drm_prime_pages_to_sg(etnaviv_obj->pages, npages); 106 if (IS_ERR(sgt)) { 107 dev_err(dev->dev, "failed to allocate sgt: %ld\n", 108 PTR_ERR(sgt)); 109 return ERR_CAST(sgt); 110 } 111 112 etnaviv_obj->sgt = sgt; 113 114 etnaviv_gem_scatter_map(etnaviv_obj); 115 } 116 117 return etnaviv_obj->pages; 118 } 119 120 void etnaviv_gem_put_pages(struct etnaviv_gem_object *etnaviv_obj) 121 { 122 lockdep_assert_held(&etnaviv_obj->lock); 123 /* when we start tracking the pin count, then do something here */ 124 } 125 126 static int etnaviv_gem_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, 127 struct vm_area_struct *vma) 128 { 129 pgprot_t vm_page_prot; 130 131 vma->vm_flags &= ~VM_PFNMAP; 132 vma->vm_flags |= VM_MIXEDMAP; 133 134 vm_page_prot = vm_get_page_prot(vma->vm_flags); 135 136 if (etnaviv_obj->flags & ETNA_BO_WC) { 137 vma->vm_page_prot = pgprot_writecombine(vm_page_prot); 138 } else if (etnaviv_obj->flags & ETNA_BO_UNCACHED) { 139 vma->vm_page_prot = pgprot_noncached(vm_page_prot); 140 } else { 141 /* 142 * Shunt off cached objs to shmem file so they have their own 143 * address_space (so unmap_mapping_range does what we want, 144 * in particular in the case of mmap'd dmabufs) 145 */ 146 fput(vma->vm_file); 147 get_file(etnaviv_obj->base.filp); 148 vma->vm_pgoff = 0; 149 vma->vm_file = etnaviv_obj->base.filp; 150 151 vma->vm_page_prot = vm_page_prot; 152 } 153 154 return 0; 155 } 156 157 int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma) 158 { 159 struct etnaviv_gem_object *obj; 160 int ret; 161 162 ret = drm_gem_mmap(filp, vma); 163 if (ret) { 164 DBG("mmap failed: %d", ret); 165 return ret; 166 } 167 168 obj = to_etnaviv_bo(vma->vm_private_data); 169 return obj->ops->mmap(obj, vma); 170 } 171 172 vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf) 173 { 174 struct vm_area_struct *vma = vmf->vma; 175 struct drm_gem_object *obj = vma->vm_private_data; 176 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 177 struct page **pages, *page; 178 pgoff_t pgoff; 179 int err; 180 181 /* 182 * Make sure we don't parallel update on a fault, nor move or remove 183 * something from beneath our feet. Note that vmf_insert_page() is 184 * specifically coded to take care of this, so we don't have to. 185 */ 186 err = mutex_lock_interruptible(&etnaviv_obj->lock); 187 if (err) 188 return VM_FAULT_NOPAGE; 189 /* make sure we have pages attached now */ 190 pages = etnaviv_gem_get_pages(etnaviv_obj); 191 mutex_unlock(&etnaviv_obj->lock); 192 193 if (IS_ERR(pages)) { 194 err = PTR_ERR(pages); 195 return vmf_error(err); 196 } 197 198 /* We don't use vmf->pgoff since that has the fake offset: */ 199 pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; 200 201 page = pages[pgoff]; 202 203 VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address, 204 page_to_pfn(page), page_to_pfn(page) << PAGE_SHIFT); 205 206 return vmf_insert_page(vma, vmf->address, page); 207 } 208 209 int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset) 210 { 211 int ret; 212 213 /* Make it mmapable */ 214 ret = drm_gem_create_mmap_offset(obj); 215 if (ret) 216 dev_err(obj->dev->dev, "could not allocate mmap offset\n"); 217 else 218 *offset = drm_vma_node_offset_addr(&obj->vma_node); 219 220 return ret; 221 } 222 223 static struct etnaviv_vram_mapping * 224 etnaviv_gem_get_vram_mapping(struct etnaviv_gem_object *obj, 225 struct etnaviv_iommu *mmu) 226 { 227 struct etnaviv_vram_mapping *mapping; 228 229 list_for_each_entry(mapping, &obj->vram_list, obj_node) { 230 if (mapping->mmu == mmu) 231 return mapping; 232 } 233 234 return NULL; 235 } 236 237 void etnaviv_gem_mapping_reference(struct etnaviv_vram_mapping *mapping) 238 { 239 struct etnaviv_gem_object *etnaviv_obj = mapping->object; 240 241 drm_gem_object_get(&etnaviv_obj->base); 242 243 mutex_lock(&etnaviv_obj->lock); 244 WARN_ON(mapping->use == 0); 245 mapping->use += 1; 246 mutex_unlock(&etnaviv_obj->lock); 247 } 248 249 void etnaviv_gem_mapping_unreference(struct etnaviv_vram_mapping *mapping) 250 { 251 struct etnaviv_gem_object *etnaviv_obj = mapping->object; 252 253 mutex_lock(&etnaviv_obj->lock); 254 WARN_ON(mapping->use == 0); 255 mapping->use -= 1; 256 mutex_unlock(&etnaviv_obj->lock); 257 258 drm_gem_object_put_unlocked(&etnaviv_obj->base); 259 } 260 261 struct etnaviv_vram_mapping *etnaviv_gem_mapping_get( 262 struct drm_gem_object *obj, struct etnaviv_gpu *gpu) 263 { 264 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 265 struct etnaviv_vram_mapping *mapping; 266 struct page **pages; 267 int ret = 0; 268 269 mutex_lock(&etnaviv_obj->lock); 270 mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, gpu->mmu); 271 if (mapping) { 272 /* 273 * Holding the object lock prevents the use count changing 274 * beneath us. If the use count is zero, the MMU might be 275 * reaping this object, so take the lock and re-check that 276 * the MMU owns this mapping to close this race. 277 */ 278 if (mapping->use == 0) { 279 mutex_lock(&gpu->mmu->lock); 280 if (mapping->mmu == gpu->mmu) 281 mapping->use += 1; 282 else 283 mapping = NULL; 284 mutex_unlock(&gpu->mmu->lock); 285 if (mapping) 286 goto out; 287 } else { 288 mapping->use += 1; 289 goto out; 290 } 291 } 292 293 pages = etnaviv_gem_get_pages(etnaviv_obj); 294 if (IS_ERR(pages)) { 295 ret = PTR_ERR(pages); 296 goto out; 297 } 298 299 /* 300 * See if we have a reaped vram mapping we can re-use before 301 * allocating a fresh mapping. 302 */ 303 mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, NULL); 304 if (!mapping) { 305 mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); 306 if (!mapping) { 307 ret = -ENOMEM; 308 goto out; 309 } 310 311 INIT_LIST_HEAD(&mapping->scan_node); 312 mapping->object = etnaviv_obj; 313 } else { 314 list_del(&mapping->obj_node); 315 } 316 317 mapping->mmu = gpu->mmu; 318 mapping->use = 1; 319 320 ret = etnaviv_iommu_map_gem(gpu->mmu, etnaviv_obj, gpu->memory_base, 321 mapping); 322 if (ret < 0) 323 kfree(mapping); 324 else 325 list_add_tail(&mapping->obj_node, &etnaviv_obj->vram_list); 326 327 out: 328 mutex_unlock(&etnaviv_obj->lock); 329 330 if (ret) 331 return ERR_PTR(ret); 332 333 /* Take a reference on the object */ 334 drm_gem_object_get(obj); 335 return mapping; 336 } 337 338 void *etnaviv_gem_vmap(struct drm_gem_object *obj) 339 { 340 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 341 342 if (etnaviv_obj->vaddr) 343 return etnaviv_obj->vaddr; 344 345 mutex_lock(&etnaviv_obj->lock); 346 /* 347 * Need to check again, as we might have raced with another thread 348 * while waiting for the mutex. 349 */ 350 if (!etnaviv_obj->vaddr) 351 etnaviv_obj->vaddr = etnaviv_obj->ops->vmap(etnaviv_obj); 352 mutex_unlock(&etnaviv_obj->lock); 353 354 return etnaviv_obj->vaddr; 355 } 356 357 static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj) 358 { 359 struct page **pages; 360 361 lockdep_assert_held(&obj->lock); 362 363 pages = etnaviv_gem_get_pages(obj); 364 if (IS_ERR(pages)) 365 return NULL; 366 367 return vmap(pages, obj->base.size >> PAGE_SHIFT, 368 VM_MAP, pgprot_writecombine(PAGE_KERNEL)); 369 } 370 371 static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op) 372 { 373 if (op & ETNA_PREP_READ) 374 return DMA_FROM_DEVICE; 375 else if (op & ETNA_PREP_WRITE) 376 return DMA_TO_DEVICE; 377 else 378 return DMA_BIDIRECTIONAL; 379 } 380 381 int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op, 382 struct timespec *timeout) 383 { 384 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 385 struct drm_device *dev = obj->dev; 386 bool write = !!(op & ETNA_PREP_WRITE); 387 int ret; 388 389 if (!etnaviv_obj->sgt) { 390 void *ret; 391 392 mutex_lock(&etnaviv_obj->lock); 393 ret = etnaviv_gem_get_pages(etnaviv_obj); 394 mutex_unlock(&etnaviv_obj->lock); 395 if (IS_ERR(ret)) 396 return PTR_ERR(ret); 397 } 398 399 if (op & ETNA_PREP_NOSYNC) { 400 if (!reservation_object_test_signaled_rcu(etnaviv_obj->resv, 401 write)) 402 return -EBUSY; 403 } else { 404 unsigned long remain = etnaviv_timeout_to_jiffies(timeout); 405 406 ret = reservation_object_wait_timeout_rcu(etnaviv_obj->resv, 407 write, true, remain); 408 if (ret <= 0) 409 return ret == 0 ? -ETIMEDOUT : ret; 410 } 411 412 if (etnaviv_obj->flags & ETNA_BO_CACHED) { 413 dma_sync_sg_for_cpu(dev->dev, etnaviv_obj->sgt->sgl, 414 etnaviv_obj->sgt->nents, 415 etnaviv_op_to_dma_dir(op)); 416 etnaviv_obj->last_cpu_prep_op = op; 417 } 418 419 return 0; 420 } 421 422 int etnaviv_gem_cpu_fini(struct drm_gem_object *obj) 423 { 424 struct drm_device *dev = obj->dev; 425 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 426 427 if (etnaviv_obj->flags & ETNA_BO_CACHED) { 428 /* fini without a prep is almost certainly a userspace error */ 429 WARN_ON(etnaviv_obj->last_cpu_prep_op == 0); 430 dma_sync_sg_for_device(dev->dev, etnaviv_obj->sgt->sgl, 431 etnaviv_obj->sgt->nents, 432 etnaviv_op_to_dma_dir(etnaviv_obj->last_cpu_prep_op)); 433 etnaviv_obj->last_cpu_prep_op = 0; 434 } 435 436 return 0; 437 } 438 439 int etnaviv_gem_wait_bo(struct etnaviv_gpu *gpu, struct drm_gem_object *obj, 440 struct timespec *timeout) 441 { 442 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 443 444 return etnaviv_gpu_wait_obj_inactive(gpu, etnaviv_obj, timeout); 445 } 446 447 #ifdef CONFIG_DEBUG_FS 448 static void etnaviv_gem_describe_fence(struct dma_fence *fence, 449 const char *type, struct seq_file *m) 450 { 451 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 452 seq_printf(m, "\t%9s: %s %s seq %llu\n", 453 type, 454 fence->ops->get_driver_name(fence), 455 fence->ops->get_timeline_name(fence), 456 fence->seqno); 457 } 458 459 static void etnaviv_gem_describe(struct drm_gem_object *obj, struct seq_file *m) 460 { 461 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 462 struct reservation_object *robj = etnaviv_obj->resv; 463 struct reservation_object_list *fobj; 464 struct dma_fence *fence; 465 unsigned long off = drm_vma_node_start(&obj->vma_node); 466 467 seq_printf(m, "%08x: %c %2d (%2d) %08lx %p %zd\n", 468 etnaviv_obj->flags, is_active(etnaviv_obj) ? 'A' : 'I', 469 obj->name, kref_read(&obj->refcount), 470 off, etnaviv_obj->vaddr, obj->size); 471 472 rcu_read_lock(); 473 fobj = rcu_dereference(robj->fence); 474 if (fobj) { 475 unsigned int i, shared_count = fobj->shared_count; 476 477 for (i = 0; i < shared_count; i++) { 478 fence = rcu_dereference(fobj->shared[i]); 479 etnaviv_gem_describe_fence(fence, "Shared", m); 480 } 481 } 482 483 fence = rcu_dereference(robj->fence_excl); 484 if (fence) 485 etnaviv_gem_describe_fence(fence, "Exclusive", m); 486 rcu_read_unlock(); 487 } 488 489 void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv, 490 struct seq_file *m) 491 { 492 struct etnaviv_gem_object *etnaviv_obj; 493 int count = 0; 494 size_t size = 0; 495 496 mutex_lock(&priv->gem_lock); 497 list_for_each_entry(etnaviv_obj, &priv->gem_list, gem_node) { 498 struct drm_gem_object *obj = &etnaviv_obj->base; 499 500 seq_puts(m, " "); 501 etnaviv_gem_describe(obj, m); 502 count++; 503 size += obj->size; 504 } 505 mutex_unlock(&priv->gem_lock); 506 507 seq_printf(m, "Total %d objects, %zu bytes\n", count, size); 508 } 509 #endif 510 511 static void etnaviv_gem_shmem_release(struct etnaviv_gem_object *etnaviv_obj) 512 { 513 vunmap(etnaviv_obj->vaddr); 514 put_pages(etnaviv_obj); 515 } 516 517 static const struct etnaviv_gem_ops etnaviv_gem_shmem_ops = { 518 .get_pages = etnaviv_gem_shmem_get_pages, 519 .release = etnaviv_gem_shmem_release, 520 .vmap = etnaviv_gem_vmap_impl, 521 .mmap = etnaviv_gem_mmap_obj, 522 }; 523 524 void etnaviv_gem_free_object(struct drm_gem_object *obj) 525 { 526 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 527 struct etnaviv_drm_private *priv = obj->dev->dev_private; 528 struct etnaviv_vram_mapping *mapping, *tmp; 529 530 /* object should not be active */ 531 WARN_ON(is_active(etnaviv_obj)); 532 533 mutex_lock(&priv->gem_lock); 534 list_del(&etnaviv_obj->gem_node); 535 mutex_unlock(&priv->gem_lock); 536 537 list_for_each_entry_safe(mapping, tmp, &etnaviv_obj->vram_list, 538 obj_node) { 539 struct etnaviv_iommu *mmu = mapping->mmu; 540 541 WARN_ON(mapping->use); 542 543 if (mmu) 544 etnaviv_iommu_unmap_gem(mmu, mapping); 545 546 list_del(&mapping->obj_node); 547 kfree(mapping); 548 } 549 550 drm_gem_free_mmap_offset(obj); 551 etnaviv_obj->ops->release(etnaviv_obj); 552 if (etnaviv_obj->resv == &etnaviv_obj->_resv) 553 reservation_object_fini(&etnaviv_obj->_resv); 554 drm_gem_object_release(obj); 555 556 kfree(etnaviv_obj); 557 } 558 559 void etnaviv_gem_obj_add(struct drm_device *dev, struct drm_gem_object *obj) 560 { 561 struct etnaviv_drm_private *priv = dev->dev_private; 562 struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); 563 564 mutex_lock(&priv->gem_lock); 565 list_add_tail(&etnaviv_obj->gem_node, &priv->gem_list); 566 mutex_unlock(&priv->gem_lock); 567 } 568 569 static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags, 570 struct reservation_object *robj, const struct etnaviv_gem_ops *ops, 571 struct drm_gem_object **obj) 572 { 573 struct etnaviv_gem_object *etnaviv_obj; 574 unsigned sz = sizeof(*etnaviv_obj); 575 bool valid = true; 576 577 /* validate flags */ 578 switch (flags & ETNA_BO_CACHE_MASK) { 579 case ETNA_BO_UNCACHED: 580 case ETNA_BO_CACHED: 581 case ETNA_BO_WC: 582 break; 583 default: 584 valid = false; 585 } 586 587 if (!valid) { 588 dev_err(dev->dev, "invalid cache flag: %x\n", 589 (flags & ETNA_BO_CACHE_MASK)); 590 return -EINVAL; 591 } 592 593 etnaviv_obj = kzalloc(sz, GFP_KERNEL); 594 if (!etnaviv_obj) 595 return -ENOMEM; 596 597 etnaviv_obj->flags = flags; 598 etnaviv_obj->ops = ops; 599 if (robj) { 600 etnaviv_obj->resv = robj; 601 } else { 602 etnaviv_obj->resv = &etnaviv_obj->_resv; 603 reservation_object_init(&etnaviv_obj->_resv); 604 } 605 606 mutex_init(&etnaviv_obj->lock); 607 INIT_LIST_HEAD(&etnaviv_obj->vram_list); 608 609 *obj = &etnaviv_obj->base; 610 611 return 0; 612 } 613 614 /* convenience method to construct a GEM buffer object, and userspace handle */ 615 int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file, 616 u32 size, u32 flags, u32 *handle) 617 { 618 struct drm_gem_object *obj = NULL; 619 int ret; 620 621 size = PAGE_ALIGN(size); 622 623 ret = etnaviv_gem_new_impl(dev, size, flags, NULL, 624 &etnaviv_gem_shmem_ops, &obj); 625 if (ret) 626 goto fail; 627 628 lockdep_set_class(&to_etnaviv_bo(obj)->lock, &etnaviv_shm_lock_class); 629 630 ret = drm_gem_object_init(dev, obj, size); 631 if (ret == 0) { 632 struct address_space *mapping; 633 634 /* 635 * Our buffers are kept pinned, so allocating them 636 * from the MOVABLE zone is a really bad idea, and 637 * conflicts with CMA. See comments above new_inode() 638 * why this is required _and_ expected if you're 639 * going to pin these pages. 640 */ 641 mapping = obj->filp->f_mapping; 642 mapping_set_gfp_mask(mapping, GFP_HIGHUSER | 643 __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 644 } 645 646 if (ret) 647 goto fail; 648 649 etnaviv_gem_obj_add(dev, obj); 650 651 ret = drm_gem_handle_create(file, obj, handle); 652 653 /* drop reference from allocate - handle holds it now */ 654 fail: 655 drm_gem_object_put_unlocked(obj); 656 657 return ret; 658 } 659 660 int etnaviv_gem_new_private(struct drm_device *dev, size_t size, u32 flags, 661 struct reservation_object *robj, const struct etnaviv_gem_ops *ops, 662 struct etnaviv_gem_object **res) 663 { 664 struct drm_gem_object *obj; 665 int ret; 666 667 ret = etnaviv_gem_new_impl(dev, size, flags, robj, ops, &obj); 668 if (ret) 669 return ret; 670 671 drm_gem_private_object_init(dev, obj, size); 672 673 *res = to_etnaviv_bo(obj); 674 675 return 0; 676 } 677 678 static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj) 679 { 680 struct page **pvec = NULL; 681 struct etnaviv_gem_userptr *userptr = &etnaviv_obj->userptr; 682 int ret, pinned = 0, npages = etnaviv_obj->base.size >> PAGE_SHIFT; 683 684 might_lock_read(¤t->mm->mmap_sem); 685 686 if (userptr->mm != current->mm) 687 return -EPERM; 688 689 pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL); 690 if (!pvec) 691 return -ENOMEM; 692 693 do { 694 unsigned num_pages = npages - pinned; 695 uint64_t ptr = userptr->ptr + pinned * PAGE_SIZE; 696 struct page **pages = pvec + pinned; 697 698 ret = get_user_pages_fast(ptr, num_pages, 699 !userptr->ro ? FOLL_WRITE : 0, pages); 700 if (ret < 0) { 701 release_pages(pvec, pinned); 702 kvfree(pvec); 703 return ret; 704 } 705 706 pinned += ret; 707 708 } while (pinned < npages); 709 710 etnaviv_obj->pages = pvec; 711 712 return 0; 713 } 714 715 static void etnaviv_gem_userptr_release(struct etnaviv_gem_object *etnaviv_obj) 716 { 717 if (etnaviv_obj->sgt) { 718 etnaviv_gem_scatterlist_unmap(etnaviv_obj); 719 sg_free_table(etnaviv_obj->sgt); 720 kfree(etnaviv_obj->sgt); 721 } 722 if (etnaviv_obj->pages) { 723 int npages = etnaviv_obj->base.size >> PAGE_SHIFT; 724 725 release_pages(etnaviv_obj->pages, npages); 726 kvfree(etnaviv_obj->pages); 727 } 728 } 729 730 static int etnaviv_gem_userptr_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, 731 struct vm_area_struct *vma) 732 { 733 return -EINVAL; 734 } 735 736 static const struct etnaviv_gem_ops etnaviv_gem_userptr_ops = { 737 .get_pages = etnaviv_gem_userptr_get_pages, 738 .release = etnaviv_gem_userptr_release, 739 .vmap = etnaviv_gem_vmap_impl, 740 .mmap = etnaviv_gem_userptr_mmap_obj, 741 }; 742 743 int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file, 744 uintptr_t ptr, u32 size, u32 flags, u32 *handle) 745 { 746 struct etnaviv_gem_object *etnaviv_obj; 747 int ret; 748 749 ret = etnaviv_gem_new_private(dev, size, ETNA_BO_CACHED, NULL, 750 &etnaviv_gem_userptr_ops, &etnaviv_obj); 751 if (ret) 752 return ret; 753 754 lockdep_set_class(&etnaviv_obj->lock, &etnaviv_userptr_lock_class); 755 756 etnaviv_obj->userptr.ptr = ptr; 757 etnaviv_obj->userptr.mm = current->mm; 758 etnaviv_obj->userptr.ro = !(flags & ETNA_USERPTR_WRITE); 759 760 etnaviv_gem_obj_add(dev, &etnaviv_obj->base); 761 762 ret = drm_gem_handle_create(file, &etnaviv_obj->base, handle); 763 764 /* drop reference from allocate - handle holds it now */ 765 drm_gem_object_put_unlocked(&etnaviv_obj->base); 766 return ret; 767 } 768