1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include <drm/drm_gem_ttm_helper.h> 28 29 #include "nouveau_drv.h" 30 #include "nouveau_dma.h" 31 #include "nouveau_fence.h" 32 #include "nouveau_abi16.h" 33 34 #include "nouveau_ttm.h" 35 #include "nouveau_gem.h" 36 #include "nouveau_mem.h" 37 #include "nouveau_vmm.h" 38 39 #include <nvif/class.h> 40 #include <nvif/push206e.h> 41 42 static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf) 43 { 44 struct vm_area_struct *vma = vmf->vma; 45 struct ttm_buffer_object *bo = vma->vm_private_data; 46 pgprot_t prot; 47 vm_fault_t ret; 48 49 ret = ttm_bo_vm_reserve(bo, vmf); 50 if (ret) 51 return ret; 52 53 ret = nouveau_ttm_fault_reserve_notify(bo); 54 if (ret) 55 goto error_unlock; 56 57 nouveau_bo_del_io_reserve_lru(bo); 58 prot = vm_get_page_prot(vma->vm_flags); 59 ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1); 60 nouveau_bo_add_io_reserve_lru(bo); 61 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 62 return ret; 63 64 error_unlock: 65 dma_resv_unlock(bo->base.resv); 66 return ret; 67 } 68 69 static const struct vm_operations_struct nouveau_ttm_vm_ops = { 70 .fault = nouveau_ttm_fault, 71 .open = ttm_bo_vm_open, 72 .close = ttm_bo_vm_close, 73 .access = ttm_bo_vm_access 74 }; 75 76 void 77 nouveau_gem_object_del(struct drm_gem_object *gem) 78 { 79 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 80 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 81 struct device *dev = drm->dev->dev; 82 int ret; 83 84 ret = pm_runtime_get_sync(dev); 85 if (WARN_ON(ret < 0 && ret != -EACCES)) { 86 pm_runtime_put_autosuspend(dev); 87 return; 88 } 89 90 if (gem->import_attach) 91 drm_prime_gem_destroy(gem, nvbo->bo.sg); 92 93 ttm_bo_put(&nvbo->bo); 94 95 pm_runtime_mark_last_busy(dev); 96 pm_runtime_put_autosuspend(dev); 97 } 98 99 int 100 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 101 { 102 struct nouveau_cli *cli = nouveau_cli(file_priv); 103 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 104 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 105 struct device *dev = drm->dev->dev; 106 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 107 struct nouveau_vma *vma; 108 int ret; 109 110 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 111 return 0; 112 113 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 114 if (ret) 115 return ret; 116 117 ret = pm_runtime_get_sync(dev); 118 if (ret < 0 && ret != -EACCES) { 119 pm_runtime_put_autosuspend(dev); 120 goto out; 121 } 122 123 ret = nouveau_vma_new(nvbo, vmm, &vma); 124 pm_runtime_mark_last_busy(dev); 125 pm_runtime_put_autosuspend(dev); 126 out: 127 ttm_bo_unreserve(&nvbo->bo); 128 return ret; 129 } 130 131 struct nouveau_gem_object_unmap { 132 struct nouveau_cli_work work; 133 struct nouveau_vma *vma; 134 }; 135 136 static void 137 nouveau_gem_object_delete(struct nouveau_vma *vma) 138 { 139 nouveau_fence_unref(&vma->fence); 140 nouveau_vma_del(&vma); 141 } 142 143 static void 144 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 145 { 146 struct nouveau_gem_object_unmap *work = 147 container_of(w, typeof(*work), work); 148 nouveau_gem_object_delete(work->vma); 149 kfree(work); 150 } 151 152 static void 153 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 154 { 155 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 156 struct nouveau_gem_object_unmap *work; 157 158 list_del_init(&vma->head); 159 160 if (!fence) { 161 nouveau_gem_object_delete(vma); 162 return; 163 } 164 165 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 166 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 167 nouveau_gem_object_delete(vma); 168 return; 169 } 170 171 work->work.func = nouveau_gem_object_delete_work; 172 work->vma = vma; 173 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 174 } 175 176 void 177 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 178 { 179 struct nouveau_cli *cli = nouveau_cli(file_priv); 180 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 181 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 182 struct device *dev = drm->dev->dev; 183 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 184 struct nouveau_vma *vma; 185 int ret; 186 187 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 188 return; 189 190 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 191 if (ret) 192 return; 193 194 vma = nouveau_vma_find(nvbo, vmm); 195 if (vma) { 196 if (--vma->refs == 0) { 197 ret = pm_runtime_get_sync(dev); 198 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 199 nouveau_gem_object_unmap(nvbo, vma); 200 pm_runtime_mark_last_busy(dev); 201 } 202 pm_runtime_put_autosuspend(dev); 203 } 204 } 205 ttm_bo_unreserve(&nvbo->bo); 206 } 207 208 const struct drm_gem_object_funcs nouveau_gem_object_funcs = { 209 .free = nouveau_gem_object_del, 210 .open = nouveau_gem_object_open, 211 .close = nouveau_gem_object_close, 212 .pin = nouveau_gem_prime_pin, 213 .unpin = nouveau_gem_prime_unpin, 214 .get_sg_table = nouveau_gem_prime_get_sg_table, 215 .vmap = drm_gem_ttm_vmap, 216 .vunmap = drm_gem_ttm_vunmap, 217 .mmap = drm_gem_ttm_mmap, 218 .vm_ops = &nouveau_ttm_vm_ops, 219 }; 220 221 int 222 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 223 uint32_t tile_mode, uint32_t tile_flags, 224 struct nouveau_bo **pnvbo) 225 { 226 struct nouveau_drm *drm = cli->drm; 227 struct nouveau_bo *nvbo; 228 int ret; 229 230 if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) 231 domain |= NOUVEAU_GEM_DOMAIN_CPU; 232 233 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 234 tile_flags); 235 if (IS_ERR(nvbo)) 236 return PTR_ERR(nvbo); 237 238 nvbo->bo.base.funcs = &nouveau_gem_object_funcs; 239 240 /* Initialize the embedded gem-object. We return a single gem-reference 241 * to the caller, instead of a normal nouveau_bo ttm reference. */ 242 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 243 if (ret) { 244 drm_gem_object_release(&nvbo->bo.base); 245 kfree(nvbo); 246 return ret; 247 } 248 249 ret = nouveau_bo_init(nvbo, size, align, domain, NULL, NULL); 250 if (ret) { 251 nouveau_bo_ref(NULL, &nvbo); 252 return ret; 253 } 254 255 /* we restrict allowed domains on nv50+ to only the types 256 * that were requested at creation time. not possibly on 257 * earlier chips without busting the ABI. 258 */ 259 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 260 NOUVEAU_GEM_DOMAIN_GART; 261 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 262 nvbo->valid_domains &= domain; 263 264 *pnvbo = nvbo; 265 return 0; 266 } 267 268 static int 269 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 270 struct drm_nouveau_gem_info *rep) 271 { 272 struct nouveau_cli *cli = nouveau_cli(file_priv); 273 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 274 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 275 struct nouveau_vma *vma; 276 277 if (is_power_of_2(nvbo->valid_domains)) 278 rep->domain = nvbo->valid_domains; 279 else if (nvbo->bo.resource->mem_type == TTM_PL_TT) 280 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 281 else 282 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 283 rep->offset = nvbo->offset; 284 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 285 vma = nouveau_vma_find(nvbo, vmm); 286 if (!vma) 287 return -EINVAL; 288 289 rep->offset = vma->addr; 290 } 291 292 rep->size = nvbo->bo.base.size; 293 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 294 rep->tile_mode = nvbo->mode; 295 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 296 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 297 rep->tile_flags |= nvbo->kind << 8; 298 else 299 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 300 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 301 else 302 rep->tile_flags |= nvbo->zeta; 303 return 0; 304 } 305 306 int 307 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 308 struct drm_file *file_priv) 309 { 310 struct nouveau_cli *cli = nouveau_cli(file_priv); 311 struct drm_nouveau_gem_new *req = data; 312 struct nouveau_bo *nvbo = NULL; 313 int ret = 0; 314 315 ret = nouveau_gem_new(cli, req->info.size, req->align, 316 req->info.domain, req->info.tile_mode, 317 req->info.tile_flags, &nvbo); 318 if (ret) 319 return ret; 320 321 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 322 &req->info.handle); 323 if (ret == 0) { 324 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 325 if (ret) 326 drm_gem_handle_delete(file_priv, req->info.handle); 327 } 328 329 /* drop reference from allocate - handle holds it now */ 330 drm_gem_object_put(&nvbo->bo.base); 331 return ret; 332 } 333 334 static int 335 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 336 uint32_t write_domains, uint32_t valid_domains) 337 { 338 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 339 struct ttm_buffer_object *bo = &nvbo->bo; 340 uint32_t domains = valid_domains & nvbo->valid_domains & 341 (write_domains ? write_domains : read_domains); 342 uint32_t pref_domains = 0;; 343 344 if (!domains) 345 return -EINVAL; 346 347 valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART); 348 349 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 350 bo->resource->mem_type == TTM_PL_VRAM) 351 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 352 353 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 354 bo->resource->mem_type == TTM_PL_TT) 355 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 356 357 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 358 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 359 360 else 361 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 362 363 nouveau_bo_placement_set(nvbo, pref_domains, valid_domains); 364 365 return 0; 366 } 367 368 struct validate_op { 369 struct list_head list; 370 struct ww_acquire_ctx ticket; 371 }; 372 373 static void 374 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 375 struct nouveau_fence *fence, 376 struct drm_nouveau_gem_pushbuf_bo *pbbo) 377 { 378 struct nouveau_bo *nvbo; 379 struct drm_nouveau_gem_pushbuf_bo *b; 380 381 while (!list_empty(&op->list)) { 382 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 383 b = &pbbo[nvbo->pbbo_index]; 384 385 if (likely(fence)) { 386 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 387 388 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 389 struct nouveau_vma *vma = 390 (void *)(unsigned long)b->user_priv; 391 nouveau_fence_unref(&vma->fence); 392 dma_fence_get(&fence->base); 393 vma->fence = fence; 394 } 395 } 396 397 if (unlikely(nvbo->validate_mapped)) { 398 ttm_bo_kunmap(&nvbo->kmap); 399 nvbo->validate_mapped = false; 400 } 401 402 list_del(&nvbo->entry); 403 nvbo->reserved_by = NULL; 404 ttm_bo_unreserve(&nvbo->bo); 405 drm_gem_object_put(&nvbo->bo.base); 406 } 407 } 408 409 static void 410 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 411 struct nouveau_fence *fence, 412 struct drm_nouveau_gem_pushbuf_bo *pbbo) 413 { 414 validate_fini_no_ticket(op, chan, fence, pbbo); 415 ww_acquire_fini(&op->ticket); 416 } 417 418 static int 419 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 420 struct drm_nouveau_gem_pushbuf_bo *pbbo, 421 int nr_buffers, struct validate_op *op) 422 { 423 struct nouveau_cli *cli = nouveau_cli(file_priv); 424 int trycnt = 0; 425 int ret = -EINVAL, i; 426 struct nouveau_bo *res_bo = NULL; 427 LIST_HEAD(gart_list); 428 LIST_HEAD(vram_list); 429 LIST_HEAD(both_list); 430 431 ww_acquire_init(&op->ticket, &reservation_ww_class); 432 retry: 433 if (++trycnt > 100000) { 434 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 435 return -EINVAL; 436 } 437 438 for (i = 0; i < nr_buffers; i++) { 439 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 440 struct drm_gem_object *gem; 441 struct nouveau_bo *nvbo; 442 443 gem = drm_gem_object_lookup(file_priv, b->handle); 444 if (!gem) { 445 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 446 ret = -ENOENT; 447 break; 448 } 449 nvbo = nouveau_gem_object(gem); 450 if (nvbo == res_bo) { 451 res_bo = NULL; 452 drm_gem_object_put(gem); 453 continue; 454 } 455 456 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 457 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 458 "validation list\n", b->handle); 459 drm_gem_object_put(gem); 460 ret = -EINVAL; 461 break; 462 } 463 464 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 465 if (ret) { 466 list_splice_tail_init(&vram_list, &op->list); 467 list_splice_tail_init(&gart_list, &op->list); 468 list_splice_tail_init(&both_list, &op->list); 469 validate_fini_no_ticket(op, chan, NULL, NULL); 470 if (unlikely(ret == -EDEADLK)) { 471 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 472 &op->ticket); 473 if (!ret) 474 res_bo = nvbo; 475 } 476 if (unlikely(ret)) { 477 if (ret != -ERESTARTSYS) 478 NV_PRINTK(err, cli, "fail reserve\n"); 479 break; 480 } 481 } 482 483 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 484 struct nouveau_vmm *vmm = chan->vmm; 485 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 486 if (!vma) { 487 NV_PRINTK(err, cli, "vma not found!\n"); 488 ret = -EINVAL; 489 break; 490 } 491 492 b->user_priv = (uint64_t)(unsigned long)vma; 493 } else { 494 b->user_priv = (uint64_t)(unsigned long)nvbo; 495 } 496 497 nvbo->reserved_by = file_priv; 498 nvbo->pbbo_index = i; 499 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 500 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 501 list_add_tail(&nvbo->entry, &both_list); 502 else 503 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 504 list_add_tail(&nvbo->entry, &vram_list); 505 else 506 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 507 list_add_tail(&nvbo->entry, &gart_list); 508 else { 509 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 510 b->valid_domains); 511 list_add_tail(&nvbo->entry, &both_list); 512 ret = -EINVAL; 513 break; 514 } 515 if (nvbo == res_bo) 516 goto retry; 517 } 518 519 ww_acquire_done(&op->ticket); 520 list_splice_tail(&vram_list, &op->list); 521 list_splice_tail(&gart_list, &op->list); 522 list_splice_tail(&both_list, &op->list); 523 if (ret) 524 validate_fini(op, chan, NULL, NULL); 525 return ret; 526 527 } 528 529 static int 530 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 531 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 532 { 533 struct nouveau_drm *drm = chan->drm; 534 struct nouveau_bo *nvbo; 535 int ret, relocs = 0; 536 537 list_for_each_entry(nvbo, list, entry) { 538 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 539 540 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 541 b->write_domains, 542 b->valid_domains); 543 if (unlikely(ret)) { 544 NV_PRINTK(err, cli, "fail set_domain\n"); 545 return ret; 546 } 547 548 ret = nouveau_bo_validate(nvbo, true, false); 549 if (unlikely(ret)) { 550 if (ret != -ERESTARTSYS) 551 NV_PRINTK(err, cli, "fail ttm_validate\n"); 552 return ret; 553 } 554 555 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 556 if (unlikely(ret)) { 557 if (ret != -ERESTARTSYS) 558 NV_PRINTK(err, cli, "fail post-validate sync\n"); 559 return ret; 560 } 561 562 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 563 if (nvbo->offset == b->presumed.offset && 564 ((nvbo->bo.resource->mem_type == TTM_PL_VRAM && 565 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 566 (nvbo->bo.resource->mem_type == TTM_PL_TT && 567 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 568 continue; 569 570 if (nvbo->bo.resource->mem_type == TTM_PL_TT) 571 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 572 else 573 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 574 b->presumed.offset = nvbo->offset; 575 b->presumed.valid = 0; 576 relocs++; 577 } 578 } 579 580 return relocs; 581 } 582 583 static int 584 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 585 struct drm_file *file_priv, 586 struct drm_nouveau_gem_pushbuf_bo *pbbo, 587 int nr_buffers, 588 struct validate_op *op, bool *apply_relocs) 589 { 590 struct nouveau_cli *cli = nouveau_cli(file_priv); 591 int ret; 592 593 INIT_LIST_HEAD(&op->list); 594 595 if (nr_buffers == 0) 596 return 0; 597 598 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 599 if (unlikely(ret)) { 600 if (ret != -ERESTARTSYS) 601 NV_PRINTK(err, cli, "validate_init\n"); 602 return ret; 603 } 604 605 ret = validate_list(chan, cli, &op->list, pbbo); 606 if (unlikely(ret < 0)) { 607 if (ret != -ERESTARTSYS) 608 NV_PRINTK(err, cli, "validating bo list\n"); 609 validate_fini(op, chan, NULL, NULL); 610 return ret; 611 } else if (ret > 0) { 612 *apply_relocs = true; 613 } 614 615 return 0; 616 } 617 618 static inline void 619 u_free(void *addr) 620 { 621 kvfree(addr); 622 } 623 624 static inline void * 625 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 626 { 627 void *mem; 628 void __user *userptr = (void __force __user *)(uintptr_t)user; 629 630 size *= nmemb; 631 632 mem = kvmalloc(size, GFP_KERNEL); 633 if (!mem) 634 return ERR_PTR(-ENOMEM); 635 636 if (copy_from_user(mem, userptr, size)) { 637 u_free(mem); 638 return ERR_PTR(-EFAULT); 639 } 640 641 return mem; 642 } 643 644 static int 645 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 646 struct drm_nouveau_gem_pushbuf *req, 647 struct drm_nouveau_gem_pushbuf_reloc *reloc, 648 struct drm_nouveau_gem_pushbuf_bo *bo) 649 { 650 int ret = 0; 651 unsigned i; 652 653 for (i = 0; i < req->nr_relocs; i++) { 654 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 655 struct drm_nouveau_gem_pushbuf_bo *b; 656 struct nouveau_bo *nvbo; 657 uint32_t data; 658 659 if (unlikely(r->bo_index >= req->nr_buffers)) { 660 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 661 ret = -EINVAL; 662 break; 663 } 664 665 b = &bo[r->bo_index]; 666 if (b->presumed.valid) 667 continue; 668 669 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 670 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 671 ret = -EINVAL; 672 break; 673 } 674 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 675 676 if (unlikely(r->reloc_bo_offset + 4 > 677 nvbo->bo.base.size)) { 678 NV_PRINTK(err, cli, "reloc outside of bo\n"); 679 ret = -EINVAL; 680 break; 681 } 682 683 if (!nvbo->kmap.virtual) { 684 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.resource->num_pages, 685 &nvbo->kmap); 686 if (ret) { 687 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 688 break; 689 } 690 nvbo->validate_mapped = true; 691 } 692 693 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 694 data = b->presumed.offset + r->data; 695 else 696 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 697 data = (b->presumed.offset + r->data) >> 32; 698 else 699 data = r->data; 700 701 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 702 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 703 data |= r->tor; 704 else 705 data |= r->vor; 706 } 707 708 ret = ttm_bo_wait(&nvbo->bo, false, false); 709 if (ret) { 710 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 711 break; 712 } 713 714 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 715 } 716 717 return ret; 718 } 719 720 int 721 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 722 struct drm_file *file_priv) 723 { 724 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 725 struct nouveau_cli *cli = nouveau_cli(file_priv); 726 struct nouveau_abi16_chan *temp; 727 struct nouveau_drm *drm = nouveau_drm(dev); 728 struct drm_nouveau_gem_pushbuf *req = data; 729 struct drm_nouveau_gem_pushbuf_push *push; 730 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 731 struct drm_nouveau_gem_pushbuf_bo *bo; 732 struct nouveau_channel *chan = NULL; 733 struct validate_op op; 734 struct nouveau_fence *fence = NULL; 735 int i, j, ret = 0; 736 bool do_reloc = false, sync = false; 737 738 if (unlikely(!abi16)) 739 return -ENOMEM; 740 741 list_for_each_entry(temp, &abi16->channels, head) { 742 if (temp->chan->chid == req->channel) { 743 chan = temp->chan; 744 break; 745 } 746 } 747 748 if (!chan) 749 return nouveau_abi16_put(abi16, -ENOENT); 750 if (unlikely(atomic_read(&chan->killed))) 751 return nouveau_abi16_put(abi16, -ENODEV); 752 753 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 754 755 req->vram_available = drm->gem.vram_available; 756 req->gart_available = drm->gem.gart_available; 757 if (unlikely(req->nr_push == 0)) 758 goto out_next; 759 760 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 761 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 762 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 763 return nouveau_abi16_put(abi16, -EINVAL); 764 } 765 766 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 767 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 768 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 769 return nouveau_abi16_put(abi16, -EINVAL); 770 } 771 772 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 773 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 774 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 775 return nouveau_abi16_put(abi16, -EINVAL); 776 } 777 778 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 779 if (IS_ERR(push)) 780 return nouveau_abi16_put(abi16, PTR_ERR(push)); 781 782 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 783 if (IS_ERR(bo)) { 784 u_free(push); 785 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 786 } 787 788 /* Ensure all push buffers are on validate list */ 789 for (i = 0; i < req->nr_push; i++) { 790 if (push[i].bo_index >= req->nr_buffers) { 791 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 792 ret = -EINVAL; 793 goto out_prevalid; 794 } 795 } 796 797 /* Validate buffer list */ 798 revalidate: 799 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 800 req->nr_buffers, &op, &do_reloc); 801 if (ret) { 802 if (ret != -ERESTARTSYS) 803 NV_PRINTK(err, cli, "validate: %d\n", ret); 804 goto out_prevalid; 805 } 806 807 /* Apply any relocations that are required */ 808 if (do_reloc) { 809 if (!reloc) { 810 validate_fini(&op, chan, NULL, bo); 811 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 812 if (IS_ERR(reloc)) { 813 ret = PTR_ERR(reloc); 814 goto out_prevalid; 815 } 816 817 goto revalidate; 818 } 819 820 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 821 if (ret) { 822 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 823 goto out; 824 } 825 } 826 827 if (chan->dma.ib_max) { 828 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 829 if (ret) { 830 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 831 goto out; 832 } 833 834 for (i = 0; i < req->nr_push; i++) { 835 struct nouveau_vma *vma = (void *)(unsigned long) 836 bo[push[i].bo_index].user_priv; 837 838 nv50_dma_push(chan, vma->addr + push[i].offset, 839 push[i].length); 840 } 841 } else 842 if (drm->client.device.info.chipset >= 0x25) { 843 ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2); 844 if (ret) { 845 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 846 goto out; 847 } 848 849 for (i = 0; i < req->nr_push; i++) { 850 struct nouveau_bo *nvbo = (void *)(unsigned long) 851 bo[push[i].bo_index].user_priv; 852 853 PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset); 854 PUSH_DATA(chan->chan.push, 0); 855 } 856 } else { 857 ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 858 if (ret) { 859 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 860 goto out; 861 } 862 863 for (i = 0; i < req->nr_push; i++) { 864 struct nouveau_bo *nvbo = (void *)(unsigned long) 865 bo[push[i].bo_index].user_priv; 866 uint32_t cmd; 867 868 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 869 cmd |= 0x20000000; 870 if (unlikely(cmd != req->suffix0)) { 871 if (!nvbo->kmap.virtual) { 872 ret = ttm_bo_kmap(&nvbo->bo, 0, 873 nvbo->bo.resource-> 874 num_pages, 875 &nvbo->kmap); 876 if (ret) { 877 WIND_RING(chan); 878 goto out; 879 } 880 nvbo->validate_mapped = true; 881 } 882 883 nouveau_bo_wr32(nvbo, (push[i].offset + 884 push[i].length - 8) / 4, cmd); 885 } 886 887 PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset); 888 PUSH_DATA(chan->chan.push, 0); 889 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 890 PUSH_DATA(chan->chan.push, 0); 891 } 892 } 893 894 ret = nouveau_fence_new(chan, false, &fence); 895 if (ret) { 896 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 897 WIND_RING(chan); 898 goto out; 899 } 900 901 if (sync) { 902 if (!(ret = nouveau_fence_wait(fence, false, false))) { 903 if ((ret = dma_fence_get_status(&fence->base)) == 1) 904 ret = 0; 905 } 906 } 907 908 out: 909 validate_fini(&op, chan, fence, bo); 910 nouveau_fence_unref(&fence); 911 912 if (do_reloc) { 913 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 914 u64_to_user_ptr(req->buffers); 915 916 for (i = 0; i < req->nr_buffers; i++) { 917 if (bo[i].presumed.valid) 918 continue; 919 920 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 921 sizeof(bo[i].presumed))) { 922 ret = -EFAULT; 923 break; 924 } 925 } 926 } 927 out_prevalid: 928 if (!IS_ERR(reloc)) 929 u_free(reloc); 930 u_free(bo); 931 u_free(push); 932 933 out_next: 934 if (chan->dma.ib_max) { 935 req->suffix0 = 0x00000000; 936 req->suffix1 = 0x00000000; 937 } else 938 if (drm->client.device.info.chipset >= 0x25) { 939 req->suffix0 = 0x00020000; 940 req->suffix1 = 0x00000000; 941 } else { 942 req->suffix0 = 0x20000000 | 943 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 944 req->suffix1 = 0x00000000; 945 } 946 947 return nouveau_abi16_put(abi16, ret); 948 } 949 950 int 951 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 952 struct drm_file *file_priv) 953 { 954 struct drm_nouveau_gem_cpu_prep *req = data; 955 struct drm_gem_object *gem; 956 struct nouveau_bo *nvbo; 957 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 958 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 959 long lret; 960 int ret; 961 962 gem = drm_gem_object_lookup(file_priv, req->handle); 963 if (!gem) 964 return -ENOENT; 965 nvbo = nouveau_gem_object(gem); 966 967 lret = dma_resv_wait_timeout(nvbo->bo.base.resv, write, true, 968 no_wait ? 0 : 30 * HZ); 969 if (!lret) 970 ret = -EBUSY; 971 else if (lret > 0) 972 ret = 0; 973 else 974 ret = lret; 975 976 nouveau_bo_sync_for_cpu(nvbo); 977 drm_gem_object_put(gem); 978 979 return ret; 980 } 981 982 int 983 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 984 struct drm_file *file_priv) 985 { 986 struct drm_nouveau_gem_cpu_fini *req = data; 987 struct drm_gem_object *gem; 988 struct nouveau_bo *nvbo; 989 990 gem = drm_gem_object_lookup(file_priv, req->handle); 991 if (!gem) 992 return -ENOENT; 993 nvbo = nouveau_gem_object(gem); 994 995 nouveau_bo_sync_for_device(nvbo); 996 drm_gem_object_put(gem); 997 return 0; 998 } 999 1000 int 1001 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 1002 struct drm_file *file_priv) 1003 { 1004 struct drm_nouveau_gem_info *req = data; 1005 struct drm_gem_object *gem; 1006 int ret; 1007 1008 gem = drm_gem_object_lookup(file_priv, req->handle); 1009 if (!gem) 1010 return -ENOENT; 1011 1012 ret = nouveau_gem_info(file_priv, gem, req); 1013 drm_gem_object_put(gem); 1014 return ret; 1015 } 1016 1017