1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include <drm/drm_gem_ttm_helper.h> 28 29 #include "nouveau_drv.h" 30 #include "nouveau_dma.h" 31 #include "nouveau_fence.h" 32 #include "nouveau_abi16.h" 33 34 #include "nouveau_ttm.h" 35 #include "nouveau_gem.h" 36 #include "nouveau_mem.h" 37 #include "nouveau_vmm.h" 38 39 #include <nvif/class.h> 40 #include <nvif/push206e.h> 41 42 static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf) 43 { 44 struct vm_area_struct *vma = vmf->vma; 45 struct ttm_buffer_object *bo = vma->vm_private_data; 46 pgprot_t prot; 47 vm_fault_t ret; 48 49 ret = ttm_bo_vm_reserve(bo, vmf); 50 if (ret) 51 return ret; 52 53 ret = nouveau_ttm_fault_reserve_notify(bo); 54 if (ret) 55 goto error_unlock; 56 57 nouveau_bo_del_io_reserve_lru(bo); 58 prot = vm_get_page_prot(vma->vm_flags); 59 ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); 60 nouveau_bo_add_io_reserve_lru(bo); 61 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 62 return ret; 63 64 error_unlock: 65 dma_resv_unlock(bo->base.resv); 66 return ret; 67 } 68 69 static const struct vm_operations_struct nouveau_ttm_vm_ops = { 70 .fault = nouveau_ttm_fault, 71 .open = ttm_bo_vm_open, 72 .close = ttm_bo_vm_close, 73 .access = ttm_bo_vm_access 74 }; 75 76 void 77 nouveau_gem_object_del(struct drm_gem_object *gem) 78 { 79 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 80 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 81 struct device *dev = drm->dev->dev; 82 int ret; 83 84 ret = pm_runtime_get_sync(dev); 85 if (WARN_ON(ret < 0 && ret != -EACCES)) { 86 pm_runtime_put_autosuspend(dev); 87 return; 88 } 89 90 if (gem->import_attach) 91 drm_prime_gem_destroy(gem, nvbo->bo.sg); 92 93 ttm_bo_put(&nvbo->bo); 94 95 pm_runtime_mark_last_busy(dev); 96 pm_runtime_put_autosuspend(dev); 97 } 98 99 int 100 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 101 { 102 struct nouveau_cli *cli = nouveau_cli(file_priv); 103 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 104 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 105 struct device *dev = drm->dev->dev; 106 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli); 107 struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); 108 struct nouveau_vma *vma; 109 int ret; 110 111 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 112 return 0; 113 114 if (nvbo->no_share && uvmm && &uvmm->resv != nvbo->bo.base.resv) 115 return -EPERM; 116 117 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 118 if (ret) 119 return ret; 120 121 ret = pm_runtime_get_sync(dev); 122 if (ret < 0 && ret != -EACCES) { 123 pm_runtime_put_autosuspend(dev); 124 goto out; 125 } 126 127 /* only create a VMA on binding */ 128 if (!nouveau_cli_uvmm(cli)) 129 ret = nouveau_vma_new(nvbo, vmm, &vma); 130 else 131 ret = 0; 132 pm_runtime_mark_last_busy(dev); 133 pm_runtime_put_autosuspend(dev); 134 out: 135 ttm_bo_unreserve(&nvbo->bo); 136 return ret; 137 } 138 139 struct nouveau_gem_object_unmap { 140 struct nouveau_cli_work work; 141 struct nouveau_vma *vma; 142 }; 143 144 static void 145 nouveau_gem_object_delete(struct nouveau_vma *vma) 146 { 147 nouveau_fence_unref(&vma->fence); 148 nouveau_vma_del(&vma); 149 } 150 151 static void 152 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 153 { 154 struct nouveau_gem_object_unmap *work = 155 container_of(w, typeof(*work), work); 156 nouveau_gem_object_delete(work->vma); 157 kfree(work); 158 } 159 160 static void 161 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 162 { 163 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 164 struct nouveau_gem_object_unmap *work; 165 166 list_del_init(&vma->head); 167 168 if (!fence) { 169 nouveau_gem_object_delete(vma); 170 return; 171 } 172 173 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 174 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 175 nouveau_gem_object_delete(vma); 176 return; 177 } 178 179 work->work.func = nouveau_gem_object_delete_work; 180 work->vma = vma; 181 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 182 } 183 184 void 185 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 186 { 187 struct nouveau_cli *cli = nouveau_cli(file_priv); 188 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 189 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 190 struct device *dev = drm->dev->dev; 191 struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); 192 struct nouveau_vma *vma; 193 int ret; 194 195 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 196 return; 197 198 if (nouveau_cli_uvmm(cli)) 199 return; 200 201 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 202 if (ret) 203 return; 204 205 vma = nouveau_vma_find(nvbo, vmm); 206 if (vma) { 207 if (--vma->refs == 0) { 208 ret = pm_runtime_get_sync(dev); 209 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 210 nouveau_gem_object_unmap(nvbo, vma); 211 pm_runtime_mark_last_busy(dev); 212 } 213 pm_runtime_put_autosuspend(dev); 214 } 215 } 216 ttm_bo_unreserve(&nvbo->bo); 217 } 218 219 const struct drm_gem_object_funcs nouveau_gem_object_funcs = { 220 .free = nouveau_gem_object_del, 221 .open = nouveau_gem_object_open, 222 .close = nouveau_gem_object_close, 223 .export = nouveau_gem_prime_export, 224 .pin = nouveau_gem_prime_pin, 225 .unpin = nouveau_gem_prime_unpin, 226 .get_sg_table = nouveau_gem_prime_get_sg_table, 227 .vmap = drm_gem_ttm_vmap, 228 .vunmap = drm_gem_ttm_vunmap, 229 .mmap = drm_gem_ttm_mmap, 230 .vm_ops = &nouveau_ttm_vm_ops, 231 }; 232 233 int 234 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 235 uint32_t tile_mode, uint32_t tile_flags, 236 struct nouveau_bo **pnvbo) 237 { 238 struct nouveau_drm *drm = cli->drm; 239 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli); 240 struct dma_resv *resv = NULL; 241 struct nouveau_bo *nvbo; 242 int ret; 243 244 if (domain & NOUVEAU_GEM_DOMAIN_NO_SHARE) { 245 if (unlikely(!uvmm)) 246 return -EINVAL; 247 248 resv = &uvmm->resv; 249 } 250 251 if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) 252 domain |= NOUVEAU_GEM_DOMAIN_CPU; 253 254 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 255 tile_flags, false); 256 if (IS_ERR(nvbo)) 257 return PTR_ERR(nvbo); 258 259 nvbo->bo.base.funcs = &nouveau_gem_object_funcs; 260 nvbo->no_share = domain & NOUVEAU_GEM_DOMAIN_NO_SHARE; 261 262 /* Initialize the embedded gem-object. We return a single gem-reference 263 * to the caller, instead of a normal nouveau_bo ttm reference. */ 264 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 265 if (ret) { 266 drm_gem_object_release(&nvbo->bo.base); 267 kfree(nvbo); 268 return ret; 269 } 270 271 if (resv) 272 dma_resv_lock(resv, NULL); 273 274 ret = nouveau_bo_init(nvbo, size, align, domain, NULL, resv); 275 276 if (resv) 277 dma_resv_unlock(resv); 278 279 if (ret) 280 return ret; 281 282 /* we restrict allowed domains on nv50+ to only the types 283 * that were requested at creation time. not possibly on 284 * earlier chips without busting the ABI. 285 */ 286 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 287 NOUVEAU_GEM_DOMAIN_GART; 288 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 289 nvbo->valid_domains &= domain; 290 291 *pnvbo = nvbo; 292 return 0; 293 } 294 295 static int 296 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 297 struct drm_nouveau_gem_info *rep) 298 { 299 struct nouveau_cli *cli = nouveau_cli(file_priv); 300 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 301 struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); 302 struct nouveau_vma *vma; 303 304 if (is_power_of_2(nvbo->valid_domains)) 305 rep->domain = nvbo->valid_domains; 306 else if (nvbo->bo.resource->mem_type == TTM_PL_TT) 307 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 308 else 309 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 310 rep->offset = nvbo->offset; 311 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50 && 312 !nouveau_cli_uvmm(cli)) { 313 vma = nouveau_vma_find(nvbo, vmm); 314 if (!vma) 315 return -EINVAL; 316 317 rep->offset = vma->addr; 318 } else 319 rep->offset = 0; 320 321 rep->size = nvbo->bo.base.size; 322 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 323 rep->tile_mode = nvbo->mode; 324 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 325 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 326 rep->tile_flags |= nvbo->kind << 8; 327 else 328 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 329 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 330 else 331 rep->tile_flags |= nvbo->zeta; 332 return 0; 333 } 334 335 int 336 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 337 struct drm_file *file_priv) 338 { 339 struct nouveau_cli *cli = nouveau_cli(file_priv); 340 struct drm_nouveau_gem_new *req = data; 341 struct nouveau_bo *nvbo = NULL; 342 int ret = 0; 343 344 /* If uvmm wasn't initialized until now disable it completely to prevent 345 * userspace from mixing up UAPIs. 346 */ 347 nouveau_cli_disable_uvmm_noinit(cli); 348 349 ret = nouveau_gem_new(cli, req->info.size, req->align, 350 req->info.domain, req->info.tile_mode, 351 req->info.tile_flags, &nvbo); 352 if (ret) 353 return ret; 354 355 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 356 &req->info.handle); 357 if (ret == 0) { 358 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 359 if (ret) 360 drm_gem_handle_delete(file_priv, req->info.handle); 361 } 362 363 /* drop reference from allocate - handle holds it now */ 364 drm_gem_object_put(&nvbo->bo.base); 365 return ret; 366 } 367 368 static int 369 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 370 uint32_t write_domains, uint32_t valid_domains) 371 { 372 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 373 struct ttm_buffer_object *bo = &nvbo->bo; 374 uint32_t domains = valid_domains & nvbo->valid_domains & 375 (write_domains ? write_domains : read_domains); 376 uint32_t pref_domains = 0; 377 378 if (!domains) 379 return -EINVAL; 380 381 valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART); 382 383 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 384 bo->resource->mem_type == TTM_PL_VRAM) 385 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 386 387 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 388 bo->resource->mem_type == TTM_PL_TT) 389 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 390 391 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 392 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 393 394 else 395 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 396 397 nouveau_bo_placement_set(nvbo, pref_domains, valid_domains); 398 399 return 0; 400 } 401 402 struct validate_op { 403 struct list_head list; 404 struct ww_acquire_ctx ticket; 405 }; 406 407 static void 408 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 409 struct nouveau_fence *fence, 410 struct drm_nouveau_gem_pushbuf_bo *pbbo) 411 { 412 struct nouveau_bo *nvbo; 413 struct drm_nouveau_gem_pushbuf_bo *b; 414 415 while (!list_empty(&op->list)) { 416 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 417 b = &pbbo[nvbo->pbbo_index]; 418 419 if (likely(fence)) { 420 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 421 422 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 423 struct nouveau_vma *vma = 424 (void *)(unsigned long)b->user_priv; 425 nouveau_fence_unref(&vma->fence); 426 dma_fence_get(&fence->base); 427 vma->fence = fence; 428 } 429 } 430 431 if (unlikely(nvbo->validate_mapped)) { 432 ttm_bo_kunmap(&nvbo->kmap); 433 nvbo->validate_mapped = false; 434 } 435 436 list_del(&nvbo->entry); 437 nvbo->reserved_by = NULL; 438 ttm_bo_unreserve(&nvbo->bo); 439 drm_gem_object_put(&nvbo->bo.base); 440 } 441 } 442 443 static void 444 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 445 struct nouveau_fence *fence, 446 struct drm_nouveau_gem_pushbuf_bo *pbbo) 447 { 448 validate_fini_no_ticket(op, chan, fence, pbbo); 449 ww_acquire_fini(&op->ticket); 450 } 451 452 static int 453 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 454 struct drm_nouveau_gem_pushbuf_bo *pbbo, 455 int nr_buffers, struct validate_op *op) 456 { 457 struct nouveau_cli *cli = nouveau_cli(file_priv); 458 int trycnt = 0; 459 int ret = -EINVAL, i; 460 struct nouveau_bo *res_bo = NULL; 461 LIST_HEAD(gart_list); 462 LIST_HEAD(vram_list); 463 LIST_HEAD(both_list); 464 465 ww_acquire_init(&op->ticket, &reservation_ww_class); 466 retry: 467 if (++trycnt > 100000) { 468 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 469 return -EINVAL; 470 } 471 472 for (i = 0; i < nr_buffers; i++) { 473 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 474 struct drm_gem_object *gem; 475 struct nouveau_bo *nvbo; 476 477 gem = drm_gem_object_lookup(file_priv, b->handle); 478 if (!gem) { 479 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 480 ret = -ENOENT; 481 break; 482 } 483 nvbo = nouveau_gem_object(gem); 484 if (nvbo == res_bo) { 485 res_bo = NULL; 486 drm_gem_object_put(gem); 487 continue; 488 } 489 490 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 491 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 492 "validation list\n", b->handle); 493 drm_gem_object_put(gem); 494 ret = -EINVAL; 495 break; 496 } 497 498 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 499 if (ret) { 500 list_splice_tail_init(&vram_list, &op->list); 501 list_splice_tail_init(&gart_list, &op->list); 502 list_splice_tail_init(&both_list, &op->list); 503 validate_fini_no_ticket(op, chan, NULL, NULL); 504 if (unlikely(ret == -EDEADLK)) { 505 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 506 &op->ticket); 507 if (!ret) 508 res_bo = nvbo; 509 } 510 if (unlikely(ret)) { 511 if (ret != -ERESTARTSYS) 512 NV_PRINTK(err, cli, "fail reserve\n"); 513 break; 514 } 515 } 516 517 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 518 struct nouveau_vmm *vmm = chan->vmm; 519 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 520 if (!vma) { 521 NV_PRINTK(err, cli, "vma not found!\n"); 522 ret = -EINVAL; 523 break; 524 } 525 526 b->user_priv = (uint64_t)(unsigned long)vma; 527 } else { 528 b->user_priv = (uint64_t)(unsigned long)nvbo; 529 } 530 531 nvbo->reserved_by = file_priv; 532 nvbo->pbbo_index = i; 533 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 534 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 535 list_add_tail(&nvbo->entry, &both_list); 536 else 537 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 538 list_add_tail(&nvbo->entry, &vram_list); 539 else 540 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 541 list_add_tail(&nvbo->entry, &gart_list); 542 else { 543 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 544 b->valid_domains); 545 list_add_tail(&nvbo->entry, &both_list); 546 ret = -EINVAL; 547 break; 548 } 549 if (nvbo == res_bo) 550 goto retry; 551 } 552 553 ww_acquire_done(&op->ticket); 554 list_splice_tail(&vram_list, &op->list); 555 list_splice_tail(&gart_list, &op->list); 556 list_splice_tail(&both_list, &op->list); 557 if (ret) 558 validate_fini(op, chan, NULL, NULL); 559 return ret; 560 561 } 562 563 static int 564 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 565 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 566 { 567 struct nouveau_drm *drm = chan->drm; 568 struct nouveau_bo *nvbo; 569 int ret, relocs = 0; 570 571 list_for_each_entry(nvbo, list, entry) { 572 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 573 574 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 575 b->write_domains, 576 b->valid_domains); 577 if (unlikely(ret)) { 578 NV_PRINTK(err, cli, "fail set_domain\n"); 579 return ret; 580 } 581 582 ret = nouveau_bo_validate(nvbo, true, false); 583 if (unlikely(ret)) { 584 if (ret != -ERESTARTSYS) 585 NV_PRINTK(err, cli, "fail ttm_validate\n"); 586 return ret; 587 } 588 589 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 590 if (unlikely(ret)) { 591 if (ret != -ERESTARTSYS) 592 NV_PRINTK(err, cli, "fail post-validate sync\n"); 593 return ret; 594 } 595 596 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 597 if (nvbo->offset == b->presumed.offset && 598 ((nvbo->bo.resource->mem_type == TTM_PL_VRAM && 599 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 600 (nvbo->bo.resource->mem_type == TTM_PL_TT && 601 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 602 continue; 603 604 if (nvbo->bo.resource->mem_type == TTM_PL_TT) 605 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 606 else 607 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 608 b->presumed.offset = nvbo->offset; 609 b->presumed.valid = 0; 610 relocs++; 611 } 612 } 613 614 return relocs; 615 } 616 617 static int 618 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 619 struct drm_file *file_priv, 620 struct drm_nouveau_gem_pushbuf_bo *pbbo, 621 int nr_buffers, 622 struct validate_op *op, bool *apply_relocs) 623 { 624 struct nouveau_cli *cli = nouveau_cli(file_priv); 625 int ret; 626 627 INIT_LIST_HEAD(&op->list); 628 629 if (nr_buffers == 0) 630 return 0; 631 632 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 633 if (unlikely(ret)) { 634 if (ret != -ERESTARTSYS) 635 NV_PRINTK(err, cli, "validate_init\n"); 636 return ret; 637 } 638 639 ret = validate_list(chan, cli, &op->list, pbbo); 640 if (unlikely(ret < 0)) { 641 if (ret != -ERESTARTSYS) 642 NV_PRINTK(err, cli, "validating bo list\n"); 643 validate_fini(op, chan, NULL, NULL); 644 return ret; 645 } else if (ret > 0) { 646 *apply_relocs = true; 647 } 648 649 return 0; 650 } 651 652 static int 653 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 654 struct drm_nouveau_gem_pushbuf *req, 655 struct drm_nouveau_gem_pushbuf_reloc *reloc, 656 struct drm_nouveau_gem_pushbuf_bo *bo) 657 { 658 int ret = 0; 659 unsigned i; 660 661 for (i = 0; i < req->nr_relocs; i++) { 662 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 663 struct drm_nouveau_gem_pushbuf_bo *b; 664 struct nouveau_bo *nvbo; 665 uint32_t data; 666 long lret; 667 668 if (unlikely(r->bo_index >= req->nr_buffers)) { 669 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 670 ret = -EINVAL; 671 break; 672 } 673 674 b = &bo[r->bo_index]; 675 if (b->presumed.valid) 676 continue; 677 678 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 679 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 680 ret = -EINVAL; 681 break; 682 } 683 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 684 685 if (unlikely(r->reloc_bo_offset + 4 > 686 nvbo->bo.base.size)) { 687 NV_PRINTK(err, cli, "reloc outside of bo\n"); 688 ret = -EINVAL; 689 break; 690 } 691 692 if (!nvbo->kmap.virtual) { 693 ret = ttm_bo_kmap(&nvbo->bo, 0, PFN_UP(nvbo->bo.base.size), 694 &nvbo->kmap); 695 if (ret) { 696 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 697 break; 698 } 699 nvbo->validate_mapped = true; 700 } 701 702 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 703 data = b->presumed.offset + r->data; 704 else 705 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 706 data = (b->presumed.offset + r->data) >> 32; 707 else 708 data = r->data; 709 710 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 711 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 712 data |= r->tor; 713 else 714 data |= r->vor; 715 } 716 717 lret = dma_resv_wait_timeout(nvbo->bo.base.resv, 718 DMA_RESV_USAGE_BOOKKEEP, 719 false, 15 * HZ); 720 if (!lret) 721 ret = -EBUSY; 722 else if (lret > 0) 723 ret = 0; 724 else 725 ret = lret; 726 727 if (ret) { 728 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", 729 ret); 730 break; 731 } 732 733 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 734 } 735 736 return ret; 737 } 738 739 int 740 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 741 struct drm_file *file_priv) 742 { 743 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 744 struct nouveau_cli *cli = nouveau_cli(file_priv); 745 struct nouveau_abi16_chan *temp; 746 struct nouveau_drm *drm = nouveau_drm(dev); 747 struct drm_nouveau_gem_pushbuf *req = data; 748 struct drm_nouveau_gem_pushbuf_push *push; 749 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 750 struct drm_nouveau_gem_pushbuf_bo *bo; 751 struct nouveau_channel *chan = NULL; 752 struct validate_op op; 753 struct nouveau_fence *fence = NULL; 754 int i, j, ret = 0; 755 bool do_reloc = false, sync = false; 756 757 if (unlikely(!abi16)) 758 return -ENOMEM; 759 760 if (unlikely(nouveau_cli_uvmm(cli))) 761 return -ENOSYS; 762 763 list_for_each_entry(temp, &abi16->channels, head) { 764 if (temp->chan->chid == req->channel) { 765 chan = temp->chan; 766 break; 767 } 768 } 769 770 if (!chan) 771 return nouveau_abi16_put(abi16, -ENOENT); 772 if (unlikely(atomic_read(&chan->killed))) 773 return nouveau_abi16_put(abi16, -ENODEV); 774 775 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 776 777 req->vram_available = drm->gem.vram_available; 778 req->gart_available = drm->gem.gart_available; 779 if (unlikely(req->nr_push == 0)) 780 goto out_next; 781 782 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 783 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 784 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 785 return nouveau_abi16_put(abi16, -EINVAL); 786 } 787 788 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 789 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 790 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 791 return nouveau_abi16_put(abi16, -EINVAL); 792 } 793 794 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 795 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 796 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 797 return nouveau_abi16_put(abi16, -EINVAL); 798 } 799 800 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 801 if (IS_ERR(push)) 802 return nouveau_abi16_put(abi16, PTR_ERR(push)); 803 804 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 805 if (IS_ERR(bo)) { 806 u_free(push); 807 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 808 } 809 810 /* Ensure all push buffers are on validate list */ 811 for (i = 0; i < req->nr_push; i++) { 812 if (push[i].bo_index >= req->nr_buffers) { 813 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 814 ret = -EINVAL; 815 goto out_prevalid; 816 } 817 } 818 819 /* Validate buffer list */ 820 revalidate: 821 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 822 req->nr_buffers, &op, &do_reloc); 823 if (ret) { 824 if (ret != -ERESTARTSYS) 825 NV_PRINTK(err, cli, "validate: %d\n", ret); 826 goto out_prevalid; 827 } 828 829 /* Apply any relocations that are required */ 830 if (do_reloc) { 831 if (!reloc) { 832 validate_fini(&op, chan, NULL, bo); 833 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 834 if (IS_ERR(reloc)) { 835 ret = PTR_ERR(reloc); 836 goto out_prevalid; 837 } 838 839 goto revalidate; 840 } 841 842 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 843 if (ret) { 844 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 845 goto out; 846 } 847 } 848 849 if (chan->dma.ib_max) { 850 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 851 if (ret) { 852 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 853 goto out; 854 } 855 856 for (i = 0; i < req->nr_push; i++) { 857 struct nouveau_vma *vma = (void *)(unsigned long) 858 bo[push[i].bo_index].user_priv; 859 u64 addr = vma->addr + push[i].offset; 860 u32 length = push[i].length & ~NOUVEAU_GEM_PUSHBUF_NO_PREFETCH; 861 bool no_prefetch = push[i].length & NOUVEAU_GEM_PUSHBUF_NO_PREFETCH; 862 863 nv50_dma_push(chan, addr, length, no_prefetch); 864 } 865 } else 866 if (drm->client.device.info.chipset >= 0x25) { 867 ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2); 868 if (ret) { 869 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 870 goto out; 871 } 872 873 for (i = 0; i < req->nr_push; i++) { 874 struct nouveau_bo *nvbo = (void *)(unsigned long) 875 bo[push[i].bo_index].user_priv; 876 877 PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset); 878 PUSH_DATA(chan->chan.push, 0); 879 } 880 } else { 881 ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 882 if (ret) { 883 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 884 goto out; 885 } 886 887 for (i = 0; i < req->nr_push; i++) { 888 struct nouveau_bo *nvbo = (void *)(unsigned long) 889 bo[push[i].bo_index].user_priv; 890 uint32_t cmd; 891 892 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 893 cmd |= 0x20000000; 894 if (unlikely(cmd != req->suffix0)) { 895 if (!nvbo->kmap.virtual) { 896 ret = ttm_bo_kmap(&nvbo->bo, 0, 897 PFN_UP(nvbo->bo.base.size), 898 &nvbo->kmap); 899 if (ret) { 900 WIND_RING(chan); 901 goto out; 902 } 903 nvbo->validate_mapped = true; 904 } 905 906 nouveau_bo_wr32(nvbo, (push[i].offset + 907 push[i].length - 8) / 4, cmd); 908 } 909 910 PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset); 911 PUSH_DATA(chan->chan.push, 0); 912 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 913 PUSH_DATA(chan->chan.push, 0); 914 } 915 } 916 917 ret = nouveau_fence_new(&fence, chan); 918 if (ret) { 919 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 920 WIND_RING(chan); 921 goto out; 922 } 923 924 if (sync) { 925 if (!(ret = nouveau_fence_wait(fence, false, false))) { 926 if ((ret = dma_fence_get_status(&fence->base)) == 1) 927 ret = 0; 928 } 929 } 930 931 out: 932 validate_fini(&op, chan, fence, bo); 933 nouveau_fence_unref(&fence); 934 935 if (do_reloc) { 936 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 937 u64_to_user_ptr(req->buffers); 938 939 for (i = 0; i < req->nr_buffers; i++) { 940 if (bo[i].presumed.valid) 941 continue; 942 943 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 944 sizeof(bo[i].presumed))) { 945 ret = -EFAULT; 946 break; 947 } 948 } 949 } 950 out_prevalid: 951 if (!IS_ERR(reloc)) 952 u_free(reloc); 953 u_free(bo); 954 u_free(push); 955 956 out_next: 957 if (chan->dma.ib_max) { 958 req->suffix0 = 0x00000000; 959 req->suffix1 = 0x00000000; 960 } else 961 if (drm->client.device.info.chipset >= 0x25) { 962 req->suffix0 = 0x00020000; 963 req->suffix1 = 0x00000000; 964 } else { 965 req->suffix0 = 0x20000000 | 966 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 967 req->suffix1 = 0x00000000; 968 } 969 970 return nouveau_abi16_put(abi16, ret); 971 } 972 973 int 974 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 975 struct drm_file *file_priv) 976 { 977 struct drm_nouveau_gem_cpu_prep *req = data; 978 struct drm_gem_object *gem; 979 struct nouveau_bo *nvbo; 980 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 981 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 982 long lret; 983 int ret; 984 985 gem = drm_gem_object_lookup(file_priv, req->handle); 986 if (!gem) 987 return -ENOENT; 988 nvbo = nouveau_gem_object(gem); 989 990 lret = dma_resv_wait_timeout(nvbo->bo.base.resv, 991 dma_resv_usage_rw(write), true, 992 no_wait ? 0 : 30 * HZ); 993 if (!lret) 994 ret = -EBUSY; 995 else if (lret > 0) 996 ret = 0; 997 else 998 ret = lret; 999 1000 nouveau_bo_sync_for_cpu(nvbo); 1001 drm_gem_object_put(gem); 1002 1003 return ret; 1004 } 1005 1006 int 1007 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 1008 struct drm_file *file_priv) 1009 { 1010 struct drm_nouveau_gem_cpu_fini *req = data; 1011 struct drm_gem_object *gem; 1012 struct nouveau_bo *nvbo; 1013 1014 gem = drm_gem_object_lookup(file_priv, req->handle); 1015 if (!gem) 1016 return -ENOENT; 1017 nvbo = nouveau_gem_object(gem); 1018 1019 nouveau_bo_sync_for_device(nvbo); 1020 drm_gem_object_put(gem); 1021 return 0; 1022 } 1023 1024 int 1025 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 1026 struct drm_file *file_priv) 1027 { 1028 struct drm_nouveau_gem_info *req = data; 1029 struct drm_gem_object *gem; 1030 int ret; 1031 1032 gem = drm_gem_object_lookup(file_priv, req->handle); 1033 if (!gem) 1034 return -ENOENT; 1035 1036 ret = nouveau_gem_info(file_priv, gem, req); 1037 drm_gem_object_put(gem); 1038 return ret; 1039 } 1040 1041