1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 #include <nvif/push206e.h> 39 40 void 41 nouveau_gem_object_del(struct drm_gem_object *gem) 42 { 43 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 44 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 45 struct device *dev = drm->dev->dev; 46 int ret; 47 48 ret = pm_runtime_get_sync(dev); 49 if (WARN_ON(ret < 0 && ret != -EACCES)) { 50 pm_runtime_put_autosuspend(dev); 51 return; 52 } 53 54 if (gem->import_attach) 55 drm_prime_gem_destroy(gem, nvbo->bo.sg); 56 57 ttm_bo_put(&nvbo->bo); 58 59 pm_runtime_mark_last_busy(dev); 60 pm_runtime_put_autosuspend(dev); 61 } 62 63 int 64 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 65 { 66 struct nouveau_cli *cli = nouveau_cli(file_priv); 67 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 68 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 69 struct device *dev = drm->dev->dev; 70 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 71 struct nouveau_vma *vma; 72 int ret; 73 74 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 75 return 0; 76 77 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 78 if (ret) 79 return ret; 80 81 ret = pm_runtime_get_sync(dev); 82 if (ret < 0 && ret != -EACCES) { 83 pm_runtime_put_autosuspend(dev); 84 goto out; 85 } 86 87 ret = nouveau_vma_new(nvbo, vmm, &vma); 88 pm_runtime_mark_last_busy(dev); 89 pm_runtime_put_autosuspend(dev); 90 out: 91 ttm_bo_unreserve(&nvbo->bo); 92 return ret; 93 } 94 95 struct nouveau_gem_object_unmap { 96 struct nouveau_cli_work work; 97 struct nouveau_vma *vma; 98 }; 99 100 static void 101 nouveau_gem_object_delete(struct nouveau_vma *vma) 102 { 103 nouveau_fence_unref(&vma->fence); 104 nouveau_vma_del(&vma); 105 } 106 107 static void 108 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 109 { 110 struct nouveau_gem_object_unmap *work = 111 container_of(w, typeof(*work), work); 112 nouveau_gem_object_delete(work->vma); 113 kfree(work); 114 } 115 116 static void 117 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 118 { 119 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 120 struct nouveau_gem_object_unmap *work; 121 122 list_del_init(&vma->head); 123 124 if (!fence) { 125 nouveau_gem_object_delete(vma); 126 return; 127 } 128 129 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 130 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 131 nouveau_gem_object_delete(vma); 132 return; 133 } 134 135 work->work.func = nouveau_gem_object_delete_work; 136 work->vma = vma; 137 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 138 } 139 140 void 141 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 142 { 143 struct nouveau_cli *cli = nouveau_cli(file_priv); 144 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 145 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 146 struct device *dev = drm->dev->dev; 147 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 148 struct nouveau_vma *vma; 149 int ret; 150 151 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 152 return; 153 154 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 155 if (ret) 156 return; 157 158 vma = nouveau_vma_find(nvbo, vmm); 159 if (vma) { 160 if (--vma->refs == 0) { 161 ret = pm_runtime_get_sync(dev); 162 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 163 nouveau_gem_object_unmap(nvbo, vma); 164 pm_runtime_mark_last_busy(dev); 165 } 166 pm_runtime_put_autosuspend(dev); 167 } 168 } 169 ttm_bo_unreserve(&nvbo->bo); 170 } 171 172 const struct drm_gem_object_funcs nouveau_gem_object_funcs = { 173 .free = nouveau_gem_object_del, 174 .open = nouveau_gem_object_open, 175 .close = nouveau_gem_object_close, 176 .pin = nouveau_gem_prime_pin, 177 .unpin = nouveau_gem_prime_unpin, 178 .get_sg_table = nouveau_gem_prime_get_sg_table, 179 .vmap = nouveau_gem_prime_vmap, 180 .vunmap = nouveau_gem_prime_vunmap, 181 }; 182 183 int 184 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 185 uint32_t tile_mode, uint32_t tile_flags, 186 struct nouveau_bo **pnvbo) 187 { 188 struct nouveau_drm *drm = cli->drm; 189 struct nouveau_bo *nvbo; 190 int ret; 191 192 if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) 193 domain |= NOUVEAU_GEM_DOMAIN_CPU; 194 195 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 196 tile_flags); 197 if (IS_ERR(nvbo)) 198 return PTR_ERR(nvbo); 199 200 nvbo->bo.base.funcs = &nouveau_gem_object_funcs; 201 202 /* Initialize the embedded gem-object. We return a single gem-reference 203 * to the caller, instead of a normal nouveau_bo ttm reference. */ 204 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 205 if (ret) { 206 drm_gem_object_release(&nvbo->bo.base); 207 kfree(nvbo); 208 return ret; 209 } 210 211 ret = nouveau_bo_init(nvbo, size, align, domain, NULL, NULL); 212 if (ret) { 213 nouveau_bo_ref(NULL, &nvbo); 214 return ret; 215 } 216 217 /* we restrict allowed domains on nv50+ to only the types 218 * that were requested at creation time. not possibly on 219 * earlier chips without busting the ABI. 220 */ 221 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 222 NOUVEAU_GEM_DOMAIN_GART; 223 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 224 nvbo->valid_domains &= domain; 225 226 *pnvbo = nvbo; 227 return 0; 228 } 229 230 static int 231 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 232 struct drm_nouveau_gem_info *rep) 233 { 234 struct nouveau_cli *cli = nouveau_cli(file_priv); 235 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 236 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 237 struct nouveau_vma *vma; 238 239 if (is_power_of_2(nvbo->valid_domains)) 240 rep->domain = nvbo->valid_domains; 241 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 242 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 243 else 244 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 245 rep->offset = nvbo->offset; 246 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 247 vma = nouveau_vma_find(nvbo, vmm); 248 if (!vma) 249 return -EINVAL; 250 251 rep->offset = vma->addr; 252 } 253 254 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 255 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 256 rep->tile_mode = nvbo->mode; 257 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 258 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 259 rep->tile_flags |= nvbo->kind << 8; 260 else 261 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 262 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 263 else 264 rep->tile_flags |= nvbo->zeta; 265 return 0; 266 } 267 268 int 269 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 270 struct drm_file *file_priv) 271 { 272 struct nouveau_cli *cli = nouveau_cli(file_priv); 273 struct drm_nouveau_gem_new *req = data; 274 struct nouveau_bo *nvbo = NULL; 275 int ret = 0; 276 277 ret = nouveau_gem_new(cli, req->info.size, req->align, 278 req->info.domain, req->info.tile_mode, 279 req->info.tile_flags, &nvbo); 280 if (ret) 281 return ret; 282 283 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 284 &req->info.handle); 285 if (ret == 0) { 286 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 287 if (ret) 288 drm_gem_handle_delete(file_priv, req->info.handle); 289 } 290 291 /* drop reference from allocate - handle holds it now */ 292 drm_gem_object_put(&nvbo->bo.base); 293 return ret; 294 } 295 296 static int 297 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 298 uint32_t write_domains, uint32_t valid_domains) 299 { 300 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 301 struct ttm_buffer_object *bo = &nvbo->bo; 302 uint32_t domains = valid_domains & nvbo->valid_domains & 303 (write_domains ? write_domains : read_domains); 304 uint32_t pref_domains = 0;; 305 306 if (!domains) 307 return -EINVAL; 308 309 valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART); 310 311 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 312 bo->mem.mem_type == TTM_PL_VRAM) 313 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 314 315 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 316 bo->mem.mem_type == TTM_PL_TT) 317 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 318 319 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 320 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 321 322 else 323 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 324 325 nouveau_bo_placement_set(nvbo, pref_domains, valid_domains); 326 327 return 0; 328 } 329 330 struct validate_op { 331 struct list_head list; 332 struct ww_acquire_ctx ticket; 333 }; 334 335 static void 336 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 337 struct nouveau_fence *fence, 338 struct drm_nouveau_gem_pushbuf_bo *pbbo) 339 { 340 struct nouveau_bo *nvbo; 341 struct drm_nouveau_gem_pushbuf_bo *b; 342 343 while (!list_empty(&op->list)) { 344 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 345 b = &pbbo[nvbo->pbbo_index]; 346 347 if (likely(fence)) { 348 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 349 350 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 351 struct nouveau_vma *vma = 352 (void *)(unsigned long)b->user_priv; 353 nouveau_fence_unref(&vma->fence); 354 dma_fence_get(&fence->base); 355 vma->fence = fence; 356 } 357 } 358 359 if (unlikely(nvbo->validate_mapped)) { 360 ttm_bo_kunmap(&nvbo->kmap); 361 nvbo->validate_mapped = false; 362 } 363 364 list_del(&nvbo->entry); 365 nvbo->reserved_by = NULL; 366 ttm_bo_unreserve(&nvbo->bo); 367 drm_gem_object_put(&nvbo->bo.base); 368 } 369 } 370 371 static void 372 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 373 struct nouveau_fence *fence, 374 struct drm_nouveau_gem_pushbuf_bo *pbbo) 375 { 376 validate_fini_no_ticket(op, chan, fence, pbbo); 377 ww_acquire_fini(&op->ticket); 378 } 379 380 static int 381 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 382 struct drm_nouveau_gem_pushbuf_bo *pbbo, 383 int nr_buffers, struct validate_op *op) 384 { 385 struct nouveau_cli *cli = nouveau_cli(file_priv); 386 int trycnt = 0; 387 int ret = -EINVAL, i; 388 struct nouveau_bo *res_bo = NULL; 389 LIST_HEAD(gart_list); 390 LIST_HEAD(vram_list); 391 LIST_HEAD(both_list); 392 393 ww_acquire_init(&op->ticket, &reservation_ww_class); 394 retry: 395 if (++trycnt > 100000) { 396 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 397 return -EINVAL; 398 } 399 400 for (i = 0; i < nr_buffers; i++) { 401 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 402 struct drm_gem_object *gem; 403 struct nouveau_bo *nvbo; 404 405 gem = drm_gem_object_lookup(file_priv, b->handle); 406 if (!gem) { 407 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 408 ret = -ENOENT; 409 break; 410 } 411 nvbo = nouveau_gem_object(gem); 412 if (nvbo == res_bo) { 413 res_bo = NULL; 414 drm_gem_object_put(gem); 415 continue; 416 } 417 418 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 419 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 420 "validation list\n", b->handle); 421 drm_gem_object_put(gem); 422 ret = -EINVAL; 423 break; 424 } 425 426 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 427 if (ret) { 428 list_splice_tail_init(&vram_list, &op->list); 429 list_splice_tail_init(&gart_list, &op->list); 430 list_splice_tail_init(&both_list, &op->list); 431 validate_fini_no_ticket(op, chan, NULL, NULL); 432 if (unlikely(ret == -EDEADLK)) { 433 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 434 &op->ticket); 435 if (!ret) 436 res_bo = nvbo; 437 } 438 if (unlikely(ret)) { 439 if (ret != -ERESTARTSYS) 440 NV_PRINTK(err, cli, "fail reserve\n"); 441 break; 442 } 443 } 444 445 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 446 struct nouveau_vmm *vmm = chan->vmm; 447 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 448 if (!vma) { 449 NV_PRINTK(err, cli, "vma not found!\n"); 450 ret = -EINVAL; 451 break; 452 } 453 454 b->user_priv = (uint64_t)(unsigned long)vma; 455 } else { 456 b->user_priv = (uint64_t)(unsigned long)nvbo; 457 } 458 459 nvbo->reserved_by = file_priv; 460 nvbo->pbbo_index = i; 461 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 462 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 463 list_add_tail(&nvbo->entry, &both_list); 464 else 465 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 466 list_add_tail(&nvbo->entry, &vram_list); 467 else 468 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 469 list_add_tail(&nvbo->entry, &gart_list); 470 else { 471 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 472 b->valid_domains); 473 list_add_tail(&nvbo->entry, &both_list); 474 ret = -EINVAL; 475 break; 476 } 477 if (nvbo == res_bo) 478 goto retry; 479 } 480 481 ww_acquire_done(&op->ticket); 482 list_splice_tail(&vram_list, &op->list); 483 list_splice_tail(&gart_list, &op->list); 484 list_splice_tail(&both_list, &op->list); 485 if (ret) 486 validate_fini(op, chan, NULL, NULL); 487 return ret; 488 489 } 490 491 static int 492 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 493 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 494 { 495 struct nouveau_drm *drm = chan->drm; 496 struct nouveau_bo *nvbo; 497 int ret, relocs = 0; 498 499 list_for_each_entry(nvbo, list, entry) { 500 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 501 502 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 503 b->write_domains, 504 b->valid_domains); 505 if (unlikely(ret)) { 506 NV_PRINTK(err, cli, "fail set_domain\n"); 507 return ret; 508 } 509 510 ret = nouveau_bo_validate(nvbo, true, false); 511 if (unlikely(ret)) { 512 if (ret != -ERESTARTSYS) 513 NV_PRINTK(err, cli, "fail ttm_validate\n"); 514 return ret; 515 } 516 517 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 518 if (unlikely(ret)) { 519 if (ret != -ERESTARTSYS) 520 NV_PRINTK(err, cli, "fail post-validate sync\n"); 521 return ret; 522 } 523 524 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 525 if (nvbo->offset == b->presumed.offset && 526 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 527 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 528 (nvbo->bo.mem.mem_type == TTM_PL_TT && 529 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 530 continue; 531 532 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 533 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 534 else 535 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 536 b->presumed.offset = nvbo->offset; 537 b->presumed.valid = 0; 538 relocs++; 539 } 540 } 541 542 return relocs; 543 } 544 545 static int 546 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 547 struct drm_file *file_priv, 548 struct drm_nouveau_gem_pushbuf_bo *pbbo, 549 int nr_buffers, 550 struct validate_op *op, bool *apply_relocs) 551 { 552 struct nouveau_cli *cli = nouveau_cli(file_priv); 553 int ret; 554 555 INIT_LIST_HEAD(&op->list); 556 557 if (nr_buffers == 0) 558 return 0; 559 560 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 561 if (unlikely(ret)) { 562 if (ret != -ERESTARTSYS) 563 NV_PRINTK(err, cli, "validate_init\n"); 564 return ret; 565 } 566 567 ret = validate_list(chan, cli, &op->list, pbbo); 568 if (unlikely(ret < 0)) { 569 if (ret != -ERESTARTSYS) 570 NV_PRINTK(err, cli, "validating bo list\n"); 571 validate_fini(op, chan, NULL, NULL); 572 return ret; 573 } 574 *apply_relocs = ret; 575 return 0; 576 } 577 578 static inline void 579 u_free(void *addr) 580 { 581 kvfree(addr); 582 } 583 584 static inline void * 585 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 586 { 587 void *mem; 588 void __user *userptr = (void __force __user *)(uintptr_t)user; 589 590 size *= nmemb; 591 592 mem = kvmalloc(size, GFP_KERNEL); 593 if (!mem) 594 return ERR_PTR(-ENOMEM); 595 596 if (copy_from_user(mem, userptr, size)) { 597 u_free(mem); 598 return ERR_PTR(-EFAULT); 599 } 600 601 return mem; 602 } 603 604 static int 605 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 606 struct drm_nouveau_gem_pushbuf *req, 607 struct drm_nouveau_gem_pushbuf_reloc *reloc, 608 struct drm_nouveau_gem_pushbuf_bo *bo) 609 { 610 int ret = 0; 611 unsigned i; 612 613 for (i = 0; i < req->nr_relocs; i++) { 614 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 615 struct drm_nouveau_gem_pushbuf_bo *b; 616 struct nouveau_bo *nvbo; 617 uint32_t data; 618 619 if (unlikely(r->bo_index >= req->nr_buffers)) { 620 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 621 ret = -EINVAL; 622 break; 623 } 624 625 b = &bo[r->bo_index]; 626 if (b->presumed.valid) 627 continue; 628 629 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 630 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 631 ret = -EINVAL; 632 break; 633 } 634 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 635 636 if (unlikely(r->reloc_bo_offset + 4 > 637 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 638 NV_PRINTK(err, cli, "reloc outside of bo\n"); 639 ret = -EINVAL; 640 break; 641 } 642 643 if (!nvbo->kmap.virtual) { 644 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 645 &nvbo->kmap); 646 if (ret) { 647 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 648 break; 649 } 650 nvbo->validate_mapped = true; 651 } 652 653 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 654 data = b->presumed.offset + r->data; 655 else 656 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 657 data = (b->presumed.offset + r->data) >> 32; 658 else 659 data = r->data; 660 661 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 662 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 663 data |= r->tor; 664 else 665 data |= r->vor; 666 } 667 668 ret = ttm_bo_wait(&nvbo->bo, false, false); 669 if (ret) { 670 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 671 break; 672 } 673 674 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 675 } 676 677 u_free(reloc); 678 return ret; 679 } 680 681 int 682 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 683 struct drm_file *file_priv) 684 { 685 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 686 struct nouveau_cli *cli = nouveau_cli(file_priv); 687 struct nouveau_abi16_chan *temp; 688 struct nouveau_drm *drm = nouveau_drm(dev); 689 struct drm_nouveau_gem_pushbuf *req = data; 690 struct drm_nouveau_gem_pushbuf_push *push; 691 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 692 struct drm_nouveau_gem_pushbuf_bo *bo; 693 struct nouveau_channel *chan = NULL; 694 struct validate_op op; 695 struct nouveau_fence *fence = NULL; 696 int i, j, ret = 0; 697 bool do_reloc = false, sync = false; 698 699 if (unlikely(!abi16)) 700 return -ENOMEM; 701 702 list_for_each_entry(temp, &abi16->channels, head) { 703 if (temp->chan->chid == req->channel) { 704 chan = temp->chan; 705 break; 706 } 707 } 708 709 if (!chan) 710 return nouveau_abi16_put(abi16, -ENOENT); 711 if (unlikely(atomic_read(&chan->killed))) 712 return nouveau_abi16_put(abi16, -ENODEV); 713 714 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 715 716 req->vram_available = drm->gem.vram_available; 717 req->gart_available = drm->gem.gart_available; 718 if (unlikely(req->nr_push == 0)) 719 goto out_next; 720 721 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 722 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 723 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 724 return nouveau_abi16_put(abi16, -EINVAL); 725 } 726 727 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 728 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 729 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 730 return nouveau_abi16_put(abi16, -EINVAL); 731 } 732 733 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 734 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 735 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 736 return nouveau_abi16_put(abi16, -EINVAL); 737 } 738 739 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 740 if (IS_ERR(push)) 741 return nouveau_abi16_put(abi16, PTR_ERR(push)); 742 743 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 744 if (IS_ERR(bo)) { 745 u_free(push); 746 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 747 } 748 749 /* Ensure all push buffers are on validate list */ 750 for (i = 0; i < req->nr_push; i++) { 751 if (push[i].bo_index >= req->nr_buffers) { 752 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 753 ret = -EINVAL; 754 goto out_prevalid; 755 } 756 } 757 758 /* Validate buffer list */ 759 revalidate: 760 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 761 req->nr_buffers, &op, &do_reloc); 762 if (ret) { 763 if (ret != -ERESTARTSYS) 764 NV_PRINTK(err, cli, "validate: %d\n", ret); 765 goto out_prevalid; 766 } 767 768 /* Apply any relocations that are required */ 769 if (do_reloc) { 770 if (!reloc) { 771 validate_fini(&op, chan, NULL, bo); 772 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 773 if (IS_ERR(reloc)) { 774 ret = PTR_ERR(reloc); 775 goto out_prevalid; 776 } 777 778 goto revalidate; 779 } 780 781 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 782 if (ret) { 783 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 784 goto out; 785 } 786 } 787 788 if (chan->dma.ib_max) { 789 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 790 if (ret) { 791 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 792 goto out; 793 } 794 795 for (i = 0; i < req->nr_push; i++) { 796 struct nouveau_vma *vma = (void *)(unsigned long) 797 bo[push[i].bo_index].user_priv; 798 799 nv50_dma_push(chan, vma->addr + push[i].offset, 800 push[i].length); 801 } 802 } else 803 if (drm->client.device.info.chipset >= 0x25) { 804 ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2); 805 if (ret) { 806 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 807 goto out; 808 } 809 810 for (i = 0; i < req->nr_push; i++) { 811 struct nouveau_bo *nvbo = (void *)(unsigned long) 812 bo[push[i].bo_index].user_priv; 813 814 PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset); 815 PUSH_DATA(chan->chan.push, 0); 816 } 817 } else { 818 ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 819 if (ret) { 820 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 821 goto out; 822 } 823 824 for (i = 0; i < req->nr_push; i++) { 825 struct nouveau_bo *nvbo = (void *)(unsigned long) 826 bo[push[i].bo_index].user_priv; 827 uint32_t cmd; 828 829 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 830 cmd |= 0x20000000; 831 if (unlikely(cmd != req->suffix0)) { 832 if (!nvbo->kmap.virtual) { 833 ret = ttm_bo_kmap(&nvbo->bo, 0, 834 nvbo->bo.mem. 835 num_pages, 836 &nvbo->kmap); 837 if (ret) { 838 WIND_RING(chan); 839 goto out; 840 } 841 nvbo->validate_mapped = true; 842 } 843 844 nouveau_bo_wr32(nvbo, (push[i].offset + 845 push[i].length - 8) / 4, cmd); 846 } 847 848 PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset); 849 PUSH_DATA(chan->chan.push, 0); 850 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 851 PUSH_DATA(chan->chan.push, 0); 852 } 853 } 854 855 ret = nouveau_fence_new(chan, false, &fence); 856 if (ret) { 857 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 858 WIND_RING(chan); 859 goto out; 860 } 861 862 if (sync) { 863 if (!(ret = nouveau_fence_wait(fence, false, false))) { 864 if ((ret = dma_fence_get_status(&fence->base)) == 1) 865 ret = 0; 866 } 867 } 868 869 out: 870 validate_fini(&op, chan, fence, bo); 871 nouveau_fence_unref(&fence); 872 873 if (do_reloc) { 874 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 875 u64_to_user_ptr(req->buffers); 876 877 for (i = 0; i < req->nr_buffers; i++) { 878 if (bo[i].presumed.valid) 879 continue; 880 881 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 882 sizeof(bo[i].presumed))) { 883 ret = -EFAULT; 884 break; 885 } 886 } 887 u_free(reloc); 888 } 889 out_prevalid: 890 u_free(bo); 891 u_free(push); 892 893 out_next: 894 if (chan->dma.ib_max) { 895 req->suffix0 = 0x00000000; 896 req->suffix1 = 0x00000000; 897 } else 898 if (drm->client.device.info.chipset >= 0x25) { 899 req->suffix0 = 0x00020000; 900 req->suffix1 = 0x00000000; 901 } else { 902 req->suffix0 = 0x20000000 | 903 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 904 req->suffix1 = 0x00000000; 905 } 906 907 return nouveau_abi16_put(abi16, ret); 908 } 909 910 int 911 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 912 struct drm_file *file_priv) 913 { 914 struct drm_nouveau_gem_cpu_prep *req = data; 915 struct drm_gem_object *gem; 916 struct nouveau_bo *nvbo; 917 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 918 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 919 long lret; 920 int ret; 921 922 gem = drm_gem_object_lookup(file_priv, req->handle); 923 if (!gem) 924 return -ENOENT; 925 nvbo = nouveau_gem_object(gem); 926 927 lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true, 928 no_wait ? 0 : 30 * HZ); 929 if (!lret) 930 ret = -EBUSY; 931 else if (lret > 0) 932 ret = 0; 933 else 934 ret = lret; 935 936 nouveau_bo_sync_for_cpu(nvbo); 937 drm_gem_object_put(gem); 938 939 return ret; 940 } 941 942 int 943 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 944 struct drm_file *file_priv) 945 { 946 struct drm_nouveau_gem_cpu_fini *req = data; 947 struct drm_gem_object *gem; 948 struct nouveau_bo *nvbo; 949 950 gem = drm_gem_object_lookup(file_priv, req->handle); 951 if (!gem) 952 return -ENOENT; 953 nvbo = nouveau_gem_object(gem); 954 955 nouveau_bo_sync_for_device(nvbo); 956 drm_gem_object_put(gem); 957 return 0; 958 } 959 960 int 961 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 962 struct drm_file *file_priv) 963 { 964 struct drm_nouveau_gem_info *req = data; 965 struct drm_gem_object *gem; 966 int ret; 967 968 gem = drm_gem_object_lookup(file_priv, req->handle); 969 if (!gem) 970 return -ENOENT; 971 972 ret = nouveau_gem_info(file_priv, gem, req); 973 drm_gem_object_put(gem); 974 return ret; 975 } 976 977