1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 39 void 40 nouveau_gem_object_del(struct drm_gem_object *gem) 41 { 42 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 43 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 44 struct device *dev = drm->dev->dev; 45 int ret; 46 47 ret = pm_runtime_get_sync(dev); 48 if (WARN_ON(ret < 0 && ret != -EACCES)) 49 return; 50 51 if (gem->import_attach) 52 drm_prime_gem_destroy(gem, nvbo->bo.sg); 53 54 drm_gem_object_release(gem); 55 56 /* reset filp so nouveau_bo_del_ttm() can test for it */ 57 gem->filp = NULL; 58 ttm_bo_put(&nvbo->bo); 59 60 pm_runtime_mark_last_busy(dev); 61 pm_runtime_put_autosuspend(dev); 62 } 63 64 int 65 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 66 { 67 struct nouveau_cli *cli = nouveau_cli(file_priv); 68 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 69 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 70 struct device *dev = drm->dev->dev; 71 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 72 struct nouveau_vma *vma; 73 int ret; 74 75 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 76 return 0; 77 78 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 79 if (ret) 80 return ret; 81 82 ret = pm_runtime_get_sync(dev); 83 if (ret < 0 && ret != -EACCES) 84 goto out; 85 86 ret = nouveau_vma_new(nvbo, vmm, &vma); 87 pm_runtime_mark_last_busy(dev); 88 pm_runtime_put_autosuspend(dev); 89 out: 90 ttm_bo_unreserve(&nvbo->bo); 91 return ret; 92 } 93 94 struct nouveau_gem_object_unmap { 95 struct nouveau_cli_work work; 96 struct nouveau_vma *vma; 97 }; 98 99 static void 100 nouveau_gem_object_delete(struct nouveau_vma *vma) 101 { 102 nouveau_fence_unref(&vma->fence); 103 nouveau_vma_del(&vma); 104 } 105 106 static void 107 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 108 { 109 struct nouveau_gem_object_unmap *work = 110 container_of(w, typeof(*work), work); 111 nouveau_gem_object_delete(work->vma); 112 kfree(work); 113 } 114 115 static void 116 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 117 { 118 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 119 struct nouveau_gem_object_unmap *work; 120 121 list_del_init(&vma->head); 122 123 if (!fence) { 124 nouveau_gem_object_delete(vma); 125 return; 126 } 127 128 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 129 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 130 nouveau_gem_object_delete(vma); 131 return; 132 } 133 134 work->work.func = nouveau_gem_object_delete_work; 135 work->vma = vma; 136 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 137 } 138 139 void 140 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 141 { 142 struct nouveau_cli *cli = nouveau_cli(file_priv); 143 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 144 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 145 struct device *dev = drm->dev->dev; 146 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 147 struct nouveau_vma *vma; 148 int ret; 149 150 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 151 return; 152 153 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 154 if (ret) 155 return; 156 157 vma = nouveau_vma_find(nvbo, vmm); 158 if (vma) { 159 if (--vma->refs == 0) { 160 ret = pm_runtime_get_sync(dev); 161 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 162 nouveau_gem_object_unmap(nvbo, vma); 163 pm_runtime_mark_last_busy(dev); 164 pm_runtime_put_autosuspend(dev); 165 } 166 } 167 } 168 ttm_bo_unreserve(&nvbo->bo); 169 } 170 171 int 172 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 173 uint32_t tile_mode, uint32_t tile_flags, 174 struct nouveau_bo **pnvbo) 175 { 176 struct nouveau_drm *drm = cli->drm; 177 struct nouveau_bo *nvbo; 178 u32 flags = 0; 179 int ret; 180 181 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 182 flags |= TTM_PL_FLAG_VRAM; 183 if (domain & NOUVEAU_GEM_DOMAIN_GART) 184 flags |= TTM_PL_FLAG_TT; 185 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 186 flags |= TTM_PL_FLAG_SYSTEM; 187 188 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) 189 flags |= TTM_PL_FLAG_UNCACHED; 190 191 ret = nouveau_bo_new(cli, size, align, flags, tile_mode, 192 tile_flags, NULL, NULL, pnvbo); 193 if (ret) 194 return ret; 195 nvbo = *pnvbo; 196 197 /* we restrict allowed domains on nv50+ to only the types 198 * that were requested at creation time. not possibly on 199 * earlier chips without busting the ABI. 200 */ 201 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 202 NOUVEAU_GEM_DOMAIN_GART; 203 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 204 nvbo->valid_domains &= domain; 205 206 /* Initialize the embedded gem-object. We return a single gem-reference 207 * to the caller, instead of a normal nouveau_bo ttm reference. */ 208 ret = drm_gem_object_init(drm->dev, &nvbo->gem, nvbo->bo.mem.size); 209 if (ret) { 210 nouveau_bo_ref(NULL, pnvbo); 211 return -ENOMEM; 212 } 213 214 nvbo->bo.persistent_swap_storage = nvbo->gem.filp; 215 return 0; 216 } 217 218 static int 219 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 220 struct drm_nouveau_gem_info *rep) 221 { 222 struct nouveau_cli *cli = nouveau_cli(file_priv); 223 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 224 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 225 struct nouveau_vma *vma; 226 227 if (is_power_of_2(nvbo->valid_domains)) 228 rep->domain = nvbo->valid_domains; 229 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 230 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 231 else 232 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 233 rep->offset = nvbo->bo.offset; 234 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 235 vma = nouveau_vma_find(nvbo, vmm); 236 if (!vma) 237 return -EINVAL; 238 239 rep->offset = vma->addr; 240 } 241 242 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 243 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); 244 rep->tile_mode = nvbo->mode; 245 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 246 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 247 rep->tile_flags |= nvbo->kind << 8; 248 else 249 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 250 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 251 else 252 rep->tile_flags |= nvbo->zeta; 253 return 0; 254 } 255 256 int 257 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 258 struct drm_file *file_priv) 259 { 260 struct nouveau_cli *cli = nouveau_cli(file_priv); 261 struct drm_nouveau_gem_new *req = data; 262 struct nouveau_bo *nvbo = NULL; 263 int ret = 0; 264 265 ret = nouveau_gem_new(cli, req->info.size, req->align, 266 req->info.domain, req->info.tile_mode, 267 req->info.tile_flags, &nvbo); 268 if (ret) 269 return ret; 270 271 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle); 272 if (ret == 0) { 273 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info); 274 if (ret) 275 drm_gem_handle_delete(file_priv, req->info.handle); 276 } 277 278 /* drop reference from allocate - handle holds it now */ 279 drm_gem_object_put_unlocked(&nvbo->gem); 280 return ret; 281 } 282 283 static int 284 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 285 uint32_t write_domains, uint32_t valid_domains) 286 { 287 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 288 struct ttm_buffer_object *bo = &nvbo->bo; 289 uint32_t domains = valid_domains & nvbo->valid_domains & 290 (write_domains ? write_domains : read_domains); 291 uint32_t pref_flags = 0, valid_flags = 0; 292 293 if (!domains) 294 return -EINVAL; 295 296 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 297 valid_flags |= TTM_PL_FLAG_VRAM; 298 299 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 300 valid_flags |= TTM_PL_FLAG_TT; 301 302 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 303 bo->mem.mem_type == TTM_PL_VRAM) 304 pref_flags |= TTM_PL_FLAG_VRAM; 305 306 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 307 bo->mem.mem_type == TTM_PL_TT) 308 pref_flags |= TTM_PL_FLAG_TT; 309 310 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 311 pref_flags |= TTM_PL_FLAG_VRAM; 312 313 else 314 pref_flags |= TTM_PL_FLAG_TT; 315 316 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 317 318 return 0; 319 } 320 321 struct validate_op { 322 struct list_head list; 323 struct ww_acquire_ctx ticket; 324 }; 325 326 static void 327 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 328 struct nouveau_fence *fence, 329 struct drm_nouveau_gem_pushbuf_bo *pbbo) 330 { 331 struct nouveau_bo *nvbo; 332 struct drm_nouveau_gem_pushbuf_bo *b; 333 334 while (!list_empty(&op->list)) { 335 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 336 b = &pbbo[nvbo->pbbo_index]; 337 338 if (likely(fence)) { 339 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 340 341 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 342 struct nouveau_vma *vma = 343 (void *)(unsigned long)b->user_priv; 344 nouveau_fence_unref(&vma->fence); 345 dma_fence_get(&fence->base); 346 vma->fence = fence; 347 } 348 } 349 350 if (unlikely(nvbo->validate_mapped)) { 351 ttm_bo_kunmap(&nvbo->kmap); 352 nvbo->validate_mapped = false; 353 } 354 355 list_del(&nvbo->entry); 356 nvbo->reserved_by = NULL; 357 ttm_bo_unreserve(&nvbo->bo); 358 drm_gem_object_put_unlocked(&nvbo->gem); 359 } 360 } 361 362 static void 363 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 364 struct nouveau_fence *fence, 365 struct drm_nouveau_gem_pushbuf_bo *pbbo) 366 { 367 validate_fini_no_ticket(op, chan, fence, pbbo); 368 ww_acquire_fini(&op->ticket); 369 } 370 371 static int 372 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 373 struct drm_nouveau_gem_pushbuf_bo *pbbo, 374 int nr_buffers, struct validate_op *op) 375 { 376 struct nouveau_cli *cli = nouveau_cli(file_priv); 377 int trycnt = 0; 378 int ret = -EINVAL, i; 379 struct nouveau_bo *res_bo = NULL; 380 LIST_HEAD(gart_list); 381 LIST_HEAD(vram_list); 382 LIST_HEAD(both_list); 383 384 ww_acquire_init(&op->ticket, &reservation_ww_class); 385 retry: 386 if (++trycnt > 100000) { 387 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 388 return -EINVAL; 389 } 390 391 for (i = 0; i < nr_buffers; i++) { 392 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 393 struct drm_gem_object *gem; 394 struct nouveau_bo *nvbo; 395 396 gem = drm_gem_object_lookup(file_priv, b->handle); 397 if (!gem) { 398 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 399 ret = -ENOENT; 400 break; 401 } 402 nvbo = nouveau_gem_object(gem); 403 if (nvbo == res_bo) { 404 res_bo = NULL; 405 drm_gem_object_put_unlocked(gem); 406 continue; 407 } 408 409 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 410 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 411 "validation list\n", b->handle); 412 drm_gem_object_put_unlocked(gem); 413 ret = -EINVAL; 414 break; 415 } 416 417 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 418 if (ret) { 419 list_splice_tail_init(&vram_list, &op->list); 420 list_splice_tail_init(&gart_list, &op->list); 421 list_splice_tail_init(&both_list, &op->list); 422 validate_fini_no_ticket(op, chan, NULL, NULL); 423 if (unlikely(ret == -EDEADLK)) { 424 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 425 &op->ticket); 426 if (!ret) 427 res_bo = nvbo; 428 } 429 if (unlikely(ret)) { 430 if (ret != -ERESTARTSYS) 431 NV_PRINTK(err, cli, "fail reserve\n"); 432 break; 433 } 434 } 435 436 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 437 struct nouveau_vmm *vmm = chan->vmm; 438 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 439 if (!vma) { 440 NV_PRINTK(err, cli, "vma not found!\n"); 441 ret = -EINVAL; 442 break; 443 } 444 445 b->user_priv = (uint64_t)(unsigned long)vma; 446 } else { 447 b->user_priv = (uint64_t)(unsigned long)nvbo; 448 } 449 450 nvbo->reserved_by = file_priv; 451 nvbo->pbbo_index = i; 452 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 453 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 454 list_add_tail(&nvbo->entry, &both_list); 455 else 456 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 457 list_add_tail(&nvbo->entry, &vram_list); 458 else 459 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 460 list_add_tail(&nvbo->entry, &gart_list); 461 else { 462 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 463 b->valid_domains); 464 list_add_tail(&nvbo->entry, &both_list); 465 ret = -EINVAL; 466 break; 467 } 468 if (nvbo == res_bo) 469 goto retry; 470 } 471 472 ww_acquire_done(&op->ticket); 473 list_splice_tail(&vram_list, &op->list); 474 list_splice_tail(&gart_list, &op->list); 475 list_splice_tail(&both_list, &op->list); 476 if (ret) 477 validate_fini(op, chan, NULL, NULL); 478 return ret; 479 480 } 481 482 static int 483 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 484 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 485 uint64_t user_pbbo_ptr) 486 { 487 struct nouveau_drm *drm = chan->drm; 488 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 489 (void __force __user *)(uintptr_t)user_pbbo_ptr; 490 struct nouveau_bo *nvbo; 491 int ret, relocs = 0; 492 493 list_for_each_entry(nvbo, list, entry) { 494 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 495 496 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains, 497 b->write_domains, 498 b->valid_domains); 499 if (unlikely(ret)) { 500 NV_PRINTK(err, cli, "fail set_domain\n"); 501 return ret; 502 } 503 504 ret = nouveau_bo_validate(nvbo, true, false); 505 if (unlikely(ret)) { 506 if (ret != -ERESTARTSYS) 507 NV_PRINTK(err, cli, "fail ttm_validate\n"); 508 return ret; 509 } 510 511 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 512 if (unlikely(ret)) { 513 if (ret != -ERESTARTSYS) 514 NV_PRINTK(err, cli, "fail post-validate sync\n"); 515 return ret; 516 } 517 518 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 519 if (nvbo->bo.offset == b->presumed.offset && 520 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 521 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 522 (nvbo->bo.mem.mem_type == TTM_PL_TT && 523 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 524 continue; 525 526 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 527 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 528 else 529 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 530 b->presumed.offset = nvbo->bo.offset; 531 b->presumed.valid = 0; 532 relocs++; 533 534 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed, 535 &b->presumed, sizeof(b->presumed))) 536 return -EFAULT; 537 } 538 } 539 540 return relocs; 541 } 542 543 static int 544 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 545 struct drm_file *file_priv, 546 struct drm_nouveau_gem_pushbuf_bo *pbbo, 547 uint64_t user_buffers, int nr_buffers, 548 struct validate_op *op, int *apply_relocs) 549 { 550 struct nouveau_cli *cli = nouveau_cli(file_priv); 551 int ret; 552 553 INIT_LIST_HEAD(&op->list); 554 555 if (nr_buffers == 0) 556 return 0; 557 558 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 559 if (unlikely(ret)) { 560 if (ret != -ERESTARTSYS) 561 NV_PRINTK(err, cli, "validate_init\n"); 562 return ret; 563 } 564 565 ret = validate_list(chan, cli, &op->list, pbbo, user_buffers); 566 if (unlikely(ret < 0)) { 567 if (ret != -ERESTARTSYS) 568 NV_PRINTK(err, cli, "validating bo list\n"); 569 validate_fini(op, chan, NULL, NULL); 570 return ret; 571 } 572 *apply_relocs = ret; 573 return 0; 574 } 575 576 static inline void 577 u_free(void *addr) 578 { 579 kvfree(addr); 580 } 581 582 static inline void * 583 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 584 { 585 void *mem; 586 void __user *userptr = (void __force __user *)(uintptr_t)user; 587 588 size *= nmemb; 589 590 mem = kvmalloc(size, GFP_KERNEL); 591 if (!mem) 592 return ERR_PTR(-ENOMEM); 593 594 if (copy_from_user(mem, userptr, size)) { 595 u_free(mem); 596 return ERR_PTR(-EFAULT); 597 } 598 599 return mem; 600 } 601 602 static int 603 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 604 struct drm_nouveau_gem_pushbuf *req, 605 struct drm_nouveau_gem_pushbuf_bo *bo) 606 { 607 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 608 int ret = 0; 609 unsigned i; 610 611 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 612 if (IS_ERR(reloc)) 613 return PTR_ERR(reloc); 614 615 for (i = 0; i < req->nr_relocs; i++) { 616 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 617 struct drm_nouveau_gem_pushbuf_bo *b; 618 struct nouveau_bo *nvbo; 619 uint32_t data; 620 621 if (unlikely(r->bo_index >= req->nr_buffers)) { 622 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 623 ret = -EINVAL; 624 break; 625 } 626 627 b = &bo[r->bo_index]; 628 if (b->presumed.valid) 629 continue; 630 631 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 632 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 633 ret = -EINVAL; 634 break; 635 } 636 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 637 638 if (unlikely(r->reloc_bo_offset + 4 > 639 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 640 NV_PRINTK(err, cli, "reloc outside of bo\n"); 641 ret = -EINVAL; 642 break; 643 } 644 645 if (!nvbo->kmap.virtual) { 646 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 647 &nvbo->kmap); 648 if (ret) { 649 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 650 break; 651 } 652 nvbo->validate_mapped = true; 653 } 654 655 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 656 data = b->presumed.offset + r->data; 657 else 658 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 659 data = (b->presumed.offset + r->data) >> 32; 660 else 661 data = r->data; 662 663 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 664 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 665 data |= r->tor; 666 else 667 data |= r->vor; 668 } 669 670 ret = ttm_bo_wait(&nvbo->bo, false, false); 671 if (ret) { 672 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 673 break; 674 } 675 676 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 677 } 678 679 u_free(reloc); 680 return ret; 681 } 682 683 int 684 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 685 struct drm_file *file_priv) 686 { 687 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 688 struct nouveau_cli *cli = nouveau_cli(file_priv); 689 struct nouveau_abi16_chan *temp; 690 struct nouveau_drm *drm = nouveau_drm(dev); 691 struct drm_nouveau_gem_pushbuf *req = data; 692 struct drm_nouveau_gem_pushbuf_push *push; 693 struct drm_nouveau_gem_pushbuf_bo *bo; 694 struct nouveau_channel *chan = NULL; 695 struct validate_op op; 696 struct nouveau_fence *fence = NULL; 697 int i, j, ret = 0, do_reloc = 0; 698 699 if (unlikely(!abi16)) 700 return -ENOMEM; 701 702 list_for_each_entry(temp, &abi16->channels, head) { 703 if (temp->chan->chid == req->channel) { 704 chan = temp->chan; 705 break; 706 } 707 } 708 709 if (!chan) 710 return nouveau_abi16_put(abi16, -ENOENT); 711 712 req->vram_available = drm->gem.vram_available; 713 req->gart_available = drm->gem.gart_available; 714 if (unlikely(req->nr_push == 0)) 715 goto out_next; 716 717 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 718 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 719 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 720 return nouveau_abi16_put(abi16, -EINVAL); 721 } 722 723 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 724 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 725 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 726 return nouveau_abi16_put(abi16, -EINVAL); 727 } 728 729 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 730 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 731 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 732 return nouveau_abi16_put(abi16, -EINVAL); 733 } 734 735 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 736 if (IS_ERR(push)) 737 return nouveau_abi16_put(abi16, PTR_ERR(push)); 738 739 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 740 if (IS_ERR(bo)) { 741 u_free(push); 742 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 743 } 744 745 /* Ensure all push buffers are on validate list */ 746 for (i = 0; i < req->nr_push; i++) { 747 if (push[i].bo_index >= req->nr_buffers) { 748 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 749 ret = -EINVAL; 750 goto out_prevalid; 751 } 752 } 753 754 /* Validate buffer list */ 755 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 756 req->nr_buffers, &op, &do_reloc); 757 if (ret) { 758 if (ret != -ERESTARTSYS) 759 NV_PRINTK(err, cli, "validate: %d\n", ret); 760 goto out_prevalid; 761 } 762 763 /* Apply any relocations that are required */ 764 if (do_reloc) { 765 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 766 if (ret) { 767 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 768 goto out; 769 } 770 } 771 772 if (chan->dma.ib_max) { 773 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 774 if (ret) { 775 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 776 goto out; 777 } 778 779 for (i = 0; i < req->nr_push; i++) { 780 struct nouveau_vma *vma = (void *)(unsigned long) 781 bo[push[i].bo_index].user_priv; 782 783 nv50_dma_push(chan, vma->addr + push[i].offset, 784 push[i].length); 785 } 786 } else 787 if (drm->client.device.info.chipset >= 0x25) { 788 ret = RING_SPACE(chan, req->nr_push * 2); 789 if (ret) { 790 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 791 goto out; 792 } 793 794 for (i = 0; i < req->nr_push; i++) { 795 struct nouveau_bo *nvbo = (void *)(unsigned long) 796 bo[push[i].bo_index].user_priv; 797 798 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 799 OUT_RING(chan, 0); 800 } 801 } else { 802 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 803 if (ret) { 804 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 805 goto out; 806 } 807 808 for (i = 0; i < req->nr_push; i++) { 809 struct nouveau_bo *nvbo = (void *)(unsigned long) 810 bo[push[i].bo_index].user_priv; 811 uint32_t cmd; 812 813 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 814 cmd |= 0x20000000; 815 if (unlikely(cmd != req->suffix0)) { 816 if (!nvbo->kmap.virtual) { 817 ret = ttm_bo_kmap(&nvbo->bo, 0, 818 nvbo->bo.mem. 819 num_pages, 820 &nvbo->kmap); 821 if (ret) { 822 WIND_RING(chan); 823 goto out; 824 } 825 nvbo->validate_mapped = true; 826 } 827 828 nouveau_bo_wr32(nvbo, (push[i].offset + 829 push[i].length - 8) / 4, cmd); 830 } 831 832 OUT_RING(chan, 0x20000000 | 833 (nvbo->bo.offset + push[i].offset)); 834 OUT_RING(chan, 0); 835 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 836 OUT_RING(chan, 0); 837 } 838 } 839 840 ret = nouveau_fence_new(chan, false, &fence); 841 if (ret) { 842 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 843 WIND_RING(chan); 844 goto out; 845 } 846 847 out: 848 validate_fini(&op, chan, fence, bo); 849 nouveau_fence_unref(&fence); 850 851 out_prevalid: 852 u_free(bo); 853 u_free(push); 854 855 out_next: 856 if (chan->dma.ib_max) { 857 req->suffix0 = 0x00000000; 858 req->suffix1 = 0x00000000; 859 } else 860 if (drm->client.device.info.chipset >= 0x25) { 861 req->suffix0 = 0x00020000; 862 req->suffix1 = 0x00000000; 863 } else { 864 req->suffix0 = 0x20000000 | 865 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 866 req->suffix1 = 0x00000000; 867 } 868 869 return nouveau_abi16_put(abi16, ret); 870 } 871 872 int 873 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 874 struct drm_file *file_priv) 875 { 876 struct drm_nouveau_gem_cpu_prep *req = data; 877 struct drm_gem_object *gem; 878 struct nouveau_bo *nvbo; 879 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 880 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 881 long lret; 882 int ret; 883 884 gem = drm_gem_object_lookup(file_priv, req->handle); 885 if (!gem) 886 return -ENOENT; 887 nvbo = nouveau_gem_object(gem); 888 889 lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true, 890 no_wait ? 0 : 30 * HZ); 891 if (!lret) 892 ret = -EBUSY; 893 else if (lret > 0) 894 ret = 0; 895 else 896 ret = lret; 897 898 nouveau_bo_sync_for_cpu(nvbo); 899 drm_gem_object_put_unlocked(gem); 900 901 return ret; 902 } 903 904 int 905 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 906 struct drm_file *file_priv) 907 { 908 struct drm_nouveau_gem_cpu_fini *req = data; 909 struct drm_gem_object *gem; 910 struct nouveau_bo *nvbo; 911 912 gem = drm_gem_object_lookup(file_priv, req->handle); 913 if (!gem) 914 return -ENOENT; 915 nvbo = nouveau_gem_object(gem); 916 917 nouveau_bo_sync_for_device(nvbo); 918 drm_gem_object_put_unlocked(gem); 919 return 0; 920 } 921 922 int 923 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 924 struct drm_file *file_priv) 925 { 926 struct drm_nouveau_gem_info *req = data; 927 struct drm_gem_object *gem; 928 int ret; 929 930 gem = drm_gem_object_lookup(file_priv, req->handle); 931 if (!gem) 932 return -ENOENT; 933 934 ret = nouveau_gem_info(file_priv, gem, req); 935 drm_gem_object_put_unlocked(gem); 936 return ret; 937 } 938 939