1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 39 void 40 nouveau_gem_object_del(struct drm_gem_object *gem) 41 { 42 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 43 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 44 struct device *dev = drm->dev->dev; 45 int ret; 46 47 ret = pm_runtime_get_sync(dev); 48 if (WARN_ON(ret < 0 && ret != -EACCES)) 49 return; 50 51 if (gem->import_attach) 52 drm_prime_gem_destroy(gem, nvbo->bo.sg); 53 54 drm_gem_object_release(gem); 55 56 /* reset filp so nouveau_bo_del_ttm() can test for it */ 57 gem->filp = NULL; 58 ttm_bo_put(&nvbo->bo); 59 60 pm_runtime_mark_last_busy(dev); 61 pm_runtime_put_autosuspend(dev); 62 } 63 64 int 65 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 66 { 67 struct nouveau_cli *cli = nouveau_cli(file_priv); 68 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 69 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 70 struct device *dev = drm->dev->dev; 71 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 72 struct nouveau_vma *vma; 73 int ret; 74 75 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 76 return 0; 77 78 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 79 if (ret) 80 return ret; 81 82 ret = pm_runtime_get_sync(dev); 83 if (ret < 0 && ret != -EACCES) 84 goto out; 85 86 ret = nouveau_vma_new(nvbo, vmm, &vma); 87 pm_runtime_mark_last_busy(dev); 88 pm_runtime_put_autosuspend(dev); 89 out: 90 ttm_bo_unreserve(&nvbo->bo); 91 return ret; 92 } 93 94 struct nouveau_gem_object_unmap { 95 struct nouveau_cli_work work; 96 struct nouveau_vma *vma; 97 }; 98 99 static void 100 nouveau_gem_object_delete(struct nouveau_vma *vma) 101 { 102 nouveau_fence_unref(&vma->fence); 103 nouveau_vma_del(&vma); 104 } 105 106 static void 107 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 108 { 109 struct nouveau_gem_object_unmap *work = 110 container_of(w, typeof(*work), work); 111 nouveau_gem_object_delete(work->vma); 112 kfree(work); 113 } 114 115 static void 116 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 117 { 118 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 119 struct nouveau_gem_object_unmap *work; 120 121 list_del_init(&vma->head); 122 123 if (!fence) { 124 nouveau_gem_object_delete(vma); 125 return; 126 } 127 128 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 129 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 130 nouveau_gem_object_delete(vma); 131 return; 132 } 133 134 work->work.func = nouveau_gem_object_delete_work; 135 work->vma = vma; 136 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 137 } 138 139 void 140 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 141 { 142 struct nouveau_cli *cli = nouveau_cli(file_priv); 143 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 144 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 145 struct device *dev = drm->dev->dev; 146 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 147 struct nouveau_vma *vma; 148 int ret; 149 150 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 151 return; 152 153 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 154 if (ret) 155 return; 156 157 vma = nouveau_vma_find(nvbo, vmm); 158 if (vma) { 159 if (--vma->refs == 0) { 160 ret = pm_runtime_get_sync(dev); 161 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 162 nouveau_gem_object_unmap(nvbo, vma); 163 pm_runtime_mark_last_busy(dev); 164 pm_runtime_put_autosuspend(dev); 165 } 166 } 167 } 168 ttm_bo_unreserve(&nvbo->bo); 169 } 170 171 int 172 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 173 uint32_t tile_mode, uint32_t tile_flags, 174 struct nouveau_bo **pnvbo) 175 { 176 struct nouveau_drm *drm = cli->drm; 177 struct nouveau_bo *nvbo; 178 u32 flags = 0; 179 int ret; 180 181 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 182 flags |= TTM_PL_FLAG_VRAM; 183 if (domain & NOUVEAU_GEM_DOMAIN_GART) 184 flags |= TTM_PL_FLAG_TT; 185 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 186 flags |= TTM_PL_FLAG_SYSTEM; 187 188 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) 189 flags |= TTM_PL_FLAG_UNCACHED; 190 191 nvbo = nouveau_bo_alloc(cli, size, flags, tile_mode, tile_flags); 192 if (IS_ERR(nvbo)) 193 return PTR_ERR(nvbo); 194 195 /* Initialize the embedded gem-object. We return a single gem-reference 196 * to the caller, instead of a normal nouveau_bo ttm reference. */ 197 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 198 if (ret) { 199 nouveau_bo_ref(NULL, &nvbo); 200 return ret; 201 } 202 203 ret = nouveau_bo_init(nvbo, size, align, flags, NULL, NULL); 204 if (ret) { 205 nouveau_bo_ref(NULL, &nvbo); 206 return ret; 207 } 208 209 /* we restrict allowed domains on nv50+ to only the types 210 * that were requested at creation time. not possibly on 211 * earlier chips without busting the ABI. 212 */ 213 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 214 NOUVEAU_GEM_DOMAIN_GART; 215 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 216 nvbo->valid_domains &= domain; 217 218 nvbo->bo.persistent_swap_storage = nvbo->bo.base.filp; 219 *pnvbo = nvbo; 220 return 0; 221 } 222 223 static int 224 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 225 struct drm_nouveau_gem_info *rep) 226 { 227 struct nouveau_cli *cli = nouveau_cli(file_priv); 228 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 229 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 230 struct nouveau_vma *vma; 231 232 if (is_power_of_2(nvbo->valid_domains)) 233 rep->domain = nvbo->valid_domains; 234 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 235 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 236 else 237 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 238 rep->offset = nvbo->bo.offset; 239 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 240 vma = nouveau_vma_find(nvbo, vmm); 241 if (!vma) 242 return -EINVAL; 243 244 rep->offset = vma->addr; 245 } 246 247 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 248 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 249 rep->tile_mode = nvbo->mode; 250 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 251 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 252 rep->tile_flags |= nvbo->kind << 8; 253 else 254 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 255 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 256 else 257 rep->tile_flags |= nvbo->zeta; 258 return 0; 259 } 260 261 int 262 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 263 struct drm_file *file_priv) 264 { 265 struct nouveau_cli *cli = nouveau_cli(file_priv); 266 struct drm_nouveau_gem_new *req = data; 267 struct nouveau_bo *nvbo = NULL; 268 int ret = 0; 269 270 ret = nouveau_gem_new(cli, req->info.size, req->align, 271 req->info.domain, req->info.tile_mode, 272 req->info.tile_flags, &nvbo); 273 if (ret) 274 return ret; 275 276 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 277 &req->info.handle); 278 if (ret == 0) { 279 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 280 if (ret) 281 drm_gem_handle_delete(file_priv, req->info.handle); 282 } 283 284 /* drop reference from allocate - handle holds it now */ 285 drm_gem_object_put_unlocked(&nvbo->bo.base); 286 return ret; 287 } 288 289 static int 290 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 291 uint32_t write_domains, uint32_t valid_domains) 292 { 293 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 294 struct ttm_buffer_object *bo = &nvbo->bo; 295 uint32_t domains = valid_domains & nvbo->valid_domains & 296 (write_domains ? write_domains : read_domains); 297 uint32_t pref_flags = 0, valid_flags = 0; 298 299 if (!domains) 300 return -EINVAL; 301 302 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 303 valid_flags |= TTM_PL_FLAG_VRAM; 304 305 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 306 valid_flags |= TTM_PL_FLAG_TT; 307 308 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 309 bo->mem.mem_type == TTM_PL_VRAM) 310 pref_flags |= TTM_PL_FLAG_VRAM; 311 312 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 313 bo->mem.mem_type == TTM_PL_TT) 314 pref_flags |= TTM_PL_FLAG_TT; 315 316 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 317 pref_flags |= TTM_PL_FLAG_VRAM; 318 319 else 320 pref_flags |= TTM_PL_FLAG_TT; 321 322 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 323 324 return 0; 325 } 326 327 struct validate_op { 328 struct list_head list; 329 struct ww_acquire_ctx ticket; 330 }; 331 332 static void 333 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 334 struct nouveau_fence *fence, 335 struct drm_nouveau_gem_pushbuf_bo *pbbo) 336 { 337 struct nouveau_bo *nvbo; 338 struct drm_nouveau_gem_pushbuf_bo *b; 339 340 while (!list_empty(&op->list)) { 341 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 342 b = &pbbo[nvbo->pbbo_index]; 343 344 if (likely(fence)) { 345 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 346 347 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 348 struct nouveau_vma *vma = 349 (void *)(unsigned long)b->user_priv; 350 nouveau_fence_unref(&vma->fence); 351 dma_fence_get(&fence->base); 352 vma->fence = fence; 353 } 354 } 355 356 if (unlikely(nvbo->validate_mapped)) { 357 ttm_bo_kunmap(&nvbo->kmap); 358 nvbo->validate_mapped = false; 359 } 360 361 list_del(&nvbo->entry); 362 nvbo->reserved_by = NULL; 363 ttm_bo_unreserve(&nvbo->bo); 364 drm_gem_object_put_unlocked(&nvbo->bo.base); 365 } 366 } 367 368 static void 369 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 370 struct nouveau_fence *fence, 371 struct drm_nouveau_gem_pushbuf_bo *pbbo) 372 { 373 validate_fini_no_ticket(op, chan, fence, pbbo); 374 ww_acquire_fini(&op->ticket); 375 } 376 377 static int 378 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 379 struct drm_nouveau_gem_pushbuf_bo *pbbo, 380 int nr_buffers, struct validate_op *op) 381 { 382 struct nouveau_cli *cli = nouveau_cli(file_priv); 383 int trycnt = 0; 384 int ret = -EINVAL, i; 385 struct nouveau_bo *res_bo = NULL; 386 LIST_HEAD(gart_list); 387 LIST_HEAD(vram_list); 388 LIST_HEAD(both_list); 389 390 ww_acquire_init(&op->ticket, &reservation_ww_class); 391 retry: 392 if (++trycnt > 100000) { 393 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 394 return -EINVAL; 395 } 396 397 for (i = 0; i < nr_buffers; i++) { 398 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 399 struct drm_gem_object *gem; 400 struct nouveau_bo *nvbo; 401 402 gem = drm_gem_object_lookup(file_priv, b->handle); 403 if (!gem) { 404 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 405 ret = -ENOENT; 406 break; 407 } 408 nvbo = nouveau_gem_object(gem); 409 if (nvbo == res_bo) { 410 res_bo = NULL; 411 drm_gem_object_put_unlocked(gem); 412 continue; 413 } 414 415 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 416 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 417 "validation list\n", b->handle); 418 drm_gem_object_put_unlocked(gem); 419 ret = -EINVAL; 420 break; 421 } 422 423 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 424 if (ret) { 425 list_splice_tail_init(&vram_list, &op->list); 426 list_splice_tail_init(&gart_list, &op->list); 427 list_splice_tail_init(&both_list, &op->list); 428 validate_fini_no_ticket(op, chan, NULL, NULL); 429 if (unlikely(ret == -EDEADLK)) { 430 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 431 &op->ticket); 432 if (!ret) 433 res_bo = nvbo; 434 } 435 if (unlikely(ret)) { 436 if (ret != -ERESTARTSYS) 437 NV_PRINTK(err, cli, "fail reserve\n"); 438 break; 439 } 440 } 441 442 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 443 struct nouveau_vmm *vmm = chan->vmm; 444 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 445 if (!vma) { 446 NV_PRINTK(err, cli, "vma not found!\n"); 447 ret = -EINVAL; 448 break; 449 } 450 451 b->user_priv = (uint64_t)(unsigned long)vma; 452 } else { 453 b->user_priv = (uint64_t)(unsigned long)nvbo; 454 } 455 456 nvbo->reserved_by = file_priv; 457 nvbo->pbbo_index = i; 458 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 459 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 460 list_add_tail(&nvbo->entry, &both_list); 461 else 462 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 463 list_add_tail(&nvbo->entry, &vram_list); 464 else 465 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 466 list_add_tail(&nvbo->entry, &gart_list); 467 else { 468 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 469 b->valid_domains); 470 list_add_tail(&nvbo->entry, &both_list); 471 ret = -EINVAL; 472 break; 473 } 474 if (nvbo == res_bo) 475 goto retry; 476 } 477 478 ww_acquire_done(&op->ticket); 479 list_splice_tail(&vram_list, &op->list); 480 list_splice_tail(&gart_list, &op->list); 481 list_splice_tail(&both_list, &op->list); 482 if (ret) 483 validate_fini(op, chan, NULL, NULL); 484 return ret; 485 486 } 487 488 static int 489 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 490 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 491 uint64_t user_pbbo_ptr) 492 { 493 struct nouveau_drm *drm = chan->drm; 494 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 495 (void __force __user *)(uintptr_t)user_pbbo_ptr; 496 struct nouveau_bo *nvbo; 497 int ret, relocs = 0; 498 499 list_for_each_entry(nvbo, list, entry) { 500 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 501 502 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 503 b->write_domains, 504 b->valid_domains); 505 if (unlikely(ret)) { 506 NV_PRINTK(err, cli, "fail set_domain\n"); 507 return ret; 508 } 509 510 ret = nouveau_bo_validate(nvbo, true, false); 511 if (unlikely(ret)) { 512 if (ret != -ERESTARTSYS) 513 NV_PRINTK(err, cli, "fail ttm_validate\n"); 514 return ret; 515 } 516 517 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 518 if (unlikely(ret)) { 519 if (ret != -ERESTARTSYS) 520 NV_PRINTK(err, cli, "fail post-validate sync\n"); 521 return ret; 522 } 523 524 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 525 if (nvbo->bo.offset == b->presumed.offset && 526 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 527 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 528 (nvbo->bo.mem.mem_type == TTM_PL_TT && 529 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 530 continue; 531 532 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 533 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 534 else 535 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 536 b->presumed.offset = nvbo->bo.offset; 537 b->presumed.valid = 0; 538 relocs++; 539 540 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed, 541 &b->presumed, sizeof(b->presumed))) 542 return -EFAULT; 543 } 544 } 545 546 return relocs; 547 } 548 549 static int 550 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 551 struct drm_file *file_priv, 552 struct drm_nouveau_gem_pushbuf_bo *pbbo, 553 uint64_t user_buffers, int nr_buffers, 554 struct validate_op *op, int *apply_relocs) 555 { 556 struct nouveau_cli *cli = nouveau_cli(file_priv); 557 int ret; 558 559 INIT_LIST_HEAD(&op->list); 560 561 if (nr_buffers == 0) 562 return 0; 563 564 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 565 if (unlikely(ret)) { 566 if (ret != -ERESTARTSYS) 567 NV_PRINTK(err, cli, "validate_init\n"); 568 return ret; 569 } 570 571 ret = validate_list(chan, cli, &op->list, pbbo, user_buffers); 572 if (unlikely(ret < 0)) { 573 if (ret != -ERESTARTSYS) 574 NV_PRINTK(err, cli, "validating bo list\n"); 575 validate_fini(op, chan, NULL, NULL); 576 return ret; 577 } 578 *apply_relocs = ret; 579 return 0; 580 } 581 582 static inline void 583 u_free(void *addr) 584 { 585 kvfree(addr); 586 } 587 588 static inline void * 589 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 590 { 591 void *mem; 592 void __user *userptr = (void __force __user *)(uintptr_t)user; 593 594 size *= nmemb; 595 596 mem = kvmalloc(size, GFP_KERNEL); 597 if (!mem) 598 return ERR_PTR(-ENOMEM); 599 600 if (copy_from_user(mem, userptr, size)) { 601 u_free(mem); 602 return ERR_PTR(-EFAULT); 603 } 604 605 return mem; 606 } 607 608 static int 609 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 610 struct drm_nouveau_gem_pushbuf *req, 611 struct drm_nouveau_gem_pushbuf_bo *bo) 612 { 613 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 614 int ret = 0; 615 unsigned i; 616 617 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 618 if (IS_ERR(reloc)) 619 return PTR_ERR(reloc); 620 621 for (i = 0; i < req->nr_relocs; i++) { 622 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 623 struct drm_nouveau_gem_pushbuf_bo *b; 624 struct nouveau_bo *nvbo; 625 uint32_t data; 626 627 if (unlikely(r->bo_index >= req->nr_buffers)) { 628 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 629 ret = -EINVAL; 630 break; 631 } 632 633 b = &bo[r->bo_index]; 634 if (b->presumed.valid) 635 continue; 636 637 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 638 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 639 ret = -EINVAL; 640 break; 641 } 642 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 643 644 if (unlikely(r->reloc_bo_offset + 4 > 645 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 646 NV_PRINTK(err, cli, "reloc outside of bo\n"); 647 ret = -EINVAL; 648 break; 649 } 650 651 if (!nvbo->kmap.virtual) { 652 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 653 &nvbo->kmap); 654 if (ret) { 655 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 656 break; 657 } 658 nvbo->validate_mapped = true; 659 } 660 661 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 662 data = b->presumed.offset + r->data; 663 else 664 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 665 data = (b->presumed.offset + r->data) >> 32; 666 else 667 data = r->data; 668 669 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 670 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 671 data |= r->tor; 672 else 673 data |= r->vor; 674 } 675 676 ret = ttm_bo_wait(&nvbo->bo, false, false); 677 if (ret) { 678 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 679 break; 680 } 681 682 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 683 } 684 685 u_free(reloc); 686 return ret; 687 } 688 689 int 690 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 691 struct drm_file *file_priv) 692 { 693 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 694 struct nouveau_cli *cli = nouveau_cli(file_priv); 695 struct nouveau_abi16_chan *temp; 696 struct nouveau_drm *drm = nouveau_drm(dev); 697 struct drm_nouveau_gem_pushbuf *req = data; 698 struct drm_nouveau_gem_pushbuf_push *push; 699 struct drm_nouveau_gem_pushbuf_bo *bo; 700 struct nouveau_channel *chan = NULL; 701 struct validate_op op; 702 struct nouveau_fence *fence = NULL; 703 int i, j, ret = 0, do_reloc = 0; 704 705 if (unlikely(!abi16)) 706 return -ENOMEM; 707 708 list_for_each_entry(temp, &abi16->channels, head) { 709 if (temp->chan->chid == req->channel) { 710 chan = temp->chan; 711 break; 712 } 713 } 714 715 if (!chan) 716 return nouveau_abi16_put(abi16, -ENOENT); 717 718 req->vram_available = drm->gem.vram_available; 719 req->gart_available = drm->gem.gart_available; 720 if (unlikely(req->nr_push == 0)) 721 goto out_next; 722 723 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 724 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 725 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 726 return nouveau_abi16_put(abi16, -EINVAL); 727 } 728 729 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 730 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 731 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 732 return nouveau_abi16_put(abi16, -EINVAL); 733 } 734 735 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 736 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 737 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 738 return nouveau_abi16_put(abi16, -EINVAL); 739 } 740 741 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 742 if (IS_ERR(push)) 743 return nouveau_abi16_put(abi16, PTR_ERR(push)); 744 745 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 746 if (IS_ERR(bo)) { 747 u_free(push); 748 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 749 } 750 751 /* Ensure all push buffers are on validate list */ 752 for (i = 0; i < req->nr_push; i++) { 753 if (push[i].bo_index >= req->nr_buffers) { 754 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 755 ret = -EINVAL; 756 goto out_prevalid; 757 } 758 } 759 760 /* Validate buffer list */ 761 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 762 req->nr_buffers, &op, &do_reloc); 763 if (ret) { 764 if (ret != -ERESTARTSYS) 765 NV_PRINTK(err, cli, "validate: %d\n", ret); 766 goto out_prevalid; 767 } 768 769 /* Apply any relocations that are required */ 770 if (do_reloc) { 771 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 772 if (ret) { 773 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 774 goto out; 775 } 776 } 777 778 if (chan->dma.ib_max) { 779 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 780 if (ret) { 781 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 782 goto out; 783 } 784 785 for (i = 0; i < req->nr_push; i++) { 786 struct nouveau_vma *vma = (void *)(unsigned long) 787 bo[push[i].bo_index].user_priv; 788 789 nv50_dma_push(chan, vma->addr + push[i].offset, 790 push[i].length); 791 } 792 } else 793 if (drm->client.device.info.chipset >= 0x25) { 794 ret = RING_SPACE(chan, req->nr_push * 2); 795 if (ret) { 796 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 797 goto out; 798 } 799 800 for (i = 0; i < req->nr_push; i++) { 801 struct nouveau_bo *nvbo = (void *)(unsigned long) 802 bo[push[i].bo_index].user_priv; 803 804 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 805 OUT_RING(chan, 0); 806 } 807 } else { 808 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 809 if (ret) { 810 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 811 goto out; 812 } 813 814 for (i = 0; i < req->nr_push; i++) { 815 struct nouveau_bo *nvbo = (void *)(unsigned long) 816 bo[push[i].bo_index].user_priv; 817 uint32_t cmd; 818 819 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 820 cmd |= 0x20000000; 821 if (unlikely(cmd != req->suffix0)) { 822 if (!nvbo->kmap.virtual) { 823 ret = ttm_bo_kmap(&nvbo->bo, 0, 824 nvbo->bo.mem. 825 num_pages, 826 &nvbo->kmap); 827 if (ret) { 828 WIND_RING(chan); 829 goto out; 830 } 831 nvbo->validate_mapped = true; 832 } 833 834 nouveau_bo_wr32(nvbo, (push[i].offset + 835 push[i].length - 8) / 4, cmd); 836 } 837 838 OUT_RING(chan, 0x20000000 | 839 (nvbo->bo.offset + push[i].offset)); 840 OUT_RING(chan, 0); 841 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 842 OUT_RING(chan, 0); 843 } 844 } 845 846 ret = nouveau_fence_new(chan, false, &fence); 847 if (ret) { 848 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 849 WIND_RING(chan); 850 goto out; 851 } 852 853 out: 854 validate_fini(&op, chan, fence, bo); 855 nouveau_fence_unref(&fence); 856 857 out_prevalid: 858 u_free(bo); 859 u_free(push); 860 861 out_next: 862 if (chan->dma.ib_max) { 863 req->suffix0 = 0x00000000; 864 req->suffix1 = 0x00000000; 865 } else 866 if (drm->client.device.info.chipset >= 0x25) { 867 req->suffix0 = 0x00020000; 868 req->suffix1 = 0x00000000; 869 } else { 870 req->suffix0 = 0x20000000 | 871 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 872 req->suffix1 = 0x00000000; 873 } 874 875 return nouveau_abi16_put(abi16, ret); 876 } 877 878 int 879 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 880 struct drm_file *file_priv) 881 { 882 struct drm_nouveau_gem_cpu_prep *req = data; 883 struct drm_gem_object *gem; 884 struct nouveau_bo *nvbo; 885 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 886 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 887 long lret; 888 int ret; 889 890 gem = drm_gem_object_lookup(file_priv, req->handle); 891 if (!gem) 892 return -ENOENT; 893 nvbo = nouveau_gem_object(gem); 894 895 lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true, 896 no_wait ? 0 : 30 * HZ); 897 if (!lret) 898 ret = -EBUSY; 899 else if (lret > 0) 900 ret = 0; 901 else 902 ret = lret; 903 904 nouveau_bo_sync_for_cpu(nvbo); 905 drm_gem_object_put_unlocked(gem); 906 907 return ret; 908 } 909 910 int 911 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 912 struct drm_file *file_priv) 913 { 914 struct drm_nouveau_gem_cpu_fini *req = data; 915 struct drm_gem_object *gem; 916 struct nouveau_bo *nvbo; 917 918 gem = drm_gem_object_lookup(file_priv, req->handle); 919 if (!gem) 920 return -ENOENT; 921 nvbo = nouveau_gem_object(gem); 922 923 nouveau_bo_sync_for_device(nvbo); 924 drm_gem_object_put_unlocked(gem); 925 return 0; 926 } 927 928 int 929 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 930 struct drm_file *file_priv) 931 { 932 struct drm_nouveau_gem_info *req = data; 933 struct drm_gem_object *gem; 934 int ret; 935 936 gem = drm_gem_object_lookup(file_priv, req->handle); 937 if (!gem) 938 return -ENOENT; 939 940 ret = nouveau_gem_info(file_priv, gem, req); 941 drm_gem_object_put_unlocked(gem); 942 return ret; 943 } 944 945