1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 #include <nvif/push206e.h> 39 40 void 41 nouveau_gem_object_del(struct drm_gem_object *gem) 42 { 43 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 44 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 45 struct device *dev = drm->dev->dev; 46 int ret; 47 48 ret = pm_runtime_get_sync(dev); 49 if (WARN_ON(ret < 0 && ret != -EACCES)) { 50 pm_runtime_put_autosuspend(dev); 51 return; 52 } 53 54 if (gem->import_attach) 55 drm_prime_gem_destroy(gem, nvbo->bo.sg); 56 57 ttm_bo_put(&nvbo->bo); 58 59 pm_runtime_mark_last_busy(dev); 60 pm_runtime_put_autosuspend(dev); 61 } 62 63 int 64 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 65 { 66 struct nouveau_cli *cli = nouveau_cli(file_priv); 67 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 68 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 69 struct device *dev = drm->dev->dev; 70 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 71 struct nouveau_vma *vma; 72 int ret; 73 74 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 75 return 0; 76 77 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 78 if (ret) 79 return ret; 80 81 ret = pm_runtime_get_sync(dev); 82 if (ret < 0 && ret != -EACCES) { 83 pm_runtime_put_autosuspend(dev); 84 goto out; 85 } 86 87 ret = nouveau_vma_new(nvbo, vmm, &vma); 88 pm_runtime_mark_last_busy(dev); 89 pm_runtime_put_autosuspend(dev); 90 out: 91 ttm_bo_unreserve(&nvbo->bo); 92 return ret; 93 } 94 95 struct nouveau_gem_object_unmap { 96 struct nouveau_cli_work work; 97 struct nouveau_vma *vma; 98 }; 99 100 static void 101 nouveau_gem_object_delete(struct nouveau_vma *vma) 102 { 103 nouveau_fence_unref(&vma->fence); 104 nouveau_vma_del(&vma); 105 } 106 107 static void 108 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 109 { 110 struct nouveau_gem_object_unmap *work = 111 container_of(w, typeof(*work), work); 112 nouveau_gem_object_delete(work->vma); 113 kfree(work); 114 } 115 116 static void 117 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 118 { 119 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 120 struct nouveau_gem_object_unmap *work; 121 122 list_del_init(&vma->head); 123 124 if (!fence) { 125 nouveau_gem_object_delete(vma); 126 return; 127 } 128 129 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 130 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 131 nouveau_gem_object_delete(vma); 132 return; 133 } 134 135 work->work.func = nouveau_gem_object_delete_work; 136 work->vma = vma; 137 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 138 } 139 140 void 141 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 142 { 143 struct nouveau_cli *cli = nouveau_cli(file_priv); 144 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 145 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 146 struct device *dev = drm->dev->dev; 147 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 148 struct nouveau_vma *vma; 149 int ret; 150 151 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 152 return; 153 154 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 155 if (ret) 156 return; 157 158 vma = nouveau_vma_find(nvbo, vmm); 159 if (vma) { 160 if (--vma->refs == 0) { 161 ret = pm_runtime_get_sync(dev); 162 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 163 nouveau_gem_object_unmap(nvbo, vma); 164 pm_runtime_mark_last_busy(dev); 165 } 166 pm_runtime_put_autosuspend(dev); 167 } 168 } 169 ttm_bo_unreserve(&nvbo->bo); 170 } 171 172 int 173 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 174 uint32_t tile_mode, uint32_t tile_flags, 175 struct nouveau_bo **pnvbo) 176 { 177 struct nouveau_drm *drm = cli->drm; 178 struct nouveau_bo *nvbo; 179 u32 flags = 0; 180 int ret; 181 182 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 183 flags |= TTM_PL_FLAG_VRAM; 184 if (domain & NOUVEAU_GEM_DOMAIN_GART) 185 flags |= TTM_PL_FLAG_TT; 186 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 187 flags |= TTM_PL_FLAG_SYSTEM; 188 189 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) 190 flags |= TTM_PL_FLAG_UNCACHED; 191 192 nvbo = nouveau_bo_alloc(cli, &size, &align, flags, tile_mode, 193 tile_flags); 194 if (IS_ERR(nvbo)) 195 return PTR_ERR(nvbo); 196 197 /* Initialize the embedded gem-object. We return a single gem-reference 198 * to the caller, instead of a normal nouveau_bo ttm reference. */ 199 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 200 if (ret) { 201 nouveau_bo_ref(NULL, &nvbo); 202 return ret; 203 } 204 205 ret = nouveau_bo_init(nvbo, size, align, flags, NULL, NULL); 206 if (ret) { 207 nouveau_bo_ref(NULL, &nvbo); 208 return ret; 209 } 210 211 /* we restrict allowed domains on nv50+ to only the types 212 * that were requested at creation time. not possibly on 213 * earlier chips without busting the ABI. 214 */ 215 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 216 NOUVEAU_GEM_DOMAIN_GART; 217 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 218 nvbo->valid_domains &= domain; 219 220 nvbo->bo.persistent_swap_storage = nvbo->bo.base.filp; 221 *pnvbo = nvbo; 222 return 0; 223 } 224 225 static int 226 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 227 struct drm_nouveau_gem_info *rep) 228 { 229 struct nouveau_cli *cli = nouveau_cli(file_priv); 230 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 231 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 232 struct nouveau_vma *vma; 233 234 if (is_power_of_2(nvbo->valid_domains)) 235 rep->domain = nvbo->valid_domains; 236 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 237 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 238 else 239 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 240 rep->offset = nvbo->offset; 241 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 242 vma = nouveau_vma_find(nvbo, vmm); 243 if (!vma) 244 return -EINVAL; 245 246 rep->offset = vma->addr; 247 } 248 249 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 250 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 251 rep->tile_mode = nvbo->mode; 252 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 253 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 254 rep->tile_flags |= nvbo->kind << 8; 255 else 256 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 257 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 258 else 259 rep->tile_flags |= nvbo->zeta; 260 return 0; 261 } 262 263 int 264 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 265 struct drm_file *file_priv) 266 { 267 struct nouveau_cli *cli = nouveau_cli(file_priv); 268 struct drm_nouveau_gem_new *req = data; 269 struct nouveau_bo *nvbo = NULL; 270 int ret = 0; 271 272 ret = nouveau_gem_new(cli, req->info.size, req->align, 273 req->info.domain, req->info.tile_mode, 274 req->info.tile_flags, &nvbo); 275 if (ret) 276 return ret; 277 278 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 279 &req->info.handle); 280 if (ret == 0) { 281 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 282 if (ret) 283 drm_gem_handle_delete(file_priv, req->info.handle); 284 } 285 286 /* drop reference from allocate - handle holds it now */ 287 drm_gem_object_put(&nvbo->bo.base); 288 return ret; 289 } 290 291 static int 292 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 293 uint32_t write_domains, uint32_t valid_domains) 294 { 295 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 296 struct ttm_buffer_object *bo = &nvbo->bo; 297 uint32_t domains = valid_domains & nvbo->valid_domains & 298 (write_domains ? write_domains : read_domains); 299 uint32_t pref_flags = 0, valid_flags = 0; 300 301 if (!domains) 302 return -EINVAL; 303 304 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 305 valid_flags |= TTM_PL_FLAG_VRAM; 306 307 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 308 valid_flags |= TTM_PL_FLAG_TT; 309 310 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 311 bo->mem.mem_type == TTM_PL_VRAM) 312 pref_flags |= TTM_PL_FLAG_VRAM; 313 314 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 315 bo->mem.mem_type == TTM_PL_TT) 316 pref_flags |= TTM_PL_FLAG_TT; 317 318 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 319 pref_flags |= TTM_PL_FLAG_VRAM; 320 321 else 322 pref_flags |= TTM_PL_FLAG_TT; 323 324 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 325 326 return 0; 327 } 328 329 struct validate_op { 330 struct list_head list; 331 struct ww_acquire_ctx ticket; 332 }; 333 334 static void 335 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 336 struct nouveau_fence *fence, 337 struct drm_nouveau_gem_pushbuf_bo *pbbo) 338 { 339 struct nouveau_bo *nvbo; 340 struct drm_nouveau_gem_pushbuf_bo *b; 341 342 while (!list_empty(&op->list)) { 343 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 344 b = &pbbo[nvbo->pbbo_index]; 345 346 if (likely(fence)) { 347 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 348 349 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 350 struct nouveau_vma *vma = 351 (void *)(unsigned long)b->user_priv; 352 nouveau_fence_unref(&vma->fence); 353 dma_fence_get(&fence->base); 354 vma->fence = fence; 355 } 356 } 357 358 if (unlikely(nvbo->validate_mapped)) { 359 ttm_bo_kunmap(&nvbo->kmap); 360 nvbo->validate_mapped = false; 361 } 362 363 list_del(&nvbo->entry); 364 nvbo->reserved_by = NULL; 365 ttm_bo_unreserve(&nvbo->bo); 366 drm_gem_object_put(&nvbo->bo.base); 367 } 368 } 369 370 static void 371 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 372 struct nouveau_fence *fence, 373 struct drm_nouveau_gem_pushbuf_bo *pbbo) 374 { 375 validate_fini_no_ticket(op, chan, fence, pbbo); 376 ww_acquire_fini(&op->ticket); 377 } 378 379 static int 380 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 381 struct drm_nouveau_gem_pushbuf_bo *pbbo, 382 int nr_buffers, struct validate_op *op) 383 { 384 struct nouveau_cli *cli = nouveau_cli(file_priv); 385 int trycnt = 0; 386 int ret = -EINVAL, i; 387 struct nouveau_bo *res_bo = NULL; 388 LIST_HEAD(gart_list); 389 LIST_HEAD(vram_list); 390 LIST_HEAD(both_list); 391 392 ww_acquire_init(&op->ticket, &reservation_ww_class); 393 retry: 394 if (++trycnt > 100000) { 395 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 396 return -EINVAL; 397 } 398 399 for (i = 0; i < nr_buffers; i++) { 400 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 401 struct drm_gem_object *gem; 402 struct nouveau_bo *nvbo; 403 404 gem = drm_gem_object_lookup(file_priv, b->handle); 405 if (!gem) { 406 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 407 ret = -ENOENT; 408 break; 409 } 410 nvbo = nouveau_gem_object(gem); 411 if (nvbo == res_bo) { 412 res_bo = NULL; 413 drm_gem_object_put(gem); 414 continue; 415 } 416 417 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 418 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 419 "validation list\n", b->handle); 420 drm_gem_object_put(gem); 421 ret = -EINVAL; 422 break; 423 } 424 425 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 426 if (ret) { 427 list_splice_tail_init(&vram_list, &op->list); 428 list_splice_tail_init(&gart_list, &op->list); 429 list_splice_tail_init(&both_list, &op->list); 430 validate_fini_no_ticket(op, chan, NULL, NULL); 431 if (unlikely(ret == -EDEADLK)) { 432 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 433 &op->ticket); 434 if (!ret) 435 res_bo = nvbo; 436 } 437 if (unlikely(ret)) { 438 if (ret != -ERESTARTSYS) 439 NV_PRINTK(err, cli, "fail reserve\n"); 440 break; 441 } 442 } 443 444 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 445 struct nouveau_vmm *vmm = chan->vmm; 446 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 447 if (!vma) { 448 NV_PRINTK(err, cli, "vma not found!\n"); 449 ret = -EINVAL; 450 break; 451 } 452 453 b->user_priv = (uint64_t)(unsigned long)vma; 454 } else { 455 b->user_priv = (uint64_t)(unsigned long)nvbo; 456 } 457 458 nvbo->reserved_by = file_priv; 459 nvbo->pbbo_index = i; 460 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 461 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 462 list_add_tail(&nvbo->entry, &both_list); 463 else 464 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 465 list_add_tail(&nvbo->entry, &vram_list); 466 else 467 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 468 list_add_tail(&nvbo->entry, &gart_list); 469 else { 470 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 471 b->valid_domains); 472 list_add_tail(&nvbo->entry, &both_list); 473 ret = -EINVAL; 474 break; 475 } 476 if (nvbo == res_bo) 477 goto retry; 478 } 479 480 ww_acquire_done(&op->ticket); 481 list_splice_tail(&vram_list, &op->list); 482 list_splice_tail(&gart_list, &op->list); 483 list_splice_tail(&both_list, &op->list); 484 if (ret) 485 validate_fini(op, chan, NULL, NULL); 486 return ret; 487 488 } 489 490 static int 491 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 492 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 493 { 494 struct nouveau_drm *drm = chan->drm; 495 struct nouveau_bo *nvbo; 496 int ret, relocs = 0; 497 498 list_for_each_entry(nvbo, list, entry) { 499 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 500 501 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 502 b->write_domains, 503 b->valid_domains); 504 if (unlikely(ret)) { 505 NV_PRINTK(err, cli, "fail set_domain\n"); 506 return ret; 507 } 508 509 ret = nouveau_bo_validate(nvbo, true, false); 510 if (unlikely(ret)) { 511 if (ret != -ERESTARTSYS) 512 NV_PRINTK(err, cli, "fail ttm_validate\n"); 513 return ret; 514 } 515 516 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 517 if (unlikely(ret)) { 518 if (ret != -ERESTARTSYS) 519 NV_PRINTK(err, cli, "fail post-validate sync\n"); 520 return ret; 521 } 522 523 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 524 if (nvbo->offset == b->presumed.offset && 525 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 526 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 527 (nvbo->bo.mem.mem_type == TTM_PL_TT && 528 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 529 continue; 530 531 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 532 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 533 else 534 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 535 b->presumed.offset = nvbo->offset; 536 b->presumed.valid = 0; 537 relocs++; 538 } 539 } 540 541 return relocs; 542 } 543 544 static int 545 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 546 struct drm_file *file_priv, 547 struct drm_nouveau_gem_pushbuf_bo *pbbo, 548 int nr_buffers, 549 struct validate_op *op, bool *apply_relocs) 550 { 551 struct nouveau_cli *cli = nouveau_cli(file_priv); 552 int ret; 553 554 INIT_LIST_HEAD(&op->list); 555 556 if (nr_buffers == 0) 557 return 0; 558 559 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 560 if (unlikely(ret)) { 561 if (ret != -ERESTARTSYS) 562 NV_PRINTK(err, cli, "validate_init\n"); 563 return ret; 564 } 565 566 ret = validate_list(chan, cli, &op->list, pbbo); 567 if (unlikely(ret < 0)) { 568 if (ret != -ERESTARTSYS) 569 NV_PRINTK(err, cli, "validating bo list\n"); 570 validate_fini(op, chan, NULL, NULL); 571 return ret; 572 } 573 *apply_relocs = ret; 574 return 0; 575 } 576 577 static inline void 578 u_free(void *addr) 579 { 580 kvfree(addr); 581 } 582 583 static inline void * 584 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 585 { 586 void *mem; 587 void __user *userptr = (void __force __user *)(uintptr_t)user; 588 589 size *= nmemb; 590 591 mem = kvmalloc(size, GFP_KERNEL); 592 if (!mem) 593 return ERR_PTR(-ENOMEM); 594 595 if (copy_from_user(mem, userptr, size)) { 596 u_free(mem); 597 return ERR_PTR(-EFAULT); 598 } 599 600 return mem; 601 } 602 603 static int 604 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 605 struct drm_nouveau_gem_pushbuf *req, 606 struct drm_nouveau_gem_pushbuf_reloc *reloc, 607 struct drm_nouveau_gem_pushbuf_bo *bo) 608 { 609 int ret = 0; 610 unsigned i; 611 612 for (i = 0; i < req->nr_relocs; i++) { 613 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 614 struct drm_nouveau_gem_pushbuf_bo *b; 615 struct nouveau_bo *nvbo; 616 uint32_t data; 617 618 if (unlikely(r->bo_index >= req->nr_buffers)) { 619 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 620 ret = -EINVAL; 621 break; 622 } 623 624 b = &bo[r->bo_index]; 625 if (b->presumed.valid) 626 continue; 627 628 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 629 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 630 ret = -EINVAL; 631 break; 632 } 633 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 634 635 if (unlikely(r->reloc_bo_offset + 4 > 636 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 637 NV_PRINTK(err, cli, "reloc outside of bo\n"); 638 ret = -EINVAL; 639 break; 640 } 641 642 if (!nvbo->kmap.virtual) { 643 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 644 &nvbo->kmap); 645 if (ret) { 646 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 647 break; 648 } 649 nvbo->validate_mapped = true; 650 } 651 652 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 653 data = b->presumed.offset + r->data; 654 else 655 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 656 data = (b->presumed.offset + r->data) >> 32; 657 else 658 data = r->data; 659 660 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 661 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 662 data |= r->tor; 663 else 664 data |= r->vor; 665 } 666 667 ret = ttm_bo_wait(&nvbo->bo, false, false); 668 if (ret) { 669 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 670 break; 671 } 672 673 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 674 } 675 676 u_free(reloc); 677 return ret; 678 } 679 680 int 681 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 682 struct drm_file *file_priv) 683 { 684 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 685 struct nouveau_cli *cli = nouveau_cli(file_priv); 686 struct nouveau_abi16_chan *temp; 687 struct nouveau_drm *drm = nouveau_drm(dev); 688 struct drm_nouveau_gem_pushbuf *req = data; 689 struct drm_nouveau_gem_pushbuf_push *push; 690 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 691 struct drm_nouveau_gem_pushbuf_bo *bo; 692 struct nouveau_channel *chan = NULL; 693 struct validate_op op; 694 struct nouveau_fence *fence = NULL; 695 int i, j, ret = 0; 696 bool do_reloc = false, sync = false; 697 698 if (unlikely(!abi16)) 699 return -ENOMEM; 700 701 list_for_each_entry(temp, &abi16->channels, head) { 702 if (temp->chan->chid == req->channel) { 703 chan = temp->chan; 704 break; 705 } 706 } 707 708 if (!chan) 709 return nouveau_abi16_put(abi16, -ENOENT); 710 if (unlikely(atomic_read(&chan->killed))) 711 return nouveau_abi16_put(abi16, -ENODEV); 712 713 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 714 715 req->vram_available = drm->gem.vram_available; 716 req->gart_available = drm->gem.gart_available; 717 if (unlikely(req->nr_push == 0)) 718 goto out_next; 719 720 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 721 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 722 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 723 return nouveau_abi16_put(abi16, -EINVAL); 724 } 725 726 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 727 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 728 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 729 return nouveau_abi16_put(abi16, -EINVAL); 730 } 731 732 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 733 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 734 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 735 return nouveau_abi16_put(abi16, -EINVAL); 736 } 737 738 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 739 if (IS_ERR(push)) 740 return nouveau_abi16_put(abi16, PTR_ERR(push)); 741 742 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 743 if (IS_ERR(bo)) { 744 u_free(push); 745 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 746 } 747 748 /* Ensure all push buffers are on validate list */ 749 for (i = 0; i < req->nr_push; i++) { 750 if (push[i].bo_index >= req->nr_buffers) { 751 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 752 ret = -EINVAL; 753 goto out_prevalid; 754 } 755 } 756 757 /* Validate buffer list */ 758 revalidate: 759 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 760 req->nr_buffers, &op, &do_reloc); 761 if (ret) { 762 if (ret != -ERESTARTSYS) 763 NV_PRINTK(err, cli, "validate: %d\n", ret); 764 goto out_prevalid; 765 } 766 767 /* Apply any relocations that are required */ 768 if (do_reloc) { 769 if (!reloc) { 770 validate_fini(&op, chan, NULL, bo); 771 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 772 if (IS_ERR(reloc)) { 773 ret = PTR_ERR(reloc); 774 goto out_prevalid; 775 } 776 777 goto revalidate; 778 } 779 780 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 781 if (ret) { 782 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 783 goto out; 784 } 785 } 786 787 if (chan->dma.ib_max) { 788 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 789 if (ret) { 790 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 791 goto out; 792 } 793 794 for (i = 0; i < req->nr_push; i++) { 795 struct nouveau_vma *vma = (void *)(unsigned long) 796 bo[push[i].bo_index].user_priv; 797 798 nv50_dma_push(chan, vma->addr + push[i].offset, 799 push[i].length); 800 } 801 } else 802 if (drm->client.device.info.chipset >= 0x25) { 803 ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2); 804 if (ret) { 805 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 806 goto out; 807 } 808 809 for (i = 0; i < req->nr_push; i++) { 810 struct nouveau_bo *nvbo = (void *)(unsigned long) 811 bo[push[i].bo_index].user_priv; 812 813 PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset); 814 PUSH_DATA(chan->chan.push, 0); 815 } 816 } else { 817 ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 818 if (ret) { 819 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 820 goto out; 821 } 822 823 for (i = 0; i < req->nr_push; i++) { 824 struct nouveau_bo *nvbo = (void *)(unsigned long) 825 bo[push[i].bo_index].user_priv; 826 uint32_t cmd; 827 828 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 829 cmd |= 0x20000000; 830 if (unlikely(cmd != req->suffix0)) { 831 if (!nvbo->kmap.virtual) { 832 ret = ttm_bo_kmap(&nvbo->bo, 0, 833 nvbo->bo.mem. 834 num_pages, 835 &nvbo->kmap); 836 if (ret) { 837 WIND_RING(chan); 838 goto out; 839 } 840 nvbo->validate_mapped = true; 841 } 842 843 nouveau_bo_wr32(nvbo, (push[i].offset + 844 push[i].length - 8) / 4, cmd); 845 } 846 847 PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset); 848 PUSH_DATA(chan->chan.push, 0); 849 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 850 PUSH_DATA(chan->chan.push, 0); 851 } 852 } 853 854 ret = nouveau_fence_new(chan, false, &fence); 855 if (ret) { 856 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 857 WIND_RING(chan); 858 goto out; 859 } 860 861 if (sync) { 862 if (!(ret = nouveau_fence_wait(fence, false, false))) { 863 if ((ret = dma_fence_get_status(&fence->base)) == 1) 864 ret = 0; 865 } 866 } 867 868 out: 869 validate_fini(&op, chan, fence, bo); 870 nouveau_fence_unref(&fence); 871 872 if (do_reloc) { 873 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 874 u64_to_user_ptr(req->buffers); 875 876 for (i = 0; i < req->nr_buffers; i++) { 877 if (bo[i].presumed.valid) 878 continue; 879 880 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 881 sizeof(bo[i].presumed))) { 882 ret = -EFAULT; 883 break; 884 } 885 } 886 u_free(reloc); 887 } 888 out_prevalid: 889 u_free(bo); 890 u_free(push); 891 892 out_next: 893 if (chan->dma.ib_max) { 894 req->suffix0 = 0x00000000; 895 req->suffix1 = 0x00000000; 896 } else 897 if (drm->client.device.info.chipset >= 0x25) { 898 req->suffix0 = 0x00020000; 899 req->suffix1 = 0x00000000; 900 } else { 901 req->suffix0 = 0x20000000 | 902 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 903 req->suffix1 = 0x00000000; 904 } 905 906 return nouveau_abi16_put(abi16, ret); 907 } 908 909 int 910 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 911 struct drm_file *file_priv) 912 { 913 struct drm_nouveau_gem_cpu_prep *req = data; 914 struct drm_gem_object *gem; 915 struct nouveau_bo *nvbo; 916 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 917 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 918 long lret; 919 int ret; 920 921 gem = drm_gem_object_lookup(file_priv, req->handle); 922 if (!gem) 923 return -ENOENT; 924 nvbo = nouveau_gem_object(gem); 925 926 lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true, 927 no_wait ? 0 : 30 * HZ); 928 if (!lret) 929 ret = -EBUSY; 930 else if (lret > 0) 931 ret = 0; 932 else 933 ret = lret; 934 935 nouveau_bo_sync_for_cpu(nvbo); 936 drm_gem_object_put(gem); 937 938 return ret; 939 } 940 941 int 942 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 943 struct drm_file *file_priv) 944 { 945 struct drm_nouveau_gem_cpu_fini *req = data; 946 struct drm_gem_object *gem; 947 struct nouveau_bo *nvbo; 948 949 gem = drm_gem_object_lookup(file_priv, req->handle); 950 if (!gem) 951 return -ENOENT; 952 nvbo = nouveau_gem_object(gem); 953 954 nouveau_bo_sync_for_device(nvbo); 955 drm_gem_object_put(gem); 956 return 0; 957 } 958 959 int 960 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 961 struct drm_file *file_priv) 962 { 963 struct drm_nouveau_gem_info *req = data; 964 struct drm_gem_object *gem; 965 int ret; 966 967 gem = drm_gem_object_lookup(file_priv, req->handle); 968 if (!gem) 969 return -ENOENT; 970 971 ret = nouveau_gem_info(file_priv, gem, req); 972 drm_gem_object_put(gem); 973 return ret; 974 } 975 976