1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 39 void 40 nouveau_gem_object_del(struct drm_gem_object *gem) 41 { 42 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 43 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 44 struct device *dev = drm->dev->dev; 45 int ret; 46 47 ret = pm_runtime_get_sync(dev); 48 if (WARN_ON(ret < 0 && ret != -EACCES)) 49 return; 50 51 if (gem->import_attach) 52 drm_prime_gem_destroy(gem, nvbo->bo.sg); 53 54 ttm_bo_put(&nvbo->bo); 55 56 pm_runtime_mark_last_busy(dev); 57 pm_runtime_put_autosuspend(dev); 58 } 59 60 int 61 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 62 { 63 struct nouveau_cli *cli = nouveau_cli(file_priv); 64 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 65 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 66 struct device *dev = drm->dev->dev; 67 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 68 struct nouveau_vma *vma; 69 int ret; 70 71 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 72 return 0; 73 74 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 75 if (ret) 76 return ret; 77 78 ret = pm_runtime_get_sync(dev); 79 if (ret < 0 && ret != -EACCES) { 80 pm_runtime_put_autosuspend(dev); 81 goto out; 82 } 83 84 ret = nouveau_vma_new(nvbo, vmm, &vma); 85 pm_runtime_mark_last_busy(dev); 86 pm_runtime_put_autosuspend(dev); 87 out: 88 ttm_bo_unreserve(&nvbo->bo); 89 return ret; 90 } 91 92 struct nouveau_gem_object_unmap { 93 struct nouveau_cli_work work; 94 struct nouveau_vma *vma; 95 }; 96 97 static void 98 nouveau_gem_object_delete(struct nouveau_vma *vma) 99 { 100 nouveau_fence_unref(&vma->fence); 101 nouveau_vma_del(&vma); 102 } 103 104 static void 105 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 106 { 107 struct nouveau_gem_object_unmap *work = 108 container_of(w, typeof(*work), work); 109 nouveau_gem_object_delete(work->vma); 110 kfree(work); 111 } 112 113 static void 114 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 115 { 116 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 117 struct nouveau_gem_object_unmap *work; 118 119 list_del_init(&vma->head); 120 121 if (!fence) { 122 nouveau_gem_object_delete(vma); 123 return; 124 } 125 126 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 127 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 128 nouveau_gem_object_delete(vma); 129 return; 130 } 131 132 work->work.func = nouveau_gem_object_delete_work; 133 work->vma = vma; 134 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 135 } 136 137 void 138 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 139 { 140 struct nouveau_cli *cli = nouveau_cli(file_priv); 141 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 142 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 143 struct device *dev = drm->dev->dev; 144 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 145 struct nouveau_vma *vma; 146 int ret; 147 148 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 149 return; 150 151 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 152 if (ret) 153 return; 154 155 vma = nouveau_vma_find(nvbo, vmm); 156 if (vma) { 157 if (--vma->refs == 0) { 158 ret = pm_runtime_get_sync(dev); 159 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 160 nouveau_gem_object_unmap(nvbo, vma); 161 pm_runtime_mark_last_busy(dev); 162 } 163 pm_runtime_put_autosuspend(dev); 164 } 165 } 166 ttm_bo_unreserve(&nvbo->bo); 167 } 168 169 int 170 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 171 uint32_t tile_mode, uint32_t tile_flags, 172 struct nouveau_bo **pnvbo) 173 { 174 struct nouveau_drm *drm = cli->drm; 175 struct nouveau_bo *nvbo; 176 u32 flags = 0; 177 int ret; 178 179 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 180 flags |= TTM_PL_FLAG_VRAM; 181 if (domain & NOUVEAU_GEM_DOMAIN_GART) 182 flags |= TTM_PL_FLAG_TT; 183 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 184 flags |= TTM_PL_FLAG_SYSTEM; 185 186 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) 187 flags |= TTM_PL_FLAG_UNCACHED; 188 189 nvbo = nouveau_bo_alloc(cli, &size, &align, flags, tile_mode, 190 tile_flags); 191 if (IS_ERR(nvbo)) 192 return PTR_ERR(nvbo); 193 194 /* Initialize the embedded gem-object. We return a single gem-reference 195 * to the caller, instead of a normal nouveau_bo ttm reference. */ 196 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 197 if (ret) { 198 nouveau_bo_ref(NULL, &nvbo); 199 return ret; 200 } 201 202 ret = nouveau_bo_init(nvbo, size, align, flags, NULL, NULL); 203 if (ret) { 204 nouveau_bo_ref(NULL, &nvbo); 205 return ret; 206 } 207 208 /* we restrict allowed domains on nv50+ to only the types 209 * that were requested at creation time. not possibly on 210 * earlier chips without busting the ABI. 211 */ 212 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 213 NOUVEAU_GEM_DOMAIN_GART; 214 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 215 nvbo->valid_domains &= domain; 216 217 nvbo->bo.persistent_swap_storage = nvbo->bo.base.filp; 218 *pnvbo = nvbo; 219 return 0; 220 } 221 222 static int 223 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 224 struct drm_nouveau_gem_info *rep) 225 { 226 struct nouveau_cli *cli = nouveau_cli(file_priv); 227 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 228 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 229 struct nouveau_vma *vma; 230 231 if (is_power_of_2(nvbo->valid_domains)) 232 rep->domain = nvbo->valid_domains; 233 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 234 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 235 else 236 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 237 rep->offset = nvbo->bo.offset; 238 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 239 vma = nouveau_vma_find(nvbo, vmm); 240 if (!vma) 241 return -EINVAL; 242 243 rep->offset = vma->addr; 244 } 245 246 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 247 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 248 rep->tile_mode = nvbo->mode; 249 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 250 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 251 rep->tile_flags |= nvbo->kind << 8; 252 else 253 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 254 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 255 else 256 rep->tile_flags |= nvbo->zeta; 257 return 0; 258 } 259 260 int 261 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 262 struct drm_file *file_priv) 263 { 264 struct nouveau_cli *cli = nouveau_cli(file_priv); 265 struct drm_nouveau_gem_new *req = data; 266 struct nouveau_bo *nvbo = NULL; 267 int ret = 0; 268 269 ret = nouveau_gem_new(cli, req->info.size, req->align, 270 req->info.domain, req->info.tile_mode, 271 req->info.tile_flags, &nvbo); 272 if (ret) 273 return ret; 274 275 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 276 &req->info.handle); 277 if (ret == 0) { 278 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 279 if (ret) 280 drm_gem_handle_delete(file_priv, req->info.handle); 281 } 282 283 /* drop reference from allocate - handle holds it now */ 284 drm_gem_object_put_unlocked(&nvbo->bo.base); 285 return ret; 286 } 287 288 static int 289 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 290 uint32_t write_domains, uint32_t valid_domains) 291 { 292 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 293 struct ttm_buffer_object *bo = &nvbo->bo; 294 uint32_t domains = valid_domains & nvbo->valid_domains & 295 (write_domains ? write_domains : read_domains); 296 uint32_t pref_flags = 0, valid_flags = 0; 297 298 if (!domains) 299 return -EINVAL; 300 301 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 302 valid_flags |= TTM_PL_FLAG_VRAM; 303 304 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 305 valid_flags |= TTM_PL_FLAG_TT; 306 307 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 308 bo->mem.mem_type == TTM_PL_VRAM) 309 pref_flags |= TTM_PL_FLAG_VRAM; 310 311 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 312 bo->mem.mem_type == TTM_PL_TT) 313 pref_flags |= TTM_PL_FLAG_TT; 314 315 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 316 pref_flags |= TTM_PL_FLAG_VRAM; 317 318 else 319 pref_flags |= TTM_PL_FLAG_TT; 320 321 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 322 323 return 0; 324 } 325 326 struct validate_op { 327 struct list_head list; 328 struct ww_acquire_ctx ticket; 329 }; 330 331 static void 332 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 333 struct nouveau_fence *fence, 334 struct drm_nouveau_gem_pushbuf_bo *pbbo) 335 { 336 struct nouveau_bo *nvbo; 337 struct drm_nouveau_gem_pushbuf_bo *b; 338 339 while (!list_empty(&op->list)) { 340 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 341 b = &pbbo[nvbo->pbbo_index]; 342 343 if (likely(fence)) { 344 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 345 346 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 347 struct nouveau_vma *vma = 348 (void *)(unsigned long)b->user_priv; 349 nouveau_fence_unref(&vma->fence); 350 dma_fence_get(&fence->base); 351 vma->fence = fence; 352 } 353 } 354 355 if (unlikely(nvbo->validate_mapped)) { 356 ttm_bo_kunmap(&nvbo->kmap); 357 nvbo->validate_mapped = false; 358 } 359 360 list_del(&nvbo->entry); 361 nvbo->reserved_by = NULL; 362 ttm_bo_unreserve(&nvbo->bo); 363 drm_gem_object_put_unlocked(&nvbo->bo.base); 364 } 365 } 366 367 static void 368 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 369 struct nouveau_fence *fence, 370 struct drm_nouveau_gem_pushbuf_bo *pbbo) 371 { 372 validate_fini_no_ticket(op, chan, fence, pbbo); 373 ww_acquire_fini(&op->ticket); 374 } 375 376 static int 377 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 378 struct drm_nouveau_gem_pushbuf_bo *pbbo, 379 int nr_buffers, struct validate_op *op) 380 { 381 struct nouveau_cli *cli = nouveau_cli(file_priv); 382 int trycnt = 0; 383 int ret = -EINVAL, i; 384 struct nouveau_bo *res_bo = NULL; 385 LIST_HEAD(gart_list); 386 LIST_HEAD(vram_list); 387 LIST_HEAD(both_list); 388 389 ww_acquire_init(&op->ticket, &reservation_ww_class); 390 retry: 391 if (++trycnt > 100000) { 392 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 393 return -EINVAL; 394 } 395 396 for (i = 0; i < nr_buffers; i++) { 397 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 398 struct drm_gem_object *gem; 399 struct nouveau_bo *nvbo; 400 401 gem = drm_gem_object_lookup(file_priv, b->handle); 402 if (!gem) { 403 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 404 ret = -ENOENT; 405 break; 406 } 407 nvbo = nouveau_gem_object(gem); 408 if (nvbo == res_bo) { 409 res_bo = NULL; 410 drm_gem_object_put_unlocked(gem); 411 continue; 412 } 413 414 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 415 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 416 "validation list\n", b->handle); 417 drm_gem_object_put_unlocked(gem); 418 ret = -EINVAL; 419 break; 420 } 421 422 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 423 if (ret) { 424 list_splice_tail_init(&vram_list, &op->list); 425 list_splice_tail_init(&gart_list, &op->list); 426 list_splice_tail_init(&both_list, &op->list); 427 validate_fini_no_ticket(op, chan, NULL, NULL); 428 if (unlikely(ret == -EDEADLK)) { 429 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 430 &op->ticket); 431 if (!ret) 432 res_bo = nvbo; 433 } 434 if (unlikely(ret)) { 435 if (ret != -ERESTARTSYS) 436 NV_PRINTK(err, cli, "fail reserve\n"); 437 break; 438 } 439 } 440 441 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 442 struct nouveau_vmm *vmm = chan->vmm; 443 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 444 if (!vma) { 445 NV_PRINTK(err, cli, "vma not found!\n"); 446 ret = -EINVAL; 447 break; 448 } 449 450 b->user_priv = (uint64_t)(unsigned long)vma; 451 } else { 452 b->user_priv = (uint64_t)(unsigned long)nvbo; 453 } 454 455 nvbo->reserved_by = file_priv; 456 nvbo->pbbo_index = i; 457 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 458 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 459 list_add_tail(&nvbo->entry, &both_list); 460 else 461 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 462 list_add_tail(&nvbo->entry, &vram_list); 463 else 464 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 465 list_add_tail(&nvbo->entry, &gart_list); 466 else { 467 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 468 b->valid_domains); 469 list_add_tail(&nvbo->entry, &both_list); 470 ret = -EINVAL; 471 break; 472 } 473 if (nvbo == res_bo) 474 goto retry; 475 } 476 477 ww_acquire_done(&op->ticket); 478 list_splice_tail(&vram_list, &op->list); 479 list_splice_tail(&gart_list, &op->list); 480 list_splice_tail(&both_list, &op->list); 481 if (ret) 482 validate_fini(op, chan, NULL, NULL); 483 return ret; 484 485 } 486 487 static int 488 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 489 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 490 { 491 struct nouveau_drm *drm = chan->drm; 492 struct nouveau_bo *nvbo; 493 int ret, relocs = 0; 494 495 list_for_each_entry(nvbo, list, entry) { 496 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 497 498 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 499 b->write_domains, 500 b->valid_domains); 501 if (unlikely(ret)) { 502 NV_PRINTK(err, cli, "fail set_domain\n"); 503 return ret; 504 } 505 506 ret = nouveau_bo_validate(nvbo, true, false); 507 if (unlikely(ret)) { 508 if (ret != -ERESTARTSYS) 509 NV_PRINTK(err, cli, "fail ttm_validate\n"); 510 return ret; 511 } 512 513 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 514 if (unlikely(ret)) { 515 if (ret != -ERESTARTSYS) 516 NV_PRINTK(err, cli, "fail post-validate sync\n"); 517 return ret; 518 } 519 520 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 521 if (nvbo->bo.offset == b->presumed.offset && 522 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 523 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 524 (nvbo->bo.mem.mem_type == TTM_PL_TT && 525 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 526 continue; 527 528 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 529 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 530 else 531 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 532 b->presumed.offset = nvbo->bo.offset; 533 b->presumed.valid = 0; 534 relocs++; 535 } 536 } 537 538 return relocs; 539 } 540 541 static int 542 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 543 struct drm_file *file_priv, 544 struct drm_nouveau_gem_pushbuf_bo *pbbo, 545 int nr_buffers, 546 struct validate_op *op, bool *apply_relocs) 547 { 548 struct nouveau_cli *cli = nouveau_cli(file_priv); 549 int ret; 550 551 INIT_LIST_HEAD(&op->list); 552 553 if (nr_buffers == 0) 554 return 0; 555 556 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 557 if (unlikely(ret)) { 558 if (ret != -ERESTARTSYS) 559 NV_PRINTK(err, cli, "validate_init\n"); 560 return ret; 561 } 562 563 ret = validate_list(chan, cli, &op->list, pbbo); 564 if (unlikely(ret < 0)) { 565 if (ret != -ERESTARTSYS) 566 NV_PRINTK(err, cli, "validating bo list\n"); 567 validate_fini(op, chan, NULL, NULL); 568 return ret; 569 } 570 *apply_relocs = ret; 571 return 0; 572 } 573 574 static inline void 575 u_free(void *addr) 576 { 577 kvfree(addr); 578 } 579 580 static inline void * 581 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 582 { 583 void *mem; 584 void __user *userptr = (void __force __user *)(uintptr_t)user; 585 586 size *= nmemb; 587 588 mem = kvmalloc(size, GFP_KERNEL); 589 if (!mem) 590 return ERR_PTR(-ENOMEM); 591 592 if (copy_from_user(mem, userptr, size)) { 593 u_free(mem); 594 return ERR_PTR(-EFAULT); 595 } 596 597 return mem; 598 } 599 600 static int 601 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 602 struct drm_nouveau_gem_pushbuf *req, 603 struct drm_nouveau_gem_pushbuf_reloc *reloc, 604 struct drm_nouveau_gem_pushbuf_bo *bo) 605 { 606 int ret = 0; 607 unsigned i; 608 609 for (i = 0; i < req->nr_relocs; i++) { 610 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 611 struct drm_nouveau_gem_pushbuf_bo *b; 612 struct nouveau_bo *nvbo; 613 uint32_t data; 614 615 if (unlikely(r->bo_index >= req->nr_buffers)) { 616 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 617 ret = -EINVAL; 618 break; 619 } 620 621 b = &bo[r->bo_index]; 622 if (b->presumed.valid) 623 continue; 624 625 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 626 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 627 ret = -EINVAL; 628 break; 629 } 630 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 631 632 if (unlikely(r->reloc_bo_offset + 4 > 633 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 634 NV_PRINTK(err, cli, "reloc outside of bo\n"); 635 ret = -EINVAL; 636 break; 637 } 638 639 if (!nvbo->kmap.virtual) { 640 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 641 &nvbo->kmap); 642 if (ret) { 643 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 644 break; 645 } 646 nvbo->validate_mapped = true; 647 } 648 649 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 650 data = b->presumed.offset + r->data; 651 else 652 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 653 data = (b->presumed.offset + r->data) >> 32; 654 else 655 data = r->data; 656 657 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 658 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 659 data |= r->tor; 660 else 661 data |= r->vor; 662 } 663 664 ret = ttm_bo_wait(&nvbo->bo, false, false); 665 if (ret) { 666 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 667 break; 668 } 669 670 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 671 } 672 673 u_free(reloc); 674 return ret; 675 } 676 677 int 678 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 679 struct drm_file *file_priv) 680 { 681 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 682 struct nouveau_cli *cli = nouveau_cli(file_priv); 683 struct nouveau_abi16_chan *temp; 684 struct nouveau_drm *drm = nouveau_drm(dev); 685 struct drm_nouveau_gem_pushbuf *req = data; 686 struct drm_nouveau_gem_pushbuf_push *push; 687 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 688 struct drm_nouveau_gem_pushbuf_bo *bo; 689 struct nouveau_channel *chan = NULL; 690 struct validate_op op; 691 struct nouveau_fence *fence = NULL; 692 int i, j, ret = 0; 693 bool do_reloc = false, sync = false; 694 695 if (unlikely(!abi16)) 696 return -ENOMEM; 697 698 list_for_each_entry(temp, &abi16->channels, head) { 699 if (temp->chan->chid == req->channel) { 700 chan = temp->chan; 701 break; 702 } 703 } 704 705 if (!chan) 706 return nouveau_abi16_put(abi16, -ENOENT); 707 if (unlikely(atomic_read(&chan->killed))) 708 return nouveau_abi16_put(abi16, -ENODEV); 709 710 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 711 712 req->vram_available = drm->gem.vram_available; 713 req->gart_available = drm->gem.gart_available; 714 if (unlikely(req->nr_push == 0)) 715 goto out_next; 716 717 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 718 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 719 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 720 return nouveau_abi16_put(abi16, -EINVAL); 721 } 722 723 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 724 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 725 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 726 return nouveau_abi16_put(abi16, -EINVAL); 727 } 728 729 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 730 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 731 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 732 return nouveau_abi16_put(abi16, -EINVAL); 733 } 734 735 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 736 if (IS_ERR(push)) 737 return nouveau_abi16_put(abi16, PTR_ERR(push)); 738 739 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 740 if (IS_ERR(bo)) { 741 u_free(push); 742 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 743 } 744 745 /* Ensure all push buffers are on validate list */ 746 for (i = 0; i < req->nr_push; i++) { 747 if (push[i].bo_index >= req->nr_buffers) { 748 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 749 ret = -EINVAL; 750 goto out_prevalid; 751 } 752 } 753 754 /* Validate buffer list */ 755 revalidate: 756 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 757 req->nr_buffers, &op, &do_reloc); 758 if (ret) { 759 if (ret != -ERESTARTSYS) 760 NV_PRINTK(err, cli, "validate: %d\n", ret); 761 goto out_prevalid; 762 } 763 764 /* Apply any relocations that are required */ 765 if (do_reloc) { 766 if (!reloc) { 767 validate_fini(&op, chan, NULL, bo); 768 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 769 if (IS_ERR(reloc)) { 770 ret = PTR_ERR(reloc); 771 goto out_prevalid; 772 } 773 774 goto revalidate; 775 } 776 777 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 778 if (ret) { 779 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 780 goto out; 781 } 782 } 783 784 if (chan->dma.ib_max) { 785 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 786 if (ret) { 787 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 788 goto out; 789 } 790 791 for (i = 0; i < req->nr_push; i++) { 792 struct nouveau_vma *vma = (void *)(unsigned long) 793 bo[push[i].bo_index].user_priv; 794 795 nv50_dma_push(chan, vma->addr + push[i].offset, 796 push[i].length); 797 } 798 } else 799 if (drm->client.device.info.chipset >= 0x25) { 800 ret = RING_SPACE(chan, req->nr_push * 2); 801 if (ret) { 802 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 803 goto out; 804 } 805 806 for (i = 0; i < req->nr_push; i++) { 807 struct nouveau_bo *nvbo = (void *)(unsigned long) 808 bo[push[i].bo_index].user_priv; 809 810 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 811 OUT_RING(chan, 0); 812 } 813 } else { 814 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 815 if (ret) { 816 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 817 goto out; 818 } 819 820 for (i = 0; i < req->nr_push; i++) { 821 struct nouveau_bo *nvbo = (void *)(unsigned long) 822 bo[push[i].bo_index].user_priv; 823 uint32_t cmd; 824 825 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 826 cmd |= 0x20000000; 827 if (unlikely(cmd != req->suffix0)) { 828 if (!nvbo->kmap.virtual) { 829 ret = ttm_bo_kmap(&nvbo->bo, 0, 830 nvbo->bo.mem. 831 num_pages, 832 &nvbo->kmap); 833 if (ret) { 834 WIND_RING(chan); 835 goto out; 836 } 837 nvbo->validate_mapped = true; 838 } 839 840 nouveau_bo_wr32(nvbo, (push[i].offset + 841 push[i].length - 8) / 4, cmd); 842 } 843 844 OUT_RING(chan, 0x20000000 | 845 (nvbo->bo.offset + push[i].offset)); 846 OUT_RING(chan, 0); 847 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 848 OUT_RING(chan, 0); 849 } 850 } 851 852 ret = nouveau_fence_new(chan, false, &fence); 853 if (ret) { 854 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 855 WIND_RING(chan); 856 goto out; 857 } 858 859 if (sync) { 860 if (!(ret = nouveau_fence_wait(fence, false, false))) { 861 if ((ret = dma_fence_get_status(&fence->base)) == 1) 862 ret = 0; 863 } 864 } 865 866 out: 867 validate_fini(&op, chan, fence, bo); 868 nouveau_fence_unref(&fence); 869 870 if (do_reloc) { 871 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 872 u64_to_user_ptr(req->buffers); 873 874 for (i = 0; i < req->nr_buffers; i++) { 875 if (bo[i].presumed.valid) 876 continue; 877 878 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 879 sizeof(bo[i].presumed))) { 880 ret = -EFAULT; 881 break; 882 } 883 } 884 u_free(reloc); 885 } 886 out_prevalid: 887 u_free(bo); 888 u_free(push); 889 890 out_next: 891 if (chan->dma.ib_max) { 892 req->suffix0 = 0x00000000; 893 req->suffix1 = 0x00000000; 894 } else 895 if (drm->client.device.info.chipset >= 0x25) { 896 req->suffix0 = 0x00020000; 897 req->suffix1 = 0x00000000; 898 } else { 899 req->suffix0 = 0x20000000 | 900 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 901 req->suffix1 = 0x00000000; 902 } 903 904 return nouveau_abi16_put(abi16, ret); 905 } 906 907 int 908 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 909 struct drm_file *file_priv) 910 { 911 struct drm_nouveau_gem_cpu_prep *req = data; 912 struct drm_gem_object *gem; 913 struct nouveau_bo *nvbo; 914 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 915 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 916 long lret; 917 int ret; 918 919 gem = drm_gem_object_lookup(file_priv, req->handle); 920 if (!gem) 921 return -ENOENT; 922 nvbo = nouveau_gem_object(gem); 923 924 lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true, 925 no_wait ? 0 : 30 * HZ); 926 if (!lret) 927 ret = -EBUSY; 928 else if (lret > 0) 929 ret = 0; 930 else 931 ret = lret; 932 933 nouveau_bo_sync_for_cpu(nvbo); 934 drm_gem_object_put_unlocked(gem); 935 936 return ret; 937 } 938 939 int 940 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 941 struct drm_file *file_priv) 942 { 943 struct drm_nouveau_gem_cpu_fini *req = data; 944 struct drm_gem_object *gem; 945 struct nouveau_bo *nvbo; 946 947 gem = drm_gem_object_lookup(file_priv, req->handle); 948 if (!gem) 949 return -ENOENT; 950 nvbo = nouveau_gem_object(gem); 951 952 nouveau_bo_sync_for_device(nvbo); 953 drm_gem_object_put_unlocked(gem); 954 return 0; 955 } 956 957 int 958 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 959 struct drm_file *file_priv) 960 { 961 struct drm_nouveau_gem_info *req = data; 962 struct drm_gem_object *gem; 963 int ret; 964 965 gem = drm_gem_object_lookup(file_priv, req->handle); 966 if (!gem) 967 return -ENOENT; 968 969 ret = nouveau_gem_info(file_priv, gem, req); 970 drm_gem_object_put_unlocked(gem); 971 return ret; 972 } 973 974