1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 #include <nvif/push206e.h> 39 40 void 41 nouveau_gem_object_del(struct drm_gem_object *gem) 42 { 43 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 44 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 45 struct device *dev = drm->dev->dev; 46 int ret; 47 48 ret = pm_runtime_get_sync(dev); 49 if (WARN_ON(ret < 0 && ret != -EACCES)) { 50 pm_runtime_put_autosuspend(dev); 51 return; 52 } 53 54 if (gem->import_attach) 55 drm_prime_gem_destroy(gem, nvbo->bo.sg); 56 57 ttm_bo_put(&nvbo->bo); 58 59 pm_runtime_mark_last_busy(dev); 60 pm_runtime_put_autosuspend(dev); 61 } 62 63 int 64 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 65 { 66 struct nouveau_cli *cli = nouveau_cli(file_priv); 67 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 68 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 69 struct device *dev = drm->dev->dev; 70 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 71 struct nouveau_vma *vma; 72 int ret; 73 74 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 75 return 0; 76 77 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 78 if (ret) 79 return ret; 80 81 ret = pm_runtime_get_sync(dev); 82 if (ret < 0 && ret != -EACCES) { 83 pm_runtime_put_autosuspend(dev); 84 goto out; 85 } 86 87 ret = nouveau_vma_new(nvbo, vmm, &vma); 88 pm_runtime_mark_last_busy(dev); 89 pm_runtime_put_autosuspend(dev); 90 out: 91 ttm_bo_unreserve(&nvbo->bo); 92 return ret; 93 } 94 95 struct nouveau_gem_object_unmap { 96 struct nouveau_cli_work work; 97 struct nouveau_vma *vma; 98 }; 99 100 static void 101 nouveau_gem_object_delete(struct nouveau_vma *vma) 102 { 103 nouveau_fence_unref(&vma->fence); 104 nouveau_vma_del(&vma); 105 } 106 107 static void 108 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 109 { 110 struct nouveau_gem_object_unmap *work = 111 container_of(w, typeof(*work), work); 112 nouveau_gem_object_delete(work->vma); 113 kfree(work); 114 } 115 116 static void 117 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 118 { 119 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 120 struct nouveau_gem_object_unmap *work; 121 122 list_del_init(&vma->head); 123 124 if (!fence) { 125 nouveau_gem_object_delete(vma); 126 return; 127 } 128 129 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 130 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 131 nouveau_gem_object_delete(vma); 132 return; 133 } 134 135 work->work.func = nouveau_gem_object_delete_work; 136 work->vma = vma; 137 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 138 } 139 140 void 141 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 142 { 143 struct nouveau_cli *cli = nouveau_cli(file_priv); 144 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 145 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 146 struct device *dev = drm->dev->dev; 147 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 148 struct nouveau_vma *vma; 149 int ret; 150 151 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 152 return; 153 154 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 155 if (ret) 156 return; 157 158 vma = nouveau_vma_find(nvbo, vmm); 159 if (vma) { 160 if (--vma->refs == 0) { 161 ret = pm_runtime_get_sync(dev); 162 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 163 nouveau_gem_object_unmap(nvbo, vma); 164 pm_runtime_mark_last_busy(dev); 165 } 166 pm_runtime_put_autosuspend(dev); 167 } 168 } 169 ttm_bo_unreserve(&nvbo->bo); 170 } 171 172 int 173 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 174 uint32_t tile_mode, uint32_t tile_flags, 175 struct nouveau_bo **pnvbo) 176 { 177 struct nouveau_drm *drm = cli->drm; 178 struct nouveau_bo *nvbo; 179 int ret; 180 181 if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) 182 domain |= NOUVEAU_GEM_DOMAIN_CPU; 183 184 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 185 tile_flags); 186 if (IS_ERR(nvbo)) 187 return PTR_ERR(nvbo); 188 189 /* Initialize the embedded gem-object. We return a single gem-reference 190 * to the caller, instead of a normal nouveau_bo ttm reference. */ 191 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 192 if (ret) { 193 drm_gem_object_release(&nvbo->bo.base); 194 kfree(nvbo); 195 return ret; 196 } 197 198 ret = nouveau_bo_init(nvbo, size, align, domain, NULL, NULL); 199 if (ret) { 200 nouveau_bo_ref(NULL, &nvbo); 201 return ret; 202 } 203 204 /* we restrict allowed domains on nv50+ to only the types 205 * that were requested at creation time. not possibly on 206 * earlier chips without busting the ABI. 207 */ 208 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 209 NOUVEAU_GEM_DOMAIN_GART; 210 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 211 nvbo->valid_domains &= domain; 212 213 nvbo->bo.persistent_swap_storage = nvbo->bo.base.filp; 214 *pnvbo = nvbo; 215 return 0; 216 } 217 218 static int 219 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 220 struct drm_nouveau_gem_info *rep) 221 { 222 struct nouveau_cli *cli = nouveau_cli(file_priv); 223 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 224 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 225 struct nouveau_vma *vma; 226 227 if (is_power_of_2(nvbo->valid_domains)) 228 rep->domain = nvbo->valid_domains; 229 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 230 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 231 else 232 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 233 rep->offset = nvbo->offset; 234 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 235 vma = nouveau_vma_find(nvbo, vmm); 236 if (!vma) 237 return -EINVAL; 238 239 rep->offset = vma->addr; 240 } 241 242 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 243 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 244 rep->tile_mode = nvbo->mode; 245 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 246 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 247 rep->tile_flags |= nvbo->kind << 8; 248 else 249 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 250 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 251 else 252 rep->tile_flags |= nvbo->zeta; 253 return 0; 254 } 255 256 int 257 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 258 struct drm_file *file_priv) 259 { 260 struct nouveau_cli *cli = nouveau_cli(file_priv); 261 struct drm_nouveau_gem_new *req = data; 262 struct nouveau_bo *nvbo = NULL; 263 int ret = 0; 264 265 ret = nouveau_gem_new(cli, req->info.size, req->align, 266 req->info.domain, req->info.tile_mode, 267 req->info.tile_flags, &nvbo); 268 if (ret) 269 return ret; 270 271 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 272 &req->info.handle); 273 if (ret == 0) { 274 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 275 if (ret) 276 drm_gem_handle_delete(file_priv, req->info.handle); 277 } 278 279 /* drop reference from allocate - handle holds it now */ 280 drm_gem_object_put(&nvbo->bo.base); 281 return ret; 282 } 283 284 static int 285 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 286 uint32_t write_domains, uint32_t valid_domains) 287 { 288 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 289 struct ttm_buffer_object *bo = &nvbo->bo; 290 uint32_t domains = valid_domains & nvbo->valid_domains & 291 (write_domains ? write_domains : read_domains); 292 uint32_t pref_domains = 0;; 293 294 if (!domains) 295 return -EINVAL; 296 297 valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART); 298 299 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 300 bo->mem.mem_type == TTM_PL_VRAM) 301 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 302 303 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 304 bo->mem.mem_type == TTM_PL_TT) 305 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 306 307 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 308 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 309 310 else 311 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 312 313 nouveau_bo_placement_set(nvbo, pref_domains, valid_domains); 314 315 return 0; 316 } 317 318 struct validate_op { 319 struct list_head list; 320 struct ww_acquire_ctx ticket; 321 }; 322 323 static void 324 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 325 struct nouveau_fence *fence, 326 struct drm_nouveau_gem_pushbuf_bo *pbbo) 327 { 328 struct nouveau_bo *nvbo; 329 struct drm_nouveau_gem_pushbuf_bo *b; 330 331 while (!list_empty(&op->list)) { 332 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 333 b = &pbbo[nvbo->pbbo_index]; 334 335 if (likely(fence)) { 336 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 337 338 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 339 struct nouveau_vma *vma = 340 (void *)(unsigned long)b->user_priv; 341 nouveau_fence_unref(&vma->fence); 342 dma_fence_get(&fence->base); 343 vma->fence = fence; 344 } 345 } 346 347 if (unlikely(nvbo->validate_mapped)) { 348 ttm_bo_kunmap(&nvbo->kmap); 349 nvbo->validate_mapped = false; 350 } 351 352 list_del(&nvbo->entry); 353 nvbo->reserved_by = NULL; 354 ttm_bo_unreserve(&nvbo->bo); 355 drm_gem_object_put(&nvbo->bo.base); 356 } 357 } 358 359 static void 360 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 361 struct nouveau_fence *fence, 362 struct drm_nouveau_gem_pushbuf_bo *pbbo) 363 { 364 validate_fini_no_ticket(op, chan, fence, pbbo); 365 ww_acquire_fini(&op->ticket); 366 } 367 368 static int 369 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 370 struct drm_nouveau_gem_pushbuf_bo *pbbo, 371 int nr_buffers, struct validate_op *op) 372 { 373 struct nouveau_cli *cli = nouveau_cli(file_priv); 374 int trycnt = 0; 375 int ret = -EINVAL, i; 376 struct nouveau_bo *res_bo = NULL; 377 LIST_HEAD(gart_list); 378 LIST_HEAD(vram_list); 379 LIST_HEAD(both_list); 380 381 ww_acquire_init(&op->ticket, &reservation_ww_class); 382 retry: 383 if (++trycnt > 100000) { 384 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 385 return -EINVAL; 386 } 387 388 for (i = 0; i < nr_buffers; i++) { 389 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 390 struct drm_gem_object *gem; 391 struct nouveau_bo *nvbo; 392 393 gem = drm_gem_object_lookup(file_priv, b->handle); 394 if (!gem) { 395 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 396 ret = -ENOENT; 397 break; 398 } 399 nvbo = nouveau_gem_object(gem); 400 if (nvbo == res_bo) { 401 res_bo = NULL; 402 drm_gem_object_put(gem); 403 continue; 404 } 405 406 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 407 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 408 "validation list\n", b->handle); 409 drm_gem_object_put(gem); 410 ret = -EINVAL; 411 break; 412 } 413 414 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 415 if (ret) { 416 list_splice_tail_init(&vram_list, &op->list); 417 list_splice_tail_init(&gart_list, &op->list); 418 list_splice_tail_init(&both_list, &op->list); 419 validate_fini_no_ticket(op, chan, NULL, NULL); 420 if (unlikely(ret == -EDEADLK)) { 421 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 422 &op->ticket); 423 if (!ret) 424 res_bo = nvbo; 425 } 426 if (unlikely(ret)) { 427 if (ret != -ERESTARTSYS) 428 NV_PRINTK(err, cli, "fail reserve\n"); 429 break; 430 } 431 } 432 433 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 434 struct nouveau_vmm *vmm = chan->vmm; 435 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 436 if (!vma) { 437 NV_PRINTK(err, cli, "vma not found!\n"); 438 ret = -EINVAL; 439 break; 440 } 441 442 b->user_priv = (uint64_t)(unsigned long)vma; 443 } else { 444 b->user_priv = (uint64_t)(unsigned long)nvbo; 445 } 446 447 nvbo->reserved_by = file_priv; 448 nvbo->pbbo_index = i; 449 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 450 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 451 list_add_tail(&nvbo->entry, &both_list); 452 else 453 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 454 list_add_tail(&nvbo->entry, &vram_list); 455 else 456 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 457 list_add_tail(&nvbo->entry, &gart_list); 458 else { 459 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 460 b->valid_domains); 461 list_add_tail(&nvbo->entry, &both_list); 462 ret = -EINVAL; 463 break; 464 } 465 if (nvbo == res_bo) 466 goto retry; 467 } 468 469 ww_acquire_done(&op->ticket); 470 list_splice_tail(&vram_list, &op->list); 471 list_splice_tail(&gart_list, &op->list); 472 list_splice_tail(&both_list, &op->list); 473 if (ret) 474 validate_fini(op, chan, NULL, NULL); 475 return ret; 476 477 } 478 479 static int 480 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 481 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 482 { 483 struct nouveau_drm *drm = chan->drm; 484 struct nouveau_bo *nvbo; 485 int ret, relocs = 0; 486 487 list_for_each_entry(nvbo, list, entry) { 488 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 489 490 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 491 b->write_domains, 492 b->valid_domains); 493 if (unlikely(ret)) { 494 NV_PRINTK(err, cli, "fail set_domain\n"); 495 return ret; 496 } 497 498 ret = nouveau_bo_validate(nvbo, true, false); 499 if (unlikely(ret)) { 500 if (ret != -ERESTARTSYS) 501 NV_PRINTK(err, cli, "fail ttm_validate\n"); 502 return ret; 503 } 504 505 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 506 if (unlikely(ret)) { 507 if (ret != -ERESTARTSYS) 508 NV_PRINTK(err, cli, "fail post-validate sync\n"); 509 return ret; 510 } 511 512 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 513 if (nvbo->offset == b->presumed.offset && 514 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 515 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 516 (nvbo->bo.mem.mem_type == TTM_PL_TT && 517 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 518 continue; 519 520 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 521 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 522 else 523 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 524 b->presumed.offset = nvbo->offset; 525 b->presumed.valid = 0; 526 relocs++; 527 } 528 } 529 530 return relocs; 531 } 532 533 static int 534 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 535 struct drm_file *file_priv, 536 struct drm_nouveau_gem_pushbuf_bo *pbbo, 537 int nr_buffers, 538 struct validate_op *op, bool *apply_relocs) 539 { 540 struct nouveau_cli *cli = nouveau_cli(file_priv); 541 int ret; 542 543 INIT_LIST_HEAD(&op->list); 544 545 if (nr_buffers == 0) 546 return 0; 547 548 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 549 if (unlikely(ret)) { 550 if (ret != -ERESTARTSYS) 551 NV_PRINTK(err, cli, "validate_init\n"); 552 return ret; 553 } 554 555 ret = validate_list(chan, cli, &op->list, pbbo); 556 if (unlikely(ret < 0)) { 557 if (ret != -ERESTARTSYS) 558 NV_PRINTK(err, cli, "validating bo list\n"); 559 validate_fini(op, chan, NULL, NULL); 560 return ret; 561 } 562 *apply_relocs = ret; 563 return 0; 564 } 565 566 static inline void 567 u_free(void *addr) 568 { 569 kvfree(addr); 570 } 571 572 static inline void * 573 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 574 { 575 void *mem; 576 void __user *userptr = (void __force __user *)(uintptr_t)user; 577 578 size *= nmemb; 579 580 mem = kvmalloc(size, GFP_KERNEL); 581 if (!mem) 582 return ERR_PTR(-ENOMEM); 583 584 if (copy_from_user(mem, userptr, size)) { 585 u_free(mem); 586 return ERR_PTR(-EFAULT); 587 } 588 589 return mem; 590 } 591 592 static int 593 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 594 struct drm_nouveau_gem_pushbuf *req, 595 struct drm_nouveau_gem_pushbuf_reloc *reloc, 596 struct drm_nouveau_gem_pushbuf_bo *bo) 597 { 598 int ret = 0; 599 unsigned i; 600 601 for (i = 0; i < req->nr_relocs; i++) { 602 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 603 struct drm_nouveau_gem_pushbuf_bo *b; 604 struct nouveau_bo *nvbo; 605 uint32_t data; 606 607 if (unlikely(r->bo_index >= req->nr_buffers)) { 608 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 609 ret = -EINVAL; 610 break; 611 } 612 613 b = &bo[r->bo_index]; 614 if (b->presumed.valid) 615 continue; 616 617 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 618 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 619 ret = -EINVAL; 620 break; 621 } 622 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 623 624 if (unlikely(r->reloc_bo_offset + 4 > 625 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 626 NV_PRINTK(err, cli, "reloc outside of bo\n"); 627 ret = -EINVAL; 628 break; 629 } 630 631 if (!nvbo->kmap.virtual) { 632 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 633 &nvbo->kmap); 634 if (ret) { 635 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 636 break; 637 } 638 nvbo->validate_mapped = true; 639 } 640 641 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 642 data = b->presumed.offset + r->data; 643 else 644 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 645 data = (b->presumed.offset + r->data) >> 32; 646 else 647 data = r->data; 648 649 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 650 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 651 data |= r->tor; 652 else 653 data |= r->vor; 654 } 655 656 ret = ttm_bo_wait(&nvbo->bo, false, false); 657 if (ret) { 658 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 659 break; 660 } 661 662 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 663 } 664 665 u_free(reloc); 666 return ret; 667 } 668 669 int 670 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 671 struct drm_file *file_priv) 672 { 673 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 674 struct nouveau_cli *cli = nouveau_cli(file_priv); 675 struct nouveau_abi16_chan *temp; 676 struct nouveau_drm *drm = nouveau_drm(dev); 677 struct drm_nouveau_gem_pushbuf *req = data; 678 struct drm_nouveau_gem_pushbuf_push *push; 679 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 680 struct drm_nouveau_gem_pushbuf_bo *bo; 681 struct nouveau_channel *chan = NULL; 682 struct validate_op op; 683 struct nouveau_fence *fence = NULL; 684 int i, j, ret = 0; 685 bool do_reloc = false, sync = false; 686 687 if (unlikely(!abi16)) 688 return -ENOMEM; 689 690 list_for_each_entry(temp, &abi16->channels, head) { 691 if (temp->chan->chid == req->channel) { 692 chan = temp->chan; 693 break; 694 } 695 } 696 697 if (!chan) 698 return nouveau_abi16_put(abi16, -ENOENT); 699 if (unlikely(atomic_read(&chan->killed))) 700 return nouveau_abi16_put(abi16, -ENODEV); 701 702 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 703 704 req->vram_available = drm->gem.vram_available; 705 req->gart_available = drm->gem.gart_available; 706 if (unlikely(req->nr_push == 0)) 707 goto out_next; 708 709 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 710 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 711 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 712 return nouveau_abi16_put(abi16, -EINVAL); 713 } 714 715 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 716 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 717 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 718 return nouveau_abi16_put(abi16, -EINVAL); 719 } 720 721 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 722 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 723 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 724 return nouveau_abi16_put(abi16, -EINVAL); 725 } 726 727 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 728 if (IS_ERR(push)) 729 return nouveau_abi16_put(abi16, PTR_ERR(push)); 730 731 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 732 if (IS_ERR(bo)) { 733 u_free(push); 734 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 735 } 736 737 /* Ensure all push buffers are on validate list */ 738 for (i = 0; i < req->nr_push; i++) { 739 if (push[i].bo_index >= req->nr_buffers) { 740 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 741 ret = -EINVAL; 742 goto out_prevalid; 743 } 744 } 745 746 /* Validate buffer list */ 747 revalidate: 748 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 749 req->nr_buffers, &op, &do_reloc); 750 if (ret) { 751 if (ret != -ERESTARTSYS) 752 NV_PRINTK(err, cli, "validate: %d\n", ret); 753 goto out_prevalid; 754 } 755 756 /* Apply any relocations that are required */ 757 if (do_reloc) { 758 if (!reloc) { 759 validate_fini(&op, chan, NULL, bo); 760 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 761 if (IS_ERR(reloc)) { 762 ret = PTR_ERR(reloc); 763 goto out_prevalid; 764 } 765 766 goto revalidate; 767 } 768 769 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 770 if (ret) { 771 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 772 goto out; 773 } 774 } 775 776 if (chan->dma.ib_max) { 777 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 778 if (ret) { 779 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 780 goto out; 781 } 782 783 for (i = 0; i < req->nr_push; i++) { 784 struct nouveau_vma *vma = (void *)(unsigned long) 785 bo[push[i].bo_index].user_priv; 786 787 nv50_dma_push(chan, vma->addr + push[i].offset, 788 push[i].length); 789 } 790 } else 791 if (drm->client.device.info.chipset >= 0x25) { 792 ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2); 793 if (ret) { 794 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 795 goto out; 796 } 797 798 for (i = 0; i < req->nr_push; i++) { 799 struct nouveau_bo *nvbo = (void *)(unsigned long) 800 bo[push[i].bo_index].user_priv; 801 802 PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset); 803 PUSH_DATA(chan->chan.push, 0); 804 } 805 } else { 806 ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 807 if (ret) { 808 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 809 goto out; 810 } 811 812 for (i = 0; i < req->nr_push; i++) { 813 struct nouveau_bo *nvbo = (void *)(unsigned long) 814 bo[push[i].bo_index].user_priv; 815 uint32_t cmd; 816 817 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 818 cmd |= 0x20000000; 819 if (unlikely(cmd != req->suffix0)) { 820 if (!nvbo->kmap.virtual) { 821 ret = ttm_bo_kmap(&nvbo->bo, 0, 822 nvbo->bo.mem. 823 num_pages, 824 &nvbo->kmap); 825 if (ret) { 826 WIND_RING(chan); 827 goto out; 828 } 829 nvbo->validate_mapped = true; 830 } 831 832 nouveau_bo_wr32(nvbo, (push[i].offset + 833 push[i].length - 8) / 4, cmd); 834 } 835 836 PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset); 837 PUSH_DATA(chan->chan.push, 0); 838 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 839 PUSH_DATA(chan->chan.push, 0); 840 } 841 } 842 843 ret = nouveau_fence_new(chan, false, &fence); 844 if (ret) { 845 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 846 WIND_RING(chan); 847 goto out; 848 } 849 850 if (sync) { 851 if (!(ret = nouveau_fence_wait(fence, false, false))) { 852 if ((ret = dma_fence_get_status(&fence->base)) == 1) 853 ret = 0; 854 } 855 } 856 857 out: 858 validate_fini(&op, chan, fence, bo); 859 nouveau_fence_unref(&fence); 860 861 if (do_reloc) { 862 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 863 u64_to_user_ptr(req->buffers); 864 865 for (i = 0; i < req->nr_buffers; i++) { 866 if (bo[i].presumed.valid) 867 continue; 868 869 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 870 sizeof(bo[i].presumed))) { 871 ret = -EFAULT; 872 break; 873 } 874 } 875 u_free(reloc); 876 } 877 out_prevalid: 878 u_free(bo); 879 u_free(push); 880 881 out_next: 882 if (chan->dma.ib_max) { 883 req->suffix0 = 0x00000000; 884 req->suffix1 = 0x00000000; 885 } else 886 if (drm->client.device.info.chipset >= 0x25) { 887 req->suffix0 = 0x00020000; 888 req->suffix1 = 0x00000000; 889 } else { 890 req->suffix0 = 0x20000000 | 891 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 892 req->suffix1 = 0x00000000; 893 } 894 895 return nouveau_abi16_put(abi16, ret); 896 } 897 898 int 899 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 900 struct drm_file *file_priv) 901 { 902 struct drm_nouveau_gem_cpu_prep *req = data; 903 struct drm_gem_object *gem; 904 struct nouveau_bo *nvbo; 905 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 906 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 907 long lret; 908 int ret; 909 910 gem = drm_gem_object_lookup(file_priv, req->handle); 911 if (!gem) 912 return -ENOENT; 913 nvbo = nouveau_gem_object(gem); 914 915 lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true, 916 no_wait ? 0 : 30 * HZ); 917 if (!lret) 918 ret = -EBUSY; 919 else if (lret > 0) 920 ret = 0; 921 else 922 ret = lret; 923 924 nouveau_bo_sync_for_cpu(nvbo); 925 drm_gem_object_put(gem); 926 927 return ret; 928 } 929 930 int 931 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 932 struct drm_file *file_priv) 933 { 934 struct drm_nouveau_gem_cpu_fini *req = data; 935 struct drm_gem_object *gem; 936 struct nouveau_bo *nvbo; 937 938 gem = drm_gem_object_lookup(file_priv, req->handle); 939 if (!gem) 940 return -ENOENT; 941 nvbo = nouveau_gem_object(gem); 942 943 nouveau_bo_sync_for_device(nvbo); 944 drm_gem_object_put(gem); 945 return 0; 946 } 947 948 int 949 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 950 struct drm_file *file_priv) 951 { 952 struct drm_nouveau_gem_info *req = data; 953 struct drm_gem_object *gem; 954 int ret; 955 956 gem = drm_gem_object_lookup(file_priv, req->handle); 957 if (!gem) 958 return -ENOENT; 959 960 ret = nouveau_gem_info(file_priv, gem, req); 961 drm_gem_object_put(gem); 962 return ret; 963 } 964 965