1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 39 void 40 nouveau_gem_object_del(struct drm_gem_object *gem) 41 { 42 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 43 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 44 struct ttm_buffer_object *bo = &nvbo->bo; 45 struct device *dev = drm->dev->dev; 46 int ret; 47 48 ret = pm_runtime_get_sync(dev); 49 if (WARN_ON(ret < 0 && ret != -EACCES)) 50 return; 51 52 if (gem->import_attach) 53 drm_prime_gem_destroy(gem, nvbo->bo.sg); 54 55 drm_gem_object_release(gem); 56 57 /* reset filp so nouveau_bo_del_ttm() can test for it */ 58 gem->filp = NULL; 59 ttm_bo_unref(&bo); 60 61 pm_runtime_mark_last_busy(dev); 62 pm_runtime_put_autosuspend(dev); 63 } 64 65 int 66 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 67 { 68 struct nouveau_cli *cli = nouveau_cli(file_priv); 69 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 70 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 71 struct device *dev = drm->dev->dev; 72 struct nouveau_vma *vma; 73 int ret; 74 75 if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50) 76 return 0; 77 78 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 79 if (ret) 80 return ret; 81 82 ret = pm_runtime_get_sync(dev); 83 if (ret < 0 && ret != -EACCES) 84 goto out; 85 86 ret = nouveau_vma_new(nvbo, &cli->vmm, &vma); 87 pm_runtime_mark_last_busy(dev); 88 pm_runtime_put_autosuspend(dev); 89 out: 90 ttm_bo_unreserve(&nvbo->bo); 91 return ret; 92 } 93 94 struct nouveau_gem_object_unmap { 95 struct nouveau_cli_work work; 96 struct nouveau_vma *vma; 97 }; 98 99 static void 100 nouveau_gem_object_delete(struct nouveau_vma *vma) 101 { 102 nouveau_vma_del(&vma); 103 } 104 105 static void 106 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 107 { 108 struct nouveau_gem_object_unmap *work = 109 container_of(w, typeof(*work), work); 110 nouveau_gem_object_delete(work->vma); 111 kfree(work); 112 } 113 114 static void 115 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 116 { 117 const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM; 118 struct reservation_object *resv = nvbo->bo.resv; 119 struct reservation_object_list *fobj; 120 struct nouveau_gem_object_unmap *work; 121 struct dma_fence *fence = NULL; 122 123 fobj = reservation_object_get_list(resv); 124 125 list_del_init(&vma->head); 126 127 if (fobj && fobj->shared_count > 1) 128 ttm_bo_wait(&nvbo->bo, false, false); 129 else if (fobj && fobj->shared_count == 1) 130 fence = rcu_dereference_protected(fobj->shared[0], 131 reservation_object_held(resv)); 132 else 133 fence = reservation_object_get_excl(nvbo->bo.resv); 134 135 if (!fence || !mapped) { 136 nouveau_gem_object_delete(vma); 137 return; 138 } 139 140 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 141 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 142 nouveau_gem_object_delete(vma); 143 return; 144 } 145 146 work->work.func = nouveau_gem_object_delete_work; 147 work->vma = vma; 148 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 149 } 150 151 void 152 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 153 { 154 struct nouveau_cli *cli = nouveau_cli(file_priv); 155 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 156 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 157 struct device *dev = drm->dev->dev; 158 struct nouveau_vma *vma; 159 int ret; 160 161 if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50) 162 return; 163 164 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 165 if (ret) 166 return; 167 168 vma = nouveau_vma_find(nvbo, &cli->vmm); 169 if (vma) { 170 if (--vma->refs == 0) { 171 ret = pm_runtime_get_sync(dev); 172 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 173 nouveau_gem_object_unmap(nvbo, vma); 174 pm_runtime_mark_last_busy(dev); 175 pm_runtime_put_autosuspend(dev); 176 } 177 } 178 } 179 ttm_bo_unreserve(&nvbo->bo); 180 } 181 182 int 183 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 184 uint32_t tile_mode, uint32_t tile_flags, 185 struct nouveau_bo **pnvbo) 186 { 187 struct nouveau_drm *drm = cli->drm; 188 struct nouveau_bo *nvbo; 189 u32 flags = 0; 190 int ret; 191 192 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 193 flags |= TTM_PL_FLAG_VRAM; 194 if (domain & NOUVEAU_GEM_DOMAIN_GART) 195 flags |= TTM_PL_FLAG_TT; 196 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 197 flags |= TTM_PL_FLAG_SYSTEM; 198 199 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) 200 flags |= TTM_PL_FLAG_UNCACHED; 201 202 ret = nouveau_bo_new(cli, size, align, flags, tile_mode, 203 tile_flags, NULL, NULL, pnvbo); 204 if (ret) 205 return ret; 206 nvbo = *pnvbo; 207 208 /* we restrict allowed domains on nv50+ to only the types 209 * that were requested at creation time. not possibly on 210 * earlier chips without busting the ABI. 211 */ 212 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 213 NOUVEAU_GEM_DOMAIN_GART; 214 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 215 nvbo->valid_domains &= domain; 216 217 /* Initialize the embedded gem-object. We return a single gem-reference 218 * to the caller, instead of a normal nouveau_bo ttm reference. */ 219 ret = drm_gem_object_init(drm->dev, &nvbo->gem, nvbo->bo.mem.size); 220 if (ret) { 221 nouveau_bo_ref(NULL, pnvbo); 222 return -ENOMEM; 223 } 224 225 nvbo->bo.persistent_swap_storage = nvbo->gem.filp; 226 return 0; 227 } 228 229 static int 230 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 231 struct drm_nouveau_gem_info *rep) 232 { 233 struct nouveau_cli *cli = nouveau_cli(file_priv); 234 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 235 struct nouveau_vma *vma; 236 237 if (is_power_of_2(nvbo->valid_domains)) 238 rep->domain = nvbo->valid_domains; 239 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 240 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 241 else 242 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 243 rep->offset = nvbo->bo.offset; 244 if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 245 vma = nouveau_vma_find(nvbo, &cli->vmm); 246 if (!vma) 247 return -EINVAL; 248 249 rep->offset = vma->addr; 250 } 251 252 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 253 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); 254 rep->tile_mode = nvbo->mode; 255 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 256 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 257 rep->tile_flags |= nvbo->kind << 8; 258 else 259 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 260 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 261 else 262 rep->tile_flags |= nvbo->zeta; 263 return 0; 264 } 265 266 int 267 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 268 struct drm_file *file_priv) 269 { 270 struct nouveau_cli *cli = nouveau_cli(file_priv); 271 struct drm_nouveau_gem_new *req = data; 272 struct nouveau_bo *nvbo = NULL; 273 int ret = 0; 274 275 ret = nouveau_gem_new(cli, req->info.size, req->align, 276 req->info.domain, req->info.tile_mode, 277 req->info.tile_flags, &nvbo); 278 if (ret) 279 return ret; 280 281 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle); 282 if (ret == 0) { 283 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info); 284 if (ret) 285 drm_gem_handle_delete(file_priv, req->info.handle); 286 } 287 288 /* drop reference from allocate - handle holds it now */ 289 drm_gem_object_unreference_unlocked(&nvbo->gem); 290 return ret; 291 } 292 293 static int 294 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 295 uint32_t write_domains, uint32_t valid_domains) 296 { 297 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 298 struct ttm_buffer_object *bo = &nvbo->bo; 299 uint32_t domains = valid_domains & nvbo->valid_domains & 300 (write_domains ? write_domains : read_domains); 301 uint32_t pref_flags = 0, valid_flags = 0; 302 303 if (!domains) 304 return -EINVAL; 305 306 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 307 valid_flags |= TTM_PL_FLAG_VRAM; 308 309 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 310 valid_flags |= TTM_PL_FLAG_TT; 311 312 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 313 bo->mem.mem_type == TTM_PL_VRAM) 314 pref_flags |= TTM_PL_FLAG_VRAM; 315 316 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 317 bo->mem.mem_type == TTM_PL_TT) 318 pref_flags |= TTM_PL_FLAG_TT; 319 320 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 321 pref_flags |= TTM_PL_FLAG_VRAM; 322 323 else 324 pref_flags |= TTM_PL_FLAG_TT; 325 326 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 327 328 return 0; 329 } 330 331 struct validate_op { 332 struct list_head list; 333 struct ww_acquire_ctx ticket; 334 }; 335 336 static void 337 validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence, 338 struct drm_nouveau_gem_pushbuf_bo *pbbo) 339 { 340 struct nouveau_bo *nvbo; 341 struct drm_nouveau_gem_pushbuf_bo *b; 342 343 while (!list_empty(&op->list)) { 344 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 345 b = &pbbo[nvbo->pbbo_index]; 346 347 if (likely(fence)) 348 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 349 350 if (unlikely(nvbo->validate_mapped)) { 351 ttm_bo_kunmap(&nvbo->kmap); 352 nvbo->validate_mapped = false; 353 } 354 355 list_del(&nvbo->entry); 356 nvbo->reserved_by = NULL; 357 ttm_bo_unreserve(&nvbo->bo); 358 drm_gem_object_unreference_unlocked(&nvbo->gem); 359 } 360 } 361 362 static void 363 validate_fini(struct validate_op *op, struct nouveau_fence *fence, 364 struct drm_nouveau_gem_pushbuf_bo *pbbo) 365 { 366 validate_fini_no_ticket(op, fence, pbbo); 367 ww_acquire_fini(&op->ticket); 368 } 369 370 static int 371 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 372 struct drm_nouveau_gem_pushbuf_bo *pbbo, 373 int nr_buffers, struct validate_op *op) 374 { 375 struct nouveau_cli *cli = nouveau_cli(file_priv); 376 int trycnt = 0; 377 int ret = -EINVAL, i; 378 struct nouveau_bo *res_bo = NULL; 379 LIST_HEAD(gart_list); 380 LIST_HEAD(vram_list); 381 LIST_HEAD(both_list); 382 383 ww_acquire_init(&op->ticket, &reservation_ww_class); 384 retry: 385 if (++trycnt > 100000) { 386 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 387 return -EINVAL; 388 } 389 390 for (i = 0; i < nr_buffers; i++) { 391 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 392 struct drm_gem_object *gem; 393 struct nouveau_bo *nvbo; 394 395 gem = drm_gem_object_lookup(file_priv, b->handle); 396 if (!gem) { 397 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 398 ret = -ENOENT; 399 break; 400 } 401 nvbo = nouveau_gem_object(gem); 402 if (nvbo == res_bo) { 403 res_bo = NULL; 404 drm_gem_object_unreference_unlocked(gem); 405 continue; 406 } 407 408 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 409 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 410 "validation list\n", b->handle); 411 drm_gem_object_unreference_unlocked(gem); 412 ret = -EINVAL; 413 break; 414 } 415 416 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 417 if (ret) { 418 list_splice_tail_init(&vram_list, &op->list); 419 list_splice_tail_init(&gart_list, &op->list); 420 list_splice_tail_init(&both_list, &op->list); 421 validate_fini_no_ticket(op, NULL, NULL); 422 if (unlikely(ret == -EDEADLK)) { 423 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 424 &op->ticket); 425 if (!ret) 426 res_bo = nvbo; 427 } 428 if (unlikely(ret)) { 429 if (ret != -ERESTARTSYS) 430 NV_PRINTK(err, cli, "fail reserve\n"); 431 break; 432 } 433 } 434 435 b->user_priv = (uint64_t)(unsigned long)nvbo; 436 nvbo->reserved_by = file_priv; 437 nvbo->pbbo_index = i; 438 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 439 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 440 list_add_tail(&nvbo->entry, &both_list); 441 else 442 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 443 list_add_tail(&nvbo->entry, &vram_list); 444 else 445 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 446 list_add_tail(&nvbo->entry, &gart_list); 447 else { 448 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 449 b->valid_domains); 450 list_add_tail(&nvbo->entry, &both_list); 451 ret = -EINVAL; 452 break; 453 } 454 if (nvbo == res_bo) 455 goto retry; 456 } 457 458 ww_acquire_done(&op->ticket); 459 list_splice_tail(&vram_list, &op->list); 460 list_splice_tail(&gart_list, &op->list); 461 list_splice_tail(&both_list, &op->list); 462 if (ret) 463 validate_fini(op, NULL, NULL); 464 return ret; 465 466 } 467 468 static int 469 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 470 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 471 uint64_t user_pbbo_ptr) 472 { 473 struct nouveau_drm *drm = chan->drm; 474 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 475 (void __force __user *)(uintptr_t)user_pbbo_ptr; 476 struct nouveau_bo *nvbo; 477 int ret, relocs = 0; 478 479 list_for_each_entry(nvbo, list, entry) { 480 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 481 482 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains, 483 b->write_domains, 484 b->valid_domains); 485 if (unlikely(ret)) { 486 NV_PRINTK(err, cli, "fail set_domain\n"); 487 return ret; 488 } 489 490 ret = nouveau_bo_validate(nvbo, true, false); 491 if (unlikely(ret)) { 492 if (ret != -ERESTARTSYS) 493 NV_PRINTK(err, cli, "fail ttm_validate\n"); 494 return ret; 495 } 496 497 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 498 if (unlikely(ret)) { 499 if (ret != -ERESTARTSYS) 500 NV_PRINTK(err, cli, "fail post-validate sync\n"); 501 return ret; 502 } 503 504 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 505 if (nvbo->bo.offset == b->presumed.offset && 506 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 507 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 508 (nvbo->bo.mem.mem_type == TTM_PL_TT && 509 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 510 continue; 511 512 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 513 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 514 else 515 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 516 b->presumed.offset = nvbo->bo.offset; 517 b->presumed.valid = 0; 518 relocs++; 519 520 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed, 521 &b->presumed, sizeof(b->presumed))) 522 return -EFAULT; 523 } 524 } 525 526 return relocs; 527 } 528 529 static int 530 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 531 struct drm_file *file_priv, 532 struct drm_nouveau_gem_pushbuf_bo *pbbo, 533 uint64_t user_buffers, int nr_buffers, 534 struct validate_op *op, int *apply_relocs) 535 { 536 struct nouveau_cli *cli = nouveau_cli(file_priv); 537 int ret; 538 539 INIT_LIST_HEAD(&op->list); 540 541 if (nr_buffers == 0) 542 return 0; 543 544 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 545 if (unlikely(ret)) { 546 if (ret != -ERESTARTSYS) 547 NV_PRINTK(err, cli, "validate_init\n"); 548 return ret; 549 } 550 551 ret = validate_list(chan, cli, &op->list, pbbo, user_buffers); 552 if (unlikely(ret < 0)) { 553 if (ret != -ERESTARTSYS) 554 NV_PRINTK(err, cli, "validating bo list\n"); 555 validate_fini(op, NULL, NULL); 556 return ret; 557 } 558 *apply_relocs = ret; 559 return 0; 560 } 561 562 static inline void 563 u_free(void *addr) 564 { 565 kvfree(addr); 566 } 567 568 static inline void * 569 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 570 { 571 void *mem; 572 void __user *userptr = (void __force __user *)(uintptr_t)user; 573 574 size *= nmemb; 575 576 mem = kvmalloc(size, GFP_KERNEL); 577 if (!mem) 578 return ERR_PTR(-ENOMEM); 579 580 if (copy_from_user(mem, userptr, size)) { 581 u_free(mem); 582 return ERR_PTR(-EFAULT); 583 } 584 585 return mem; 586 } 587 588 static int 589 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 590 struct drm_nouveau_gem_pushbuf *req, 591 struct drm_nouveau_gem_pushbuf_bo *bo) 592 { 593 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 594 int ret = 0; 595 unsigned i; 596 597 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 598 if (IS_ERR(reloc)) 599 return PTR_ERR(reloc); 600 601 for (i = 0; i < req->nr_relocs; i++) { 602 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 603 struct drm_nouveau_gem_pushbuf_bo *b; 604 struct nouveau_bo *nvbo; 605 uint32_t data; 606 607 if (unlikely(r->bo_index > req->nr_buffers)) { 608 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 609 ret = -EINVAL; 610 break; 611 } 612 613 b = &bo[r->bo_index]; 614 if (b->presumed.valid) 615 continue; 616 617 if (unlikely(r->reloc_bo_index > req->nr_buffers)) { 618 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 619 ret = -EINVAL; 620 break; 621 } 622 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 623 624 if (unlikely(r->reloc_bo_offset + 4 > 625 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 626 NV_PRINTK(err, cli, "reloc outside of bo\n"); 627 ret = -EINVAL; 628 break; 629 } 630 631 if (!nvbo->kmap.virtual) { 632 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 633 &nvbo->kmap); 634 if (ret) { 635 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 636 break; 637 } 638 nvbo->validate_mapped = true; 639 } 640 641 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 642 data = b->presumed.offset + r->data; 643 else 644 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 645 data = (b->presumed.offset + r->data) >> 32; 646 else 647 data = r->data; 648 649 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 650 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 651 data |= r->tor; 652 else 653 data |= r->vor; 654 } 655 656 ret = ttm_bo_wait(&nvbo->bo, false, false); 657 if (ret) { 658 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 659 break; 660 } 661 662 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 663 } 664 665 u_free(reloc); 666 return ret; 667 } 668 669 int 670 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 671 struct drm_file *file_priv) 672 { 673 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 674 struct nouveau_cli *cli = nouveau_cli(file_priv); 675 struct nouveau_abi16_chan *temp; 676 struct nouveau_drm *drm = nouveau_drm(dev); 677 struct drm_nouveau_gem_pushbuf *req = data; 678 struct drm_nouveau_gem_pushbuf_push *push; 679 struct drm_nouveau_gem_pushbuf_bo *bo; 680 struct nouveau_channel *chan = NULL; 681 struct validate_op op; 682 struct nouveau_fence *fence = NULL; 683 int i, j, ret = 0, do_reloc = 0; 684 685 if (unlikely(!abi16)) 686 return -ENOMEM; 687 688 list_for_each_entry(temp, &abi16->channels, head) { 689 if (temp->chan->chid == req->channel) { 690 chan = temp->chan; 691 break; 692 } 693 } 694 695 if (!chan) 696 return nouveau_abi16_put(abi16, -ENOENT); 697 698 req->vram_available = drm->gem.vram_available; 699 req->gart_available = drm->gem.gart_available; 700 if (unlikely(req->nr_push == 0)) 701 goto out_next; 702 703 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 704 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 705 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 706 return nouveau_abi16_put(abi16, -EINVAL); 707 } 708 709 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 710 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 711 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 712 return nouveau_abi16_put(abi16, -EINVAL); 713 } 714 715 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 716 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 717 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 718 return nouveau_abi16_put(abi16, -EINVAL); 719 } 720 721 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 722 if (IS_ERR(push)) 723 return nouveau_abi16_put(abi16, PTR_ERR(push)); 724 725 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 726 if (IS_ERR(bo)) { 727 u_free(push); 728 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 729 } 730 731 /* Ensure all push buffers are on validate list */ 732 for (i = 0; i < req->nr_push; i++) { 733 if (push[i].bo_index >= req->nr_buffers) { 734 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 735 ret = -EINVAL; 736 goto out_prevalid; 737 } 738 } 739 740 /* Validate buffer list */ 741 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 742 req->nr_buffers, &op, &do_reloc); 743 if (ret) { 744 if (ret != -ERESTARTSYS) 745 NV_PRINTK(err, cli, "validate: %d\n", ret); 746 goto out_prevalid; 747 } 748 749 /* Apply any relocations that are required */ 750 if (do_reloc) { 751 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 752 if (ret) { 753 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 754 goto out; 755 } 756 } 757 758 if (chan->dma.ib_max) { 759 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 760 if (ret) { 761 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 762 goto out; 763 } 764 765 for (i = 0; i < req->nr_push; i++) { 766 struct nouveau_bo *nvbo = (void *)(unsigned long) 767 bo[push[i].bo_index].user_priv; 768 769 nv50_dma_push(chan, nvbo, push[i].offset, 770 push[i].length); 771 } 772 } else 773 if (drm->client.device.info.chipset >= 0x25) { 774 ret = RING_SPACE(chan, req->nr_push * 2); 775 if (ret) { 776 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 777 goto out; 778 } 779 780 for (i = 0; i < req->nr_push; i++) { 781 struct nouveau_bo *nvbo = (void *)(unsigned long) 782 bo[push[i].bo_index].user_priv; 783 784 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 785 OUT_RING(chan, 0); 786 } 787 } else { 788 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 789 if (ret) { 790 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 791 goto out; 792 } 793 794 for (i = 0; i < req->nr_push; i++) { 795 struct nouveau_bo *nvbo = (void *)(unsigned long) 796 bo[push[i].bo_index].user_priv; 797 uint32_t cmd; 798 799 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 800 cmd |= 0x20000000; 801 if (unlikely(cmd != req->suffix0)) { 802 if (!nvbo->kmap.virtual) { 803 ret = ttm_bo_kmap(&nvbo->bo, 0, 804 nvbo->bo.mem. 805 num_pages, 806 &nvbo->kmap); 807 if (ret) { 808 WIND_RING(chan); 809 goto out; 810 } 811 nvbo->validate_mapped = true; 812 } 813 814 nouveau_bo_wr32(nvbo, (push[i].offset + 815 push[i].length - 8) / 4, cmd); 816 } 817 818 OUT_RING(chan, 0x20000000 | 819 (nvbo->bo.offset + push[i].offset)); 820 OUT_RING(chan, 0); 821 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 822 OUT_RING(chan, 0); 823 } 824 } 825 826 ret = nouveau_fence_new(chan, false, &fence); 827 if (ret) { 828 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 829 WIND_RING(chan); 830 goto out; 831 } 832 833 out: 834 validate_fini(&op, fence, bo); 835 nouveau_fence_unref(&fence); 836 837 out_prevalid: 838 u_free(bo); 839 u_free(push); 840 841 out_next: 842 if (chan->dma.ib_max) { 843 req->suffix0 = 0x00000000; 844 req->suffix1 = 0x00000000; 845 } else 846 if (drm->client.device.info.chipset >= 0x25) { 847 req->suffix0 = 0x00020000; 848 req->suffix1 = 0x00000000; 849 } else { 850 req->suffix0 = 0x20000000 | 851 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 852 req->suffix1 = 0x00000000; 853 } 854 855 return nouveau_abi16_put(abi16, ret); 856 } 857 858 int 859 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 860 struct drm_file *file_priv) 861 { 862 struct drm_nouveau_gem_cpu_prep *req = data; 863 struct drm_gem_object *gem; 864 struct nouveau_bo *nvbo; 865 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 866 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 867 long lret; 868 int ret; 869 870 gem = drm_gem_object_lookup(file_priv, req->handle); 871 if (!gem) 872 return -ENOENT; 873 nvbo = nouveau_gem_object(gem); 874 875 lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true, 876 no_wait ? 0 : 30 * HZ); 877 if (!lret) 878 ret = -EBUSY; 879 else if (lret > 0) 880 ret = 0; 881 else 882 ret = lret; 883 884 nouveau_bo_sync_for_cpu(nvbo); 885 drm_gem_object_unreference_unlocked(gem); 886 887 return ret; 888 } 889 890 int 891 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 892 struct drm_file *file_priv) 893 { 894 struct drm_nouveau_gem_cpu_fini *req = data; 895 struct drm_gem_object *gem; 896 struct nouveau_bo *nvbo; 897 898 gem = drm_gem_object_lookup(file_priv, req->handle); 899 if (!gem) 900 return -ENOENT; 901 nvbo = nouveau_gem_object(gem); 902 903 nouveau_bo_sync_for_device(nvbo); 904 drm_gem_object_unreference_unlocked(gem); 905 return 0; 906 } 907 908 int 909 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 910 struct drm_file *file_priv) 911 { 912 struct drm_nouveau_gem_info *req = data; 913 struct drm_gem_object *gem; 914 int ret; 915 916 gem = drm_gem_object_lookup(file_priv, req->handle); 917 if (!gem) 918 return -ENOENT; 919 920 ret = nouveau_gem_info(file_priv, gem, req); 921 drm_gem_object_unreference_unlocked(gem); 922 return ret; 923 } 924 925