1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include <subdev/fb.h> 28 29 #include "nouveau_drm.h" 30 #include "nouveau_dma.h" 31 #include "nouveau_fence.h" 32 #include "nouveau_abi16.h" 33 34 #include "nouveau_ttm.h" 35 #include "nouveau_gem.h" 36 37 void 38 nouveau_gem_object_del(struct drm_gem_object *gem) 39 { 40 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 41 struct ttm_buffer_object *bo = &nvbo->bo; 42 43 if (gem->import_attach) 44 drm_prime_gem_destroy(gem, nvbo->bo.sg); 45 46 drm_gem_object_release(gem); 47 48 /* reset filp so nouveau_bo_del_ttm() can test for it */ 49 gem->filp = NULL; 50 ttm_bo_unref(&bo); 51 } 52 53 int 54 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 55 { 56 struct nouveau_cli *cli = nouveau_cli(file_priv); 57 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 58 struct nouveau_vma *vma; 59 int ret; 60 61 if (!cli->base.vm) 62 return 0; 63 64 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 65 if (ret) 66 return ret; 67 68 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 69 if (!vma) { 70 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 71 if (!vma) { 72 ret = -ENOMEM; 73 goto out; 74 } 75 76 ret = nouveau_bo_vma_add(nvbo, cli->base.vm, vma); 77 if (ret) { 78 kfree(vma); 79 goto out; 80 } 81 } else { 82 vma->refcount++; 83 } 84 85 out: 86 ttm_bo_unreserve(&nvbo->bo); 87 return ret; 88 } 89 90 static void 91 nouveau_gem_object_delete(void *data) 92 { 93 struct nouveau_vma *vma = data; 94 nouveau_vm_unmap(vma); 95 nouveau_vm_put(vma); 96 kfree(vma); 97 } 98 99 static void 100 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 101 { 102 const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM; 103 struct nouveau_fence *fence = NULL; 104 105 list_del(&vma->head); 106 107 if (mapped) { 108 spin_lock(&nvbo->bo.bdev->fence_lock); 109 fence = nouveau_fence_ref(nvbo->bo.sync_obj); 110 spin_unlock(&nvbo->bo.bdev->fence_lock); 111 } 112 113 if (fence) { 114 nouveau_fence_work(fence, nouveau_gem_object_delete, vma); 115 } else { 116 if (mapped) 117 nouveau_vm_unmap(vma); 118 nouveau_vm_put(vma); 119 kfree(vma); 120 } 121 nouveau_fence_unref(&fence); 122 } 123 124 void 125 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 126 { 127 struct nouveau_cli *cli = nouveau_cli(file_priv); 128 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 129 struct nouveau_vma *vma; 130 int ret; 131 132 if (!cli->base.vm) 133 return; 134 135 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 136 if (ret) 137 return; 138 139 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 140 if (vma) { 141 if (--vma->refcount == 0) 142 nouveau_gem_object_unmap(nvbo, vma); 143 } 144 ttm_bo_unreserve(&nvbo->bo); 145 } 146 147 int 148 nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain, 149 uint32_t tile_mode, uint32_t tile_flags, 150 struct nouveau_bo **pnvbo) 151 { 152 struct nouveau_drm *drm = nouveau_drm(dev); 153 struct nouveau_bo *nvbo; 154 u32 flags = 0; 155 int ret; 156 157 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 158 flags |= TTM_PL_FLAG_VRAM; 159 if (domain & NOUVEAU_GEM_DOMAIN_GART) 160 flags |= TTM_PL_FLAG_TT; 161 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 162 flags |= TTM_PL_FLAG_SYSTEM; 163 164 ret = nouveau_bo_new(dev, size, align, flags, tile_mode, 165 tile_flags, NULL, pnvbo); 166 if (ret) 167 return ret; 168 nvbo = *pnvbo; 169 170 /* we restrict allowed domains on nv50+ to only the types 171 * that were requested at creation time. not possibly on 172 * earlier chips without busting the ABI. 173 */ 174 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 175 NOUVEAU_GEM_DOMAIN_GART; 176 if (nv_device(drm->device)->card_type >= NV_50) 177 nvbo->valid_domains &= domain; 178 179 /* Initialize the embedded gem-object. We return a single gem-reference 180 * to the caller, instead of a normal nouveau_bo ttm reference. */ 181 ret = drm_gem_object_init(dev, &nvbo->gem, nvbo->bo.mem.size); 182 if (ret) { 183 nouveau_bo_ref(NULL, pnvbo); 184 return -ENOMEM; 185 } 186 187 nvbo->bo.persistent_swap_storage = nvbo->gem.filp; 188 return 0; 189 } 190 191 static int 192 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 193 struct drm_nouveau_gem_info *rep) 194 { 195 struct nouveau_cli *cli = nouveau_cli(file_priv); 196 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 197 struct nouveau_vma *vma; 198 199 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 200 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 201 else 202 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 203 204 rep->offset = nvbo->bo.offset; 205 if (cli->base.vm) { 206 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 207 if (!vma) 208 return -EINVAL; 209 210 rep->offset = vma->offset; 211 } 212 213 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 214 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); 215 rep->tile_mode = nvbo->tile_mode; 216 rep->tile_flags = nvbo->tile_flags; 217 return 0; 218 } 219 220 int 221 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 222 struct drm_file *file_priv) 223 { 224 struct nouveau_drm *drm = nouveau_drm(dev); 225 struct nouveau_cli *cli = nouveau_cli(file_priv); 226 struct nouveau_fb *pfb = nouveau_fb(drm->device); 227 struct drm_nouveau_gem_new *req = data; 228 struct nouveau_bo *nvbo = NULL; 229 int ret = 0; 230 231 drm->ttm.bdev.dev_mapping = drm->dev->dev_mapping; 232 233 if (!pfb->memtype_valid(pfb, req->info.tile_flags)) { 234 NV_ERROR(cli, "bad page flags: 0x%08x\n", req->info.tile_flags); 235 return -EINVAL; 236 } 237 238 ret = nouveau_gem_new(dev, req->info.size, req->align, 239 req->info.domain, req->info.tile_mode, 240 req->info.tile_flags, &nvbo); 241 if (ret) 242 return ret; 243 244 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle); 245 if (ret == 0) { 246 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info); 247 if (ret) 248 drm_gem_handle_delete(file_priv, req->info.handle); 249 } 250 251 /* drop reference from allocate - handle holds it now */ 252 drm_gem_object_unreference_unlocked(&nvbo->gem); 253 return ret; 254 } 255 256 static int 257 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 258 uint32_t write_domains, uint32_t valid_domains) 259 { 260 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 261 struct ttm_buffer_object *bo = &nvbo->bo; 262 uint32_t domains = valid_domains & nvbo->valid_domains & 263 (write_domains ? write_domains : read_domains); 264 uint32_t pref_flags = 0, valid_flags = 0; 265 266 if (!domains) 267 return -EINVAL; 268 269 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 270 valid_flags |= TTM_PL_FLAG_VRAM; 271 272 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 273 valid_flags |= TTM_PL_FLAG_TT; 274 275 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 276 bo->mem.mem_type == TTM_PL_VRAM) 277 pref_flags |= TTM_PL_FLAG_VRAM; 278 279 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 280 bo->mem.mem_type == TTM_PL_TT) 281 pref_flags |= TTM_PL_FLAG_TT; 282 283 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 284 pref_flags |= TTM_PL_FLAG_VRAM; 285 286 else 287 pref_flags |= TTM_PL_FLAG_TT; 288 289 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 290 291 return 0; 292 } 293 294 struct validate_op { 295 struct list_head vram_list; 296 struct list_head gart_list; 297 struct list_head both_list; 298 struct ww_acquire_ctx ticket; 299 }; 300 301 static void 302 validate_fini_list(struct list_head *list, struct nouveau_fence *fence, 303 struct ww_acquire_ctx *ticket) 304 { 305 struct list_head *entry, *tmp; 306 struct nouveau_bo *nvbo; 307 308 list_for_each_safe(entry, tmp, list) { 309 nvbo = list_entry(entry, struct nouveau_bo, entry); 310 311 if (likely(fence)) 312 nouveau_bo_fence(nvbo, fence); 313 314 if (unlikely(nvbo->validate_mapped)) { 315 ttm_bo_kunmap(&nvbo->kmap); 316 nvbo->validate_mapped = false; 317 } 318 319 list_del(&nvbo->entry); 320 nvbo->reserved_by = NULL; 321 ttm_bo_unreserve_ticket(&nvbo->bo, ticket); 322 drm_gem_object_unreference_unlocked(&nvbo->gem); 323 } 324 } 325 326 static void 327 validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence) 328 { 329 validate_fini_list(&op->vram_list, fence, &op->ticket); 330 validate_fini_list(&op->gart_list, fence, &op->ticket); 331 validate_fini_list(&op->both_list, fence, &op->ticket); 332 } 333 334 static void 335 validate_fini(struct validate_op *op, struct nouveau_fence *fence) 336 { 337 validate_fini_no_ticket(op, fence); 338 ww_acquire_fini(&op->ticket); 339 } 340 341 static int 342 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 343 struct drm_nouveau_gem_pushbuf_bo *pbbo, 344 int nr_buffers, struct validate_op *op) 345 { 346 struct nouveau_cli *cli = nouveau_cli(file_priv); 347 struct drm_device *dev = chan->drm->dev; 348 int trycnt = 0; 349 int ret, i; 350 struct nouveau_bo *res_bo = NULL; 351 352 ww_acquire_init(&op->ticket, &reservation_ww_class); 353 retry: 354 if (++trycnt > 100000) { 355 NV_ERROR(cli, "%s failed and gave up.\n", __func__); 356 return -EINVAL; 357 } 358 359 for (i = 0; i < nr_buffers; i++) { 360 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 361 struct drm_gem_object *gem; 362 struct nouveau_bo *nvbo; 363 364 gem = drm_gem_object_lookup(dev, file_priv, b->handle); 365 if (!gem) { 366 NV_ERROR(cli, "Unknown handle 0x%08x\n", b->handle); 367 ww_acquire_done(&op->ticket); 368 validate_fini(op, NULL); 369 return -ENOENT; 370 } 371 nvbo = nouveau_gem_object(gem); 372 if (nvbo == res_bo) { 373 res_bo = NULL; 374 drm_gem_object_unreference_unlocked(gem); 375 continue; 376 } 377 378 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 379 NV_ERROR(cli, "multiple instances of buffer %d on " 380 "validation list\n", b->handle); 381 drm_gem_object_unreference_unlocked(gem); 382 ww_acquire_done(&op->ticket); 383 validate_fini(op, NULL); 384 return -EINVAL; 385 } 386 387 ret = ttm_bo_reserve(&nvbo->bo, true, false, true, &op->ticket); 388 if (ret) { 389 validate_fini_no_ticket(op, NULL); 390 if (unlikely(ret == -EDEADLK)) { 391 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 392 &op->ticket); 393 if (!ret) 394 res_bo = nvbo; 395 } 396 if (unlikely(ret)) { 397 ww_acquire_done(&op->ticket); 398 ww_acquire_fini(&op->ticket); 399 drm_gem_object_unreference_unlocked(gem); 400 if (ret != -ERESTARTSYS) 401 NV_ERROR(cli, "fail reserve\n"); 402 return ret; 403 } 404 } 405 406 b->user_priv = (uint64_t)(unsigned long)nvbo; 407 nvbo->reserved_by = file_priv; 408 nvbo->pbbo_index = i; 409 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 410 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 411 list_add_tail(&nvbo->entry, &op->both_list); 412 else 413 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 414 list_add_tail(&nvbo->entry, &op->vram_list); 415 else 416 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 417 list_add_tail(&nvbo->entry, &op->gart_list); 418 else { 419 NV_ERROR(cli, "invalid valid domains: 0x%08x\n", 420 b->valid_domains); 421 list_add_tail(&nvbo->entry, &op->both_list); 422 ww_acquire_done(&op->ticket); 423 validate_fini(op, NULL); 424 return -EINVAL; 425 } 426 if (nvbo == res_bo) 427 goto retry; 428 } 429 430 ww_acquire_done(&op->ticket); 431 return 0; 432 } 433 434 static int 435 validate_sync(struct nouveau_channel *chan, struct nouveau_bo *nvbo) 436 { 437 struct nouveau_fence *fence = NULL; 438 int ret = 0; 439 440 spin_lock(&nvbo->bo.bdev->fence_lock); 441 fence = nouveau_fence_ref(nvbo->bo.sync_obj); 442 spin_unlock(&nvbo->bo.bdev->fence_lock); 443 444 if (fence) { 445 ret = nouveau_fence_sync(fence, chan); 446 nouveau_fence_unref(&fence); 447 } 448 449 return ret; 450 } 451 452 static int 453 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 454 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 455 uint64_t user_pbbo_ptr) 456 { 457 struct nouveau_drm *drm = chan->drm; 458 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 459 (void __force __user *)(uintptr_t)user_pbbo_ptr; 460 struct nouveau_bo *nvbo; 461 int ret, relocs = 0; 462 463 list_for_each_entry(nvbo, list, entry) { 464 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 465 466 ret = validate_sync(chan, nvbo); 467 if (unlikely(ret)) { 468 NV_ERROR(cli, "fail pre-validate sync\n"); 469 return ret; 470 } 471 472 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains, 473 b->write_domains, 474 b->valid_domains); 475 if (unlikely(ret)) { 476 NV_ERROR(cli, "fail set_domain\n"); 477 return ret; 478 } 479 480 ret = nouveau_bo_validate(nvbo, true, false); 481 if (unlikely(ret)) { 482 if (ret != -ERESTARTSYS) 483 NV_ERROR(cli, "fail ttm_validate\n"); 484 return ret; 485 } 486 487 ret = validate_sync(chan, nvbo); 488 if (unlikely(ret)) { 489 NV_ERROR(cli, "fail post-validate sync\n"); 490 return ret; 491 } 492 493 if (nv_device(drm->device)->card_type < NV_50) { 494 if (nvbo->bo.offset == b->presumed.offset && 495 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 496 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 497 (nvbo->bo.mem.mem_type == TTM_PL_TT && 498 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 499 continue; 500 501 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 502 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 503 else 504 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 505 b->presumed.offset = nvbo->bo.offset; 506 b->presumed.valid = 0; 507 relocs++; 508 509 if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index].presumed, 510 &b->presumed, sizeof(b->presumed))) 511 return -EFAULT; 512 } 513 } 514 515 return relocs; 516 } 517 518 static int 519 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 520 struct drm_file *file_priv, 521 struct drm_nouveau_gem_pushbuf_bo *pbbo, 522 uint64_t user_buffers, int nr_buffers, 523 struct validate_op *op, int *apply_relocs) 524 { 525 struct nouveau_cli *cli = nouveau_cli(file_priv); 526 int ret, relocs = 0; 527 528 INIT_LIST_HEAD(&op->vram_list); 529 INIT_LIST_HEAD(&op->gart_list); 530 INIT_LIST_HEAD(&op->both_list); 531 532 if (nr_buffers == 0) 533 return 0; 534 535 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 536 if (unlikely(ret)) { 537 if (ret != -ERESTARTSYS) 538 NV_ERROR(cli, "validate_init\n"); 539 return ret; 540 } 541 542 ret = validate_list(chan, cli, &op->vram_list, pbbo, user_buffers); 543 if (unlikely(ret < 0)) { 544 if (ret != -ERESTARTSYS) 545 NV_ERROR(cli, "validate vram_list\n"); 546 validate_fini(op, NULL); 547 return ret; 548 } 549 relocs += ret; 550 551 ret = validate_list(chan, cli, &op->gart_list, pbbo, user_buffers); 552 if (unlikely(ret < 0)) { 553 if (ret != -ERESTARTSYS) 554 NV_ERROR(cli, "validate gart_list\n"); 555 validate_fini(op, NULL); 556 return ret; 557 } 558 relocs += ret; 559 560 ret = validate_list(chan, cli, &op->both_list, pbbo, user_buffers); 561 if (unlikely(ret < 0)) { 562 if (ret != -ERESTARTSYS) 563 NV_ERROR(cli, "validate both_list\n"); 564 validate_fini(op, NULL); 565 return ret; 566 } 567 relocs += ret; 568 569 *apply_relocs = relocs; 570 return 0; 571 } 572 573 static inline void 574 u_free(void *addr) 575 { 576 if (!is_vmalloc_addr(addr)) 577 kfree(addr); 578 else 579 vfree(addr); 580 } 581 582 static inline void * 583 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 584 { 585 void *mem; 586 void __user *userptr = (void __force __user *)(uintptr_t)user; 587 588 size *= nmemb; 589 590 mem = kmalloc(size, GFP_KERNEL | __GFP_NOWARN); 591 if (!mem) 592 mem = vmalloc(size); 593 if (!mem) 594 return ERR_PTR(-ENOMEM); 595 596 if (DRM_COPY_FROM_USER(mem, userptr, size)) { 597 u_free(mem); 598 return ERR_PTR(-EFAULT); 599 } 600 601 return mem; 602 } 603 604 static int 605 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 606 struct drm_nouveau_gem_pushbuf *req, 607 struct drm_nouveau_gem_pushbuf_bo *bo) 608 { 609 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 610 int ret = 0; 611 unsigned i; 612 613 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 614 if (IS_ERR(reloc)) 615 return PTR_ERR(reloc); 616 617 for (i = 0; i < req->nr_relocs; i++) { 618 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 619 struct drm_nouveau_gem_pushbuf_bo *b; 620 struct nouveau_bo *nvbo; 621 uint32_t data; 622 623 if (unlikely(r->bo_index > req->nr_buffers)) { 624 NV_ERROR(cli, "reloc bo index invalid\n"); 625 ret = -EINVAL; 626 break; 627 } 628 629 b = &bo[r->bo_index]; 630 if (b->presumed.valid) 631 continue; 632 633 if (unlikely(r->reloc_bo_index > req->nr_buffers)) { 634 NV_ERROR(cli, "reloc container bo index invalid\n"); 635 ret = -EINVAL; 636 break; 637 } 638 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 639 640 if (unlikely(r->reloc_bo_offset + 4 > 641 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 642 NV_ERROR(cli, "reloc outside of bo\n"); 643 ret = -EINVAL; 644 break; 645 } 646 647 if (!nvbo->kmap.virtual) { 648 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 649 &nvbo->kmap); 650 if (ret) { 651 NV_ERROR(cli, "failed kmap for reloc\n"); 652 break; 653 } 654 nvbo->validate_mapped = true; 655 } 656 657 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 658 data = b->presumed.offset + r->data; 659 else 660 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 661 data = (b->presumed.offset + r->data) >> 32; 662 else 663 data = r->data; 664 665 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 666 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 667 data |= r->tor; 668 else 669 data |= r->vor; 670 } 671 672 spin_lock(&nvbo->bo.bdev->fence_lock); 673 ret = ttm_bo_wait(&nvbo->bo, false, false, false); 674 spin_unlock(&nvbo->bo.bdev->fence_lock); 675 if (ret) { 676 NV_ERROR(cli, "reloc wait_idle failed: %d\n", ret); 677 break; 678 } 679 680 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 681 } 682 683 u_free(reloc); 684 return ret; 685 } 686 687 int 688 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 689 struct drm_file *file_priv) 690 { 691 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev); 692 struct nouveau_cli *cli = nouveau_cli(file_priv); 693 struct nouveau_abi16_chan *temp; 694 struct nouveau_drm *drm = nouveau_drm(dev); 695 struct drm_nouveau_gem_pushbuf *req = data; 696 struct drm_nouveau_gem_pushbuf_push *push; 697 struct drm_nouveau_gem_pushbuf_bo *bo; 698 struct nouveau_channel *chan = NULL; 699 struct validate_op op; 700 struct nouveau_fence *fence = NULL; 701 int i, j, ret = 0, do_reloc = 0; 702 703 if (unlikely(!abi16)) 704 return -ENOMEM; 705 706 list_for_each_entry(temp, &abi16->channels, head) { 707 if (temp->chan->handle == (NVDRM_CHAN | req->channel)) { 708 chan = temp->chan; 709 break; 710 } 711 } 712 713 if (!chan) 714 return nouveau_abi16_put(abi16, -ENOENT); 715 716 req->vram_available = drm->gem.vram_available; 717 req->gart_available = drm->gem.gart_available; 718 if (unlikely(req->nr_push == 0)) 719 goto out_next; 720 721 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 722 NV_ERROR(cli, "pushbuf push count exceeds limit: %d max %d\n", 723 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 724 return nouveau_abi16_put(abi16, -EINVAL); 725 } 726 727 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 728 NV_ERROR(cli, "pushbuf bo count exceeds limit: %d max %d\n", 729 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 730 return nouveau_abi16_put(abi16, -EINVAL); 731 } 732 733 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 734 NV_ERROR(cli, "pushbuf reloc count exceeds limit: %d max %d\n", 735 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 736 return nouveau_abi16_put(abi16, -EINVAL); 737 } 738 739 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 740 if (IS_ERR(push)) 741 return nouveau_abi16_put(abi16, PTR_ERR(push)); 742 743 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 744 if (IS_ERR(bo)) { 745 u_free(push); 746 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 747 } 748 749 /* Ensure all push buffers are on validate list */ 750 for (i = 0; i < req->nr_push; i++) { 751 if (push[i].bo_index >= req->nr_buffers) { 752 NV_ERROR(cli, "push %d buffer not in list\n", i); 753 ret = -EINVAL; 754 goto out_prevalid; 755 } 756 } 757 758 /* Validate buffer list */ 759 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 760 req->nr_buffers, &op, &do_reloc); 761 if (ret) { 762 if (ret != -ERESTARTSYS) 763 NV_ERROR(cli, "validate: %d\n", ret); 764 goto out_prevalid; 765 } 766 767 /* Apply any relocations that are required */ 768 if (do_reloc) { 769 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 770 if (ret) { 771 NV_ERROR(cli, "reloc apply: %d\n", ret); 772 goto out; 773 } 774 } 775 776 if (chan->dma.ib_max) { 777 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 778 if (ret) { 779 NV_ERROR(cli, "nv50cal_space: %d\n", ret); 780 goto out; 781 } 782 783 for (i = 0; i < req->nr_push; i++) { 784 struct nouveau_bo *nvbo = (void *)(unsigned long) 785 bo[push[i].bo_index].user_priv; 786 787 nv50_dma_push(chan, nvbo, push[i].offset, 788 push[i].length); 789 } 790 } else 791 if (nv_device(drm->device)->chipset >= 0x25) { 792 ret = RING_SPACE(chan, req->nr_push * 2); 793 if (ret) { 794 NV_ERROR(cli, "cal_space: %d\n", ret); 795 goto out; 796 } 797 798 for (i = 0; i < req->nr_push; i++) { 799 struct nouveau_bo *nvbo = (void *)(unsigned long) 800 bo[push[i].bo_index].user_priv; 801 802 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 803 OUT_RING(chan, 0); 804 } 805 } else { 806 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 807 if (ret) { 808 NV_ERROR(cli, "jmp_space: %d\n", ret); 809 goto out; 810 } 811 812 for (i = 0; i < req->nr_push; i++) { 813 struct nouveau_bo *nvbo = (void *)(unsigned long) 814 bo[push[i].bo_index].user_priv; 815 uint32_t cmd; 816 817 cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2); 818 cmd |= 0x20000000; 819 if (unlikely(cmd != req->suffix0)) { 820 if (!nvbo->kmap.virtual) { 821 ret = ttm_bo_kmap(&nvbo->bo, 0, 822 nvbo->bo.mem. 823 num_pages, 824 &nvbo->kmap); 825 if (ret) { 826 WIND_RING(chan); 827 goto out; 828 } 829 nvbo->validate_mapped = true; 830 } 831 832 nouveau_bo_wr32(nvbo, (push[i].offset + 833 push[i].length - 8) / 4, cmd); 834 } 835 836 OUT_RING(chan, 0x20000000 | 837 (nvbo->bo.offset + push[i].offset)); 838 OUT_RING(chan, 0); 839 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 840 OUT_RING(chan, 0); 841 } 842 } 843 844 ret = nouveau_fence_new(chan, false, &fence); 845 if (ret) { 846 NV_ERROR(cli, "error fencing pushbuf: %d\n", ret); 847 WIND_RING(chan); 848 goto out; 849 } 850 851 out: 852 validate_fini(&op, fence); 853 nouveau_fence_unref(&fence); 854 855 out_prevalid: 856 u_free(bo); 857 u_free(push); 858 859 out_next: 860 if (chan->dma.ib_max) { 861 req->suffix0 = 0x00000000; 862 req->suffix1 = 0x00000000; 863 } else 864 if (nv_device(drm->device)->chipset >= 0x25) { 865 req->suffix0 = 0x00020000; 866 req->suffix1 = 0x00000000; 867 } else { 868 req->suffix0 = 0x20000000 | 869 (chan->push.vma.offset + ((chan->dma.cur + 2) << 2)); 870 req->suffix1 = 0x00000000; 871 } 872 873 return nouveau_abi16_put(abi16, ret); 874 } 875 876 static inline uint32_t 877 domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain) 878 { 879 uint32_t flags = 0; 880 881 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 882 flags |= TTM_PL_FLAG_VRAM; 883 if (domain & NOUVEAU_GEM_DOMAIN_GART) 884 flags |= TTM_PL_FLAG_TT; 885 886 return flags; 887 } 888 889 int 890 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 891 struct drm_file *file_priv) 892 { 893 struct drm_nouveau_gem_cpu_prep *req = data; 894 struct drm_gem_object *gem; 895 struct nouveau_bo *nvbo; 896 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 897 int ret = -EINVAL; 898 899 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 900 if (!gem) 901 return -ENOENT; 902 nvbo = nouveau_gem_object(gem); 903 904 spin_lock(&nvbo->bo.bdev->fence_lock); 905 ret = ttm_bo_wait(&nvbo->bo, true, true, no_wait); 906 spin_unlock(&nvbo->bo.bdev->fence_lock); 907 drm_gem_object_unreference_unlocked(gem); 908 return ret; 909 } 910 911 int 912 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 913 struct drm_file *file_priv) 914 { 915 return 0; 916 } 917 918 int 919 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 920 struct drm_file *file_priv) 921 { 922 struct drm_nouveau_gem_info *req = data; 923 struct drm_gem_object *gem; 924 int ret; 925 926 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 927 if (!gem) 928 return -ENOENT; 929 930 ret = nouveau_gem_info(file_priv, gem, req); 931 drm_gem_object_unreference_unlocked(gem); 932 return ret; 933 } 934 935