1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drm.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 35 void 36 nouveau_gem_object_del(struct drm_gem_object *gem) 37 { 38 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 39 struct ttm_buffer_object *bo = &nvbo->bo; 40 41 if (gem->import_attach) 42 drm_prime_gem_destroy(gem, nvbo->bo.sg); 43 44 drm_gem_object_release(gem); 45 46 /* reset filp so nouveau_bo_del_ttm() can test for it */ 47 gem->filp = NULL; 48 ttm_bo_unref(&bo); 49 } 50 51 int 52 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 53 { 54 struct nouveau_cli *cli = nouveau_cli(file_priv); 55 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 56 struct nouveau_vma *vma; 57 int ret; 58 59 if (!cli->vm) 60 return 0; 61 62 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL); 63 if (ret) 64 return ret; 65 66 vma = nouveau_bo_vma_find(nvbo, cli->vm); 67 if (!vma) { 68 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 69 if (!vma) { 70 ret = -ENOMEM; 71 goto out; 72 } 73 74 ret = nouveau_bo_vma_add(nvbo, cli->vm, vma); 75 if (ret) { 76 kfree(vma); 77 goto out; 78 } 79 } else { 80 vma->refcount++; 81 } 82 83 out: 84 ttm_bo_unreserve(&nvbo->bo); 85 return ret; 86 } 87 88 static void 89 nouveau_gem_object_delete(void *data) 90 { 91 struct nouveau_vma *vma = data; 92 nouveau_vm_unmap(vma); 93 nouveau_vm_put(vma); 94 kfree(vma); 95 } 96 97 static void 98 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 99 { 100 const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM; 101 struct reservation_object *resv = nvbo->bo.resv; 102 struct reservation_object_list *fobj; 103 struct fence *fence = NULL; 104 105 fobj = reservation_object_get_list(resv); 106 107 list_del(&vma->head); 108 109 if (fobj && fobj->shared_count > 1) 110 ttm_bo_wait(&nvbo->bo, true, false, false); 111 else if (fobj && fobj->shared_count == 1) 112 fence = rcu_dereference_protected(fobj->shared[0], 113 reservation_object_held(resv)); 114 else 115 fence = reservation_object_get_excl(nvbo->bo.resv); 116 117 if (fence && mapped) { 118 nouveau_fence_work(fence, nouveau_gem_object_delete, vma); 119 } else { 120 if (mapped) 121 nouveau_vm_unmap(vma); 122 nouveau_vm_put(vma); 123 kfree(vma); 124 } 125 } 126 127 void 128 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 129 { 130 struct nouveau_cli *cli = nouveau_cli(file_priv); 131 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 132 struct nouveau_vma *vma; 133 int ret; 134 135 if (!cli->vm) 136 return; 137 138 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL); 139 if (ret) 140 return; 141 142 vma = nouveau_bo_vma_find(nvbo, cli->vm); 143 if (vma) { 144 if (--vma->refcount == 0) 145 nouveau_gem_object_unmap(nvbo, vma); 146 } 147 ttm_bo_unreserve(&nvbo->bo); 148 } 149 150 int 151 nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain, 152 uint32_t tile_mode, uint32_t tile_flags, 153 struct nouveau_bo **pnvbo) 154 { 155 struct nouveau_drm *drm = nouveau_drm(dev); 156 struct nouveau_bo *nvbo; 157 u32 flags = 0; 158 int ret; 159 160 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 161 flags |= TTM_PL_FLAG_VRAM; 162 if (domain & NOUVEAU_GEM_DOMAIN_GART) 163 flags |= TTM_PL_FLAG_TT; 164 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 165 flags |= TTM_PL_FLAG_SYSTEM; 166 167 ret = nouveau_bo_new(dev, size, align, flags, tile_mode, 168 tile_flags, NULL, NULL, pnvbo); 169 if (ret) 170 return ret; 171 nvbo = *pnvbo; 172 173 /* we restrict allowed domains on nv50+ to only the types 174 * that were requested at creation time. not possibly on 175 * earlier chips without busting the ABI. 176 */ 177 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 178 NOUVEAU_GEM_DOMAIN_GART; 179 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 180 nvbo->valid_domains &= domain; 181 182 /* Initialize the embedded gem-object. We return a single gem-reference 183 * to the caller, instead of a normal nouveau_bo ttm reference. */ 184 ret = drm_gem_object_init(dev, &nvbo->gem, nvbo->bo.mem.size); 185 if (ret) { 186 nouveau_bo_ref(NULL, pnvbo); 187 return -ENOMEM; 188 } 189 190 nvbo->bo.persistent_swap_storage = nvbo->gem.filp; 191 return 0; 192 } 193 194 static int 195 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 196 struct drm_nouveau_gem_info *rep) 197 { 198 struct nouveau_cli *cli = nouveau_cli(file_priv); 199 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 200 struct nouveau_vma *vma; 201 202 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 203 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 204 else 205 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 206 207 rep->offset = nvbo->bo.offset; 208 if (cli->vm) { 209 vma = nouveau_bo_vma_find(nvbo, cli->vm); 210 if (!vma) 211 return -EINVAL; 212 213 rep->offset = vma->offset; 214 } 215 216 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 217 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); 218 rep->tile_mode = nvbo->tile_mode; 219 rep->tile_flags = nvbo->tile_flags; 220 return 0; 221 } 222 223 int 224 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 225 struct drm_file *file_priv) 226 { 227 struct nouveau_drm *drm = nouveau_drm(dev); 228 struct nouveau_cli *cli = nouveau_cli(file_priv); 229 struct nouveau_fb *pfb = nvkm_fb(&drm->device); 230 struct drm_nouveau_gem_new *req = data; 231 struct nouveau_bo *nvbo = NULL; 232 int ret = 0; 233 234 if (!pfb->memtype_valid(pfb, req->info.tile_flags)) { 235 NV_PRINTK(error, cli, "bad page flags: 0x%08x\n", req->info.tile_flags); 236 return -EINVAL; 237 } 238 239 ret = nouveau_gem_new(dev, req->info.size, req->align, 240 req->info.domain, req->info.tile_mode, 241 req->info.tile_flags, &nvbo); 242 if (ret) 243 return ret; 244 245 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle); 246 if (ret == 0) { 247 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info); 248 if (ret) 249 drm_gem_handle_delete(file_priv, req->info.handle); 250 } 251 252 /* drop reference from allocate - handle holds it now */ 253 drm_gem_object_unreference_unlocked(&nvbo->gem); 254 return ret; 255 } 256 257 static int 258 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 259 uint32_t write_domains, uint32_t valid_domains) 260 { 261 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 262 struct ttm_buffer_object *bo = &nvbo->bo; 263 uint32_t domains = valid_domains & nvbo->valid_domains & 264 (write_domains ? write_domains : read_domains); 265 uint32_t pref_flags = 0, valid_flags = 0; 266 267 if (!domains) 268 return -EINVAL; 269 270 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 271 valid_flags |= TTM_PL_FLAG_VRAM; 272 273 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 274 valid_flags |= TTM_PL_FLAG_TT; 275 276 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 277 bo->mem.mem_type == TTM_PL_VRAM) 278 pref_flags |= TTM_PL_FLAG_VRAM; 279 280 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 281 bo->mem.mem_type == TTM_PL_TT) 282 pref_flags |= TTM_PL_FLAG_TT; 283 284 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 285 pref_flags |= TTM_PL_FLAG_VRAM; 286 287 else 288 pref_flags |= TTM_PL_FLAG_TT; 289 290 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 291 292 return 0; 293 } 294 295 struct validate_op { 296 struct list_head list; 297 struct ww_acquire_ctx ticket; 298 }; 299 300 static void 301 validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence, 302 struct drm_nouveau_gem_pushbuf_bo *pbbo) 303 { 304 struct nouveau_bo *nvbo; 305 struct drm_nouveau_gem_pushbuf_bo *b; 306 307 while (!list_empty(&op->list)) { 308 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 309 b = &pbbo[nvbo->pbbo_index]; 310 311 if (likely(fence)) 312 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 313 314 if (unlikely(nvbo->validate_mapped)) { 315 ttm_bo_kunmap(&nvbo->kmap); 316 nvbo->validate_mapped = false; 317 } 318 319 list_del(&nvbo->entry); 320 nvbo->reserved_by = NULL; 321 ttm_bo_unreserve_ticket(&nvbo->bo, &op->ticket); 322 drm_gem_object_unreference_unlocked(&nvbo->gem); 323 } 324 } 325 326 static void 327 validate_fini(struct validate_op *op, struct nouveau_fence *fence, 328 struct drm_nouveau_gem_pushbuf_bo *pbbo) 329 { 330 validate_fini_no_ticket(op, fence, pbbo); 331 ww_acquire_fini(&op->ticket); 332 } 333 334 static int 335 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 336 struct drm_nouveau_gem_pushbuf_bo *pbbo, 337 int nr_buffers, struct validate_op *op) 338 { 339 struct nouveau_cli *cli = nouveau_cli(file_priv); 340 struct drm_device *dev = chan->drm->dev; 341 int trycnt = 0; 342 int ret, i; 343 struct nouveau_bo *res_bo = NULL; 344 LIST_HEAD(gart_list); 345 LIST_HEAD(vram_list); 346 LIST_HEAD(both_list); 347 348 ww_acquire_init(&op->ticket, &reservation_ww_class); 349 retry: 350 if (++trycnt > 100000) { 351 NV_PRINTK(error, cli, "%s failed and gave up.\n", __func__); 352 return -EINVAL; 353 } 354 355 for (i = 0; i < nr_buffers; i++) { 356 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 357 struct drm_gem_object *gem; 358 struct nouveau_bo *nvbo; 359 360 gem = drm_gem_object_lookup(dev, file_priv, b->handle); 361 if (!gem) { 362 NV_PRINTK(error, cli, "Unknown handle 0x%08x\n", b->handle); 363 ret = -ENOENT; 364 break; 365 } 366 nvbo = nouveau_gem_object(gem); 367 if (nvbo == res_bo) { 368 res_bo = NULL; 369 drm_gem_object_unreference_unlocked(gem); 370 continue; 371 } 372 373 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 374 NV_PRINTK(error, cli, "multiple instances of buffer %d on " 375 "validation list\n", b->handle); 376 drm_gem_object_unreference_unlocked(gem); 377 ret = -EINVAL; 378 break; 379 } 380 381 ret = ttm_bo_reserve(&nvbo->bo, true, false, true, &op->ticket); 382 if (ret) { 383 list_splice_tail_init(&vram_list, &op->list); 384 list_splice_tail_init(&gart_list, &op->list); 385 list_splice_tail_init(&both_list, &op->list); 386 validate_fini_no_ticket(op, NULL, NULL); 387 if (unlikely(ret == -EDEADLK)) { 388 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 389 &op->ticket); 390 if (!ret) 391 res_bo = nvbo; 392 } 393 if (unlikely(ret)) { 394 if (ret != -ERESTARTSYS) 395 NV_PRINTK(error, cli, "fail reserve\n"); 396 break; 397 } 398 } 399 400 b->user_priv = (uint64_t)(unsigned long)nvbo; 401 nvbo->reserved_by = file_priv; 402 nvbo->pbbo_index = i; 403 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 404 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 405 list_add_tail(&nvbo->entry, &both_list); 406 else 407 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 408 list_add_tail(&nvbo->entry, &vram_list); 409 else 410 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 411 list_add_tail(&nvbo->entry, &gart_list); 412 else { 413 NV_PRINTK(error, cli, "invalid valid domains: 0x%08x\n", 414 b->valid_domains); 415 list_add_tail(&nvbo->entry, &both_list); 416 ret = -EINVAL; 417 break; 418 } 419 if (nvbo == res_bo) 420 goto retry; 421 } 422 423 ww_acquire_done(&op->ticket); 424 list_splice_tail(&vram_list, &op->list); 425 list_splice_tail(&gart_list, &op->list); 426 list_splice_tail(&both_list, &op->list); 427 if (ret) 428 validate_fini(op, NULL, NULL); 429 return ret; 430 431 } 432 433 static int 434 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 435 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 436 uint64_t user_pbbo_ptr) 437 { 438 struct nouveau_drm *drm = chan->drm; 439 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 440 (void __force __user *)(uintptr_t)user_pbbo_ptr; 441 struct nouveau_bo *nvbo; 442 int ret, relocs = 0; 443 444 list_for_each_entry(nvbo, list, entry) { 445 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 446 447 WARN_ONCE(nvbo->gem.dumb, 448 "GPU use of dumb buffer is illegal.\n"); 449 450 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains, 451 b->write_domains, 452 b->valid_domains); 453 if (unlikely(ret)) { 454 NV_PRINTK(error, cli, "fail set_domain\n"); 455 return ret; 456 } 457 458 ret = nouveau_bo_validate(nvbo, true, false); 459 if (unlikely(ret)) { 460 if (ret != -ERESTARTSYS) 461 NV_PRINTK(error, cli, "fail ttm_validate\n"); 462 return ret; 463 } 464 465 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 466 if (unlikely(ret)) { 467 if (ret != -ERESTARTSYS) 468 NV_PRINTK(error, cli, "fail post-validate sync\n"); 469 return ret; 470 } 471 472 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) { 473 if (nvbo->bo.offset == b->presumed.offset && 474 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 475 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 476 (nvbo->bo.mem.mem_type == TTM_PL_TT && 477 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 478 continue; 479 480 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 481 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 482 else 483 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 484 b->presumed.offset = nvbo->bo.offset; 485 b->presumed.valid = 0; 486 relocs++; 487 488 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed, 489 &b->presumed, sizeof(b->presumed))) 490 return -EFAULT; 491 } 492 } 493 494 return relocs; 495 } 496 497 static int 498 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 499 struct drm_file *file_priv, 500 struct drm_nouveau_gem_pushbuf_bo *pbbo, 501 uint64_t user_buffers, int nr_buffers, 502 struct validate_op *op, int *apply_relocs) 503 { 504 struct nouveau_cli *cli = nouveau_cli(file_priv); 505 int ret; 506 507 INIT_LIST_HEAD(&op->list); 508 509 if (nr_buffers == 0) 510 return 0; 511 512 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 513 if (unlikely(ret)) { 514 if (ret != -ERESTARTSYS) 515 NV_PRINTK(error, cli, "validate_init\n"); 516 return ret; 517 } 518 519 ret = validate_list(chan, cli, &op->list, pbbo, user_buffers); 520 if (unlikely(ret < 0)) { 521 if (ret != -ERESTARTSYS) 522 NV_PRINTK(error, cli, "validating bo list\n"); 523 validate_fini(op, NULL, NULL); 524 return ret; 525 } 526 *apply_relocs = ret; 527 return 0; 528 } 529 530 static inline void 531 u_free(void *addr) 532 { 533 if (!is_vmalloc_addr(addr)) 534 kfree(addr); 535 else 536 vfree(addr); 537 } 538 539 static inline void * 540 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 541 { 542 void *mem; 543 void __user *userptr = (void __force __user *)(uintptr_t)user; 544 545 size *= nmemb; 546 547 mem = kmalloc(size, GFP_KERNEL | __GFP_NOWARN); 548 if (!mem) 549 mem = vmalloc(size); 550 if (!mem) 551 return ERR_PTR(-ENOMEM); 552 553 if (copy_from_user(mem, userptr, size)) { 554 u_free(mem); 555 return ERR_PTR(-EFAULT); 556 } 557 558 return mem; 559 } 560 561 static int 562 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 563 struct drm_nouveau_gem_pushbuf *req, 564 struct drm_nouveau_gem_pushbuf_bo *bo) 565 { 566 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 567 int ret = 0; 568 unsigned i; 569 570 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 571 if (IS_ERR(reloc)) 572 return PTR_ERR(reloc); 573 574 for (i = 0; i < req->nr_relocs; i++) { 575 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 576 struct drm_nouveau_gem_pushbuf_bo *b; 577 struct nouveau_bo *nvbo; 578 uint32_t data; 579 580 if (unlikely(r->bo_index > req->nr_buffers)) { 581 NV_PRINTK(error, cli, "reloc bo index invalid\n"); 582 ret = -EINVAL; 583 break; 584 } 585 586 b = &bo[r->bo_index]; 587 if (b->presumed.valid) 588 continue; 589 590 if (unlikely(r->reloc_bo_index > req->nr_buffers)) { 591 NV_PRINTK(error, cli, "reloc container bo index invalid\n"); 592 ret = -EINVAL; 593 break; 594 } 595 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 596 597 if (unlikely(r->reloc_bo_offset + 4 > 598 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 599 NV_PRINTK(error, cli, "reloc outside of bo\n"); 600 ret = -EINVAL; 601 break; 602 } 603 604 if (!nvbo->kmap.virtual) { 605 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 606 &nvbo->kmap); 607 if (ret) { 608 NV_PRINTK(error, cli, "failed kmap for reloc\n"); 609 break; 610 } 611 nvbo->validate_mapped = true; 612 } 613 614 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 615 data = b->presumed.offset + r->data; 616 else 617 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 618 data = (b->presumed.offset + r->data) >> 32; 619 else 620 data = r->data; 621 622 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 623 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 624 data |= r->tor; 625 else 626 data |= r->vor; 627 } 628 629 ret = ttm_bo_wait(&nvbo->bo, true, false, false); 630 if (ret) { 631 NV_PRINTK(error, cli, "reloc wait_idle failed: %d\n", ret); 632 break; 633 } 634 635 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 636 } 637 638 u_free(reloc); 639 return ret; 640 } 641 642 int 643 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 644 struct drm_file *file_priv) 645 { 646 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev); 647 struct nouveau_cli *cli = nouveau_cli(file_priv); 648 struct nouveau_abi16_chan *temp; 649 struct nouveau_drm *drm = nouveau_drm(dev); 650 struct drm_nouveau_gem_pushbuf *req = data; 651 struct drm_nouveau_gem_pushbuf_push *push; 652 struct drm_nouveau_gem_pushbuf_bo *bo; 653 struct nouveau_channel *chan = NULL; 654 struct validate_op op; 655 struct nouveau_fence *fence = NULL; 656 int i, j, ret = 0, do_reloc = 0; 657 658 if (unlikely(!abi16)) 659 return -ENOMEM; 660 661 list_for_each_entry(temp, &abi16->channels, head) { 662 if (temp->chan->object->handle == (NVDRM_CHAN | req->channel)) { 663 chan = temp->chan; 664 break; 665 } 666 } 667 668 if (!chan) 669 return nouveau_abi16_put(abi16, -ENOENT); 670 671 req->vram_available = drm->gem.vram_available; 672 req->gart_available = drm->gem.gart_available; 673 if (unlikely(req->nr_push == 0)) 674 goto out_next; 675 676 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 677 NV_PRINTK(error, cli, "pushbuf push count exceeds limit: %d max %d\n", 678 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 679 return nouveau_abi16_put(abi16, -EINVAL); 680 } 681 682 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 683 NV_PRINTK(error, cli, "pushbuf bo count exceeds limit: %d max %d\n", 684 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 685 return nouveau_abi16_put(abi16, -EINVAL); 686 } 687 688 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 689 NV_PRINTK(error, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 690 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 691 return nouveau_abi16_put(abi16, -EINVAL); 692 } 693 694 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 695 if (IS_ERR(push)) 696 return nouveau_abi16_put(abi16, PTR_ERR(push)); 697 698 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 699 if (IS_ERR(bo)) { 700 u_free(push); 701 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 702 } 703 704 /* Ensure all push buffers are on validate list */ 705 for (i = 0; i < req->nr_push; i++) { 706 if (push[i].bo_index >= req->nr_buffers) { 707 NV_PRINTK(error, cli, "push %d buffer not in list\n", i); 708 ret = -EINVAL; 709 goto out_prevalid; 710 } 711 } 712 713 /* Validate buffer list */ 714 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 715 req->nr_buffers, &op, &do_reloc); 716 if (ret) { 717 if (ret != -ERESTARTSYS) 718 NV_PRINTK(error, cli, "validate: %d\n", ret); 719 goto out_prevalid; 720 } 721 722 /* Apply any relocations that are required */ 723 if (do_reloc) { 724 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 725 if (ret) { 726 NV_PRINTK(error, cli, "reloc apply: %d\n", ret); 727 goto out; 728 } 729 } 730 731 if (chan->dma.ib_max) { 732 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 733 if (ret) { 734 NV_PRINTK(error, cli, "nv50cal_space: %d\n", ret); 735 goto out; 736 } 737 738 for (i = 0; i < req->nr_push; i++) { 739 struct nouveau_bo *nvbo = (void *)(unsigned long) 740 bo[push[i].bo_index].user_priv; 741 742 nv50_dma_push(chan, nvbo, push[i].offset, 743 push[i].length); 744 } 745 } else 746 if (drm->device.info.chipset >= 0x25) { 747 ret = RING_SPACE(chan, req->nr_push * 2); 748 if (ret) { 749 NV_PRINTK(error, cli, "cal_space: %d\n", ret); 750 goto out; 751 } 752 753 for (i = 0; i < req->nr_push; i++) { 754 struct nouveau_bo *nvbo = (void *)(unsigned long) 755 bo[push[i].bo_index].user_priv; 756 757 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 758 OUT_RING(chan, 0); 759 } 760 } else { 761 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 762 if (ret) { 763 NV_PRINTK(error, cli, "jmp_space: %d\n", ret); 764 goto out; 765 } 766 767 for (i = 0; i < req->nr_push; i++) { 768 struct nouveau_bo *nvbo = (void *)(unsigned long) 769 bo[push[i].bo_index].user_priv; 770 uint32_t cmd; 771 772 cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2); 773 cmd |= 0x20000000; 774 if (unlikely(cmd != req->suffix0)) { 775 if (!nvbo->kmap.virtual) { 776 ret = ttm_bo_kmap(&nvbo->bo, 0, 777 nvbo->bo.mem. 778 num_pages, 779 &nvbo->kmap); 780 if (ret) { 781 WIND_RING(chan); 782 goto out; 783 } 784 nvbo->validate_mapped = true; 785 } 786 787 nouveau_bo_wr32(nvbo, (push[i].offset + 788 push[i].length - 8) / 4, cmd); 789 } 790 791 OUT_RING(chan, 0x20000000 | 792 (nvbo->bo.offset + push[i].offset)); 793 OUT_RING(chan, 0); 794 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 795 OUT_RING(chan, 0); 796 } 797 } 798 799 ret = nouveau_fence_new(chan, false, &fence); 800 if (ret) { 801 NV_PRINTK(error, cli, "error fencing pushbuf: %d\n", ret); 802 WIND_RING(chan); 803 goto out; 804 } 805 806 out: 807 validate_fini(&op, fence, bo); 808 nouveau_fence_unref(&fence); 809 810 out_prevalid: 811 u_free(bo); 812 u_free(push); 813 814 out_next: 815 if (chan->dma.ib_max) { 816 req->suffix0 = 0x00000000; 817 req->suffix1 = 0x00000000; 818 } else 819 if (drm->device.info.chipset >= 0x25) { 820 req->suffix0 = 0x00020000; 821 req->suffix1 = 0x00000000; 822 } else { 823 req->suffix0 = 0x20000000 | 824 (chan->push.vma.offset + ((chan->dma.cur + 2) << 2)); 825 req->suffix1 = 0x00000000; 826 } 827 828 return nouveau_abi16_put(abi16, ret); 829 } 830 831 static inline uint32_t 832 domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain) 833 { 834 uint32_t flags = 0; 835 836 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 837 flags |= TTM_PL_FLAG_VRAM; 838 if (domain & NOUVEAU_GEM_DOMAIN_GART) 839 flags |= TTM_PL_FLAG_TT; 840 841 return flags; 842 } 843 844 int 845 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 846 struct drm_file *file_priv) 847 { 848 struct drm_nouveau_gem_cpu_prep *req = data; 849 struct drm_gem_object *gem; 850 struct nouveau_bo *nvbo; 851 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 852 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 853 int ret; 854 855 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 856 if (!gem) 857 return -ENOENT; 858 nvbo = nouveau_gem_object(gem); 859 860 if (no_wait) 861 ret = reservation_object_test_signaled_rcu(nvbo->bo.resv, write) ? 0 : -EBUSY; 862 else { 863 long lret; 864 865 lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true, 30 * HZ); 866 if (!lret) 867 ret = -EBUSY; 868 else if (lret > 0) 869 ret = 0; 870 else 871 ret = lret; 872 } 873 nouveau_bo_sync_for_cpu(nvbo); 874 drm_gem_object_unreference_unlocked(gem); 875 876 return ret; 877 } 878 879 int 880 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 881 struct drm_file *file_priv) 882 { 883 struct drm_nouveau_gem_cpu_fini *req = data; 884 struct drm_gem_object *gem; 885 struct nouveau_bo *nvbo; 886 887 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 888 if (!gem) 889 return -ENOENT; 890 nvbo = nouveau_gem_object(gem); 891 892 nouveau_bo_sync_for_device(nvbo); 893 drm_gem_object_unreference_unlocked(gem); 894 return 0; 895 } 896 897 int 898 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 899 struct drm_file *file_priv) 900 { 901 struct drm_nouveau_gem_info *req = data; 902 struct drm_gem_object *gem; 903 int ret; 904 905 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 906 if (!gem) 907 return -ENOENT; 908 909 ret = nouveau_gem_info(file_priv, gem, req); 910 drm_gem_object_unreference_unlocked(gem); 911 return ret; 912 } 913 914