1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 39 void 40 nouveau_gem_object_del(struct drm_gem_object *gem) 41 { 42 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 43 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 44 struct device *dev = drm->dev->dev; 45 int ret; 46 47 ret = pm_runtime_get_sync(dev); 48 if (WARN_ON(ret < 0 && ret != -EACCES)) 49 return; 50 51 if (gem->import_attach) 52 drm_prime_gem_destroy(gem, nvbo->bo.sg); 53 54 drm_gem_object_release(gem); 55 56 /* reset filp so nouveau_bo_del_ttm() can test for it */ 57 gem->filp = NULL; 58 ttm_bo_put(&nvbo->bo); 59 60 pm_runtime_mark_last_busy(dev); 61 pm_runtime_put_autosuspend(dev); 62 } 63 64 int 65 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 66 { 67 struct nouveau_cli *cli = nouveau_cli(file_priv); 68 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 69 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 70 struct device *dev = drm->dev->dev; 71 struct nouveau_vma *vma; 72 int ret; 73 74 if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50) 75 return 0; 76 77 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 78 if (ret) 79 return ret; 80 81 ret = pm_runtime_get_sync(dev); 82 if (ret < 0 && ret != -EACCES) 83 goto out; 84 85 ret = nouveau_vma_new(nvbo, &cli->vmm, &vma); 86 pm_runtime_mark_last_busy(dev); 87 pm_runtime_put_autosuspend(dev); 88 out: 89 ttm_bo_unreserve(&nvbo->bo); 90 return ret; 91 } 92 93 struct nouveau_gem_object_unmap { 94 struct nouveau_cli_work work; 95 struct nouveau_vma *vma; 96 }; 97 98 static void 99 nouveau_gem_object_delete(struct nouveau_vma *vma) 100 { 101 nouveau_fence_unref(&vma->fence); 102 nouveau_vma_del(&vma); 103 } 104 105 static void 106 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 107 { 108 struct nouveau_gem_object_unmap *work = 109 container_of(w, typeof(*work), work); 110 nouveau_gem_object_delete(work->vma); 111 kfree(work); 112 } 113 114 static void 115 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 116 { 117 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 118 struct nouveau_gem_object_unmap *work; 119 120 list_del_init(&vma->head); 121 122 if (!fence) { 123 nouveau_gem_object_delete(vma); 124 return; 125 } 126 127 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 128 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 129 nouveau_gem_object_delete(vma); 130 return; 131 } 132 133 work->work.func = nouveau_gem_object_delete_work; 134 work->vma = vma; 135 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 136 } 137 138 void 139 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 140 { 141 struct nouveau_cli *cli = nouveau_cli(file_priv); 142 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 143 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 144 struct device *dev = drm->dev->dev; 145 struct nouveau_vma *vma; 146 int ret; 147 148 if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50) 149 return; 150 151 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 152 if (ret) 153 return; 154 155 vma = nouveau_vma_find(nvbo, &cli->vmm); 156 if (vma) { 157 if (--vma->refs == 0) { 158 ret = pm_runtime_get_sync(dev); 159 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 160 nouveau_gem_object_unmap(nvbo, vma); 161 pm_runtime_mark_last_busy(dev); 162 pm_runtime_put_autosuspend(dev); 163 } 164 } 165 } 166 ttm_bo_unreserve(&nvbo->bo); 167 } 168 169 int 170 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 171 uint32_t tile_mode, uint32_t tile_flags, 172 struct nouveau_bo **pnvbo) 173 { 174 struct nouveau_drm *drm = cli->drm; 175 struct nouveau_bo *nvbo; 176 u32 flags = 0; 177 int ret; 178 179 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 180 flags |= TTM_PL_FLAG_VRAM; 181 if (domain & NOUVEAU_GEM_DOMAIN_GART) 182 flags |= TTM_PL_FLAG_TT; 183 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 184 flags |= TTM_PL_FLAG_SYSTEM; 185 186 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) 187 flags |= TTM_PL_FLAG_UNCACHED; 188 189 ret = nouveau_bo_new(cli, size, align, flags, tile_mode, 190 tile_flags, NULL, NULL, pnvbo); 191 if (ret) 192 return ret; 193 nvbo = *pnvbo; 194 195 /* we restrict allowed domains on nv50+ to only the types 196 * that were requested at creation time. not possibly on 197 * earlier chips without busting the ABI. 198 */ 199 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 200 NOUVEAU_GEM_DOMAIN_GART; 201 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 202 nvbo->valid_domains &= domain; 203 204 /* Initialize the embedded gem-object. We return a single gem-reference 205 * to the caller, instead of a normal nouveau_bo ttm reference. */ 206 ret = drm_gem_object_init(drm->dev, &nvbo->gem, nvbo->bo.mem.size); 207 if (ret) { 208 nouveau_bo_ref(NULL, pnvbo); 209 return -ENOMEM; 210 } 211 212 nvbo->bo.persistent_swap_storage = nvbo->gem.filp; 213 return 0; 214 } 215 216 static int 217 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 218 struct drm_nouveau_gem_info *rep) 219 { 220 struct nouveau_cli *cli = nouveau_cli(file_priv); 221 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 222 struct nouveau_vma *vma; 223 224 if (is_power_of_2(nvbo->valid_domains)) 225 rep->domain = nvbo->valid_domains; 226 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 227 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 228 else 229 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 230 rep->offset = nvbo->bo.offset; 231 if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 232 vma = nouveau_vma_find(nvbo, &cli->vmm); 233 if (!vma) 234 return -EINVAL; 235 236 rep->offset = vma->addr; 237 } 238 239 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 240 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); 241 rep->tile_mode = nvbo->mode; 242 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 243 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 244 rep->tile_flags |= nvbo->kind << 8; 245 else 246 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 247 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 248 else 249 rep->tile_flags |= nvbo->zeta; 250 return 0; 251 } 252 253 int 254 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 255 struct drm_file *file_priv) 256 { 257 struct nouveau_cli *cli = nouveau_cli(file_priv); 258 struct drm_nouveau_gem_new *req = data; 259 struct nouveau_bo *nvbo = NULL; 260 int ret = 0; 261 262 ret = nouveau_gem_new(cli, req->info.size, req->align, 263 req->info.domain, req->info.tile_mode, 264 req->info.tile_flags, &nvbo); 265 if (ret) 266 return ret; 267 268 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle); 269 if (ret == 0) { 270 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info); 271 if (ret) 272 drm_gem_handle_delete(file_priv, req->info.handle); 273 } 274 275 /* drop reference from allocate - handle holds it now */ 276 drm_gem_object_put_unlocked(&nvbo->gem); 277 return ret; 278 } 279 280 static int 281 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 282 uint32_t write_domains, uint32_t valid_domains) 283 { 284 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 285 struct ttm_buffer_object *bo = &nvbo->bo; 286 uint32_t domains = valid_domains & nvbo->valid_domains & 287 (write_domains ? write_domains : read_domains); 288 uint32_t pref_flags = 0, valid_flags = 0; 289 290 if (!domains) 291 return -EINVAL; 292 293 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 294 valid_flags |= TTM_PL_FLAG_VRAM; 295 296 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 297 valid_flags |= TTM_PL_FLAG_TT; 298 299 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 300 bo->mem.mem_type == TTM_PL_VRAM) 301 pref_flags |= TTM_PL_FLAG_VRAM; 302 303 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 304 bo->mem.mem_type == TTM_PL_TT) 305 pref_flags |= TTM_PL_FLAG_TT; 306 307 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 308 pref_flags |= TTM_PL_FLAG_VRAM; 309 310 else 311 pref_flags |= TTM_PL_FLAG_TT; 312 313 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 314 315 return 0; 316 } 317 318 struct validate_op { 319 struct list_head list; 320 struct ww_acquire_ctx ticket; 321 }; 322 323 static void 324 validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence, 325 struct drm_nouveau_gem_pushbuf_bo *pbbo) 326 { 327 struct nouveau_bo *nvbo; 328 struct drm_nouveau_gem_pushbuf_bo *b; 329 330 while (!list_empty(&op->list)) { 331 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 332 b = &pbbo[nvbo->pbbo_index]; 333 334 if (likely(fence)) { 335 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 336 struct nouveau_vma *vma; 337 338 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 339 340 if (drm->client.vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 341 vma = (void *)(unsigned long)b->user_priv; 342 nouveau_fence_unref(&vma->fence); 343 dma_fence_get(&fence->base); 344 vma->fence = fence; 345 } 346 } 347 348 if (unlikely(nvbo->validate_mapped)) { 349 ttm_bo_kunmap(&nvbo->kmap); 350 nvbo->validate_mapped = false; 351 } 352 353 list_del(&nvbo->entry); 354 nvbo->reserved_by = NULL; 355 ttm_bo_unreserve(&nvbo->bo); 356 drm_gem_object_put_unlocked(&nvbo->gem); 357 } 358 } 359 360 static void 361 validate_fini(struct validate_op *op, struct nouveau_fence *fence, 362 struct drm_nouveau_gem_pushbuf_bo *pbbo) 363 { 364 validate_fini_no_ticket(op, fence, pbbo); 365 ww_acquire_fini(&op->ticket); 366 } 367 368 static int 369 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 370 struct drm_nouveau_gem_pushbuf_bo *pbbo, 371 int nr_buffers, struct validate_op *op) 372 { 373 struct nouveau_cli *cli = nouveau_cli(file_priv); 374 int trycnt = 0; 375 int ret = -EINVAL, i; 376 struct nouveau_bo *res_bo = NULL; 377 LIST_HEAD(gart_list); 378 LIST_HEAD(vram_list); 379 LIST_HEAD(both_list); 380 381 ww_acquire_init(&op->ticket, &reservation_ww_class); 382 retry: 383 if (++trycnt > 100000) { 384 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 385 return -EINVAL; 386 } 387 388 for (i = 0; i < nr_buffers; i++) { 389 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 390 struct drm_gem_object *gem; 391 struct nouveau_bo *nvbo; 392 393 gem = drm_gem_object_lookup(file_priv, b->handle); 394 if (!gem) { 395 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 396 ret = -ENOENT; 397 break; 398 } 399 nvbo = nouveau_gem_object(gem); 400 if (nvbo == res_bo) { 401 res_bo = NULL; 402 drm_gem_object_put_unlocked(gem); 403 continue; 404 } 405 406 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 407 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 408 "validation list\n", b->handle); 409 drm_gem_object_put_unlocked(gem); 410 ret = -EINVAL; 411 break; 412 } 413 414 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 415 if (ret) { 416 list_splice_tail_init(&vram_list, &op->list); 417 list_splice_tail_init(&gart_list, &op->list); 418 list_splice_tail_init(&both_list, &op->list); 419 validate_fini_no_ticket(op, NULL, NULL); 420 if (unlikely(ret == -EDEADLK)) { 421 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 422 &op->ticket); 423 if (!ret) 424 res_bo = nvbo; 425 } 426 if (unlikely(ret)) { 427 if (ret != -ERESTARTSYS) 428 NV_PRINTK(err, cli, "fail reserve\n"); 429 break; 430 } 431 } 432 433 if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 434 struct nouveau_vmm *vmm = &cli->vmm; 435 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 436 if (!vma) { 437 NV_PRINTK(err, cli, "vma not found!\n"); 438 ret = -EINVAL; 439 break; 440 } 441 442 b->user_priv = (uint64_t)(unsigned long)vma; 443 } else { 444 b->user_priv = (uint64_t)(unsigned long)nvbo; 445 } 446 447 nvbo->reserved_by = file_priv; 448 nvbo->pbbo_index = i; 449 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 450 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 451 list_add_tail(&nvbo->entry, &both_list); 452 else 453 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 454 list_add_tail(&nvbo->entry, &vram_list); 455 else 456 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 457 list_add_tail(&nvbo->entry, &gart_list); 458 else { 459 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 460 b->valid_domains); 461 list_add_tail(&nvbo->entry, &both_list); 462 ret = -EINVAL; 463 break; 464 } 465 if (nvbo == res_bo) 466 goto retry; 467 } 468 469 ww_acquire_done(&op->ticket); 470 list_splice_tail(&vram_list, &op->list); 471 list_splice_tail(&gart_list, &op->list); 472 list_splice_tail(&both_list, &op->list); 473 if (ret) 474 validate_fini(op, NULL, NULL); 475 return ret; 476 477 } 478 479 static int 480 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 481 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 482 uint64_t user_pbbo_ptr) 483 { 484 struct nouveau_drm *drm = chan->drm; 485 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 486 (void __force __user *)(uintptr_t)user_pbbo_ptr; 487 struct nouveau_bo *nvbo; 488 int ret, relocs = 0; 489 490 list_for_each_entry(nvbo, list, entry) { 491 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 492 493 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains, 494 b->write_domains, 495 b->valid_domains); 496 if (unlikely(ret)) { 497 NV_PRINTK(err, cli, "fail set_domain\n"); 498 return ret; 499 } 500 501 ret = nouveau_bo_validate(nvbo, true, false); 502 if (unlikely(ret)) { 503 if (ret != -ERESTARTSYS) 504 NV_PRINTK(err, cli, "fail ttm_validate\n"); 505 return ret; 506 } 507 508 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 509 if (unlikely(ret)) { 510 if (ret != -ERESTARTSYS) 511 NV_PRINTK(err, cli, "fail post-validate sync\n"); 512 return ret; 513 } 514 515 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 516 if (nvbo->bo.offset == b->presumed.offset && 517 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 518 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 519 (nvbo->bo.mem.mem_type == TTM_PL_TT && 520 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 521 continue; 522 523 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 524 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 525 else 526 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 527 b->presumed.offset = nvbo->bo.offset; 528 b->presumed.valid = 0; 529 relocs++; 530 531 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed, 532 &b->presumed, sizeof(b->presumed))) 533 return -EFAULT; 534 } 535 } 536 537 return relocs; 538 } 539 540 static int 541 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 542 struct drm_file *file_priv, 543 struct drm_nouveau_gem_pushbuf_bo *pbbo, 544 uint64_t user_buffers, int nr_buffers, 545 struct validate_op *op, int *apply_relocs) 546 { 547 struct nouveau_cli *cli = nouveau_cli(file_priv); 548 int ret; 549 550 INIT_LIST_HEAD(&op->list); 551 552 if (nr_buffers == 0) 553 return 0; 554 555 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 556 if (unlikely(ret)) { 557 if (ret != -ERESTARTSYS) 558 NV_PRINTK(err, cli, "validate_init\n"); 559 return ret; 560 } 561 562 ret = validate_list(chan, cli, &op->list, pbbo, user_buffers); 563 if (unlikely(ret < 0)) { 564 if (ret != -ERESTARTSYS) 565 NV_PRINTK(err, cli, "validating bo list\n"); 566 validate_fini(op, NULL, NULL); 567 return ret; 568 } 569 *apply_relocs = ret; 570 return 0; 571 } 572 573 static inline void 574 u_free(void *addr) 575 { 576 kvfree(addr); 577 } 578 579 static inline void * 580 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 581 { 582 void *mem; 583 void __user *userptr = (void __force __user *)(uintptr_t)user; 584 585 size *= nmemb; 586 587 mem = kvmalloc(size, GFP_KERNEL); 588 if (!mem) 589 return ERR_PTR(-ENOMEM); 590 591 if (copy_from_user(mem, userptr, size)) { 592 u_free(mem); 593 return ERR_PTR(-EFAULT); 594 } 595 596 return mem; 597 } 598 599 static int 600 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 601 struct drm_nouveau_gem_pushbuf *req, 602 struct drm_nouveau_gem_pushbuf_bo *bo) 603 { 604 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 605 int ret = 0; 606 unsigned i; 607 608 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 609 if (IS_ERR(reloc)) 610 return PTR_ERR(reloc); 611 612 for (i = 0; i < req->nr_relocs; i++) { 613 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 614 struct drm_nouveau_gem_pushbuf_bo *b; 615 struct nouveau_bo *nvbo; 616 uint32_t data; 617 618 if (unlikely(r->bo_index >= req->nr_buffers)) { 619 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 620 ret = -EINVAL; 621 break; 622 } 623 624 b = &bo[r->bo_index]; 625 if (b->presumed.valid) 626 continue; 627 628 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 629 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 630 ret = -EINVAL; 631 break; 632 } 633 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 634 635 if (unlikely(r->reloc_bo_offset + 4 > 636 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 637 NV_PRINTK(err, cli, "reloc outside of bo\n"); 638 ret = -EINVAL; 639 break; 640 } 641 642 if (!nvbo->kmap.virtual) { 643 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 644 &nvbo->kmap); 645 if (ret) { 646 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 647 break; 648 } 649 nvbo->validate_mapped = true; 650 } 651 652 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 653 data = b->presumed.offset + r->data; 654 else 655 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 656 data = (b->presumed.offset + r->data) >> 32; 657 else 658 data = r->data; 659 660 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 661 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 662 data |= r->tor; 663 else 664 data |= r->vor; 665 } 666 667 ret = ttm_bo_wait(&nvbo->bo, false, false); 668 if (ret) { 669 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 670 break; 671 } 672 673 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 674 } 675 676 u_free(reloc); 677 return ret; 678 } 679 680 int 681 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 682 struct drm_file *file_priv) 683 { 684 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 685 struct nouveau_cli *cli = nouveau_cli(file_priv); 686 struct nouveau_abi16_chan *temp; 687 struct nouveau_drm *drm = nouveau_drm(dev); 688 struct drm_nouveau_gem_pushbuf *req = data; 689 struct drm_nouveau_gem_pushbuf_push *push; 690 struct drm_nouveau_gem_pushbuf_bo *bo; 691 struct nouveau_channel *chan = NULL; 692 struct validate_op op; 693 struct nouveau_fence *fence = NULL; 694 int i, j, ret = 0, do_reloc = 0; 695 696 if (unlikely(!abi16)) 697 return -ENOMEM; 698 699 list_for_each_entry(temp, &abi16->channels, head) { 700 if (temp->chan->chid == req->channel) { 701 chan = temp->chan; 702 break; 703 } 704 } 705 706 if (!chan) 707 return nouveau_abi16_put(abi16, -ENOENT); 708 709 req->vram_available = drm->gem.vram_available; 710 req->gart_available = drm->gem.gart_available; 711 if (unlikely(req->nr_push == 0)) 712 goto out_next; 713 714 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 715 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 716 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 717 return nouveau_abi16_put(abi16, -EINVAL); 718 } 719 720 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 721 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 722 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 723 return nouveau_abi16_put(abi16, -EINVAL); 724 } 725 726 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 727 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 728 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 729 return nouveau_abi16_put(abi16, -EINVAL); 730 } 731 732 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 733 if (IS_ERR(push)) 734 return nouveau_abi16_put(abi16, PTR_ERR(push)); 735 736 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 737 if (IS_ERR(bo)) { 738 u_free(push); 739 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 740 } 741 742 /* Ensure all push buffers are on validate list */ 743 for (i = 0; i < req->nr_push; i++) { 744 if (push[i].bo_index >= req->nr_buffers) { 745 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 746 ret = -EINVAL; 747 goto out_prevalid; 748 } 749 } 750 751 /* Validate buffer list */ 752 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 753 req->nr_buffers, &op, &do_reloc); 754 if (ret) { 755 if (ret != -ERESTARTSYS) 756 NV_PRINTK(err, cli, "validate: %d\n", ret); 757 goto out_prevalid; 758 } 759 760 /* Apply any relocations that are required */ 761 if (do_reloc) { 762 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 763 if (ret) { 764 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 765 goto out; 766 } 767 } 768 769 if (chan->dma.ib_max) { 770 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 771 if (ret) { 772 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 773 goto out; 774 } 775 776 for (i = 0; i < req->nr_push; i++) { 777 struct nouveau_vma *vma = (void *)(unsigned long) 778 bo[push[i].bo_index].user_priv; 779 780 nv50_dma_push(chan, vma->addr + push[i].offset, 781 push[i].length); 782 } 783 } else 784 if (drm->client.device.info.chipset >= 0x25) { 785 ret = RING_SPACE(chan, req->nr_push * 2); 786 if (ret) { 787 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 788 goto out; 789 } 790 791 for (i = 0; i < req->nr_push; i++) { 792 struct nouveau_bo *nvbo = (void *)(unsigned long) 793 bo[push[i].bo_index].user_priv; 794 795 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 796 OUT_RING(chan, 0); 797 } 798 } else { 799 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 800 if (ret) { 801 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 802 goto out; 803 } 804 805 for (i = 0; i < req->nr_push; i++) { 806 struct nouveau_bo *nvbo = (void *)(unsigned long) 807 bo[push[i].bo_index].user_priv; 808 uint32_t cmd; 809 810 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 811 cmd |= 0x20000000; 812 if (unlikely(cmd != req->suffix0)) { 813 if (!nvbo->kmap.virtual) { 814 ret = ttm_bo_kmap(&nvbo->bo, 0, 815 nvbo->bo.mem. 816 num_pages, 817 &nvbo->kmap); 818 if (ret) { 819 WIND_RING(chan); 820 goto out; 821 } 822 nvbo->validate_mapped = true; 823 } 824 825 nouveau_bo_wr32(nvbo, (push[i].offset + 826 push[i].length - 8) / 4, cmd); 827 } 828 829 OUT_RING(chan, 0x20000000 | 830 (nvbo->bo.offset + push[i].offset)); 831 OUT_RING(chan, 0); 832 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 833 OUT_RING(chan, 0); 834 } 835 } 836 837 ret = nouveau_fence_new(chan, false, &fence); 838 if (ret) { 839 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 840 WIND_RING(chan); 841 goto out; 842 } 843 844 out: 845 validate_fini(&op, fence, bo); 846 nouveau_fence_unref(&fence); 847 848 out_prevalid: 849 u_free(bo); 850 u_free(push); 851 852 out_next: 853 if (chan->dma.ib_max) { 854 req->suffix0 = 0x00000000; 855 req->suffix1 = 0x00000000; 856 } else 857 if (drm->client.device.info.chipset >= 0x25) { 858 req->suffix0 = 0x00020000; 859 req->suffix1 = 0x00000000; 860 } else { 861 req->suffix0 = 0x20000000 | 862 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 863 req->suffix1 = 0x00000000; 864 } 865 866 return nouveau_abi16_put(abi16, ret); 867 } 868 869 int 870 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 871 struct drm_file *file_priv) 872 { 873 struct drm_nouveau_gem_cpu_prep *req = data; 874 struct drm_gem_object *gem; 875 struct nouveau_bo *nvbo; 876 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 877 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 878 long lret; 879 int ret; 880 881 gem = drm_gem_object_lookup(file_priv, req->handle); 882 if (!gem) 883 return -ENOENT; 884 nvbo = nouveau_gem_object(gem); 885 886 lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true, 887 no_wait ? 0 : 30 * HZ); 888 if (!lret) 889 ret = -EBUSY; 890 else if (lret > 0) 891 ret = 0; 892 else 893 ret = lret; 894 895 nouveau_bo_sync_for_cpu(nvbo); 896 drm_gem_object_put_unlocked(gem); 897 898 return ret; 899 } 900 901 int 902 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 903 struct drm_file *file_priv) 904 { 905 struct drm_nouveau_gem_cpu_fini *req = data; 906 struct drm_gem_object *gem; 907 struct nouveau_bo *nvbo; 908 909 gem = drm_gem_object_lookup(file_priv, req->handle); 910 if (!gem) 911 return -ENOENT; 912 nvbo = nouveau_gem_object(gem); 913 914 nouveau_bo_sync_for_device(nvbo); 915 drm_gem_object_put_unlocked(gem); 916 return 0; 917 } 918 919 int 920 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 921 struct drm_file *file_priv) 922 { 923 struct drm_nouveau_gem_info *req = data; 924 struct drm_gem_object *gem; 925 int ret; 926 927 gem = drm_gem_object_lookup(file_priv, req->handle); 928 if (!gem) 929 return -ENOENT; 930 931 ret = nouveau_gem_info(file_priv, gem, req); 932 drm_gem_object_put_unlocked(gem); 933 return ret; 934 } 935 936