1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 39 void 40 nouveau_gem_object_del(struct drm_gem_object *gem) 41 { 42 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 43 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 44 struct ttm_buffer_object *bo = &nvbo->bo; 45 struct device *dev = drm->dev->dev; 46 int ret; 47 48 ret = pm_runtime_get_sync(dev); 49 if (WARN_ON(ret < 0 && ret != -EACCES)) 50 return; 51 52 if (gem->import_attach) 53 drm_prime_gem_destroy(gem, nvbo->bo.sg); 54 55 drm_gem_object_release(gem); 56 57 /* reset filp so nouveau_bo_del_ttm() can test for it */ 58 gem->filp = NULL; 59 ttm_bo_unref(&bo); 60 61 pm_runtime_mark_last_busy(dev); 62 pm_runtime_put_autosuspend(dev); 63 } 64 65 int 66 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 67 { 68 struct nouveau_cli *cli = nouveau_cli(file_priv); 69 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 70 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 71 struct device *dev = drm->dev->dev; 72 struct nouveau_vma *vma; 73 int ret; 74 75 if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50) 76 return 0; 77 78 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 79 if (ret) 80 return ret; 81 82 ret = pm_runtime_get_sync(dev); 83 if (ret < 0 && ret != -EACCES) 84 goto out; 85 86 ret = nouveau_vma_new(nvbo, &cli->vmm, &vma); 87 pm_runtime_mark_last_busy(dev); 88 pm_runtime_put_autosuspend(dev); 89 out: 90 ttm_bo_unreserve(&nvbo->bo); 91 return ret; 92 } 93 94 struct nouveau_gem_object_unmap { 95 struct nouveau_cli_work work; 96 struct nouveau_vma *vma; 97 }; 98 99 static void 100 nouveau_gem_object_delete(struct nouveau_vma *vma) 101 { 102 nouveau_fence_unref(&vma->fence); 103 nouveau_vma_del(&vma); 104 } 105 106 static void 107 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 108 { 109 struct nouveau_gem_object_unmap *work = 110 container_of(w, typeof(*work), work); 111 nouveau_gem_object_delete(work->vma); 112 kfree(work); 113 } 114 115 static void 116 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 117 { 118 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 119 struct nouveau_gem_object_unmap *work; 120 121 list_del_init(&vma->head); 122 123 if (!fence) { 124 nouveau_gem_object_delete(vma); 125 return; 126 } 127 128 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 129 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 130 nouveau_gem_object_delete(vma); 131 return; 132 } 133 134 work->work.func = nouveau_gem_object_delete_work; 135 work->vma = vma; 136 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 137 } 138 139 void 140 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 141 { 142 struct nouveau_cli *cli = nouveau_cli(file_priv); 143 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 144 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 145 struct device *dev = drm->dev->dev; 146 struct nouveau_vma *vma; 147 int ret; 148 149 if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50) 150 return; 151 152 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 153 if (ret) 154 return; 155 156 vma = nouveau_vma_find(nvbo, &cli->vmm); 157 if (vma) { 158 if (--vma->refs == 0) { 159 ret = pm_runtime_get_sync(dev); 160 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 161 nouveau_gem_object_unmap(nvbo, vma); 162 pm_runtime_mark_last_busy(dev); 163 pm_runtime_put_autosuspend(dev); 164 } 165 } 166 } 167 ttm_bo_unreserve(&nvbo->bo); 168 } 169 170 int 171 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 172 uint32_t tile_mode, uint32_t tile_flags, 173 struct nouveau_bo **pnvbo) 174 { 175 struct nouveau_drm *drm = cli->drm; 176 struct nouveau_bo *nvbo; 177 u32 flags = 0; 178 int ret; 179 180 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 181 flags |= TTM_PL_FLAG_VRAM; 182 if (domain & NOUVEAU_GEM_DOMAIN_GART) 183 flags |= TTM_PL_FLAG_TT; 184 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 185 flags |= TTM_PL_FLAG_SYSTEM; 186 187 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) 188 flags |= TTM_PL_FLAG_UNCACHED; 189 190 ret = nouveau_bo_new(cli, size, align, flags, tile_mode, 191 tile_flags, NULL, NULL, pnvbo); 192 if (ret) 193 return ret; 194 nvbo = *pnvbo; 195 196 /* we restrict allowed domains on nv50+ to only the types 197 * that were requested at creation time. not possibly on 198 * earlier chips without busting the ABI. 199 */ 200 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 201 NOUVEAU_GEM_DOMAIN_GART; 202 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 203 nvbo->valid_domains &= domain; 204 205 /* Initialize the embedded gem-object. We return a single gem-reference 206 * to the caller, instead of a normal nouveau_bo ttm reference. */ 207 ret = drm_gem_object_init(drm->dev, &nvbo->gem, nvbo->bo.mem.size); 208 if (ret) { 209 nouveau_bo_ref(NULL, pnvbo); 210 return -ENOMEM; 211 } 212 213 nvbo->bo.persistent_swap_storage = nvbo->gem.filp; 214 return 0; 215 } 216 217 static int 218 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 219 struct drm_nouveau_gem_info *rep) 220 { 221 struct nouveau_cli *cli = nouveau_cli(file_priv); 222 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 223 struct nouveau_vma *vma; 224 225 if (is_power_of_2(nvbo->valid_domains)) 226 rep->domain = nvbo->valid_domains; 227 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 228 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 229 else 230 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 231 rep->offset = nvbo->bo.offset; 232 if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 233 vma = nouveau_vma_find(nvbo, &cli->vmm); 234 if (!vma) 235 return -EINVAL; 236 237 rep->offset = vma->addr; 238 } 239 240 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 241 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); 242 rep->tile_mode = nvbo->mode; 243 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 244 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 245 rep->tile_flags |= nvbo->kind << 8; 246 else 247 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 248 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 249 else 250 rep->tile_flags |= nvbo->zeta; 251 return 0; 252 } 253 254 int 255 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 256 struct drm_file *file_priv) 257 { 258 struct nouveau_cli *cli = nouveau_cli(file_priv); 259 struct drm_nouveau_gem_new *req = data; 260 struct nouveau_bo *nvbo = NULL; 261 int ret = 0; 262 263 ret = nouveau_gem_new(cli, req->info.size, req->align, 264 req->info.domain, req->info.tile_mode, 265 req->info.tile_flags, &nvbo); 266 if (ret) 267 return ret; 268 269 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle); 270 if (ret == 0) { 271 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info); 272 if (ret) 273 drm_gem_handle_delete(file_priv, req->info.handle); 274 } 275 276 /* drop reference from allocate - handle holds it now */ 277 drm_gem_object_unreference_unlocked(&nvbo->gem); 278 return ret; 279 } 280 281 static int 282 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 283 uint32_t write_domains, uint32_t valid_domains) 284 { 285 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 286 struct ttm_buffer_object *bo = &nvbo->bo; 287 uint32_t domains = valid_domains & nvbo->valid_domains & 288 (write_domains ? write_domains : read_domains); 289 uint32_t pref_flags = 0, valid_flags = 0; 290 291 if (!domains) 292 return -EINVAL; 293 294 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 295 valid_flags |= TTM_PL_FLAG_VRAM; 296 297 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 298 valid_flags |= TTM_PL_FLAG_TT; 299 300 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 301 bo->mem.mem_type == TTM_PL_VRAM) 302 pref_flags |= TTM_PL_FLAG_VRAM; 303 304 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 305 bo->mem.mem_type == TTM_PL_TT) 306 pref_flags |= TTM_PL_FLAG_TT; 307 308 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 309 pref_flags |= TTM_PL_FLAG_VRAM; 310 311 else 312 pref_flags |= TTM_PL_FLAG_TT; 313 314 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 315 316 return 0; 317 } 318 319 struct validate_op { 320 struct list_head list; 321 struct ww_acquire_ctx ticket; 322 }; 323 324 static void 325 validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence, 326 struct drm_nouveau_gem_pushbuf_bo *pbbo) 327 { 328 struct nouveau_bo *nvbo; 329 struct drm_nouveau_gem_pushbuf_bo *b; 330 331 while (!list_empty(&op->list)) { 332 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 333 b = &pbbo[nvbo->pbbo_index]; 334 335 if (likely(fence)) { 336 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 337 struct nouveau_vma *vma; 338 339 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 340 341 if (drm->client.vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 342 vma = (void *)(unsigned long)b->user_priv; 343 nouveau_fence_unref(&vma->fence); 344 dma_fence_get(&fence->base); 345 vma->fence = fence; 346 } 347 } 348 349 if (unlikely(nvbo->validate_mapped)) { 350 ttm_bo_kunmap(&nvbo->kmap); 351 nvbo->validate_mapped = false; 352 } 353 354 list_del(&nvbo->entry); 355 nvbo->reserved_by = NULL; 356 ttm_bo_unreserve(&nvbo->bo); 357 drm_gem_object_unreference_unlocked(&nvbo->gem); 358 } 359 } 360 361 static void 362 validate_fini(struct validate_op *op, struct nouveau_fence *fence, 363 struct drm_nouveau_gem_pushbuf_bo *pbbo) 364 { 365 validate_fini_no_ticket(op, fence, pbbo); 366 ww_acquire_fini(&op->ticket); 367 } 368 369 static int 370 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 371 struct drm_nouveau_gem_pushbuf_bo *pbbo, 372 int nr_buffers, struct validate_op *op) 373 { 374 struct nouveau_cli *cli = nouveau_cli(file_priv); 375 int trycnt = 0; 376 int ret = -EINVAL, i; 377 struct nouveau_bo *res_bo = NULL; 378 LIST_HEAD(gart_list); 379 LIST_HEAD(vram_list); 380 LIST_HEAD(both_list); 381 382 ww_acquire_init(&op->ticket, &reservation_ww_class); 383 retry: 384 if (++trycnt > 100000) { 385 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 386 return -EINVAL; 387 } 388 389 for (i = 0; i < nr_buffers; i++) { 390 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 391 struct drm_gem_object *gem; 392 struct nouveau_bo *nvbo; 393 394 gem = drm_gem_object_lookup(file_priv, b->handle); 395 if (!gem) { 396 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 397 ret = -ENOENT; 398 break; 399 } 400 nvbo = nouveau_gem_object(gem); 401 if (nvbo == res_bo) { 402 res_bo = NULL; 403 drm_gem_object_unreference_unlocked(gem); 404 continue; 405 } 406 407 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 408 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 409 "validation list\n", b->handle); 410 drm_gem_object_unreference_unlocked(gem); 411 ret = -EINVAL; 412 break; 413 } 414 415 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 416 if (ret) { 417 list_splice_tail_init(&vram_list, &op->list); 418 list_splice_tail_init(&gart_list, &op->list); 419 list_splice_tail_init(&both_list, &op->list); 420 validate_fini_no_ticket(op, NULL, NULL); 421 if (unlikely(ret == -EDEADLK)) { 422 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 423 &op->ticket); 424 if (!ret) 425 res_bo = nvbo; 426 } 427 if (unlikely(ret)) { 428 if (ret != -ERESTARTSYS) 429 NV_PRINTK(err, cli, "fail reserve\n"); 430 break; 431 } 432 } 433 434 if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 435 struct nouveau_vmm *vmm = &cli->vmm; 436 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 437 if (!vma) { 438 NV_PRINTK(err, cli, "vma not found!\n"); 439 ret = -EINVAL; 440 break; 441 } 442 443 b->user_priv = (uint64_t)(unsigned long)vma; 444 } else { 445 b->user_priv = (uint64_t)(unsigned long)nvbo; 446 } 447 448 nvbo->reserved_by = file_priv; 449 nvbo->pbbo_index = i; 450 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 451 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 452 list_add_tail(&nvbo->entry, &both_list); 453 else 454 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 455 list_add_tail(&nvbo->entry, &vram_list); 456 else 457 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 458 list_add_tail(&nvbo->entry, &gart_list); 459 else { 460 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 461 b->valid_domains); 462 list_add_tail(&nvbo->entry, &both_list); 463 ret = -EINVAL; 464 break; 465 } 466 if (nvbo == res_bo) 467 goto retry; 468 } 469 470 ww_acquire_done(&op->ticket); 471 list_splice_tail(&vram_list, &op->list); 472 list_splice_tail(&gart_list, &op->list); 473 list_splice_tail(&both_list, &op->list); 474 if (ret) 475 validate_fini(op, NULL, NULL); 476 return ret; 477 478 } 479 480 static int 481 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 482 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 483 uint64_t user_pbbo_ptr) 484 { 485 struct nouveau_drm *drm = chan->drm; 486 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 487 (void __force __user *)(uintptr_t)user_pbbo_ptr; 488 struct nouveau_bo *nvbo; 489 int ret, relocs = 0; 490 491 list_for_each_entry(nvbo, list, entry) { 492 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 493 494 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains, 495 b->write_domains, 496 b->valid_domains); 497 if (unlikely(ret)) { 498 NV_PRINTK(err, cli, "fail set_domain\n"); 499 return ret; 500 } 501 502 ret = nouveau_bo_validate(nvbo, true, false); 503 if (unlikely(ret)) { 504 if (ret != -ERESTARTSYS) 505 NV_PRINTK(err, cli, "fail ttm_validate\n"); 506 return ret; 507 } 508 509 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 510 if (unlikely(ret)) { 511 if (ret != -ERESTARTSYS) 512 NV_PRINTK(err, cli, "fail post-validate sync\n"); 513 return ret; 514 } 515 516 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 517 if (nvbo->bo.offset == b->presumed.offset && 518 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 519 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 520 (nvbo->bo.mem.mem_type == TTM_PL_TT && 521 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 522 continue; 523 524 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 525 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 526 else 527 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 528 b->presumed.offset = nvbo->bo.offset; 529 b->presumed.valid = 0; 530 relocs++; 531 532 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed, 533 &b->presumed, sizeof(b->presumed))) 534 return -EFAULT; 535 } 536 } 537 538 return relocs; 539 } 540 541 static int 542 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 543 struct drm_file *file_priv, 544 struct drm_nouveau_gem_pushbuf_bo *pbbo, 545 uint64_t user_buffers, int nr_buffers, 546 struct validate_op *op, int *apply_relocs) 547 { 548 struct nouveau_cli *cli = nouveau_cli(file_priv); 549 int ret; 550 551 INIT_LIST_HEAD(&op->list); 552 553 if (nr_buffers == 0) 554 return 0; 555 556 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 557 if (unlikely(ret)) { 558 if (ret != -ERESTARTSYS) 559 NV_PRINTK(err, cli, "validate_init\n"); 560 return ret; 561 } 562 563 ret = validate_list(chan, cli, &op->list, pbbo, user_buffers); 564 if (unlikely(ret < 0)) { 565 if (ret != -ERESTARTSYS) 566 NV_PRINTK(err, cli, "validating bo list\n"); 567 validate_fini(op, NULL, NULL); 568 return ret; 569 } 570 *apply_relocs = ret; 571 return 0; 572 } 573 574 static inline void 575 u_free(void *addr) 576 { 577 kvfree(addr); 578 } 579 580 static inline void * 581 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 582 { 583 void *mem; 584 void __user *userptr = (void __force __user *)(uintptr_t)user; 585 586 size *= nmemb; 587 588 mem = kvmalloc(size, GFP_KERNEL); 589 if (!mem) 590 return ERR_PTR(-ENOMEM); 591 592 if (copy_from_user(mem, userptr, size)) { 593 u_free(mem); 594 return ERR_PTR(-EFAULT); 595 } 596 597 return mem; 598 } 599 600 static int 601 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 602 struct drm_nouveau_gem_pushbuf *req, 603 struct drm_nouveau_gem_pushbuf_bo *bo) 604 { 605 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 606 int ret = 0; 607 unsigned i; 608 609 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 610 if (IS_ERR(reloc)) 611 return PTR_ERR(reloc); 612 613 for (i = 0; i < req->nr_relocs; i++) { 614 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 615 struct drm_nouveau_gem_pushbuf_bo *b; 616 struct nouveau_bo *nvbo; 617 uint32_t data; 618 619 if (unlikely(r->bo_index > req->nr_buffers)) { 620 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 621 ret = -EINVAL; 622 break; 623 } 624 625 b = &bo[r->bo_index]; 626 if (b->presumed.valid) 627 continue; 628 629 if (unlikely(r->reloc_bo_index > req->nr_buffers)) { 630 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 631 ret = -EINVAL; 632 break; 633 } 634 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 635 636 if (unlikely(r->reloc_bo_offset + 4 > 637 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 638 NV_PRINTK(err, cli, "reloc outside of bo\n"); 639 ret = -EINVAL; 640 break; 641 } 642 643 if (!nvbo->kmap.virtual) { 644 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 645 &nvbo->kmap); 646 if (ret) { 647 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 648 break; 649 } 650 nvbo->validate_mapped = true; 651 } 652 653 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 654 data = b->presumed.offset + r->data; 655 else 656 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 657 data = (b->presumed.offset + r->data) >> 32; 658 else 659 data = r->data; 660 661 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 662 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 663 data |= r->tor; 664 else 665 data |= r->vor; 666 } 667 668 ret = ttm_bo_wait(&nvbo->bo, false, false); 669 if (ret) { 670 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 671 break; 672 } 673 674 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 675 } 676 677 u_free(reloc); 678 return ret; 679 } 680 681 int 682 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 683 struct drm_file *file_priv) 684 { 685 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 686 struct nouveau_cli *cli = nouveau_cli(file_priv); 687 struct nouveau_abi16_chan *temp; 688 struct nouveau_drm *drm = nouveau_drm(dev); 689 struct drm_nouveau_gem_pushbuf *req = data; 690 struct drm_nouveau_gem_pushbuf_push *push; 691 struct drm_nouveau_gem_pushbuf_bo *bo; 692 struct nouveau_channel *chan = NULL; 693 struct validate_op op; 694 struct nouveau_fence *fence = NULL; 695 int i, j, ret = 0, do_reloc = 0; 696 697 if (unlikely(!abi16)) 698 return -ENOMEM; 699 700 list_for_each_entry(temp, &abi16->channels, head) { 701 if (temp->chan->chid == req->channel) { 702 chan = temp->chan; 703 break; 704 } 705 } 706 707 if (!chan) 708 return nouveau_abi16_put(abi16, -ENOENT); 709 710 req->vram_available = drm->gem.vram_available; 711 req->gart_available = drm->gem.gart_available; 712 if (unlikely(req->nr_push == 0)) 713 goto out_next; 714 715 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 716 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 717 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 718 return nouveau_abi16_put(abi16, -EINVAL); 719 } 720 721 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 722 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 723 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 724 return nouveau_abi16_put(abi16, -EINVAL); 725 } 726 727 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 728 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 729 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 730 return nouveau_abi16_put(abi16, -EINVAL); 731 } 732 733 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 734 if (IS_ERR(push)) 735 return nouveau_abi16_put(abi16, PTR_ERR(push)); 736 737 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 738 if (IS_ERR(bo)) { 739 u_free(push); 740 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 741 } 742 743 /* Ensure all push buffers are on validate list */ 744 for (i = 0; i < req->nr_push; i++) { 745 if (push[i].bo_index >= req->nr_buffers) { 746 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 747 ret = -EINVAL; 748 goto out_prevalid; 749 } 750 } 751 752 /* Validate buffer list */ 753 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 754 req->nr_buffers, &op, &do_reloc); 755 if (ret) { 756 if (ret != -ERESTARTSYS) 757 NV_PRINTK(err, cli, "validate: %d\n", ret); 758 goto out_prevalid; 759 } 760 761 /* Apply any relocations that are required */ 762 if (do_reloc) { 763 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 764 if (ret) { 765 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 766 goto out; 767 } 768 } 769 770 if (chan->dma.ib_max) { 771 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 772 if (ret) { 773 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 774 goto out; 775 } 776 777 for (i = 0; i < req->nr_push; i++) { 778 struct nouveau_vma *vma = (void *)(unsigned long) 779 bo[push[i].bo_index].user_priv; 780 781 nv50_dma_push(chan, vma->addr + push[i].offset, 782 push[i].length); 783 } 784 } else 785 if (drm->client.device.info.chipset >= 0x25) { 786 ret = RING_SPACE(chan, req->nr_push * 2); 787 if (ret) { 788 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 789 goto out; 790 } 791 792 for (i = 0; i < req->nr_push; i++) { 793 struct nouveau_bo *nvbo = (void *)(unsigned long) 794 bo[push[i].bo_index].user_priv; 795 796 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 797 OUT_RING(chan, 0); 798 } 799 } else { 800 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 801 if (ret) { 802 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 803 goto out; 804 } 805 806 for (i = 0; i < req->nr_push; i++) { 807 struct nouveau_bo *nvbo = (void *)(unsigned long) 808 bo[push[i].bo_index].user_priv; 809 uint32_t cmd; 810 811 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 812 cmd |= 0x20000000; 813 if (unlikely(cmd != req->suffix0)) { 814 if (!nvbo->kmap.virtual) { 815 ret = ttm_bo_kmap(&nvbo->bo, 0, 816 nvbo->bo.mem. 817 num_pages, 818 &nvbo->kmap); 819 if (ret) { 820 WIND_RING(chan); 821 goto out; 822 } 823 nvbo->validate_mapped = true; 824 } 825 826 nouveau_bo_wr32(nvbo, (push[i].offset + 827 push[i].length - 8) / 4, cmd); 828 } 829 830 OUT_RING(chan, 0x20000000 | 831 (nvbo->bo.offset + push[i].offset)); 832 OUT_RING(chan, 0); 833 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 834 OUT_RING(chan, 0); 835 } 836 } 837 838 ret = nouveau_fence_new(chan, false, &fence); 839 if (ret) { 840 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 841 WIND_RING(chan); 842 goto out; 843 } 844 845 out: 846 validate_fini(&op, fence, bo); 847 nouveau_fence_unref(&fence); 848 849 out_prevalid: 850 u_free(bo); 851 u_free(push); 852 853 out_next: 854 if (chan->dma.ib_max) { 855 req->suffix0 = 0x00000000; 856 req->suffix1 = 0x00000000; 857 } else 858 if (drm->client.device.info.chipset >= 0x25) { 859 req->suffix0 = 0x00020000; 860 req->suffix1 = 0x00000000; 861 } else { 862 req->suffix0 = 0x20000000 | 863 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 864 req->suffix1 = 0x00000000; 865 } 866 867 return nouveau_abi16_put(abi16, ret); 868 } 869 870 int 871 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 872 struct drm_file *file_priv) 873 { 874 struct drm_nouveau_gem_cpu_prep *req = data; 875 struct drm_gem_object *gem; 876 struct nouveau_bo *nvbo; 877 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 878 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 879 long lret; 880 int ret; 881 882 gem = drm_gem_object_lookup(file_priv, req->handle); 883 if (!gem) 884 return -ENOENT; 885 nvbo = nouveau_gem_object(gem); 886 887 lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true, 888 no_wait ? 0 : 30 * HZ); 889 if (!lret) 890 ret = -EBUSY; 891 else if (lret > 0) 892 ret = 0; 893 else 894 ret = lret; 895 896 nouveau_bo_sync_for_cpu(nvbo); 897 drm_gem_object_unreference_unlocked(gem); 898 899 return ret; 900 } 901 902 int 903 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 904 struct drm_file *file_priv) 905 { 906 struct drm_nouveau_gem_cpu_fini *req = data; 907 struct drm_gem_object *gem; 908 struct nouveau_bo *nvbo; 909 910 gem = drm_gem_object_lookup(file_priv, req->handle); 911 if (!gem) 912 return -ENOENT; 913 nvbo = nouveau_gem_object(gem); 914 915 nouveau_bo_sync_for_device(nvbo); 916 drm_gem_object_unreference_unlocked(gem); 917 return 0; 918 } 919 920 int 921 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 922 struct drm_file *file_priv) 923 { 924 struct drm_nouveau_gem_info *req = data; 925 struct drm_gem_object *gem; 926 int ret; 927 928 gem = drm_gem_object_lookup(file_priv, req->handle); 929 if (!gem) 930 return -ENOENT; 931 932 ret = nouveau_gem_info(file_priv, gem, req); 933 drm_gem_object_unreference_unlocked(gem); 934 return ret; 935 } 936 937