1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include <drm/drm_gem_ttm_helper.h> 28 29 #include "nouveau_drv.h" 30 #include "nouveau_dma.h" 31 #include "nouveau_fence.h" 32 #include "nouveau_abi16.h" 33 34 #include "nouveau_ttm.h" 35 #include "nouveau_gem.h" 36 #include "nouveau_mem.h" 37 #include "nouveau_vmm.h" 38 39 #include <nvif/class.h> 40 #include <nvif/push206e.h> 41 42 void 43 nouveau_gem_object_del(struct drm_gem_object *gem) 44 { 45 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 46 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 47 struct device *dev = drm->dev->dev; 48 int ret; 49 50 ret = pm_runtime_get_sync(dev); 51 if (WARN_ON(ret < 0 && ret != -EACCES)) { 52 pm_runtime_put_autosuspend(dev); 53 return; 54 } 55 56 if (gem->import_attach) 57 drm_prime_gem_destroy(gem, nvbo->bo.sg); 58 59 ttm_bo_put(&nvbo->bo); 60 61 pm_runtime_mark_last_busy(dev); 62 pm_runtime_put_autosuspend(dev); 63 } 64 65 int 66 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 67 { 68 struct nouveau_cli *cli = nouveau_cli(file_priv); 69 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 70 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 71 struct device *dev = drm->dev->dev; 72 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 73 struct nouveau_vma *vma; 74 int ret; 75 76 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 77 return 0; 78 79 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 80 if (ret) 81 return ret; 82 83 ret = pm_runtime_get_sync(dev); 84 if (ret < 0 && ret != -EACCES) { 85 pm_runtime_put_autosuspend(dev); 86 goto out; 87 } 88 89 ret = nouveau_vma_new(nvbo, vmm, &vma); 90 pm_runtime_mark_last_busy(dev); 91 pm_runtime_put_autosuspend(dev); 92 out: 93 ttm_bo_unreserve(&nvbo->bo); 94 return ret; 95 } 96 97 struct nouveau_gem_object_unmap { 98 struct nouveau_cli_work work; 99 struct nouveau_vma *vma; 100 }; 101 102 static void 103 nouveau_gem_object_delete(struct nouveau_vma *vma) 104 { 105 nouveau_fence_unref(&vma->fence); 106 nouveau_vma_del(&vma); 107 } 108 109 static void 110 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 111 { 112 struct nouveau_gem_object_unmap *work = 113 container_of(w, typeof(*work), work); 114 nouveau_gem_object_delete(work->vma); 115 kfree(work); 116 } 117 118 static void 119 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 120 { 121 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 122 struct nouveau_gem_object_unmap *work; 123 124 list_del_init(&vma->head); 125 126 if (!fence) { 127 nouveau_gem_object_delete(vma); 128 return; 129 } 130 131 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 132 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 133 nouveau_gem_object_delete(vma); 134 return; 135 } 136 137 work->work.func = nouveau_gem_object_delete_work; 138 work->vma = vma; 139 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 140 } 141 142 void 143 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 144 { 145 struct nouveau_cli *cli = nouveau_cli(file_priv); 146 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 147 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 148 struct device *dev = drm->dev->dev; 149 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 150 struct nouveau_vma *vma; 151 int ret; 152 153 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 154 return; 155 156 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 157 if (ret) 158 return; 159 160 vma = nouveau_vma_find(nvbo, vmm); 161 if (vma) { 162 if (--vma->refs == 0) { 163 ret = pm_runtime_get_sync(dev); 164 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 165 nouveau_gem_object_unmap(nvbo, vma); 166 pm_runtime_mark_last_busy(dev); 167 } 168 pm_runtime_put_autosuspend(dev); 169 } 170 } 171 ttm_bo_unreserve(&nvbo->bo); 172 } 173 174 const struct drm_gem_object_funcs nouveau_gem_object_funcs = { 175 .free = nouveau_gem_object_del, 176 .open = nouveau_gem_object_open, 177 .close = nouveau_gem_object_close, 178 .pin = nouveau_gem_prime_pin, 179 .unpin = nouveau_gem_prime_unpin, 180 .get_sg_table = nouveau_gem_prime_get_sg_table, 181 .vmap = drm_gem_ttm_vmap, 182 .vunmap = drm_gem_ttm_vunmap, 183 }; 184 185 int 186 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 187 uint32_t tile_mode, uint32_t tile_flags, 188 struct nouveau_bo **pnvbo) 189 { 190 struct nouveau_drm *drm = cli->drm; 191 struct nouveau_bo *nvbo; 192 int ret; 193 194 if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) 195 domain |= NOUVEAU_GEM_DOMAIN_CPU; 196 197 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 198 tile_flags); 199 if (IS_ERR(nvbo)) 200 return PTR_ERR(nvbo); 201 202 nvbo->bo.base.funcs = &nouveau_gem_object_funcs; 203 204 /* Initialize the embedded gem-object. We return a single gem-reference 205 * to the caller, instead of a normal nouveau_bo ttm reference. */ 206 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 207 if (ret) { 208 drm_gem_object_release(&nvbo->bo.base); 209 kfree(nvbo); 210 return ret; 211 } 212 213 ret = nouveau_bo_init(nvbo, size, align, domain, NULL, NULL); 214 if (ret) { 215 nouveau_bo_ref(NULL, &nvbo); 216 return ret; 217 } 218 219 /* we restrict allowed domains on nv50+ to only the types 220 * that were requested at creation time. not possibly on 221 * earlier chips without busting the ABI. 222 */ 223 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 224 NOUVEAU_GEM_DOMAIN_GART; 225 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 226 nvbo->valid_domains &= domain; 227 228 *pnvbo = nvbo; 229 return 0; 230 } 231 232 static int 233 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 234 struct drm_nouveau_gem_info *rep) 235 { 236 struct nouveau_cli *cli = nouveau_cli(file_priv); 237 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 238 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 239 struct nouveau_vma *vma; 240 241 if (is_power_of_2(nvbo->valid_domains)) 242 rep->domain = nvbo->valid_domains; 243 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 244 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 245 else 246 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 247 rep->offset = nvbo->offset; 248 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 249 vma = nouveau_vma_find(nvbo, vmm); 250 if (!vma) 251 return -EINVAL; 252 253 rep->offset = vma->addr; 254 } 255 256 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 257 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 258 rep->tile_mode = nvbo->mode; 259 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 260 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 261 rep->tile_flags |= nvbo->kind << 8; 262 else 263 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 264 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 265 else 266 rep->tile_flags |= nvbo->zeta; 267 return 0; 268 } 269 270 int 271 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 272 struct drm_file *file_priv) 273 { 274 struct nouveau_cli *cli = nouveau_cli(file_priv); 275 struct drm_nouveau_gem_new *req = data; 276 struct nouveau_bo *nvbo = NULL; 277 int ret = 0; 278 279 ret = nouveau_gem_new(cli, req->info.size, req->align, 280 req->info.domain, req->info.tile_mode, 281 req->info.tile_flags, &nvbo); 282 if (ret) 283 return ret; 284 285 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 286 &req->info.handle); 287 if (ret == 0) { 288 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 289 if (ret) 290 drm_gem_handle_delete(file_priv, req->info.handle); 291 } 292 293 /* drop reference from allocate - handle holds it now */ 294 drm_gem_object_put(&nvbo->bo.base); 295 return ret; 296 } 297 298 static int 299 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 300 uint32_t write_domains, uint32_t valid_domains) 301 { 302 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 303 struct ttm_buffer_object *bo = &nvbo->bo; 304 uint32_t domains = valid_domains & nvbo->valid_domains & 305 (write_domains ? write_domains : read_domains); 306 uint32_t pref_domains = 0;; 307 308 if (!domains) 309 return -EINVAL; 310 311 valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART); 312 313 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 314 bo->mem.mem_type == TTM_PL_VRAM) 315 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 316 317 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 318 bo->mem.mem_type == TTM_PL_TT) 319 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 320 321 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 322 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 323 324 else 325 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 326 327 nouveau_bo_placement_set(nvbo, pref_domains, valid_domains); 328 329 return 0; 330 } 331 332 struct validate_op { 333 struct list_head list; 334 struct ww_acquire_ctx ticket; 335 }; 336 337 static void 338 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 339 struct nouveau_fence *fence, 340 struct drm_nouveau_gem_pushbuf_bo *pbbo) 341 { 342 struct nouveau_bo *nvbo; 343 struct drm_nouveau_gem_pushbuf_bo *b; 344 345 while (!list_empty(&op->list)) { 346 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 347 b = &pbbo[nvbo->pbbo_index]; 348 349 if (likely(fence)) { 350 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 351 352 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 353 struct nouveau_vma *vma = 354 (void *)(unsigned long)b->user_priv; 355 nouveau_fence_unref(&vma->fence); 356 dma_fence_get(&fence->base); 357 vma->fence = fence; 358 } 359 } 360 361 if (unlikely(nvbo->validate_mapped)) { 362 ttm_bo_kunmap(&nvbo->kmap); 363 nvbo->validate_mapped = false; 364 } 365 366 list_del(&nvbo->entry); 367 nvbo->reserved_by = NULL; 368 ttm_bo_unreserve(&nvbo->bo); 369 drm_gem_object_put(&nvbo->bo.base); 370 } 371 } 372 373 static void 374 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 375 struct nouveau_fence *fence, 376 struct drm_nouveau_gem_pushbuf_bo *pbbo) 377 { 378 validate_fini_no_ticket(op, chan, fence, pbbo); 379 ww_acquire_fini(&op->ticket); 380 } 381 382 static int 383 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 384 struct drm_nouveau_gem_pushbuf_bo *pbbo, 385 int nr_buffers, struct validate_op *op) 386 { 387 struct nouveau_cli *cli = nouveau_cli(file_priv); 388 int trycnt = 0; 389 int ret = -EINVAL, i; 390 struct nouveau_bo *res_bo = NULL; 391 LIST_HEAD(gart_list); 392 LIST_HEAD(vram_list); 393 LIST_HEAD(both_list); 394 395 ww_acquire_init(&op->ticket, &reservation_ww_class); 396 retry: 397 if (++trycnt > 100000) { 398 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 399 return -EINVAL; 400 } 401 402 for (i = 0; i < nr_buffers; i++) { 403 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 404 struct drm_gem_object *gem; 405 struct nouveau_bo *nvbo; 406 407 gem = drm_gem_object_lookup(file_priv, b->handle); 408 if (!gem) { 409 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 410 ret = -ENOENT; 411 break; 412 } 413 nvbo = nouveau_gem_object(gem); 414 if (nvbo == res_bo) { 415 res_bo = NULL; 416 drm_gem_object_put(gem); 417 continue; 418 } 419 420 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 421 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 422 "validation list\n", b->handle); 423 drm_gem_object_put(gem); 424 ret = -EINVAL; 425 break; 426 } 427 428 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 429 if (ret) { 430 list_splice_tail_init(&vram_list, &op->list); 431 list_splice_tail_init(&gart_list, &op->list); 432 list_splice_tail_init(&both_list, &op->list); 433 validate_fini_no_ticket(op, chan, NULL, NULL); 434 if (unlikely(ret == -EDEADLK)) { 435 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 436 &op->ticket); 437 if (!ret) 438 res_bo = nvbo; 439 } 440 if (unlikely(ret)) { 441 if (ret != -ERESTARTSYS) 442 NV_PRINTK(err, cli, "fail reserve\n"); 443 break; 444 } 445 } 446 447 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 448 struct nouveau_vmm *vmm = chan->vmm; 449 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 450 if (!vma) { 451 NV_PRINTK(err, cli, "vma not found!\n"); 452 ret = -EINVAL; 453 break; 454 } 455 456 b->user_priv = (uint64_t)(unsigned long)vma; 457 } else { 458 b->user_priv = (uint64_t)(unsigned long)nvbo; 459 } 460 461 nvbo->reserved_by = file_priv; 462 nvbo->pbbo_index = i; 463 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 464 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 465 list_add_tail(&nvbo->entry, &both_list); 466 else 467 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 468 list_add_tail(&nvbo->entry, &vram_list); 469 else 470 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 471 list_add_tail(&nvbo->entry, &gart_list); 472 else { 473 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 474 b->valid_domains); 475 list_add_tail(&nvbo->entry, &both_list); 476 ret = -EINVAL; 477 break; 478 } 479 if (nvbo == res_bo) 480 goto retry; 481 } 482 483 ww_acquire_done(&op->ticket); 484 list_splice_tail(&vram_list, &op->list); 485 list_splice_tail(&gart_list, &op->list); 486 list_splice_tail(&both_list, &op->list); 487 if (ret) 488 validate_fini(op, chan, NULL, NULL); 489 return ret; 490 491 } 492 493 static int 494 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 495 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 496 { 497 struct nouveau_drm *drm = chan->drm; 498 struct nouveau_bo *nvbo; 499 int ret, relocs = 0; 500 501 list_for_each_entry(nvbo, list, entry) { 502 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 503 504 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 505 b->write_domains, 506 b->valid_domains); 507 if (unlikely(ret)) { 508 NV_PRINTK(err, cli, "fail set_domain\n"); 509 return ret; 510 } 511 512 ret = nouveau_bo_validate(nvbo, true, false); 513 if (unlikely(ret)) { 514 if (ret != -ERESTARTSYS) 515 NV_PRINTK(err, cli, "fail ttm_validate\n"); 516 return ret; 517 } 518 519 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 520 if (unlikely(ret)) { 521 if (ret != -ERESTARTSYS) 522 NV_PRINTK(err, cli, "fail post-validate sync\n"); 523 return ret; 524 } 525 526 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 527 if (nvbo->offset == b->presumed.offset && 528 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 529 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 530 (nvbo->bo.mem.mem_type == TTM_PL_TT && 531 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 532 continue; 533 534 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 535 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 536 else 537 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 538 b->presumed.offset = nvbo->offset; 539 b->presumed.valid = 0; 540 relocs++; 541 } 542 } 543 544 return relocs; 545 } 546 547 static int 548 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 549 struct drm_file *file_priv, 550 struct drm_nouveau_gem_pushbuf_bo *pbbo, 551 int nr_buffers, 552 struct validate_op *op, bool *apply_relocs) 553 { 554 struct nouveau_cli *cli = nouveau_cli(file_priv); 555 int ret; 556 557 INIT_LIST_HEAD(&op->list); 558 559 if (nr_buffers == 0) 560 return 0; 561 562 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 563 if (unlikely(ret)) { 564 if (ret != -ERESTARTSYS) 565 NV_PRINTK(err, cli, "validate_init\n"); 566 return ret; 567 } 568 569 ret = validate_list(chan, cli, &op->list, pbbo); 570 if (unlikely(ret < 0)) { 571 if (ret != -ERESTARTSYS) 572 NV_PRINTK(err, cli, "validating bo list\n"); 573 validate_fini(op, chan, NULL, NULL); 574 return ret; 575 } 576 *apply_relocs = ret; 577 return 0; 578 } 579 580 static inline void 581 u_free(void *addr) 582 { 583 kvfree(addr); 584 } 585 586 static inline void * 587 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 588 { 589 void *mem; 590 void __user *userptr = (void __force __user *)(uintptr_t)user; 591 592 size *= nmemb; 593 594 mem = kvmalloc(size, GFP_KERNEL); 595 if (!mem) 596 return ERR_PTR(-ENOMEM); 597 598 if (copy_from_user(mem, userptr, size)) { 599 u_free(mem); 600 return ERR_PTR(-EFAULT); 601 } 602 603 return mem; 604 } 605 606 static int 607 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 608 struct drm_nouveau_gem_pushbuf *req, 609 struct drm_nouveau_gem_pushbuf_reloc *reloc, 610 struct drm_nouveau_gem_pushbuf_bo *bo) 611 { 612 int ret = 0; 613 unsigned i; 614 615 for (i = 0; i < req->nr_relocs; i++) { 616 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 617 struct drm_nouveau_gem_pushbuf_bo *b; 618 struct nouveau_bo *nvbo; 619 uint32_t data; 620 621 if (unlikely(r->bo_index >= req->nr_buffers)) { 622 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 623 ret = -EINVAL; 624 break; 625 } 626 627 b = &bo[r->bo_index]; 628 if (b->presumed.valid) 629 continue; 630 631 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 632 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 633 ret = -EINVAL; 634 break; 635 } 636 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 637 638 if (unlikely(r->reloc_bo_offset + 4 > 639 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 640 NV_PRINTK(err, cli, "reloc outside of bo\n"); 641 ret = -EINVAL; 642 break; 643 } 644 645 if (!nvbo->kmap.virtual) { 646 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 647 &nvbo->kmap); 648 if (ret) { 649 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 650 break; 651 } 652 nvbo->validate_mapped = true; 653 } 654 655 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 656 data = b->presumed.offset + r->data; 657 else 658 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 659 data = (b->presumed.offset + r->data) >> 32; 660 else 661 data = r->data; 662 663 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 664 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 665 data |= r->tor; 666 else 667 data |= r->vor; 668 } 669 670 ret = ttm_bo_wait(&nvbo->bo, false, false); 671 if (ret) { 672 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 673 break; 674 } 675 676 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 677 } 678 679 u_free(reloc); 680 return ret; 681 } 682 683 int 684 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 685 struct drm_file *file_priv) 686 { 687 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 688 struct nouveau_cli *cli = nouveau_cli(file_priv); 689 struct nouveau_abi16_chan *temp; 690 struct nouveau_drm *drm = nouveau_drm(dev); 691 struct drm_nouveau_gem_pushbuf *req = data; 692 struct drm_nouveau_gem_pushbuf_push *push; 693 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 694 struct drm_nouveau_gem_pushbuf_bo *bo; 695 struct nouveau_channel *chan = NULL; 696 struct validate_op op; 697 struct nouveau_fence *fence = NULL; 698 int i, j, ret = 0; 699 bool do_reloc = false, sync = false; 700 701 if (unlikely(!abi16)) 702 return -ENOMEM; 703 704 list_for_each_entry(temp, &abi16->channels, head) { 705 if (temp->chan->chid == req->channel) { 706 chan = temp->chan; 707 break; 708 } 709 } 710 711 if (!chan) 712 return nouveau_abi16_put(abi16, -ENOENT); 713 if (unlikely(atomic_read(&chan->killed))) 714 return nouveau_abi16_put(abi16, -ENODEV); 715 716 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 717 718 req->vram_available = drm->gem.vram_available; 719 req->gart_available = drm->gem.gart_available; 720 if (unlikely(req->nr_push == 0)) 721 goto out_next; 722 723 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 724 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 725 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 726 return nouveau_abi16_put(abi16, -EINVAL); 727 } 728 729 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 730 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 731 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 732 return nouveau_abi16_put(abi16, -EINVAL); 733 } 734 735 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 736 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 737 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 738 return nouveau_abi16_put(abi16, -EINVAL); 739 } 740 741 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 742 if (IS_ERR(push)) 743 return nouveau_abi16_put(abi16, PTR_ERR(push)); 744 745 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 746 if (IS_ERR(bo)) { 747 u_free(push); 748 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 749 } 750 751 /* Ensure all push buffers are on validate list */ 752 for (i = 0; i < req->nr_push; i++) { 753 if (push[i].bo_index >= req->nr_buffers) { 754 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 755 ret = -EINVAL; 756 goto out_prevalid; 757 } 758 } 759 760 /* Validate buffer list */ 761 revalidate: 762 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 763 req->nr_buffers, &op, &do_reloc); 764 if (ret) { 765 if (ret != -ERESTARTSYS) 766 NV_PRINTK(err, cli, "validate: %d\n", ret); 767 goto out_prevalid; 768 } 769 770 /* Apply any relocations that are required */ 771 if (do_reloc) { 772 if (!reloc) { 773 validate_fini(&op, chan, NULL, bo); 774 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 775 if (IS_ERR(reloc)) { 776 ret = PTR_ERR(reloc); 777 goto out_prevalid; 778 } 779 780 goto revalidate; 781 } 782 783 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 784 if (ret) { 785 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 786 goto out; 787 } 788 } 789 790 if (chan->dma.ib_max) { 791 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 792 if (ret) { 793 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 794 goto out; 795 } 796 797 for (i = 0; i < req->nr_push; i++) { 798 struct nouveau_vma *vma = (void *)(unsigned long) 799 bo[push[i].bo_index].user_priv; 800 801 nv50_dma_push(chan, vma->addr + push[i].offset, 802 push[i].length); 803 } 804 } else 805 if (drm->client.device.info.chipset >= 0x25) { 806 ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2); 807 if (ret) { 808 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 809 goto out; 810 } 811 812 for (i = 0; i < req->nr_push; i++) { 813 struct nouveau_bo *nvbo = (void *)(unsigned long) 814 bo[push[i].bo_index].user_priv; 815 816 PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset); 817 PUSH_DATA(chan->chan.push, 0); 818 } 819 } else { 820 ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 821 if (ret) { 822 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 823 goto out; 824 } 825 826 for (i = 0; i < req->nr_push; i++) { 827 struct nouveau_bo *nvbo = (void *)(unsigned long) 828 bo[push[i].bo_index].user_priv; 829 uint32_t cmd; 830 831 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 832 cmd |= 0x20000000; 833 if (unlikely(cmd != req->suffix0)) { 834 if (!nvbo->kmap.virtual) { 835 ret = ttm_bo_kmap(&nvbo->bo, 0, 836 nvbo->bo.mem. 837 num_pages, 838 &nvbo->kmap); 839 if (ret) { 840 WIND_RING(chan); 841 goto out; 842 } 843 nvbo->validate_mapped = true; 844 } 845 846 nouveau_bo_wr32(nvbo, (push[i].offset + 847 push[i].length - 8) / 4, cmd); 848 } 849 850 PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset); 851 PUSH_DATA(chan->chan.push, 0); 852 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 853 PUSH_DATA(chan->chan.push, 0); 854 } 855 } 856 857 ret = nouveau_fence_new(chan, false, &fence); 858 if (ret) { 859 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 860 WIND_RING(chan); 861 goto out; 862 } 863 864 if (sync) { 865 if (!(ret = nouveau_fence_wait(fence, false, false))) { 866 if ((ret = dma_fence_get_status(&fence->base)) == 1) 867 ret = 0; 868 } 869 } 870 871 out: 872 validate_fini(&op, chan, fence, bo); 873 nouveau_fence_unref(&fence); 874 875 if (do_reloc) { 876 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 877 u64_to_user_ptr(req->buffers); 878 879 for (i = 0; i < req->nr_buffers; i++) { 880 if (bo[i].presumed.valid) 881 continue; 882 883 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 884 sizeof(bo[i].presumed))) { 885 ret = -EFAULT; 886 break; 887 } 888 } 889 u_free(reloc); 890 } 891 out_prevalid: 892 u_free(bo); 893 u_free(push); 894 895 out_next: 896 if (chan->dma.ib_max) { 897 req->suffix0 = 0x00000000; 898 req->suffix1 = 0x00000000; 899 } else 900 if (drm->client.device.info.chipset >= 0x25) { 901 req->suffix0 = 0x00020000; 902 req->suffix1 = 0x00000000; 903 } else { 904 req->suffix0 = 0x20000000 | 905 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 906 req->suffix1 = 0x00000000; 907 } 908 909 return nouveau_abi16_put(abi16, ret); 910 } 911 912 int 913 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 914 struct drm_file *file_priv) 915 { 916 struct drm_nouveau_gem_cpu_prep *req = data; 917 struct drm_gem_object *gem; 918 struct nouveau_bo *nvbo; 919 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 920 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 921 long lret; 922 int ret; 923 924 gem = drm_gem_object_lookup(file_priv, req->handle); 925 if (!gem) 926 return -ENOENT; 927 nvbo = nouveau_gem_object(gem); 928 929 lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true, 930 no_wait ? 0 : 30 * HZ); 931 if (!lret) 932 ret = -EBUSY; 933 else if (lret > 0) 934 ret = 0; 935 else 936 ret = lret; 937 938 nouveau_bo_sync_for_cpu(nvbo); 939 drm_gem_object_put(gem); 940 941 return ret; 942 } 943 944 int 945 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 946 struct drm_file *file_priv) 947 { 948 struct drm_nouveau_gem_cpu_fini *req = data; 949 struct drm_gem_object *gem; 950 struct nouveau_bo *nvbo; 951 952 gem = drm_gem_object_lookup(file_priv, req->handle); 953 if (!gem) 954 return -ENOENT; 955 nvbo = nouveau_gem_object(gem); 956 957 nouveau_bo_sync_for_device(nvbo); 958 drm_gem_object_put(gem); 959 return 0; 960 } 961 962 int 963 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 964 struct drm_file *file_priv) 965 { 966 struct drm_nouveau_gem_info *req = data; 967 struct drm_gem_object *gem; 968 int ret; 969 970 gem = drm_gem_object_lookup(file_priv, req->handle); 971 if (!gem) 972 return -ENOENT; 973 974 ret = nouveau_gem_info(file_priv, gem, req); 975 drm_gem_object_put(gem); 976 return ret; 977 } 978 979