1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 #include "drmP.h" 27 #include "drm.h" 28 29 #include "nouveau_drv.h" 30 #include "nouveau_drm.h" 31 #include "nouveau_dma.h" 32 33 #define nouveau_gem_pushbuf_sync(chan) 0 34 35 int 36 nouveau_gem_object_new(struct drm_gem_object *gem) 37 { 38 return 0; 39 } 40 41 void 42 nouveau_gem_object_del(struct drm_gem_object *gem) 43 { 44 struct nouveau_bo *nvbo = gem->driver_private; 45 struct ttm_buffer_object *bo = &nvbo->bo; 46 47 if (!nvbo) 48 return; 49 nvbo->gem = NULL; 50 51 if (unlikely(nvbo->cpu_filp)) 52 ttm_bo_synccpu_write_release(bo); 53 54 if (unlikely(nvbo->pin_refcnt)) { 55 nvbo->pin_refcnt = 1; 56 nouveau_bo_unpin(nvbo); 57 } 58 59 ttm_bo_unref(&bo); 60 } 61 62 int 63 nouveau_gem_new(struct drm_device *dev, struct nouveau_channel *chan, 64 int size, int align, uint32_t flags, uint32_t tile_mode, 65 uint32_t tile_flags, bool no_vm, bool mappable, 66 struct nouveau_bo **pnvbo) 67 { 68 struct nouveau_bo *nvbo; 69 int ret; 70 71 ret = nouveau_bo_new(dev, chan, size, align, flags, tile_mode, 72 tile_flags, no_vm, mappable, pnvbo); 73 if (ret) 74 return ret; 75 nvbo = *pnvbo; 76 77 nvbo->gem = drm_gem_object_alloc(dev, nvbo->bo.mem.size); 78 if (!nvbo->gem) { 79 nouveau_bo_ref(NULL, pnvbo); 80 return -ENOMEM; 81 } 82 83 nvbo->bo.persistant_swap_storage = nvbo->gem->filp; 84 nvbo->gem->driver_private = nvbo; 85 return 0; 86 } 87 88 static int 89 nouveau_gem_info(struct drm_gem_object *gem, struct drm_nouveau_gem_info *rep) 90 { 91 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 92 93 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 94 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 95 else 96 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 97 98 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 99 rep->offset = nvbo->bo.offset; 100 rep->map_handle = nvbo->mappable ? nvbo->bo.addr_space_offset : 0; 101 rep->tile_mode = nvbo->tile_mode; 102 rep->tile_flags = nvbo->tile_flags; 103 return 0; 104 } 105 106 static bool 107 nouveau_gem_tile_flags_valid(struct drm_device *dev, uint32_t tile_flags) { 108 switch (tile_flags) { 109 case 0x0000: 110 case 0x1800: 111 case 0x2800: 112 case 0x4800: 113 case 0x7000: 114 case 0x7400: 115 case 0x7a00: 116 case 0xe000: 117 break; 118 default: 119 NV_ERROR(dev, "bad page flags: 0x%08x\n", tile_flags); 120 return false; 121 } 122 123 return true; 124 } 125 126 int 127 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 128 struct drm_file *file_priv) 129 { 130 struct drm_nouveau_private *dev_priv = dev->dev_private; 131 struct drm_nouveau_gem_new *req = data; 132 struct nouveau_bo *nvbo = NULL; 133 struct nouveau_channel *chan = NULL; 134 uint32_t flags = 0; 135 int ret = 0; 136 137 NOUVEAU_CHECK_INITIALISED_WITH_RETURN; 138 139 if (unlikely(dev_priv->ttm.bdev.dev_mapping == NULL)) 140 dev_priv->ttm.bdev.dev_mapping = dev_priv->dev->dev_mapping; 141 142 if (req->channel_hint) { 143 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel_hint, 144 file_priv, chan); 145 } 146 147 if (req->info.domain & NOUVEAU_GEM_DOMAIN_VRAM) 148 flags |= TTM_PL_FLAG_VRAM; 149 if (req->info.domain & NOUVEAU_GEM_DOMAIN_GART) 150 flags |= TTM_PL_FLAG_TT; 151 if (!flags || req->info.domain & NOUVEAU_GEM_DOMAIN_CPU) 152 flags |= TTM_PL_FLAG_SYSTEM; 153 154 if (!nouveau_gem_tile_flags_valid(dev, req->info.tile_flags)) 155 return -EINVAL; 156 157 ret = nouveau_gem_new(dev, chan, req->info.size, req->align, flags, 158 req->info.tile_mode, req->info.tile_flags, false, 159 (req->info.domain & NOUVEAU_GEM_DOMAIN_MAPPABLE), 160 &nvbo); 161 if (ret) 162 return ret; 163 164 ret = nouveau_gem_info(nvbo->gem, &req->info); 165 if (ret) 166 goto out; 167 168 ret = drm_gem_handle_create(file_priv, nvbo->gem, &req->info.handle); 169 out: 170 mutex_lock(&dev->struct_mutex); 171 drm_gem_object_handle_unreference(nvbo->gem); 172 mutex_unlock(&dev->struct_mutex); 173 174 if (ret) 175 drm_gem_object_unreference(nvbo->gem); 176 return ret; 177 } 178 179 static int 180 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 181 uint32_t write_domains, uint32_t valid_domains) 182 { 183 struct nouveau_bo *nvbo = gem->driver_private; 184 struct ttm_buffer_object *bo = &nvbo->bo; 185 uint64_t flags; 186 187 if (!valid_domains || (!read_domains && !write_domains)) 188 return -EINVAL; 189 190 if (write_domains) { 191 if ((valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 192 (write_domains & NOUVEAU_GEM_DOMAIN_VRAM)) 193 flags = TTM_PL_FLAG_VRAM; 194 else 195 if ((valid_domains & NOUVEAU_GEM_DOMAIN_GART) && 196 (write_domains & NOUVEAU_GEM_DOMAIN_GART)) 197 flags = TTM_PL_FLAG_TT; 198 else 199 return -EINVAL; 200 } else { 201 if ((valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 202 (read_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 203 bo->mem.mem_type == TTM_PL_VRAM) 204 flags = TTM_PL_FLAG_VRAM; 205 else 206 if ((valid_domains & NOUVEAU_GEM_DOMAIN_GART) && 207 (read_domains & NOUVEAU_GEM_DOMAIN_GART) && 208 bo->mem.mem_type == TTM_PL_TT) 209 flags = TTM_PL_FLAG_TT; 210 else 211 if ((valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 212 (read_domains & NOUVEAU_GEM_DOMAIN_VRAM)) 213 flags = TTM_PL_FLAG_VRAM; 214 else 215 flags = TTM_PL_FLAG_TT; 216 } 217 218 nouveau_bo_placement_set(nvbo, flags); 219 return 0; 220 } 221 222 struct validate_op { 223 struct list_head vram_list; 224 struct list_head gart_list; 225 struct list_head both_list; 226 }; 227 228 static void 229 validate_fini_list(struct list_head *list, struct nouveau_fence *fence) 230 { 231 struct list_head *entry, *tmp; 232 struct nouveau_bo *nvbo; 233 234 list_for_each_safe(entry, tmp, list) { 235 nvbo = list_entry(entry, struct nouveau_bo, entry); 236 if (likely(fence)) { 237 struct nouveau_fence *prev_fence; 238 239 spin_lock(&nvbo->bo.lock); 240 prev_fence = nvbo->bo.sync_obj; 241 nvbo->bo.sync_obj = nouveau_fence_ref(fence); 242 spin_unlock(&nvbo->bo.lock); 243 nouveau_fence_unref((void *)&prev_fence); 244 } 245 246 list_del(&nvbo->entry); 247 nvbo->reserved_by = NULL; 248 ttm_bo_unreserve(&nvbo->bo); 249 drm_gem_object_unreference(nvbo->gem); 250 } 251 } 252 253 static void 254 validate_fini(struct validate_op *op, struct nouveau_fence* fence) 255 { 256 validate_fini_list(&op->vram_list, fence); 257 validate_fini_list(&op->gart_list, fence); 258 validate_fini_list(&op->both_list, fence); 259 } 260 261 static int 262 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 263 struct drm_nouveau_gem_pushbuf_bo *pbbo, 264 int nr_buffers, struct validate_op *op) 265 { 266 struct drm_device *dev = chan->dev; 267 struct drm_nouveau_private *dev_priv = dev->dev_private; 268 uint32_t sequence; 269 int trycnt = 0; 270 int ret, i; 271 272 sequence = atomic_add_return(1, &dev_priv->ttm.validate_sequence); 273 retry: 274 if (++trycnt > 100000) { 275 NV_ERROR(dev, "%s failed and gave up.\n", __func__); 276 return -EINVAL; 277 } 278 279 for (i = 0; i < nr_buffers; i++) { 280 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 281 struct drm_gem_object *gem; 282 struct nouveau_bo *nvbo; 283 284 gem = drm_gem_object_lookup(dev, file_priv, b->handle); 285 if (!gem) { 286 NV_ERROR(dev, "Unknown handle 0x%08x\n", b->handle); 287 validate_fini(op, NULL); 288 return -EINVAL; 289 } 290 nvbo = gem->driver_private; 291 292 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 293 NV_ERROR(dev, "multiple instances of buffer %d on " 294 "validation list\n", b->handle); 295 validate_fini(op, NULL); 296 return -EINVAL; 297 } 298 299 ret = ttm_bo_reserve(&nvbo->bo, false, false, true, sequence); 300 if (ret) { 301 validate_fini(op, NULL); 302 if (ret == -EAGAIN) 303 ret = ttm_bo_wait_unreserved(&nvbo->bo, false); 304 drm_gem_object_unreference(gem); 305 if (ret) 306 return ret; 307 goto retry; 308 } 309 310 nvbo->reserved_by = file_priv; 311 nvbo->pbbo_index = i; 312 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 313 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 314 list_add_tail(&nvbo->entry, &op->both_list); 315 else 316 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 317 list_add_tail(&nvbo->entry, &op->vram_list); 318 else 319 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 320 list_add_tail(&nvbo->entry, &op->gart_list); 321 else { 322 NV_ERROR(dev, "invalid valid domains: 0x%08x\n", 323 b->valid_domains); 324 validate_fini(op, NULL); 325 return -EINVAL; 326 } 327 328 if (unlikely(atomic_read(&nvbo->bo.cpu_writers) > 0)) { 329 validate_fini(op, NULL); 330 331 if (nvbo->cpu_filp == file_priv) { 332 NV_ERROR(dev, "bo %p mapped by process trying " 333 "to validate it!\n", nvbo); 334 return -EINVAL; 335 } 336 337 ret = ttm_bo_wait_cpu(&nvbo->bo, false); 338 if (ret) 339 return ret; 340 goto retry; 341 } 342 } 343 344 return 0; 345 } 346 347 static int 348 validate_list(struct nouveau_channel *chan, struct list_head *list, 349 struct drm_nouveau_gem_pushbuf_bo *pbbo, uint64_t user_pbbo_ptr) 350 { 351 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 352 (void __force __user *)(uintptr_t)user_pbbo_ptr; 353 struct nouveau_bo *nvbo; 354 int ret, relocs = 0; 355 356 list_for_each_entry(nvbo, list, entry) { 357 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 358 struct nouveau_fence *prev_fence = nvbo->bo.sync_obj; 359 360 if (prev_fence && nouveau_fence_channel(prev_fence) != chan) { 361 spin_lock(&nvbo->bo.lock); 362 ret = ttm_bo_wait(&nvbo->bo, false, false, false); 363 spin_unlock(&nvbo->bo.lock); 364 if (unlikely(ret)) 365 return ret; 366 } 367 368 ret = nouveau_gem_set_domain(nvbo->gem, b->read_domains, 369 b->write_domains, 370 b->valid_domains); 371 if (unlikely(ret)) 372 return ret; 373 374 nvbo->channel = chan; 375 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, 376 false, false); 377 nvbo->channel = NULL; 378 if (unlikely(ret)) 379 return ret; 380 381 if (nvbo->bo.offset == b->presumed_offset && 382 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 383 b->presumed_domain & NOUVEAU_GEM_DOMAIN_VRAM) || 384 (nvbo->bo.mem.mem_type == TTM_PL_TT && 385 b->presumed_domain & NOUVEAU_GEM_DOMAIN_GART))) 386 continue; 387 388 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 389 b->presumed_domain = NOUVEAU_GEM_DOMAIN_GART; 390 else 391 b->presumed_domain = NOUVEAU_GEM_DOMAIN_VRAM; 392 b->presumed_offset = nvbo->bo.offset; 393 b->presumed_ok = 0; 394 relocs++; 395 396 if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index], b, sizeof(*b))) 397 return -EFAULT; 398 } 399 400 return relocs; 401 } 402 403 static int 404 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 405 struct drm_file *file_priv, 406 struct drm_nouveau_gem_pushbuf_bo *pbbo, 407 uint64_t user_buffers, int nr_buffers, 408 struct validate_op *op, int *apply_relocs) 409 { 410 int ret, relocs = 0; 411 412 INIT_LIST_HEAD(&op->vram_list); 413 INIT_LIST_HEAD(&op->gart_list); 414 INIT_LIST_HEAD(&op->both_list); 415 416 if (nr_buffers == 0) 417 return 0; 418 419 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 420 if (unlikely(ret)) 421 return ret; 422 423 ret = validate_list(chan, &op->vram_list, pbbo, user_buffers); 424 if (unlikely(ret < 0)) { 425 validate_fini(op, NULL); 426 return ret; 427 } 428 relocs += ret; 429 430 ret = validate_list(chan, &op->gart_list, pbbo, user_buffers); 431 if (unlikely(ret < 0)) { 432 validate_fini(op, NULL); 433 return ret; 434 } 435 relocs += ret; 436 437 ret = validate_list(chan, &op->both_list, pbbo, user_buffers); 438 if (unlikely(ret < 0)) { 439 validate_fini(op, NULL); 440 return ret; 441 } 442 relocs += ret; 443 444 *apply_relocs = relocs; 445 return 0; 446 } 447 448 static inline void * 449 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 450 { 451 void *mem; 452 void __user *userptr = (void __force __user *)(uintptr_t)user; 453 454 mem = kmalloc(nmemb * size, GFP_KERNEL); 455 if (!mem) 456 return ERR_PTR(-ENOMEM); 457 458 if (DRM_COPY_FROM_USER(mem, userptr, nmemb * size)) { 459 kfree(mem); 460 return ERR_PTR(-EFAULT); 461 } 462 463 return mem; 464 } 465 466 static int 467 nouveau_gem_pushbuf_reloc_apply(struct nouveau_channel *chan, int nr_bo, 468 struct drm_nouveau_gem_pushbuf_bo *bo, 469 int nr_relocs, uint64_t ptr_relocs, 470 int nr_dwords, int first_dword, 471 uint32_t *pushbuf, bool is_iomem) 472 { 473 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 474 struct drm_device *dev = chan->dev; 475 int ret = 0, i; 476 477 reloc = u_memcpya(ptr_relocs, nr_relocs, sizeof(*reloc)); 478 if (IS_ERR(reloc)) 479 return PTR_ERR(reloc); 480 481 for (i = 0; i < nr_relocs; i++) { 482 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 483 struct drm_nouveau_gem_pushbuf_bo *b; 484 uint32_t data; 485 486 if (r->bo_index >= nr_bo || r->reloc_index < first_dword || 487 r->reloc_index >= first_dword + nr_dwords) { 488 NV_ERROR(dev, "Bad relocation %d\n", i); 489 NV_ERROR(dev, " bo: %d max %d\n", r->bo_index, nr_bo); 490 NV_ERROR(dev, " id: %d max %d\n", r->reloc_index, nr_dwords); 491 ret = -EINVAL; 492 break; 493 } 494 495 b = &bo[r->bo_index]; 496 if (b->presumed_ok) 497 continue; 498 499 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 500 data = b->presumed_offset + r->data; 501 else 502 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 503 data = (b->presumed_offset + r->data) >> 32; 504 else 505 data = r->data; 506 507 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 508 if (b->presumed_domain == NOUVEAU_GEM_DOMAIN_GART) 509 data |= r->tor; 510 else 511 data |= r->vor; 512 } 513 514 if (is_iomem) 515 iowrite32_native(data, (void __force __iomem *) 516 &pushbuf[r->reloc_index]); 517 else 518 pushbuf[r->reloc_index] = data; 519 } 520 521 kfree(reloc); 522 return ret; 523 } 524 525 int 526 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 527 struct drm_file *file_priv) 528 { 529 struct drm_nouveau_gem_pushbuf *req = data; 530 struct drm_nouveau_gem_pushbuf_bo *bo = NULL; 531 struct nouveau_channel *chan; 532 struct validate_op op; 533 struct nouveau_fence* fence = 0; 534 uint32_t *pushbuf = NULL; 535 int ret = 0, do_reloc = 0, i; 536 537 NOUVEAU_CHECK_INITIALISED_WITH_RETURN; 538 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel, file_priv, chan); 539 540 if (req->nr_dwords >= chan->dma.max || 541 req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS || 542 req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS) { 543 NV_ERROR(dev, "Pushbuf config exceeds limits:\n"); 544 NV_ERROR(dev, " dwords : %d max %d\n", req->nr_dwords, 545 chan->dma.max - 1); 546 NV_ERROR(dev, " buffers: %d max %d\n", req->nr_buffers, 547 NOUVEAU_GEM_MAX_BUFFERS); 548 NV_ERROR(dev, " relocs : %d max %d\n", req->nr_relocs, 549 NOUVEAU_GEM_MAX_RELOCS); 550 return -EINVAL; 551 } 552 553 pushbuf = u_memcpya(req->dwords, req->nr_dwords, sizeof(uint32_t)); 554 if (IS_ERR(pushbuf)) 555 return PTR_ERR(pushbuf); 556 557 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 558 if (IS_ERR(bo)) { 559 kfree(pushbuf); 560 return PTR_ERR(bo); 561 } 562 563 mutex_lock(&dev->struct_mutex); 564 565 /* Validate buffer list */ 566 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 567 req->nr_buffers, &op, &do_reloc); 568 if (ret) 569 goto out; 570 571 /* Apply any relocations that are required */ 572 if (do_reloc) { 573 ret = nouveau_gem_pushbuf_reloc_apply(chan, req->nr_buffers, 574 bo, req->nr_relocs, 575 req->relocs, 576 req->nr_dwords, 0, 577 pushbuf, false); 578 if (ret) 579 goto out; 580 } 581 582 /* Emit push buffer to the hw 583 */ 584 ret = RING_SPACE(chan, req->nr_dwords); 585 if (ret) 586 goto out; 587 588 OUT_RINGp(chan, pushbuf, req->nr_dwords); 589 590 ret = nouveau_fence_new(chan, &fence, true); 591 if (ret) { 592 NV_ERROR(dev, "error fencing pushbuf: %d\n", ret); 593 WIND_RING(chan); 594 goto out; 595 } 596 597 if (nouveau_gem_pushbuf_sync(chan)) { 598 ret = nouveau_fence_wait(fence, NULL, false, false); 599 if (ret) { 600 for (i = 0; i < req->nr_dwords; i++) 601 NV_ERROR(dev, "0x%08x\n", pushbuf[i]); 602 NV_ERROR(dev, "^^ above push buffer is fail :(\n"); 603 } 604 } 605 606 out: 607 validate_fini(&op, fence); 608 nouveau_fence_unref((void**)&fence); 609 mutex_unlock(&dev->struct_mutex); 610 kfree(pushbuf); 611 kfree(bo); 612 return ret; 613 } 614 615 #define PUSHBUF_CAL (dev_priv->card_type >= NV_20) 616 617 int 618 nouveau_gem_ioctl_pushbuf_call(struct drm_device *dev, void *data, 619 struct drm_file *file_priv) 620 { 621 struct drm_nouveau_private *dev_priv = dev->dev_private; 622 struct drm_nouveau_gem_pushbuf_call *req = data; 623 struct drm_nouveau_gem_pushbuf_bo *bo = NULL; 624 struct nouveau_channel *chan; 625 struct drm_gem_object *gem; 626 struct nouveau_bo *pbbo; 627 struct validate_op op; 628 struct nouveau_fence* fence = 0; 629 int i, ret = 0, do_reloc = 0; 630 631 NOUVEAU_CHECK_INITIALISED_WITH_RETURN; 632 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel, file_priv, chan); 633 634 if (unlikely(req->handle == 0)) 635 goto out_next; 636 637 if (req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS || 638 req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS) { 639 NV_ERROR(dev, "Pushbuf config exceeds limits:\n"); 640 NV_ERROR(dev, " buffers: %d max %d\n", req->nr_buffers, 641 NOUVEAU_GEM_MAX_BUFFERS); 642 NV_ERROR(dev, " relocs : %d max %d\n", req->nr_relocs, 643 NOUVEAU_GEM_MAX_RELOCS); 644 return -EINVAL; 645 } 646 647 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 648 if (IS_ERR(bo)) 649 return PTR_ERR(bo); 650 651 mutex_lock(&dev->struct_mutex); 652 653 /* Validate buffer list */ 654 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 655 req->nr_buffers, &op, &do_reloc); 656 if (ret) { 657 NV_ERROR(dev, "validate: %d\n", ret); 658 goto out; 659 } 660 661 /* Validate DMA push buffer */ 662 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 663 if (!gem) { 664 NV_ERROR(dev, "Unknown pb handle 0x%08x\n", req->handle); 665 ret = -EINVAL; 666 goto out; 667 } 668 pbbo = nouveau_gem_object(gem); 669 670 ret = ttm_bo_reserve(&pbbo->bo, false, false, true, 671 chan->fence.sequence); 672 if (ret) { 673 NV_ERROR(dev, "resv pb: %d\n", ret); 674 drm_gem_object_unreference(gem); 675 goto out; 676 } 677 678 nouveau_bo_placement_set(pbbo, 1 << chan->pushbuf_bo->bo.mem.mem_type); 679 ret = ttm_bo_validate(&pbbo->bo, &pbbo->placement, false, false); 680 if (ret) { 681 NV_ERROR(dev, "validate pb: %d\n", ret); 682 ttm_bo_unreserve(&pbbo->bo); 683 drm_gem_object_unreference(gem); 684 goto out; 685 } 686 687 list_add_tail(&pbbo->entry, &op.both_list); 688 689 /* If presumed return address doesn't match, we need to map the 690 * push buffer and fix it.. 691 */ 692 if (!PUSHBUF_CAL) { 693 uint32_t retaddy; 694 695 if (chan->dma.free < 4 + NOUVEAU_DMA_SKIPS) { 696 ret = nouveau_dma_wait(chan, 4 + NOUVEAU_DMA_SKIPS); 697 if (ret) { 698 NV_ERROR(dev, "jmp_space: %d\n", ret); 699 goto out; 700 } 701 } 702 703 retaddy = chan->pushbuf_base + ((chan->dma.cur + 2) << 2); 704 retaddy |= 0x20000000; 705 if (retaddy != req->suffix0) { 706 req->suffix0 = retaddy; 707 do_reloc = 1; 708 } 709 } 710 711 /* Apply any relocations that are required */ 712 if (do_reloc) { 713 void *pbvirt; 714 bool is_iomem; 715 ret = ttm_bo_kmap(&pbbo->bo, 0, pbbo->bo.mem.num_pages, 716 &pbbo->kmap); 717 if (ret) { 718 NV_ERROR(dev, "kmap pb: %d\n", ret); 719 goto out; 720 } 721 722 pbvirt = ttm_kmap_obj_virtual(&pbbo->kmap, &is_iomem); 723 ret = nouveau_gem_pushbuf_reloc_apply(chan, req->nr_buffers, bo, 724 req->nr_relocs, 725 req->relocs, 726 req->nr_dwords, 727 req->offset / 4, 728 pbvirt, is_iomem); 729 730 if (!PUSHBUF_CAL) { 731 nouveau_bo_wr32(pbbo, 732 req->offset / 4 + req->nr_dwords - 2, 733 req->suffix0); 734 } 735 736 ttm_bo_kunmap(&pbbo->kmap); 737 if (ret) { 738 NV_ERROR(dev, "reloc apply: %d\n", ret); 739 goto out; 740 } 741 } 742 743 if (PUSHBUF_CAL) { 744 ret = RING_SPACE(chan, 2); 745 if (ret) { 746 NV_ERROR(dev, "cal_space: %d\n", ret); 747 goto out; 748 } 749 OUT_RING(chan, ((pbbo->bo.mem.mm_node->start << PAGE_SHIFT) + 750 req->offset) | 2); 751 OUT_RING(chan, 0); 752 } else { 753 ret = RING_SPACE(chan, 2 + NOUVEAU_DMA_SKIPS); 754 if (ret) { 755 NV_ERROR(dev, "jmp_space: %d\n", ret); 756 goto out; 757 } 758 OUT_RING(chan, ((pbbo->bo.mem.mm_node->start << PAGE_SHIFT) + 759 req->offset) | 0x20000000); 760 OUT_RING(chan, 0); 761 762 /* Space the jumps apart with NOPs. */ 763 for (i = 0; i < NOUVEAU_DMA_SKIPS; i++) 764 OUT_RING(chan, 0); 765 } 766 767 ret = nouveau_fence_new(chan, &fence, true); 768 if (ret) { 769 NV_ERROR(dev, "error fencing pushbuf: %d\n", ret); 770 WIND_RING(chan); 771 goto out; 772 } 773 774 out: 775 validate_fini(&op, fence); 776 nouveau_fence_unref((void**)&fence); 777 mutex_unlock(&dev->struct_mutex); 778 kfree(bo); 779 780 out_next: 781 if (PUSHBUF_CAL) { 782 req->suffix0 = 0x00020000; 783 req->suffix1 = 0x00000000; 784 } else { 785 req->suffix0 = 0x20000000 | 786 (chan->pushbuf_base + ((chan->dma.cur + 2) << 2)); 787 req->suffix1 = 0x00000000; 788 } 789 790 return ret; 791 } 792 793 int 794 nouveau_gem_ioctl_pushbuf_call2(struct drm_device *dev, void *data, 795 struct drm_file *file_priv) 796 { 797 struct drm_nouveau_private *dev_priv = dev->dev_private; 798 struct drm_nouveau_gem_pushbuf_call *req = data; 799 800 req->vram_available = dev_priv->fb_aper_free; 801 req->gart_available = dev_priv->gart_info.aper_free; 802 803 return nouveau_gem_ioctl_pushbuf_call(dev, data, file_priv); 804 } 805 806 static inline uint32_t 807 domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain) 808 { 809 uint32_t flags = 0; 810 811 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 812 flags |= TTM_PL_FLAG_VRAM; 813 if (domain & NOUVEAU_GEM_DOMAIN_GART) 814 flags |= TTM_PL_FLAG_TT; 815 816 return flags; 817 } 818 819 int 820 nouveau_gem_ioctl_pin(struct drm_device *dev, void *data, 821 struct drm_file *file_priv) 822 { 823 struct drm_nouveau_gem_pin *req = data; 824 struct drm_gem_object *gem; 825 struct nouveau_bo *nvbo; 826 int ret = 0; 827 828 NOUVEAU_CHECK_INITIALISED_WITH_RETURN; 829 830 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 831 NV_ERROR(dev, "pin only allowed without kernel modesetting\n"); 832 return -EINVAL; 833 } 834 835 if (!DRM_SUSER(DRM_CURPROC)) 836 return -EPERM; 837 838 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 839 if (!gem) 840 return -EINVAL; 841 nvbo = nouveau_gem_object(gem); 842 843 ret = nouveau_bo_pin(nvbo, domain_to_ttm(nvbo, req->domain)); 844 if (ret) 845 goto out; 846 847 req->offset = nvbo->bo.offset; 848 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 849 req->domain = NOUVEAU_GEM_DOMAIN_GART; 850 else 851 req->domain = NOUVEAU_GEM_DOMAIN_VRAM; 852 853 out: 854 mutex_lock(&dev->struct_mutex); 855 drm_gem_object_unreference(gem); 856 mutex_unlock(&dev->struct_mutex); 857 858 return ret; 859 } 860 861 int 862 nouveau_gem_ioctl_unpin(struct drm_device *dev, void *data, 863 struct drm_file *file_priv) 864 { 865 struct drm_nouveau_gem_pin *req = data; 866 struct drm_gem_object *gem; 867 int ret; 868 869 NOUVEAU_CHECK_INITIALISED_WITH_RETURN; 870 871 if (drm_core_check_feature(dev, DRIVER_MODESET)) 872 return -EINVAL; 873 874 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 875 if (!gem) 876 return -EINVAL; 877 878 ret = nouveau_bo_unpin(nouveau_gem_object(gem)); 879 880 mutex_lock(&dev->struct_mutex); 881 drm_gem_object_unreference(gem); 882 mutex_unlock(&dev->struct_mutex); 883 884 return ret; 885 } 886 887 int 888 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 889 struct drm_file *file_priv) 890 { 891 struct drm_nouveau_gem_cpu_prep *req = data; 892 struct drm_gem_object *gem; 893 struct nouveau_bo *nvbo; 894 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 895 int ret = -EINVAL; 896 897 NOUVEAU_CHECK_INITIALISED_WITH_RETURN; 898 899 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 900 if (!gem) 901 return ret; 902 nvbo = nouveau_gem_object(gem); 903 904 if (nvbo->cpu_filp) { 905 if (nvbo->cpu_filp == file_priv) 906 goto out; 907 908 ret = ttm_bo_wait_cpu(&nvbo->bo, no_wait); 909 if (ret) 910 goto out; 911 } 912 913 if (req->flags & NOUVEAU_GEM_CPU_PREP_NOBLOCK) { 914 ret = ttm_bo_wait(&nvbo->bo, false, false, no_wait); 915 } else { 916 ret = ttm_bo_synccpu_write_grab(&nvbo->bo, no_wait); 917 if (ret == 0) 918 nvbo->cpu_filp = file_priv; 919 } 920 921 out: 922 mutex_lock(&dev->struct_mutex); 923 drm_gem_object_unreference(gem); 924 mutex_unlock(&dev->struct_mutex); 925 return ret; 926 } 927 928 int 929 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 930 struct drm_file *file_priv) 931 { 932 struct drm_nouveau_gem_cpu_prep *req = data; 933 struct drm_gem_object *gem; 934 struct nouveau_bo *nvbo; 935 int ret = -EINVAL; 936 937 NOUVEAU_CHECK_INITIALISED_WITH_RETURN; 938 939 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 940 if (!gem) 941 return ret; 942 nvbo = nouveau_gem_object(gem); 943 944 if (nvbo->cpu_filp != file_priv) 945 goto out; 946 nvbo->cpu_filp = NULL; 947 948 ttm_bo_synccpu_write_release(&nvbo->bo); 949 ret = 0; 950 951 out: 952 mutex_lock(&dev->struct_mutex); 953 drm_gem_object_unreference(gem); 954 mutex_unlock(&dev->struct_mutex); 955 return ret; 956 } 957 958 int 959 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 960 struct drm_file *file_priv) 961 { 962 struct drm_nouveau_gem_info *req = data; 963 struct drm_gem_object *gem; 964 int ret; 965 966 NOUVEAU_CHECK_INITIALISED_WITH_RETURN; 967 968 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 969 if (!gem) 970 return -EINVAL; 971 972 ret = nouveau_gem_info(gem, req); 973 mutex_lock(&dev->struct_mutex); 974 drm_gem_object_unreference(gem); 975 mutex_unlock(&dev->struct_mutex); 976 return ret; 977 } 978 979