1 /* 2 * Copyright 2007 Dave Airlied 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 /* 25 * Authors: Dave Airlied <airlied@linux.ie> 26 * Ben Skeggs <darktama@iinet.net.au> 27 * Jeremy Kolb <jkolb@brandeis.edu> 28 */ 29 30 #include "drmP.h" 31 32 #include "nouveau_drm.h" 33 #include "nouveau_drv.h" 34 #include "nouveau_dma.h" 35 #include "nouveau_mm.h" 36 #include "nouveau_vm.h" 37 38 #include <linux/log2.h> 39 #include <linux/slab.h> 40 41 static void 42 nouveau_bo_del_ttm(struct ttm_buffer_object *bo) 43 { 44 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 45 struct drm_device *dev = dev_priv->dev; 46 struct nouveau_bo *nvbo = nouveau_bo(bo); 47 48 if (unlikely(nvbo->gem)) 49 DRM_ERROR("bo %p still attached to GEM object\n", bo); 50 51 nv10_mem_put_tile_region(dev, nvbo->tile, NULL); 52 if (nvbo->vma.node) { 53 nouveau_vm_unmap(&nvbo->vma); 54 nouveau_vm_put(&nvbo->vma); 55 } 56 kfree(nvbo); 57 } 58 59 static void 60 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, int *size, 61 int *page_shift) 62 { 63 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 64 65 if (dev_priv->card_type < NV_50) { 66 if (nvbo->tile_mode) { 67 if (dev_priv->chipset >= 0x40) { 68 *align = 65536; 69 *size = roundup(*size, 64 * nvbo->tile_mode); 70 71 } else if (dev_priv->chipset >= 0x30) { 72 *align = 32768; 73 *size = roundup(*size, 64 * nvbo->tile_mode); 74 75 } else if (dev_priv->chipset >= 0x20) { 76 *align = 16384; 77 *size = roundup(*size, 64 * nvbo->tile_mode); 78 79 } else if (dev_priv->chipset >= 0x10) { 80 *align = 16384; 81 *size = roundup(*size, 32 * nvbo->tile_mode); 82 } 83 } 84 } else { 85 if (likely(dev_priv->chan_vm)) { 86 if (*size > 256 * 1024) 87 *page_shift = dev_priv->chan_vm->lpg_shift; 88 else 89 *page_shift = dev_priv->chan_vm->spg_shift; 90 } else { 91 *page_shift = 12; 92 } 93 94 *size = roundup(*size, (1 << *page_shift)); 95 *align = max((1 << *page_shift), *align); 96 } 97 98 *size = roundup(*size, PAGE_SIZE); 99 } 100 101 int 102 nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan, 103 int size, int align, uint32_t flags, uint32_t tile_mode, 104 uint32_t tile_flags, bool no_vm, bool mappable, 105 struct nouveau_bo **pnvbo) 106 { 107 struct drm_nouveau_private *dev_priv = dev->dev_private; 108 struct nouveau_bo *nvbo; 109 int ret = 0, page_shift = 0; 110 111 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); 112 if (!nvbo) 113 return -ENOMEM; 114 INIT_LIST_HEAD(&nvbo->head); 115 INIT_LIST_HEAD(&nvbo->entry); 116 nvbo->mappable = mappable; 117 nvbo->no_vm = no_vm; 118 nvbo->tile_mode = tile_mode; 119 nvbo->tile_flags = tile_flags; 120 nvbo->bo.bdev = &dev_priv->ttm.bdev; 121 122 nouveau_bo_fixup_align(nvbo, &align, &size, &page_shift); 123 align >>= PAGE_SHIFT; 124 125 if (!nvbo->no_vm && dev_priv->chan_vm) { 126 ret = nouveau_vm_get(dev_priv->chan_vm, size, page_shift, 127 NV_MEM_ACCESS_RW, &nvbo->vma); 128 if (ret) { 129 kfree(nvbo); 130 return ret; 131 } 132 } 133 134 nvbo->bo.mem.num_pages = size >> PAGE_SHIFT; 135 nouveau_bo_placement_set(nvbo, flags, 0); 136 137 nvbo->channel = chan; 138 ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size, 139 ttm_bo_type_device, &nvbo->placement, align, 0, 140 false, NULL, size, nouveau_bo_del_ttm); 141 if (ret) { 142 /* ttm will call nouveau_bo_del_ttm if it fails.. */ 143 return ret; 144 } 145 nvbo->channel = NULL; 146 147 if (nvbo->vma.node) { 148 if (nvbo->bo.mem.mem_type == TTM_PL_VRAM) 149 nvbo->bo.offset = nvbo->vma.offset; 150 } 151 152 *pnvbo = nvbo; 153 return 0; 154 } 155 156 static void 157 set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags) 158 { 159 *n = 0; 160 161 if (type & TTM_PL_FLAG_VRAM) 162 pl[(*n)++] = TTM_PL_FLAG_VRAM | flags; 163 if (type & TTM_PL_FLAG_TT) 164 pl[(*n)++] = TTM_PL_FLAG_TT | flags; 165 if (type & TTM_PL_FLAG_SYSTEM) 166 pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags; 167 } 168 169 static void 170 set_placement_range(struct nouveau_bo *nvbo, uint32_t type) 171 { 172 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 173 int vram_pages = dev_priv->vram_size >> PAGE_SHIFT; 174 175 if (dev_priv->card_type == NV_10 && 176 nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) && 177 nvbo->bo.mem.num_pages < vram_pages / 2) { 178 /* 179 * Make sure that the color and depth buffers are handled 180 * by independent memory controller units. Up to a 9x 181 * speed up when alpha-blending and depth-test are enabled 182 * at the same time. 183 */ 184 if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) { 185 nvbo->placement.fpfn = vram_pages / 2; 186 nvbo->placement.lpfn = ~0; 187 } else { 188 nvbo->placement.fpfn = 0; 189 nvbo->placement.lpfn = vram_pages / 2; 190 } 191 } 192 } 193 194 void 195 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy) 196 { 197 struct ttm_placement *pl = &nvbo->placement; 198 uint32_t flags = TTM_PL_MASK_CACHING | 199 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0); 200 201 pl->placement = nvbo->placements; 202 set_placement_list(nvbo->placements, &pl->num_placement, 203 type, flags); 204 205 pl->busy_placement = nvbo->busy_placements; 206 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement, 207 type | busy, flags); 208 209 set_placement_range(nvbo, type); 210 } 211 212 int 213 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype) 214 { 215 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 216 struct ttm_buffer_object *bo = &nvbo->bo; 217 int ret; 218 219 if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) { 220 NV_ERROR(nouveau_bdev(bo->bdev)->dev, 221 "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo, 222 1 << bo->mem.mem_type, memtype); 223 return -EINVAL; 224 } 225 226 if (nvbo->pin_refcnt++) 227 return 0; 228 229 ret = ttm_bo_reserve(bo, false, false, false, 0); 230 if (ret) 231 goto out; 232 233 nouveau_bo_placement_set(nvbo, memtype, 0); 234 235 ret = nouveau_bo_validate(nvbo, false, false, false); 236 if (ret == 0) { 237 switch (bo->mem.mem_type) { 238 case TTM_PL_VRAM: 239 dev_priv->fb_aper_free -= bo->mem.size; 240 break; 241 case TTM_PL_TT: 242 dev_priv->gart_info.aper_free -= bo->mem.size; 243 break; 244 default: 245 break; 246 } 247 } 248 ttm_bo_unreserve(bo); 249 out: 250 if (unlikely(ret)) 251 nvbo->pin_refcnt--; 252 return ret; 253 } 254 255 int 256 nouveau_bo_unpin(struct nouveau_bo *nvbo) 257 { 258 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 259 struct ttm_buffer_object *bo = &nvbo->bo; 260 int ret; 261 262 if (--nvbo->pin_refcnt) 263 return 0; 264 265 ret = ttm_bo_reserve(bo, false, false, false, 0); 266 if (ret) 267 return ret; 268 269 nouveau_bo_placement_set(nvbo, bo->mem.placement, 0); 270 271 ret = nouveau_bo_validate(nvbo, false, false, false); 272 if (ret == 0) { 273 switch (bo->mem.mem_type) { 274 case TTM_PL_VRAM: 275 dev_priv->fb_aper_free += bo->mem.size; 276 break; 277 case TTM_PL_TT: 278 dev_priv->gart_info.aper_free += bo->mem.size; 279 break; 280 default: 281 break; 282 } 283 } 284 285 ttm_bo_unreserve(bo); 286 return ret; 287 } 288 289 int 290 nouveau_bo_map(struct nouveau_bo *nvbo) 291 { 292 int ret; 293 294 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 295 if (ret) 296 return ret; 297 298 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap); 299 ttm_bo_unreserve(&nvbo->bo); 300 return ret; 301 } 302 303 void 304 nouveau_bo_unmap(struct nouveau_bo *nvbo) 305 { 306 if (nvbo) 307 ttm_bo_kunmap(&nvbo->kmap); 308 } 309 310 int 311 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, 312 bool no_wait_reserve, bool no_wait_gpu) 313 { 314 int ret; 315 316 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, interruptible, 317 no_wait_reserve, no_wait_gpu); 318 if (ret) 319 return ret; 320 321 if (nvbo->vma.node) { 322 if (nvbo->bo.mem.mem_type == TTM_PL_VRAM) 323 nvbo->bo.offset = nvbo->vma.offset; 324 } 325 326 return 0; 327 } 328 329 u16 330 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index) 331 { 332 bool is_iomem; 333 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 334 mem = &mem[index]; 335 if (is_iomem) 336 return ioread16_native((void __force __iomem *)mem); 337 else 338 return *mem; 339 } 340 341 void 342 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) 343 { 344 bool is_iomem; 345 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 346 mem = &mem[index]; 347 if (is_iomem) 348 iowrite16_native(val, (void __force __iomem *)mem); 349 else 350 *mem = val; 351 } 352 353 u32 354 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) 355 { 356 bool is_iomem; 357 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 358 mem = &mem[index]; 359 if (is_iomem) 360 return ioread32_native((void __force __iomem *)mem); 361 else 362 return *mem; 363 } 364 365 void 366 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) 367 { 368 bool is_iomem; 369 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 370 mem = &mem[index]; 371 if (is_iomem) 372 iowrite32_native(val, (void __force __iomem *)mem); 373 else 374 *mem = val; 375 } 376 377 static struct ttm_backend * 378 nouveau_bo_create_ttm_backend_entry(struct ttm_bo_device *bdev) 379 { 380 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 381 struct drm_device *dev = dev_priv->dev; 382 383 switch (dev_priv->gart_info.type) { 384 #if __OS_HAS_AGP 385 case NOUVEAU_GART_AGP: 386 return ttm_agp_backend_init(bdev, dev->agp->bridge); 387 #endif 388 case NOUVEAU_GART_SGDMA: 389 return nouveau_sgdma_init_ttm(dev); 390 default: 391 NV_ERROR(dev, "Unknown GART type %d\n", 392 dev_priv->gart_info.type); 393 break; 394 } 395 396 return NULL; 397 } 398 399 static int 400 nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) 401 { 402 /* We'll do this from user space. */ 403 return 0; 404 } 405 406 static int 407 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 408 struct ttm_mem_type_manager *man) 409 { 410 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 411 struct drm_device *dev = dev_priv->dev; 412 413 switch (type) { 414 case TTM_PL_SYSTEM: 415 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 416 man->available_caching = TTM_PL_MASK_CACHING; 417 man->default_caching = TTM_PL_FLAG_CACHED; 418 break; 419 case TTM_PL_VRAM: 420 if (dev_priv->card_type >= NV_50) { 421 man->func = &nouveau_vram_manager; 422 man->io_reserve_fastpath = false; 423 man->use_io_reserve_lru = true; 424 } else { 425 man->func = &ttm_bo_manager_func; 426 } 427 man->flags = TTM_MEMTYPE_FLAG_FIXED | 428 TTM_MEMTYPE_FLAG_MAPPABLE; 429 man->available_caching = TTM_PL_FLAG_UNCACHED | 430 TTM_PL_FLAG_WC; 431 man->default_caching = TTM_PL_FLAG_WC; 432 break; 433 case TTM_PL_TT: 434 man->func = &ttm_bo_manager_func; 435 switch (dev_priv->gart_info.type) { 436 case NOUVEAU_GART_AGP: 437 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 438 man->available_caching = TTM_PL_FLAG_UNCACHED | 439 TTM_PL_FLAG_WC; 440 man->default_caching = TTM_PL_FLAG_WC; 441 break; 442 case NOUVEAU_GART_SGDMA: 443 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | 444 TTM_MEMTYPE_FLAG_CMA; 445 man->available_caching = TTM_PL_MASK_CACHING; 446 man->default_caching = TTM_PL_FLAG_CACHED; 447 man->gpu_offset = dev_priv->gart_info.aper_base; 448 break; 449 default: 450 NV_ERROR(dev, "Unknown GART type: %d\n", 451 dev_priv->gart_info.type); 452 return -EINVAL; 453 } 454 break; 455 default: 456 NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type); 457 return -EINVAL; 458 } 459 return 0; 460 } 461 462 static void 463 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) 464 { 465 struct nouveau_bo *nvbo = nouveau_bo(bo); 466 467 switch (bo->mem.mem_type) { 468 case TTM_PL_VRAM: 469 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 470 TTM_PL_FLAG_SYSTEM); 471 break; 472 default: 473 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0); 474 break; 475 } 476 477 *pl = nvbo->placement; 478 } 479 480 481 /* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access 482 * TTM_PL_{VRAM,TT} directly. 483 */ 484 485 static int 486 nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan, 487 struct nouveau_bo *nvbo, bool evict, 488 bool no_wait_reserve, bool no_wait_gpu, 489 struct ttm_mem_reg *new_mem) 490 { 491 struct nouveau_fence *fence = NULL; 492 int ret; 493 494 ret = nouveau_fence_new(chan, &fence, true); 495 if (ret) 496 return ret; 497 498 ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict, 499 no_wait_reserve, no_wait_gpu, new_mem); 500 nouveau_fence_unref(&fence); 501 return ret; 502 } 503 504 static inline uint32_t 505 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo, 506 struct nouveau_channel *chan, struct ttm_mem_reg *mem) 507 { 508 struct nouveau_bo *nvbo = nouveau_bo(bo); 509 510 if (nvbo->no_vm) { 511 if (mem->mem_type == TTM_PL_TT) 512 return NvDmaGART; 513 return NvDmaVRAM; 514 } 515 516 if (mem->mem_type == TTM_PL_TT) 517 return chan->gart_handle; 518 return chan->vram_handle; 519 } 520 521 static int 522 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 523 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 524 { 525 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 526 struct nouveau_bo *nvbo = nouveau_bo(bo); 527 u64 src_offset = old_mem->start << PAGE_SHIFT; 528 u64 dst_offset = new_mem->start << PAGE_SHIFT; 529 u32 page_count = new_mem->num_pages; 530 int ret; 531 532 if (!nvbo->no_vm) { 533 if (old_mem->mem_type == TTM_PL_VRAM) 534 src_offset = nvbo->vma.offset; 535 else 536 src_offset += dev_priv->gart_info.aper_base; 537 538 if (new_mem->mem_type == TTM_PL_VRAM) 539 dst_offset = nvbo->vma.offset; 540 else 541 dst_offset += dev_priv->gart_info.aper_base; 542 } 543 544 page_count = new_mem->num_pages; 545 while (page_count) { 546 int line_count = (page_count > 2047) ? 2047 : page_count; 547 548 ret = RING_SPACE(chan, 12); 549 if (ret) 550 return ret; 551 552 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0238, 2); 553 OUT_RING (chan, upper_32_bits(dst_offset)); 554 OUT_RING (chan, lower_32_bits(dst_offset)); 555 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x030c, 6); 556 OUT_RING (chan, upper_32_bits(src_offset)); 557 OUT_RING (chan, lower_32_bits(src_offset)); 558 OUT_RING (chan, PAGE_SIZE); /* src_pitch */ 559 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */ 560 OUT_RING (chan, PAGE_SIZE); /* line_length */ 561 OUT_RING (chan, line_count); 562 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0300, 1); 563 OUT_RING (chan, 0x00100110); 564 565 page_count -= line_count; 566 src_offset += (PAGE_SIZE * line_count); 567 dst_offset += (PAGE_SIZE * line_count); 568 } 569 570 return 0; 571 } 572 573 static int 574 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 575 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 576 { 577 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 578 struct nouveau_bo *nvbo = nouveau_bo(bo); 579 u64 length = (new_mem->num_pages << PAGE_SHIFT); 580 u64 src_offset, dst_offset; 581 int ret; 582 583 src_offset = old_mem->start << PAGE_SHIFT; 584 dst_offset = new_mem->start << PAGE_SHIFT; 585 if (!nvbo->no_vm) { 586 if (old_mem->mem_type == TTM_PL_VRAM) 587 src_offset = nvbo->vma.offset; 588 else 589 src_offset += dev_priv->gart_info.aper_base; 590 591 if (new_mem->mem_type == TTM_PL_VRAM) 592 dst_offset = nvbo->vma.offset; 593 else 594 dst_offset += dev_priv->gart_info.aper_base; 595 } 596 597 ret = RING_SPACE(chan, 3); 598 if (ret) 599 return ret; 600 601 BEGIN_RING(chan, NvSubM2MF, 0x0184, 2); 602 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem)); 603 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem)); 604 605 while (length) { 606 u32 amount, stride, height; 607 608 amount = min(length, (u64)(4 * 1024 * 1024)); 609 stride = 16 * 4; 610 height = amount / stride; 611 612 if (new_mem->mem_type == TTM_PL_VRAM && 613 nouveau_bo_tile_layout(nvbo)) { 614 ret = RING_SPACE(chan, 8); 615 if (ret) 616 return ret; 617 618 BEGIN_RING(chan, NvSubM2MF, 0x0200, 7); 619 OUT_RING (chan, 0); 620 OUT_RING (chan, 0); 621 OUT_RING (chan, stride); 622 OUT_RING (chan, height); 623 OUT_RING (chan, 1); 624 OUT_RING (chan, 0); 625 OUT_RING (chan, 0); 626 } else { 627 ret = RING_SPACE(chan, 2); 628 if (ret) 629 return ret; 630 631 BEGIN_RING(chan, NvSubM2MF, 0x0200, 1); 632 OUT_RING (chan, 1); 633 } 634 if (old_mem->mem_type == TTM_PL_VRAM && 635 nouveau_bo_tile_layout(nvbo)) { 636 ret = RING_SPACE(chan, 8); 637 if (ret) 638 return ret; 639 640 BEGIN_RING(chan, NvSubM2MF, 0x021c, 7); 641 OUT_RING (chan, 0); 642 OUT_RING (chan, 0); 643 OUT_RING (chan, stride); 644 OUT_RING (chan, height); 645 OUT_RING (chan, 1); 646 OUT_RING (chan, 0); 647 OUT_RING (chan, 0); 648 } else { 649 ret = RING_SPACE(chan, 2); 650 if (ret) 651 return ret; 652 653 BEGIN_RING(chan, NvSubM2MF, 0x021c, 1); 654 OUT_RING (chan, 1); 655 } 656 657 ret = RING_SPACE(chan, 14); 658 if (ret) 659 return ret; 660 661 BEGIN_RING(chan, NvSubM2MF, 0x0238, 2); 662 OUT_RING (chan, upper_32_bits(src_offset)); 663 OUT_RING (chan, upper_32_bits(dst_offset)); 664 BEGIN_RING(chan, NvSubM2MF, 0x030c, 8); 665 OUT_RING (chan, lower_32_bits(src_offset)); 666 OUT_RING (chan, lower_32_bits(dst_offset)); 667 OUT_RING (chan, stride); 668 OUT_RING (chan, stride); 669 OUT_RING (chan, stride); 670 OUT_RING (chan, height); 671 OUT_RING (chan, 0x00000101); 672 OUT_RING (chan, 0x00000000); 673 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); 674 OUT_RING (chan, 0); 675 676 length -= amount; 677 src_offset += amount; 678 dst_offset += amount; 679 } 680 681 return 0; 682 } 683 684 static int 685 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 686 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 687 { 688 u32 src_offset = old_mem->start << PAGE_SHIFT; 689 u32 dst_offset = new_mem->start << PAGE_SHIFT; 690 u32 page_count = new_mem->num_pages; 691 int ret; 692 693 ret = RING_SPACE(chan, 3); 694 if (ret) 695 return ret; 696 697 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2); 698 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem)); 699 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem)); 700 701 page_count = new_mem->num_pages; 702 while (page_count) { 703 int line_count = (page_count > 2047) ? 2047 : page_count; 704 705 ret = RING_SPACE(chan, 11); 706 if (ret) 707 return ret; 708 709 BEGIN_RING(chan, NvSubM2MF, 710 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8); 711 OUT_RING (chan, src_offset); 712 OUT_RING (chan, dst_offset); 713 OUT_RING (chan, PAGE_SIZE); /* src_pitch */ 714 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */ 715 OUT_RING (chan, PAGE_SIZE); /* line_length */ 716 OUT_RING (chan, line_count); 717 OUT_RING (chan, 0x00000101); 718 OUT_RING (chan, 0x00000000); 719 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); 720 OUT_RING (chan, 0); 721 722 page_count -= line_count; 723 src_offset += (PAGE_SIZE * line_count); 724 dst_offset += (PAGE_SIZE * line_count); 725 } 726 727 return 0; 728 } 729 730 static int 731 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr, 732 bool no_wait_reserve, bool no_wait_gpu, 733 struct ttm_mem_reg *new_mem) 734 { 735 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 736 struct nouveau_bo *nvbo = nouveau_bo(bo); 737 struct nouveau_channel *chan; 738 int ret; 739 740 chan = nvbo->channel; 741 if (!chan || nvbo->no_vm) { 742 chan = dev_priv->channel; 743 mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX); 744 } 745 746 if (dev_priv->card_type < NV_50) 747 ret = nv04_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 748 else 749 if (dev_priv->card_type < NV_C0) 750 ret = nv50_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 751 else 752 ret = nvc0_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 753 if (ret == 0) { 754 ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict, 755 no_wait_reserve, 756 no_wait_gpu, new_mem); 757 } 758 759 if (chan == dev_priv->channel) 760 mutex_unlock(&chan->mutex); 761 return ret; 762 } 763 764 static int 765 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, 766 bool no_wait_reserve, bool no_wait_gpu, 767 struct ttm_mem_reg *new_mem) 768 { 769 u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; 770 struct ttm_placement placement; 771 struct ttm_mem_reg tmp_mem; 772 int ret; 773 774 placement.fpfn = placement.lpfn = 0; 775 placement.num_placement = placement.num_busy_placement = 1; 776 placement.placement = placement.busy_placement = &placement_memtype; 777 778 tmp_mem = *new_mem; 779 tmp_mem.mm_node = NULL; 780 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu); 781 if (ret) 782 return ret; 783 784 ret = ttm_tt_bind(bo->ttm, &tmp_mem); 785 if (ret) 786 goto out; 787 788 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem); 789 if (ret) 790 goto out; 791 792 ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem); 793 out: 794 ttm_bo_mem_put(bo, &tmp_mem); 795 return ret; 796 } 797 798 static int 799 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr, 800 bool no_wait_reserve, bool no_wait_gpu, 801 struct ttm_mem_reg *new_mem) 802 { 803 u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; 804 struct ttm_placement placement; 805 struct ttm_mem_reg tmp_mem; 806 int ret; 807 808 placement.fpfn = placement.lpfn = 0; 809 placement.num_placement = placement.num_busy_placement = 1; 810 placement.placement = placement.busy_placement = &placement_memtype; 811 812 tmp_mem = *new_mem; 813 tmp_mem.mm_node = NULL; 814 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu); 815 if (ret) 816 return ret; 817 818 ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem); 819 if (ret) 820 goto out; 821 822 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, new_mem); 823 if (ret) 824 goto out; 825 826 out: 827 ttm_bo_mem_put(bo, &tmp_mem); 828 return ret; 829 } 830 831 static int 832 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem, 833 struct nouveau_tile_reg **new_tile) 834 { 835 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 836 struct drm_device *dev = dev_priv->dev; 837 struct nouveau_bo *nvbo = nouveau_bo(bo); 838 uint64_t offset; 839 840 if (nvbo->no_vm || new_mem->mem_type != TTM_PL_VRAM) { 841 /* Nothing to do. */ 842 *new_tile = NULL; 843 return 0; 844 } 845 846 offset = new_mem->start << PAGE_SHIFT; 847 848 if (dev_priv->chan_vm) { 849 nouveau_vm_map(&nvbo->vma, new_mem->mm_node); 850 } else if (dev_priv->card_type >= NV_10) { 851 *new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size, 852 nvbo->tile_mode, 853 nvbo->tile_flags); 854 } 855 856 return 0; 857 } 858 859 static void 860 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, 861 struct nouveau_tile_reg *new_tile, 862 struct nouveau_tile_reg **old_tile) 863 { 864 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 865 struct drm_device *dev = dev_priv->dev; 866 867 if (dev_priv->card_type >= NV_10 && 868 dev_priv->card_type < NV_50) { 869 nv10_mem_put_tile_region(dev, *old_tile, bo->sync_obj); 870 *old_tile = new_tile; 871 } 872 } 873 874 static int 875 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr, 876 bool no_wait_reserve, bool no_wait_gpu, 877 struct ttm_mem_reg *new_mem) 878 { 879 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 880 struct nouveau_bo *nvbo = nouveau_bo(bo); 881 struct ttm_mem_reg *old_mem = &bo->mem; 882 struct nouveau_tile_reg *new_tile = NULL; 883 int ret = 0; 884 885 ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile); 886 if (ret) 887 return ret; 888 889 /* Fake bo copy. */ 890 if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) { 891 BUG_ON(bo->mem.mm_node != NULL); 892 bo->mem = *new_mem; 893 new_mem->mm_node = NULL; 894 goto out; 895 } 896 897 /* Software copy if the card isn't up and running yet. */ 898 if (!dev_priv->channel) { 899 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem); 900 goto out; 901 } 902 903 /* Hardware assisted copy. */ 904 if (new_mem->mem_type == TTM_PL_SYSTEM) 905 ret = nouveau_bo_move_flipd(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 906 else if (old_mem->mem_type == TTM_PL_SYSTEM) 907 ret = nouveau_bo_move_flips(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 908 else 909 ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 910 911 if (!ret) 912 goto out; 913 914 /* Fallback to software copy. */ 915 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem); 916 917 out: 918 if (ret) 919 nouveau_bo_vm_cleanup(bo, NULL, &new_tile); 920 else 921 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile); 922 923 return ret; 924 } 925 926 static int 927 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp) 928 { 929 return 0; 930 } 931 932 static int 933 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 934 { 935 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 936 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 937 struct drm_device *dev = dev_priv->dev; 938 int ret; 939 940 mem->bus.addr = NULL; 941 mem->bus.offset = 0; 942 mem->bus.size = mem->num_pages << PAGE_SHIFT; 943 mem->bus.base = 0; 944 mem->bus.is_iomem = false; 945 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 946 return -EINVAL; 947 switch (mem->mem_type) { 948 case TTM_PL_SYSTEM: 949 /* System memory */ 950 return 0; 951 case TTM_PL_TT: 952 #if __OS_HAS_AGP 953 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) { 954 mem->bus.offset = mem->start << PAGE_SHIFT; 955 mem->bus.base = dev_priv->gart_info.aper_base; 956 mem->bus.is_iomem = true; 957 } 958 #endif 959 break; 960 case TTM_PL_VRAM: 961 { 962 struct nouveau_vram *vram = mem->mm_node; 963 u8 page_shift; 964 965 if (!dev_priv->bar1_vm) { 966 mem->bus.offset = mem->start << PAGE_SHIFT; 967 mem->bus.base = pci_resource_start(dev->pdev, 1); 968 mem->bus.is_iomem = true; 969 break; 970 } 971 972 if (dev_priv->card_type == NV_C0) 973 page_shift = vram->page_shift; 974 else 975 page_shift = 12; 976 977 ret = nouveau_vm_get(dev_priv->bar1_vm, mem->bus.size, 978 page_shift, NV_MEM_ACCESS_RW, 979 &vram->bar_vma); 980 if (ret) 981 return ret; 982 983 nouveau_vm_map(&vram->bar_vma, vram); 984 if (ret) { 985 nouveau_vm_put(&vram->bar_vma); 986 return ret; 987 } 988 989 mem->bus.offset = vram->bar_vma.offset; 990 if (dev_priv->card_type == NV_50) /*XXX*/ 991 mem->bus.offset -= 0x0020000000ULL; 992 mem->bus.base = pci_resource_start(dev->pdev, 1); 993 mem->bus.is_iomem = true; 994 } 995 break; 996 default: 997 return -EINVAL; 998 } 999 return 0; 1000 } 1001 1002 static void 1003 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1004 { 1005 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 1006 struct nouveau_vram *vram = mem->mm_node; 1007 1008 if (!dev_priv->bar1_vm || mem->mem_type != TTM_PL_VRAM) 1009 return; 1010 1011 if (!vram->bar_vma.node) 1012 return; 1013 1014 nouveau_vm_unmap(&vram->bar_vma); 1015 nouveau_vm_put(&vram->bar_vma); 1016 } 1017 1018 static int 1019 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) 1020 { 1021 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 1022 struct nouveau_bo *nvbo = nouveau_bo(bo); 1023 1024 /* as long as the bo isn't in vram, and isn't tiled, we've got 1025 * nothing to do here. 1026 */ 1027 if (bo->mem.mem_type != TTM_PL_VRAM) { 1028 if (dev_priv->card_type < NV_50 || 1029 !nouveau_bo_tile_layout(nvbo)) 1030 return 0; 1031 } 1032 1033 /* make sure bo is in mappable vram */ 1034 if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages) 1035 return 0; 1036 1037 1038 nvbo->placement.fpfn = 0; 1039 nvbo->placement.lpfn = dev_priv->fb_mappable_pages; 1040 nouveau_bo_placement_set(nvbo, TTM_PL_VRAM, 0); 1041 return nouveau_bo_validate(nvbo, false, true, false); 1042 } 1043 1044 void 1045 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence) 1046 { 1047 struct nouveau_fence *old_fence; 1048 1049 if (likely(fence)) 1050 nouveau_fence_ref(fence); 1051 1052 spin_lock(&nvbo->bo.bdev->fence_lock); 1053 old_fence = nvbo->bo.sync_obj; 1054 nvbo->bo.sync_obj = fence; 1055 spin_unlock(&nvbo->bo.bdev->fence_lock); 1056 1057 nouveau_fence_unref(&old_fence); 1058 } 1059 1060 struct ttm_bo_driver nouveau_bo_driver = { 1061 .create_ttm_backend_entry = nouveau_bo_create_ttm_backend_entry, 1062 .invalidate_caches = nouveau_bo_invalidate_caches, 1063 .init_mem_type = nouveau_bo_init_mem_type, 1064 .evict_flags = nouveau_bo_evict_flags, 1065 .move = nouveau_bo_move, 1066 .verify_access = nouveau_bo_verify_access, 1067 .sync_obj_signaled = __nouveau_fence_signalled, 1068 .sync_obj_wait = __nouveau_fence_wait, 1069 .sync_obj_flush = __nouveau_fence_flush, 1070 .sync_obj_unref = __nouveau_fence_unref, 1071 .sync_obj_ref = __nouveau_fence_ref, 1072 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify, 1073 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1074 .io_mem_free = &nouveau_ttm_io_mem_free, 1075 }; 1076 1077