1 /* 2 * Copyright 2007 Dave Airlied 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 /* 25 * Authors: Dave Airlied <airlied@linux.ie> 26 * Ben Skeggs <darktama@iinet.net.au> 27 * Jeremy Kolb <jkolb@brandeis.edu> 28 */ 29 30 #include "drmP.h" 31 32 #include "nouveau_drm.h" 33 #include "nouveau_drv.h" 34 #include "nouveau_dma.h" 35 #include "nouveau_mm.h" 36 #include "nouveau_vm.h" 37 38 #include <linux/log2.h> 39 #include <linux/slab.h> 40 41 static void 42 nouveau_bo_del_ttm(struct ttm_buffer_object *bo) 43 { 44 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 45 struct drm_device *dev = dev_priv->dev; 46 struct nouveau_bo *nvbo = nouveau_bo(bo); 47 48 if (unlikely(nvbo->gem)) 49 DRM_ERROR("bo %p still attached to GEM object\n", bo); 50 51 nv10_mem_put_tile_region(dev, nvbo->tile, NULL); 52 nouveau_vm_put(&nvbo->vma); 53 kfree(nvbo); 54 } 55 56 static void 57 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, int *size, 58 int *page_shift) 59 { 60 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 61 62 if (dev_priv->card_type < NV_50) { 63 if (nvbo->tile_mode) { 64 if (dev_priv->chipset >= 0x40) { 65 *align = 65536; 66 *size = roundup(*size, 64 * nvbo->tile_mode); 67 68 } else if (dev_priv->chipset >= 0x30) { 69 *align = 32768; 70 *size = roundup(*size, 64 * nvbo->tile_mode); 71 72 } else if (dev_priv->chipset >= 0x20) { 73 *align = 16384; 74 *size = roundup(*size, 64 * nvbo->tile_mode); 75 76 } else if (dev_priv->chipset >= 0x10) { 77 *align = 16384; 78 *size = roundup(*size, 32 * nvbo->tile_mode); 79 } 80 } 81 } else { 82 if (likely(dev_priv->chan_vm)) { 83 if (*size > 256 * 1024) 84 *page_shift = dev_priv->chan_vm->lpg_shift; 85 else 86 *page_shift = dev_priv->chan_vm->spg_shift; 87 } else { 88 *page_shift = 12; 89 } 90 91 *size = roundup(*size, (1 << *page_shift)); 92 *align = max((1 << *page_shift), *align); 93 } 94 95 *size = roundup(*size, PAGE_SIZE); 96 } 97 98 int 99 nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan, 100 int size, int align, uint32_t flags, uint32_t tile_mode, 101 uint32_t tile_flags, bool no_vm, bool mappable, 102 struct nouveau_bo **pnvbo) 103 { 104 struct drm_nouveau_private *dev_priv = dev->dev_private; 105 struct nouveau_bo *nvbo; 106 int ret = 0, page_shift = 0; 107 108 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); 109 if (!nvbo) 110 return -ENOMEM; 111 INIT_LIST_HEAD(&nvbo->head); 112 INIT_LIST_HEAD(&nvbo->entry); 113 nvbo->mappable = mappable; 114 nvbo->no_vm = no_vm; 115 nvbo->tile_mode = tile_mode; 116 nvbo->tile_flags = tile_flags; 117 nvbo->bo.bdev = &dev_priv->ttm.bdev; 118 119 nouveau_bo_fixup_align(nvbo, &align, &size, &page_shift); 120 align >>= PAGE_SHIFT; 121 122 if (!nvbo->no_vm && dev_priv->chan_vm) { 123 ret = nouveau_vm_get(dev_priv->chan_vm, size, page_shift, 124 NV_MEM_ACCESS_RW, &nvbo->vma); 125 if (ret) { 126 kfree(nvbo); 127 return ret; 128 } 129 } 130 131 nvbo->bo.mem.num_pages = size >> PAGE_SHIFT; 132 nouveau_bo_placement_set(nvbo, flags, 0); 133 134 nvbo->channel = chan; 135 ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size, 136 ttm_bo_type_device, &nvbo->placement, align, 0, 137 false, NULL, size, nouveau_bo_del_ttm); 138 if (ret) { 139 /* ttm will call nouveau_bo_del_ttm if it fails.. */ 140 return ret; 141 } 142 nvbo->channel = NULL; 143 144 if (nvbo->vma.node) { 145 if (nvbo->bo.mem.mem_type == TTM_PL_VRAM) 146 nvbo->bo.offset = nvbo->vma.offset; 147 } 148 149 *pnvbo = nvbo; 150 return 0; 151 } 152 153 static void 154 set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags) 155 { 156 *n = 0; 157 158 if (type & TTM_PL_FLAG_VRAM) 159 pl[(*n)++] = TTM_PL_FLAG_VRAM | flags; 160 if (type & TTM_PL_FLAG_TT) 161 pl[(*n)++] = TTM_PL_FLAG_TT | flags; 162 if (type & TTM_PL_FLAG_SYSTEM) 163 pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags; 164 } 165 166 static void 167 set_placement_range(struct nouveau_bo *nvbo, uint32_t type) 168 { 169 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 170 int vram_pages = dev_priv->vram_size >> PAGE_SHIFT; 171 172 if (dev_priv->card_type == NV_10 && 173 nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) && 174 nvbo->bo.mem.num_pages < vram_pages / 2) { 175 /* 176 * Make sure that the color and depth buffers are handled 177 * by independent memory controller units. Up to a 9x 178 * speed up when alpha-blending and depth-test are enabled 179 * at the same time. 180 */ 181 if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) { 182 nvbo->placement.fpfn = vram_pages / 2; 183 nvbo->placement.lpfn = ~0; 184 } else { 185 nvbo->placement.fpfn = 0; 186 nvbo->placement.lpfn = vram_pages / 2; 187 } 188 } 189 } 190 191 void 192 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy) 193 { 194 struct ttm_placement *pl = &nvbo->placement; 195 uint32_t flags = TTM_PL_MASK_CACHING | 196 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0); 197 198 pl->placement = nvbo->placements; 199 set_placement_list(nvbo->placements, &pl->num_placement, 200 type, flags); 201 202 pl->busy_placement = nvbo->busy_placements; 203 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement, 204 type | busy, flags); 205 206 set_placement_range(nvbo, type); 207 } 208 209 int 210 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype) 211 { 212 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 213 struct ttm_buffer_object *bo = &nvbo->bo; 214 int ret; 215 216 if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) { 217 NV_ERROR(nouveau_bdev(bo->bdev)->dev, 218 "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo, 219 1 << bo->mem.mem_type, memtype); 220 return -EINVAL; 221 } 222 223 if (nvbo->pin_refcnt++) 224 return 0; 225 226 ret = ttm_bo_reserve(bo, false, false, false, 0); 227 if (ret) 228 goto out; 229 230 nouveau_bo_placement_set(nvbo, memtype, 0); 231 232 ret = nouveau_bo_validate(nvbo, false, false, false); 233 if (ret == 0) { 234 switch (bo->mem.mem_type) { 235 case TTM_PL_VRAM: 236 dev_priv->fb_aper_free -= bo->mem.size; 237 break; 238 case TTM_PL_TT: 239 dev_priv->gart_info.aper_free -= bo->mem.size; 240 break; 241 default: 242 break; 243 } 244 } 245 ttm_bo_unreserve(bo); 246 out: 247 if (unlikely(ret)) 248 nvbo->pin_refcnt--; 249 return ret; 250 } 251 252 int 253 nouveau_bo_unpin(struct nouveau_bo *nvbo) 254 { 255 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 256 struct ttm_buffer_object *bo = &nvbo->bo; 257 int ret; 258 259 if (--nvbo->pin_refcnt) 260 return 0; 261 262 ret = ttm_bo_reserve(bo, false, false, false, 0); 263 if (ret) 264 return ret; 265 266 nouveau_bo_placement_set(nvbo, bo->mem.placement, 0); 267 268 ret = nouveau_bo_validate(nvbo, false, false, false); 269 if (ret == 0) { 270 switch (bo->mem.mem_type) { 271 case TTM_PL_VRAM: 272 dev_priv->fb_aper_free += bo->mem.size; 273 break; 274 case TTM_PL_TT: 275 dev_priv->gart_info.aper_free += bo->mem.size; 276 break; 277 default: 278 break; 279 } 280 } 281 282 ttm_bo_unreserve(bo); 283 return ret; 284 } 285 286 int 287 nouveau_bo_map(struct nouveau_bo *nvbo) 288 { 289 int ret; 290 291 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 292 if (ret) 293 return ret; 294 295 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap); 296 ttm_bo_unreserve(&nvbo->bo); 297 return ret; 298 } 299 300 void 301 nouveau_bo_unmap(struct nouveau_bo *nvbo) 302 { 303 if (nvbo) 304 ttm_bo_kunmap(&nvbo->kmap); 305 } 306 307 int 308 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, 309 bool no_wait_reserve, bool no_wait_gpu) 310 { 311 int ret; 312 313 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, interruptible, 314 no_wait_reserve, no_wait_gpu); 315 if (ret) 316 return ret; 317 318 if (nvbo->vma.node) { 319 if (nvbo->bo.mem.mem_type == TTM_PL_VRAM) 320 nvbo->bo.offset = nvbo->vma.offset; 321 } 322 323 return 0; 324 } 325 326 u16 327 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index) 328 { 329 bool is_iomem; 330 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 331 mem = &mem[index]; 332 if (is_iomem) 333 return ioread16_native((void __force __iomem *)mem); 334 else 335 return *mem; 336 } 337 338 void 339 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) 340 { 341 bool is_iomem; 342 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 343 mem = &mem[index]; 344 if (is_iomem) 345 iowrite16_native(val, (void __force __iomem *)mem); 346 else 347 *mem = val; 348 } 349 350 u32 351 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) 352 { 353 bool is_iomem; 354 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 355 mem = &mem[index]; 356 if (is_iomem) 357 return ioread32_native((void __force __iomem *)mem); 358 else 359 return *mem; 360 } 361 362 void 363 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) 364 { 365 bool is_iomem; 366 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 367 mem = &mem[index]; 368 if (is_iomem) 369 iowrite32_native(val, (void __force __iomem *)mem); 370 else 371 *mem = val; 372 } 373 374 static struct ttm_backend * 375 nouveau_bo_create_ttm_backend_entry(struct ttm_bo_device *bdev) 376 { 377 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 378 struct drm_device *dev = dev_priv->dev; 379 380 switch (dev_priv->gart_info.type) { 381 #if __OS_HAS_AGP 382 case NOUVEAU_GART_AGP: 383 return ttm_agp_backend_init(bdev, dev->agp->bridge); 384 #endif 385 case NOUVEAU_GART_SGDMA: 386 return nouveau_sgdma_init_ttm(dev); 387 default: 388 NV_ERROR(dev, "Unknown GART type %d\n", 389 dev_priv->gart_info.type); 390 break; 391 } 392 393 return NULL; 394 } 395 396 static int 397 nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) 398 { 399 /* We'll do this from user space. */ 400 return 0; 401 } 402 403 static int 404 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 405 struct ttm_mem_type_manager *man) 406 { 407 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 408 struct drm_device *dev = dev_priv->dev; 409 410 switch (type) { 411 case TTM_PL_SYSTEM: 412 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 413 man->available_caching = TTM_PL_MASK_CACHING; 414 man->default_caching = TTM_PL_FLAG_CACHED; 415 break; 416 case TTM_PL_VRAM: 417 if (dev_priv->card_type >= NV_50) { 418 man->func = &nouveau_vram_manager; 419 man->io_reserve_fastpath = false; 420 man->use_io_reserve_lru = true; 421 } else { 422 man->func = &ttm_bo_manager_func; 423 } 424 man->flags = TTM_MEMTYPE_FLAG_FIXED | 425 TTM_MEMTYPE_FLAG_MAPPABLE; 426 man->available_caching = TTM_PL_FLAG_UNCACHED | 427 TTM_PL_FLAG_WC; 428 man->default_caching = TTM_PL_FLAG_WC; 429 break; 430 case TTM_PL_TT: 431 man->func = &ttm_bo_manager_func; 432 switch (dev_priv->gart_info.type) { 433 case NOUVEAU_GART_AGP: 434 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 435 man->available_caching = TTM_PL_FLAG_UNCACHED | 436 TTM_PL_FLAG_WC; 437 man->default_caching = TTM_PL_FLAG_WC; 438 break; 439 case NOUVEAU_GART_SGDMA: 440 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | 441 TTM_MEMTYPE_FLAG_CMA; 442 man->available_caching = TTM_PL_MASK_CACHING; 443 man->default_caching = TTM_PL_FLAG_CACHED; 444 man->gpu_offset = dev_priv->gart_info.aper_base; 445 break; 446 default: 447 NV_ERROR(dev, "Unknown GART type: %d\n", 448 dev_priv->gart_info.type); 449 return -EINVAL; 450 } 451 break; 452 default: 453 NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type); 454 return -EINVAL; 455 } 456 return 0; 457 } 458 459 static void 460 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) 461 { 462 struct nouveau_bo *nvbo = nouveau_bo(bo); 463 464 switch (bo->mem.mem_type) { 465 case TTM_PL_VRAM: 466 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 467 TTM_PL_FLAG_SYSTEM); 468 break; 469 default: 470 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0); 471 break; 472 } 473 474 *pl = nvbo->placement; 475 } 476 477 478 /* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access 479 * TTM_PL_{VRAM,TT} directly. 480 */ 481 482 static int 483 nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan, 484 struct nouveau_bo *nvbo, bool evict, 485 bool no_wait_reserve, bool no_wait_gpu, 486 struct ttm_mem_reg *new_mem) 487 { 488 struct nouveau_fence *fence = NULL; 489 int ret; 490 491 ret = nouveau_fence_new(chan, &fence, true); 492 if (ret) 493 return ret; 494 495 ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict, 496 no_wait_reserve, no_wait_gpu, new_mem); 497 nouveau_fence_unref(&fence); 498 return ret; 499 } 500 501 static inline uint32_t 502 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo, 503 struct nouveau_channel *chan, struct ttm_mem_reg *mem) 504 { 505 struct nouveau_bo *nvbo = nouveau_bo(bo); 506 507 if (nvbo->no_vm) { 508 if (mem->mem_type == TTM_PL_TT) 509 return NvDmaGART; 510 return NvDmaVRAM; 511 } 512 513 if (mem->mem_type == TTM_PL_TT) 514 return chan->gart_handle; 515 return chan->vram_handle; 516 } 517 518 static int 519 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 520 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 521 { 522 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 523 struct nouveau_bo *nvbo = nouveau_bo(bo); 524 u64 src_offset = old_mem->start << PAGE_SHIFT; 525 u64 dst_offset = new_mem->start << PAGE_SHIFT; 526 u32 page_count = new_mem->num_pages; 527 int ret; 528 529 if (!nvbo->no_vm) { 530 if (old_mem->mem_type == TTM_PL_VRAM) 531 src_offset = nvbo->vma.offset; 532 else 533 src_offset += dev_priv->gart_info.aper_base; 534 535 if (new_mem->mem_type == TTM_PL_VRAM) 536 dst_offset = nvbo->vma.offset; 537 else 538 dst_offset += dev_priv->gart_info.aper_base; 539 } 540 541 page_count = new_mem->num_pages; 542 while (page_count) { 543 int line_count = (page_count > 2047) ? 2047 : page_count; 544 545 ret = RING_SPACE(chan, 12); 546 if (ret) 547 return ret; 548 549 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0238, 2); 550 OUT_RING (chan, upper_32_bits(dst_offset)); 551 OUT_RING (chan, lower_32_bits(dst_offset)); 552 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x030c, 6); 553 OUT_RING (chan, upper_32_bits(src_offset)); 554 OUT_RING (chan, lower_32_bits(src_offset)); 555 OUT_RING (chan, PAGE_SIZE); /* src_pitch */ 556 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */ 557 OUT_RING (chan, PAGE_SIZE); /* line_length */ 558 OUT_RING (chan, line_count); 559 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0300, 1); 560 OUT_RING (chan, 0x00100110); 561 562 page_count -= line_count; 563 src_offset += (PAGE_SIZE * line_count); 564 dst_offset += (PAGE_SIZE * line_count); 565 } 566 567 return 0; 568 } 569 570 static int 571 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 572 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 573 { 574 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 575 struct nouveau_bo *nvbo = nouveau_bo(bo); 576 u64 length = (new_mem->num_pages << PAGE_SHIFT); 577 u64 src_offset, dst_offset; 578 int ret; 579 580 src_offset = old_mem->start << PAGE_SHIFT; 581 dst_offset = new_mem->start << PAGE_SHIFT; 582 if (!nvbo->no_vm) { 583 if (old_mem->mem_type == TTM_PL_VRAM) 584 src_offset = nvbo->vma.offset; 585 else 586 src_offset += dev_priv->gart_info.aper_base; 587 588 if (new_mem->mem_type == TTM_PL_VRAM) 589 dst_offset = nvbo->vma.offset; 590 else 591 dst_offset += dev_priv->gart_info.aper_base; 592 } 593 594 ret = RING_SPACE(chan, 3); 595 if (ret) 596 return ret; 597 598 BEGIN_RING(chan, NvSubM2MF, 0x0184, 2); 599 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem)); 600 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem)); 601 602 while (length) { 603 u32 amount, stride, height; 604 605 amount = min(length, (u64)(4 * 1024 * 1024)); 606 stride = 16 * 4; 607 height = amount / stride; 608 609 if (new_mem->mem_type == TTM_PL_VRAM && 610 nouveau_bo_tile_layout(nvbo)) { 611 ret = RING_SPACE(chan, 8); 612 if (ret) 613 return ret; 614 615 BEGIN_RING(chan, NvSubM2MF, 0x0200, 7); 616 OUT_RING (chan, 0); 617 OUT_RING (chan, 0); 618 OUT_RING (chan, stride); 619 OUT_RING (chan, height); 620 OUT_RING (chan, 1); 621 OUT_RING (chan, 0); 622 OUT_RING (chan, 0); 623 } else { 624 ret = RING_SPACE(chan, 2); 625 if (ret) 626 return ret; 627 628 BEGIN_RING(chan, NvSubM2MF, 0x0200, 1); 629 OUT_RING (chan, 1); 630 } 631 if (old_mem->mem_type == TTM_PL_VRAM && 632 nouveau_bo_tile_layout(nvbo)) { 633 ret = RING_SPACE(chan, 8); 634 if (ret) 635 return ret; 636 637 BEGIN_RING(chan, NvSubM2MF, 0x021c, 7); 638 OUT_RING (chan, 0); 639 OUT_RING (chan, 0); 640 OUT_RING (chan, stride); 641 OUT_RING (chan, height); 642 OUT_RING (chan, 1); 643 OUT_RING (chan, 0); 644 OUT_RING (chan, 0); 645 } else { 646 ret = RING_SPACE(chan, 2); 647 if (ret) 648 return ret; 649 650 BEGIN_RING(chan, NvSubM2MF, 0x021c, 1); 651 OUT_RING (chan, 1); 652 } 653 654 ret = RING_SPACE(chan, 14); 655 if (ret) 656 return ret; 657 658 BEGIN_RING(chan, NvSubM2MF, 0x0238, 2); 659 OUT_RING (chan, upper_32_bits(src_offset)); 660 OUT_RING (chan, upper_32_bits(dst_offset)); 661 BEGIN_RING(chan, NvSubM2MF, 0x030c, 8); 662 OUT_RING (chan, lower_32_bits(src_offset)); 663 OUT_RING (chan, lower_32_bits(dst_offset)); 664 OUT_RING (chan, stride); 665 OUT_RING (chan, stride); 666 OUT_RING (chan, stride); 667 OUT_RING (chan, height); 668 OUT_RING (chan, 0x00000101); 669 OUT_RING (chan, 0x00000000); 670 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); 671 OUT_RING (chan, 0); 672 673 length -= amount; 674 src_offset += amount; 675 dst_offset += amount; 676 } 677 678 return 0; 679 } 680 681 static int 682 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 683 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 684 { 685 u32 src_offset = old_mem->start << PAGE_SHIFT; 686 u32 dst_offset = new_mem->start << PAGE_SHIFT; 687 u32 page_count = new_mem->num_pages; 688 int ret; 689 690 ret = RING_SPACE(chan, 3); 691 if (ret) 692 return ret; 693 694 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2); 695 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem)); 696 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem)); 697 698 page_count = new_mem->num_pages; 699 while (page_count) { 700 int line_count = (page_count > 2047) ? 2047 : page_count; 701 702 ret = RING_SPACE(chan, 11); 703 if (ret) 704 return ret; 705 706 BEGIN_RING(chan, NvSubM2MF, 707 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8); 708 OUT_RING (chan, src_offset); 709 OUT_RING (chan, dst_offset); 710 OUT_RING (chan, PAGE_SIZE); /* src_pitch */ 711 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */ 712 OUT_RING (chan, PAGE_SIZE); /* line_length */ 713 OUT_RING (chan, line_count); 714 OUT_RING (chan, 0x00000101); 715 OUT_RING (chan, 0x00000000); 716 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); 717 OUT_RING (chan, 0); 718 719 page_count -= line_count; 720 src_offset += (PAGE_SIZE * line_count); 721 dst_offset += (PAGE_SIZE * line_count); 722 } 723 724 return 0; 725 } 726 727 static int 728 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr, 729 bool no_wait_reserve, bool no_wait_gpu, 730 struct ttm_mem_reg *new_mem) 731 { 732 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 733 struct nouveau_bo *nvbo = nouveau_bo(bo); 734 struct nouveau_channel *chan; 735 int ret; 736 737 chan = nvbo->channel; 738 if (!chan || nvbo->no_vm) { 739 chan = dev_priv->channel; 740 mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX); 741 } 742 743 if (dev_priv->card_type < NV_50) 744 ret = nv04_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 745 else 746 if (dev_priv->card_type < NV_C0) 747 ret = nv50_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 748 else 749 ret = nvc0_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 750 if (ret == 0) { 751 ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict, 752 no_wait_reserve, 753 no_wait_gpu, new_mem); 754 } 755 756 if (chan == dev_priv->channel) 757 mutex_unlock(&chan->mutex); 758 return ret; 759 } 760 761 static int 762 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, 763 bool no_wait_reserve, bool no_wait_gpu, 764 struct ttm_mem_reg *new_mem) 765 { 766 u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; 767 struct ttm_placement placement; 768 struct ttm_mem_reg tmp_mem; 769 int ret; 770 771 placement.fpfn = placement.lpfn = 0; 772 placement.num_placement = placement.num_busy_placement = 1; 773 placement.placement = placement.busy_placement = &placement_memtype; 774 775 tmp_mem = *new_mem; 776 tmp_mem.mm_node = NULL; 777 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu); 778 if (ret) 779 return ret; 780 781 ret = ttm_tt_bind(bo->ttm, &tmp_mem); 782 if (ret) 783 goto out; 784 785 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem); 786 if (ret) 787 goto out; 788 789 ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem); 790 out: 791 ttm_bo_mem_put(bo, &tmp_mem); 792 return ret; 793 } 794 795 static int 796 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr, 797 bool no_wait_reserve, bool no_wait_gpu, 798 struct ttm_mem_reg *new_mem) 799 { 800 u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; 801 struct ttm_placement placement; 802 struct ttm_mem_reg tmp_mem; 803 int ret; 804 805 placement.fpfn = placement.lpfn = 0; 806 placement.num_placement = placement.num_busy_placement = 1; 807 placement.placement = placement.busy_placement = &placement_memtype; 808 809 tmp_mem = *new_mem; 810 tmp_mem.mm_node = NULL; 811 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu); 812 if (ret) 813 return ret; 814 815 ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem); 816 if (ret) 817 goto out; 818 819 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, new_mem); 820 if (ret) 821 goto out; 822 823 out: 824 ttm_bo_mem_put(bo, &tmp_mem); 825 return ret; 826 } 827 828 static int 829 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem, 830 struct nouveau_tile_reg **new_tile) 831 { 832 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 833 struct drm_device *dev = dev_priv->dev; 834 struct nouveau_bo *nvbo = nouveau_bo(bo); 835 uint64_t offset; 836 837 if (nvbo->no_vm || new_mem->mem_type != TTM_PL_VRAM) { 838 /* Nothing to do. */ 839 *new_tile = NULL; 840 return 0; 841 } 842 843 offset = new_mem->start << PAGE_SHIFT; 844 845 if (dev_priv->chan_vm) { 846 nouveau_vm_map(&nvbo->vma, new_mem->mm_node); 847 } else if (dev_priv->card_type >= NV_10) { 848 *new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size, 849 nvbo->tile_mode, 850 nvbo->tile_flags); 851 } 852 853 return 0; 854 } 855 856 static void 857 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, 858 struct nouveau_tile_reg *new_tile, 859 struct nouveau_tile_reg **old_tile) 860 { 861 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 862 struct drm_device *dev = dev_priv->dev; 863 864 if (dev_priv->card_type >= NV_10 && 865 dev_priv->card_type < NV_50) { 866 nv10_mem_put_tile_region(dev, *old_tile, bo->sync_obj); 867 *old_tile = new_tile; 868 } 869 } 870 871 static int 872 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr, 873 bool no_wait_reserve, bool no_wait_gpu, 874 struct ttm_mem_reg *new_mem) 875 { 876 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 877 struct nouveau_bo *nvbo = nouveau_bo(bo); 878 struct ttm_mem_reg *old_mem = &bo->mem; 879 struct nouveau_tile_reg *new_tile = NULL; 880 int ret = 0; 881 882 ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile); 883 if (ret) 884 return ret; 885 886 /* Fake bo copy. */ 887 if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) { 888 BUG_ON(bo->mem.mm_node != NULL); 889 bo->mem = *new_mem; 890 new_mem->mm_node = NULL; 891 goto out; 892 } 893 894 /* Software copy if the card isn't up and running yet. */ 895 if (!dev_priv->channel) { 896 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem); 897 goto out; 898 } 899 900 /* Hardware assisted copy. */ 901 if (new_mem->mem_type == TTM_PL_SYSTEM) 902 ret = nouveau_bo_move_flipd(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 903 else if (old_mem->mem_type == TTM_PL_SYSTEM) 904 ret = nouveau_bo_move_flips(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 905 else 906 ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 907 908 if (!ret) 909 goto out; 910 911 /* Fallback to software copy. */ 912 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem); 913 914 out: 915 if (ret) 916 nouveau_bo_vm_cleanup(bo, NULL, &new_tile); 917 else 918 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile); 919 920 return ret; 921 } 922 923 static int 924 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp) 925 { 926 return 0; 927 } 928 929 static int 930 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 931 { 932 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 933 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 934 struct drm_device *dev = dev_priv->dev; 935 int ret; 936 937 mem->bus.addr = NULL; 938 mem->bus.offset = 0; 939 mem->bus.size = mem->num_pages << PAGE_SHIFT; 940 mem->bus.base = 0; 941 mem->bus.is_iomem = false; 942 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 943 return -EINVAL; 944 switch (mem->mem_type) { 945 case TTM_PL_SYSTEM: 946 /* System memory */ 947 return 0; 948 case TTM_PL_TT: 949 #if __OS_HAS_AGP 950 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) { 951 mem->bus.offset = mem->start << PAGE_SHIFT; 952 mem->bus.base = dev_priv->gart_info.aper_base; 953 mem->bus.is_iomem = true; 954 } 955 #endif 956 break; 957 case TTM_PL_VRAM: 958 { 959 struct nouveau_vram *vram = mem->mm_node; 960 u8 page_shift; 961 962 if (!dev_priv->bar1_vm) { 963 mem->bus.offset = mem->start << PAGE_SHIFT; 964 mem->bus.base = pci_resource_start(dev->pdev, 1); 965 mem->bus.is_iomem = true; 966 break; 967 } 968 969 if (dev_priv->card_type == NV_C0) 970 page_shift = vram->page_shift; 971 else 972 page_shift = 12; 973 974 ret = nouveau_vm_get(dev_priv->bar1_vm, mem->bus.size, 975 page_shift, NV_MEM_ACCESS_RW, 976 &vram->bar_vma); 977 if (ret) 978 return ret; 979 980 nouveau_vm_map(&vram->bar_vma, vram); 981 if (ret) { 982 nouveau_vm_put(&vram->bar_vma); 983 return ret; 984 } 985 986 mem->bus.offset = vram->bar_vma.offset; 987 if (dev_priv->card_type == NV_50) /*XXX*/ 988 mem->bus.offset -= 0x0020000000ULL; 989 mem->bus.base = pci_resource_start(dev->pdev, 1); 990 mem->bus.is_iomem = true; 991 } 992 break; 993 default: 994 return -EINVAL; 995 } 996 return 0; 997 } 998 999 static void 1000 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1001 { 1002 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 1003 struct nouveau_vram *vram = mem->mm_node; 1004 1005 if (!dev_priv->bar1_vm || mem->mem_type != TTM_PL_VRAM) 1006 return; 1007 1008 if (!vram->bar_vma.node) 1009 return; 1010 1011 nouveau_vm_unmap(&vram->bar_vma); 1012 nouveau_vm_put(&vram->bar_vma); 1013 } 1014 1015 static int 1016 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) 1017 { 1018 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 1019 struct nouveau_bo *nvbo = nouveau_bo(bo); 1020 1021 /* as long as the bo isn't in vram, and isn't tiled, we've got 1022 * nothing to do here. 1023 */ 1024 if (bo->mem.mem_type != TTM_PL_VRAM) { 1025 if (dev_priv->card_type < NV_50 || 1026 !nouveau_bo_tile_layout(nvbo)) 1027 return 0; 1028 } 1029 1030 /* make sure bo is in mappable vram */ 1031 if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages) 1032 return 0; 1033 1034 1035 nvbo->placement.fpfn = 0; 1036 nvbo->placement.lpfn = dev_priv->fb_mappable_pages; 1037 nouveau_bo_placement_set(nvbo, TTM_PL_VRAM, 0); 1038 return nouveau_bo_validate(nvbo, false, true, false); 1039 } 1040 1041 void 1042 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence) 1043 { 1044 struct nouveau_fence *old_fence; 1045 1046 if (likely(fence)) 1047 nouveau_fence_ref(fence); 1048 1049 spin_lock(&nvbo->bo.bdev->fence_lock); 1050 old_fence = nvbo->bo.sync_obj; 1051 nvbo->bo.sync_obj = fence; 1052 spin_unlock(&nvbo->bo.bdev->fence_lock); 1053 1054 nouveau_fence_unref(&old_fence); 1055 } 1056 1057 struct ttm_bo_driver nouveau_bo_driver = { 1058 .create_ttm_backend_entry = nouveau_bo_create_ttm_backend_entry, 1059 .invalidate_caches = nouveau_bo_invalidate_caches, 1060 .init_mem_type = nouveau_bo_init_mem_type, 1061 .evict_flags = nouveau_bo_evict_flags, 1062 .move = nouveau_bo_move, 1063 .verify_access = nouveau_bo_verify_access, 1064 .sync_obj_signaled = __nouveau_fence_signalled, 1065 .sync_obj_wait = __nouveau_fence_wait, 1066 .sync_obj_flush = __nouveau_fence_flush, 1067 .sync_obj_unref = __nouveau_fence_unref, 1068 .sync_obj_ref = __nouveau_fence_ref, 1069 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify, 1070 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1071 .io_mem_free = &nouveau_ttm_io_mem_free, 1072 }; 1073 1074