1 /* 2 * Copyright 2007 Dave Airlied 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 /* 25 * Authors: Dave Airlied <airlied@linux.ie> 26 * Ben Skeggs <darktama@iinet.net.au> 27 * Jeremy Kolb <jkolb@brandeis.edu> 28 */ 29 30 #include "drmP.h" 31 #include "ttm/ttm_page_alloc.h" 32 33 #include "nouveau_drm.h" 34 #include "nouveau_drv.h" 35 #include "nouveau_dma.h" 36 #include "nouveau_mm.h" 37 #include "nouveau_vm.h" 38 39 #include <linux/log2.h> 40 #include <linux/slab.h> 41 42 static void 43 nouveau_bo_del_ttm(struct ttm_buffer_object *bo) 44 { 45 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 46 struct drm_device *dev = dev_priv->dev; 47 struct nouveau_bo *nvbo = nouveau_bo(bo); 48 49 if (unlikely(nvbo->gem)) 50 DRM_ERROR("bo %p still attached to GEM object\n", bo); 51 52 nv10_mem_put_tile_region(dev, nvbo->tile, NULL); 53 kfree(nvbo); 54 } 55 56 static void 57 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags, 58 int *align, int *size) 59 { 60 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 61 62 if (dev_priv->card_type < NV_50) { 63 if (nvbo->tile_mode) { 64 if (dev_priv->chipset >= 0x40) { 65 *align = 65536; 66 *size = roundup(*size, 64 * nvbo->tile_mode); 67 68 } else if (dev_priv->chipset >= 0x30) { 69 *align = 32768; 70 *size = roundup(*size, 64 * nvbo->tile_mode); 71 72 } else if (dev_priv->chipset >= 0x20) { 73 *align = 16384; 74 *size = roundup(*size, 64 * nvbo->tile_mode); 75 76 } else if (dev_priv->chipset >= 0x10) { 77 *align = 16384; 78 *size = roundup(*size, 32 * nvbo->tile_mode); 79 } 80 } 81 } else { 82 *size = roundup(*size, (1 << nvbo->page_shift)); 83 *align = max((1 << nvbo->page_shift), *align); 84 } 85 86 *size = roundup(*size, PAGE_SIZE); 87 } 88 89 int 90 nouveau_bo_new(struct drm_device *dev, int size, int align, 91 uint32_t flags, uint32_t tile_mode, uint32_t tile_flags, 92 struct nouveau_bo **pnvbo) 93 { 94 struct drm_nouveau_private *dev_priv = dev->dev_private; 95 struct nouveau_bo *nvbo; 96 size_t acc_size; 97 int ret; 98 99 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); 100 if (!nvbo) 101 return -ENOMEM; 102 INIT_LIST_HEAD(&nvbo->head); 103 INIT_LIST_HEAD(&nvbo->entry); 104 INIT_LIST_HEAD(&nvbo->vma_list); 105 nvbo->tile_mode = tile_mode; 106 nvbo->tile_flags = tile_flags; 107 nvbo->bo.bdev = &dev_priv->ttm.bdev; 108 109 nvbo->page_shift = 12; 110 if (dev_priv->bar1_vm) { 111 if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024) 112 nvbo->page_shift = dev_priv->bar1_vm->lpg_shift; 113 } 114 115 nouveau_bo_fixup_align(nvbo, flags, &align, &size); 116 nvbo->bo.mem.num_pages = size >> PAGE_SHIFT; 117 nouveau_bo_placement_set(nvbo, flags, 0); 118 119 acc_size = ttm_bo_dma_acc_size(&dev_priv->ttm.bdev, size, 120 sizeof(struct nouveau_bo)); 121 122 ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size, 123 ttm_bo_type_device, &nvbo->placement, 124 align >> PAGE_SHIFT, 0, false, NULL, acc_size, 125 nouveau_bo_del_ttm); 126 if (ret) { 127 /* ttm will call nouveau_bo_del_ttm if it fails.. */ 128 return ret; 129 } 130 131 *pnvbo = nvbo; 132 return 0; 133 } 134 135 static void 136 set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags) 137 { 138 *n = 0; 139 140 if (type & TTM_PL_FLAG_VRAM) 141 pl[(*n)++] = TTM_PL_FLAG_VRAM | flags; 142 if (type & TTM_PL_FLAG_TT) 143 pl[(*n)++] = TTM_PL_FLAG_TT | flags; 144 if (type & TTM_PL_FLAG_SYSTEM) 145 pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags; 146 } 147 148 static void 149 set_placement_range(struct nouveau_bo *nvbo, uint32_t type) 150 { 151 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 152 int vram_pages = dev_priv->vram_size >> PAGE_SHIFT; 153 154 if (dev_priv->card_type == NV_10 && 155 nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) && 156 nvbo->bo.mem.num_pages < vram_pages / 4) { 157 /* 158 * Make sure that the color and depth buffers are handled 159 * by independent memory controller units. Up to a 9x 160 * speed up when alpha-blending and depth-test are enabled 161 * at the same time. 162 */ 163 if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) { 164 nvbo->placement.fpfn = vram_pages / 2; 165 nvbo->placement.lpfn = ~0; 166 } else { 167 nvbo->placement.fpfn = 0; 168 nvbo->placement.lpfn = vram_pages / 2; 169 } 170 } 171 } 172 173 void 174 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy) 175 { 176 struct ttm_placement *pl = &nvbo->placement; 177 uint32_t flags = TTM_PL_MASK_CACHING | 178 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0); 179 180 pl->placement = nvbo->placements; 181 set_placement_list(nvbo->placements, &pl->num_placement, 182 type, flags); 183 184 pl->busy_placement = nvbo->busy_placements; 185 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement, 186 type | busy, flags); 187 188 set_placement_range(nvbo, type); 189 } 190 191 int 192 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype) 193 { 194 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 195 struct ttm_buffer_object *bo = &nvbo->bo; 196 int ret; 197 198 if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) { 199 NV_ERROR(nouveau_bdev(bo->bdev)->dev, 200 "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo, 201 1 << bo->mem.mem_type, memtype); 202 return -EINVAL; 203 } 204 205 if (nvbo->pin_refcnt++) 206 return 0; 207 208 ret = ttm_bo_reserve(bo, false, false, false, 0); 209 if (ret) 210 goto out; 211 212 nouveau_bo_placement_set(nvbo, memtype, 0); 213 214 ret = nouveau_bo_validate(nvbo, false, false, false); 215 if (ret == 0) { 216 switch (bo->mem.mem_type) { 217 case TTM_PL_VRAM: 218 dev_priv->fb_aper_free -= bo->mem.size; 219 break; 220 case TTM_PL_TT: 221 dev_priv->gart_info.aper_free -= bo->mem.size; 222 break; 223 default: 224 break; 225 } 226 } 227 ttm_bo_unreserve(bo); 228 out: 229 if (unlikely(ret)) 230 nvbo->pin_refcnt--; 231 return ret; 232 } 233 234 int 235 nouveau_bo_unpin(struct nouveau_bo *nvbo) 236 { 237 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 238 struct ttm_buffer_object *bo = &nvbo->bo; 239 int ret; 240 241 if (--nvbo->pin_refcnt) 242 return 0; 243 244 ret = ttm_bo_reserve(bo, false, false, false, 0); 245 if (ret) 246 return ret; 247 248 nouveau_bo_placement_set(nvbo, bo->mem.placement, 0); 249 250 ret = nouveau_bo_validate(nvbo, false, false, false); 251 if (ret == 0) { 252 switch (bo->mem.mem_type) { 253 case TTM_PL_VRAM: 254 dev_priv->fb_aper_free += bo->mem.size; 255 break; 256 case TTM_PL_TT: 257 dev_priv->gart_info.aper_free += bo->mem.size; 258 break; 259 default: 260 break; 261 } 262 } 263 264 ttm_bo_unreserve(bo); 265 return ret; 266 } 267 268 int 269 nouveau_bo_map(struct nouveau_bo *nvbo) 270 { 271 int ret; 272 273 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 274 if (ret) 275 return ret; 276 277 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap); 278 ttm_bo_unreserve(&nvbo->bo); 279 return ret; 280 } 281 282 void 283 nouveau_bo_unmap(struct nouveau_bo *nvbo) 284 { 285 if (nvbo) 286 ttm_bo_kunmap(&nvbo->kmap); 287 } 288 289 int 290 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, 291 bool no_wait_reserve, bool no_wait_gpu) 292 { 293 int ret; 294 295 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, interruptible, 296 no_wait_reserve, no_wait_gpu); 297 if (ret) 298 return ret; 299 300 return 0; 301 } 302 303 u16 304 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index) 305 { 306 bool is_iomem; 307 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 308 mem = &mem[index]; 309 if (is_iomem) 310 return ioread16_native((void __force __iomem *)mem); 311 else 312 return *mem; 313 } 314 315 void 316 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) 317 { 318 bool is_iomem; 319 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 320 mem = &mem[index]; 321 if (is_iomem) 322 iowrite16_native(val, (void __force __iomem *)mem); 323 else 324 *mem = val; 325 } 326 327 u32 328 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) 329 { 330 bool is_iomem; 331 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 332 mem = &mem[index]; 333 if (is_iomem) 334 return ioread32_native((void __force __iomem *)mem); 335 else 336 return *mem; 337 } 338 339 void 340 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) 341 { 342 bool is_iomem; 343 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 344 mem = &mem[index]; 345 if (is_iomem) 346 iowrite32_native(val, (void __force __iomem *)mem); 347 else 348 *mem = val; 349 } 350 351 static struct ttm_tt * 352 nouveau_ttm_tt_create(struct ttm_bo_device *bdev, 353 unsigned long size, uint32_t page_flags, 354 struct page *dummy_read_page) 355 { 356 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 357 struct drm_device *dev = dev_priv->dev; 358 359 switch (dev_priv->gart_info.type) { 360 #if __OS_HAS_AGP 361 case NOUVEAU_GART_AGP: 362 return ttm_agp_tt_create(bdev, dev->agp->bridge, 363 size, page_flags, dummy_read_page); 364 #endif 365 case NOUVEAU_GART_PDMA: 366 case NOUVEAU_GART_HW: 367 return nouveau_sgdma_create_ttm(bdev, size, page_flags, 368 dummy_read_page); 369 default: 370 NV_ERROR(dev, "Unknown GART type %d\n", 371 dev_priv->gart_info.type); 372 break; 373 } 374 375 return NULL; 376 } 377 378 static int 379 nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) 380 { 381 /* We'll do this from user space. */ 382 return 0; 383 } 384 385 static int 386 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 387 struct ttm_mem_type_manager *man) 388 { 389 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 390 struct drm_device *dev = dev_priv->dev; 391 392 switch (type) { 393 case TTM_PL_SYSTEM: 394 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 395 man->available_caching = TTM_PL_MASK_CACHING; 396 man->default_caching = TTM_PL_FLAG_CACHED; 397 break; 398 case TTM_PL_VRAM: 399 if (dev_priv->card_type >= NV_50) { 400 man->func = &nouveau_vram_manager; 401 man->io_reserve_fastpath = false; 402 man->use_io_reserve_lru = true; 403 } else { 404 man->func = &ttm_bo_manager_func; 405 } 406 man->flags = TTM_MEMTYPE_FLAG_FIXED | 407 TTM_MEMTYPE_FLAG_MAPPABLE; 408 man->available_caching = TTM_PL_FLAG_UNCACHED | 409 TTM_PL_FLAG_WC; 410 man->default_caching = TTM_PL_FLAG_WC; 411 break; 412 case TTM_PL_TT: 413 if (dev_priv->card_type >= NV_50) 414 man->func = &nouveau_gart_manager; 415 else 416 man->func = &ttm_bo_manager_func; 417 switch (dev_priv->gart_info.type) { 418 case NOUVEAU_GART_AGP: 419 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 420 man->available_caching = TTM_PL_FLAG_UNCACHED | 421 TTM_PL_FLAG_WC; 422 man->default_caching = TTM_PL_FLAG_WC; 423 break; 424 case NOUVEAU_GART_PDMA: 425 case NOUVEAU_GART_HW: 426 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | 427 TTM_MEMTYPE_FLAG_CMA; 428 man->available_caching = TTM_PL_MASK_CACHING; 429 man->default_caching = TTM_PL_FLAG_CACHED; 430 break; 431 default: 432 NV_ERROR(dev, "Unknown GART type: %d\n", 433 dev_priv->gart_info.type); 434 return -EINVAL; 435 } 436 break; 437 default: 438 NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type); 439 return -EINVAL; 440 } 441 return 0; 442 } 443 444 static void 445 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) 446 { 447 struct nouveau_bo *nvbo = nouveau_bo(bo); 448 449 switch (bo->mem.mem_type) { 450 case TTM_PL_VRAM: 451 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 452 TTM_PL_FLAG_SYSTEM); 453 break; 454 default: 455 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0); 456 break; 457 } 458 459 *pl = nvbo->placement; 460 } 461 462 463 /* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access 464 * TTM_PL_{VRAM,TT} directly. 465 */ 466 467 static int 468 nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan, 469 struct nouveau_bo *nvbo, bool evict, 470 bool no_wait_reserve, bool no_wait_gpu, 471 struct ttm_mem_reg *new_mem) 472 { 473 struct nouveau_fence *fence = NULL; 474 int ret; 475 476 ret = nouveau_fence_new(chan, &fence, true); 477 if (ret) 478 return ret; 479 480 ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict, 481 no_wait_reserve, no_wait_gpu, new_mem); 482 nouveau_fence_unref(&fence); 483 return ret; 484 } 485 486 static int 487 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 488 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 489 { 490 struct nouveau_mem *node = old_mem->mm_node; 491 u64 src_offset = node->vma[0].offset; 492 u64 dst_offset = node->vma[1].offset; 493 u32 page_count = new_mem->num_pages; 494 int ret; 495 496 page_count = new_mem->num_pages; 497 while (page_count) { 498 int line_count = (page_count > 2047) ? 2047 : page_count; 499 500 ret = RING_SPACE(chan, 12); 501 if (ret) 502 return ret; 503 504 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0238, 2); 505 OUT_RING (chan, upper_32_bits(dst_offset)); 506 OUT_RING (chan, lower_32_bits(dst_offset)); 507 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x030c, 6); 508 OUT_RING (chan, upper_32_bits(src_offset)); 509 OUT_RING (chan, lower_32_bits(src_offset)); 510 OUT_RING (chan, PAGE_SIZE); /* src_pitch */ 511 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */ 512 OUT_RING (chan, PAGE_SIZE); /* line_length */ 513 OUT_RING (chan, line_count); 514 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0300, 1); 515 OUT_RING (chan, 0x00100110); 516 517 page_count -= line_count; 518 src_offset += (PAGE_SIZE * line_count); 519 dst_offset += (PAGE_SIZE * line_count); 520 } 521 522 return 0; 523 } 524 525 static int 526 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 527 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 528 { 529 struct nouveau_mem *node = old_mem->mm_node; 530 struct nouveau_bo *nvbo = nouveau_bo(bo); 531 u64 length = (new_mem->num_pages << PAGE_SHIFT); 532 u64 src_offset = node->vma[0].offset; 533 u64 dst_offset = node->vma[1].offset; 534 int ret; 535 536 while (length) { 537 u32 amount, stride, height; 538 539 amount = min(length, (u64)(4 * 1024 * 1024)); 540 stride = 16 * 4; 541 height = amount / stride; 542 543 if (new_mem->mem_type == TTM_PL_VRAM && 544 nouveau_bo_tile_layout(nvbo)) { 545 ret = RING_SPACE(chan, 8); 546 if (ret) 547 return ret; 548 549 BEGIN_RING(chan, NvSubM2MF, 0x0200, 7); 550 OUT_RING (chan, 0); 551 OUT_RING (chan, 0); 552 OUT_RING (chan, stride); 553 OUT_RING (chan, height); 554 OUT_RING (chan, 1); 555 OUT_RING (chan, 0); 556 OUT_RING (chan, 0); 557 } else { 558 ret = RING_SPACE(chan, 2); 559 if (ret) 560 return ret; 561 562 BEGIN_RING(chan, NvSubM2MF, 0x0200, 1); 563 OUT_RING (chan, 1); 564 } 565 if (old_mem->mem_type == TTM_PL_VRAM && 566 nouveau_bo_tile_layout(nvbo)) { 567 ret = RING_SPACE(chan, 8); 568 if (ret) 569 return ret; 570 571 BEGIN_RING(chan, NvSubM2MF, 0x021c, 7); 572 OUT_RING (chan, 0); 573 OUT_RING (chan, 0); 574 OUT_RING (chan, stride); 575 OUT_RING (chan, height); 576 OUT_RING (chan, 1); 577 OUT_RING (chan, 0); 578 OUT_RING (chan, 0); 579 } else { 580 ret = RING_SPACE(chan, 2); 581 if (ret) 582 return ret; 583 584 BEGIN_RING(chan, NvSubM2MF, 0x021c, 1); 585 OUT_RING (chan, 1); 586 } 587 588 ret = RING_SPACE(chan, 14); 589 if (ret) 590 return ret; 591 592 BEGIN_RING(chan, NvSubM2MF, 0x0238, 2); 593 OUT_RING (chan, upper_32_bits(src_offset)); 594 OUT_RING (chan, upper_32_bits(dst_offset)); 595 BEGIN_RING(chan, NvSubM2MF, 0x030c, 8); 596 OUT_RING (chan, lower_32_bits(src_offset)); 597 OUT_RING (chan, lower_32_bits(dst_offset)); 598 OUT_RING (chan, stride); 599 OUT_RING (chan, stride); 600 OUT_RING (chan, stride); 601 OUT_RING (chan, height); 602 OUT_RING (chan, 0x00000101); 603 OUT_RING (chan, 0x00000000); 604 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); 605 OUT_RING (chan, 0); 606 607 length -= amount; 608 src_offset += amount; 609 dst_offset += amount; 610 } 611 612 return 0; 613 } 614 615 static inline uint32_t 616 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo, 617 struct nouveau_channel *chan, struct ttm_mem_reg *mem) 618 { 619 if (mem->mem_type == TTM_PL_TT) 620 return chan->gart_handle; 621 return chan->vram_handle; 622 } 623 624 static int 625 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 626 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 627 { 628 u32 src_offset = old_mem->start << PAGE_SHIFT; 629 u32 dst_offset = new_mem->start << PAGE_SHIFT; 630 u32 page_count = new_mem->num_pages; 631 int ret; 632 633 ret = RING_SPACE(chan, 3); 634 if (ret) 635 return ret; 636 637 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2); 638 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem)); 639 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem)); 640 641 page_count = new_mem->num_pages; 642 while (page_count) { 643 int line_count = (page_count > 2047) ? 2047 : page_count; 644 645 ret = RING_SPACE(chan, 11); 646 if (ret) 647 return ret; 648 649 BEGIN_RING(chan, NvSubM2MF, 650 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8); 651 OUT_RING (chan, src_offset); 652 OUT_RING (chan, dst_offset); 653 OUT_RING (chan, PAGE_SIZE); /* src_pitch */ 654 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */ 655 OUT_RING (chan, PAGE_SIZE); /* line_length */ 656 OUT_RING (chan, line_count); 657 OUT_RING (chan, 0x00000101); 658 OUT_RING (chan, 0x00000000); 659 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); 660 OUT_RING (chan, 0); 661 662 page_count -= line_count; 663 src_offset += (PAGE_SIZE * line_count); 664 dst_offset += (PAGE_SIZE * line_count); 665 } 666 667 return 0; 668 } 669 670 static int 671 nouveau_vma_getmap(struct nouveau_channel *chan, struct nouveau_bo *nvbo, 672 struct ttm_mem_reg *mem, struct nouveau_vma *vma) 673 { 674 struct nouveau_mem *node = mem->mm_node; 675 int ret; 676 677 ret = nouveau_vm_get(chan->vm, mem->num_pages << PAGE_SHIFT, 678 node->page_shift, NV_MEM_ACCESS_RO, vma); 679 if (ret) 680 return ret; 681 682 if (mem->mem_type == TTM_PL_VRAM) 683 nouveau_vm_map(vma, node); 684 else 685 nouveau_vm_map_sg(vma, 0, mem->num_pages << PAGE_SHIFT, node); 686 687 return 0; 688 } 689 690 static int 691 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr, 692 bool no_wait_reserve, bool no_wait_gpu, 693 struct ttm_mem_reg *new_mem) 694 { 695 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 696 struct nouveau_channel *chan = chan = dev_priv->channel; 697 struct nouveau_bo *nvbo = nouveau_bo(bo); 698 struct ttm_mem_reg *old_mem = &bo->mem; 699 int ret; 700 701 mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX); 702 703 /* create temporary vmas for the transfer and attach them to the 704 * old nouveau_mem node, these will get cleaned up after ttm has 705 * destroyed the ttm_mem_reg 706 */ 707 if (dev_priv->card_type >= NV_50) { 708 struct nouveau_mem *node = old_mem->mm_node; 709 710 ret = nouveau_vma_getmap(chan, nvbo, old_mem, &node->vma[0]); 711 if (ret) 712 goto out; 713 714 ret = nouveau_vma_getmap(chan, nvbo, new_mem, &node->vma[1]); 715 if (ret) 716 goto out; 717 } 718 719 if (dev_priv->card_type < NV_50) 720 ret = nv04_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 721 else 722 if (dev_priv->card_type < NV_C0) 723 ret = nv50_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 724 else 725 ret = nvc0_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 726 if (ret == 0) { 727 ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict, 728 no_wait_reserve, 729 no_wait_gpu, new_mem); 730 } 731 732 out: 733 mutex_unlock(&chan->mutex); 734 return ret; 735 } 736 737 static int 738 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, 739 bool no_wait_reserve, bool no_wait_gpu, 740 struct ttm_mem_reg *new_mem) 741 { 742 u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; 743 struct ttm_placement placement; 744 struct ttm_mem_reg tmp_mem; 745 int ret; 746 747 placement.fpfn = placement.lpfn = 0; 748 placement.num_placement = placement.num_busy_placement = 1; 749 placement.placement = placement.busy_placement = &placement_memtype; 750 751 tmp_mem = *new_mem; 752 tmp_mem.mm_node = NULL; 753 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu); 754 if (ret) 755 return ret; 756 757 ret = ttm_tt_bind(bo->ttm, &tmp_mem); 758 if (ret) 759 goto out; 760 761 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem); 762 if (ret) 763 goto out; 764 765 ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem); 766 out: 767 ttm_bo_mem_put(bo, &tmp_mem); 768 return ret; 769 } 770 771 static int 772 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr, 773 bool no_wait_reserve, bool no_wait_gpu, 774 struct ttm_mem_reg *new_mem) 775 { 776 u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; 777 struct ttm_placement placement; 778 struct ttm_mem_reg tmp_mem; 779 int ret; 780 781 placement.fpfn = placement.lpfn = 0; 782 placement.num_placement = placement.num_busy_placement = 1; 783 placement.placement = placement.busy_placement = &placement_memtype; 784 785 tmp_mem = *new_mem; 786 tmp_mem.mm_node = NULL; 787 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu); 788 if (ret) 789 return ret; 790 791 ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem); 792 if (ret) 793 goto out; 794 795 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, new_mem); 796 if (ret) 797 goto out; 798 799 out: 800 ttm_bo_mem_put(bo, &tmp_mem); 801 return ret; 802 } 803 804 static void 805 nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem) 806 { 807 struct nouveau_bo *nvbo = nouveau_bo(bo); 808 struct nouveau_vma *vma; 809 810 /* ttm can now (stupidly) pass the driver bos it didn't create... */ 811 if (bo->destroy != nouveau_bo_del_ttm) 812 return; 813 814 list_for_each_entry(vma, &nvbo->vma_list, head) { 815 if (new_mem && new_mem->mem_type == TTM_PL_VRAM) { 816 nouveau_vm_map(vma, new_mem->mm_node); 817 } else 818 if (new_mem && new_mem->mem_type == TTM_PL_TT && 819 nvbo->page_shift == vma->vm->spg_shift) { 820 nouveau_vm_map_sg(vma, 0, new_mem-> 821 num_pages << PAGE_SHIFT, 822 new_mem->mm_node); 823 } else { 824 nouveau_vm_unmap(vma); 825 } 826 } 827 } 828 829 static int 830 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem, 831 struct nouveau_tile_reg **new_tile) 832 { 833 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 834 struct drm_device *dev = dev_priv->dev; 835 struct nouveau_bo *nvbo = nouveau_bo(bo); 836 u64 offset = new_mem->start << PAGE_SHIFT; 837 838 *new_tile = NULL; 839 if (new_mem->mem_type != TTM_PL_VRAM) 840 return 0; 841 842 if (dev_priv->card_type >= NV_10) { 843 *new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size, 844 nvbo->tile_mode, 845 nvbo->tile_flags); 846 } 847 848 return 0; 849 } 850 851 static void 852 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, 853 struct nouveau_tile_reg *new_tile, 854 struct nouveau_tile_reg **old_tile) 855 { 856 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 857 struct drm_device *dev = dev_priv->dev; 858 859 nv10_mem_put_tile_region(dev, *old_tile, bo->sync_obj); 860 *old_tile = new_tile; 861 } 862 863 static int 864 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr, 865 bool no_wait_reserve, bool no_wait_gpu, 866 struct ttm_mem_reg *new_mem) 867 { 868 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 869 struct nouveau_bo *nvbo = nouveau_bo(bo); 870 struct ttm_mem_reg *old_mem = &bo->mem; 871 struct nouveau_tile_reg *new_tile = NULL; 872 int ret = 0; 873 874 if (dev_priv->card_type < NV_50) { 875 ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile); 876 if (ret) 877 return ret; 878 } 879 880 /* Fake bo copy. */ 881 if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) { 882 BUG_ON(bo->mem.mm_node != NULL); 883 bo->mem = *new_mem; 884 new_mem->mm_node = NULL; 885 goto out; 886 } 887 888 /* Software copy if the card isn't up and running yet. */ 889 if (!dev_priv->channel) { 890 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem); 891 goto out; 892 } 893 894 /* Hardware assisted copy. */ 895 if (new_mem->mem_type == TTM_PL_SYSTEM) 896 ret = nouveau_bo_move_flipd(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 897 else if (old_mem->mem_type == TTM_PL_SYSTEM) 898 ret = nouveau_bo_move_flips(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 899 else 900 ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 901 902 if (!ret) 903 goto out; 904 905 /* Fallback to software copy. */ 906 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem); 907 908 out: 909 if (dev_priv->card_type < NV_50) { 910 if (ret) 911 nouveau_bo_vm_cleanup(bo, NULL, &new_tile); 912 else 913 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile); 914 } 915 916 return ret; 917 } 918 919 static int 920 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp) 921 { 922 return 0; 923 } 924 925 static int 926 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 927 { 928 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 929 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 930 struct drm_device *dev = dev_priv->dev; 931 int ret; 932 933 mem->bus.addr = NULL; 934 mem->bus.offset = 0; 935 mem->bus.size = mem->num_pages << PAGE_SHIFT; 936 mem->bus.base = 0; 937 mem->bus.is_iomem = false; 938 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 939 return -EINVAL; 940 switch (mem->mem_type) { 941 case TTM_PL_SYSTEM: 942 /* System memory */ 943 return 0; 944 case TTM_PL_TT: 945 #if __OS_HAS_AGP 946 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) { 947 mem->bus.offset = mem->start << PAGE_SHIFT; 948 mem->bus.base = dev_priv->gart_info.aper_base; 949 mem->bus.is_iomem = true; 950 } 951 #endif 952 break; 953 case TTM_PL_VRAM: 954 { 955 struct nouveau_mem *node = mem->mm_node; 956 u8 page_shift; 957 958 if (!dev_priv->bar1_vm) { 959 mem->bus.offset = mem->start << PAGE_SHIFT; 960 mem->bus.base = pci_resource_start(dev->pdev, 1); 961 mem->bus.is_iomem = true; 962 break; 963 } 964 965 if (dev_priv->card_type >= NV_C0) 966 page_shift = node->page_shift; 967 else 968 page_shift = 12; 969 970 ret = nouveau_vm_get(dev_priv->bar1_vm, mem->bus.size, 971 page_shift, NV_MEM_ACCESS_RW, 972 &node->bar_vma); 973 if (ret) 974 return ret; 975 976 nouveau_vm_map(&node->bar_vma, node); 977 if (ret) { 978 nouveau_vm_put(&node->bar_vma); 979 return ret; 980 } 981 982 mem->bus.offset = node->bar_vma.offset; 983 if (dev_priv->card_type == NV_50) /*XXX*/ 984 mem->bus.offset -= 0x0020000000ULL; 985 mem->bus.base = pci_resource_start(dev->pdev, 1); 986 mem->bus.is_iomem = true; 987 } 988 break; 989 default: 990 return -EINVAL; 991 } 992 return 0; 993 } 994 995 static void 996 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 997 { 998 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 999 struct nouveau_mem *node = mem->mm_node; 1000 1001 if (!dev_priv->bar1_vm || mem->mem_type != TTM_PL_VRAM) 1002 return; 1003 1004 if (!node->bar_vma.node) 1005 return; 1006 1007 nouveau_vm_unmap(&node->bar_vma); 1008 nouveau_vm_put(&node->bar_vma); 1009 } 1010 1011 static int 1012 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) 1013 { 1014 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 1015 struct nouveau_bo *nvbo = nouveau_bo(bo); 1016 1017 /* as long as the bo isn't in vram, and isn't tiled, we've got 1018 * nothing to do here. 1019 */ 1020 if (bo->mem.mem_type != TTM_PL_VRAM) { 1021 if (dev_priv->card_type < NV_50 || 1022 !nouveau_bo_tile_layout(nvbo)) 1023 return 0; 1024 } 1025 1026 /* make sure bo is in mappable vram */ 1027 if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages) 1028 return 0; 1029 1030 1031 nvbo->placement.fpfn = 0; 1032 nvbo->placement.lpfn = dev_priv->fb_mappable_pages; 1033 nouveau_bo_placement_set(nvbo, TTM_PL_VRAM, 0); 1034 return nouveau_bo_validate(nvbo, false, true, false); 1035 } 1036 1037 void 1038 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence) 1039 { 1040 struct nouveau_fence *old_fence; 1041 1042 if (likely(fence)) 1043 nouveau_fence_ref(fence); 1044 1045 spin_lock(&nvbo->bo.bdev->fence_lock); 1046 old_fence = nvbo->bo.sync_obj; 1047 nvbo->bo.sync_obj = fence; 1048 spin_unlock(&nvbo->bo.bdev->fence_lock); 1049 1050 nouveau_fence_unref(&old_fence); 1051 } 1052 1053 static int 1054 nouveau_ttm_tt_populate(struct ttm_tt *ttm) 1055 { 1056 struct ttm_dma_tt *ttm_dma = (void *)ttm; 1057 struct drm_nouveau_private *dev_priv; 1058 struct drm_device *dev; 1059 unsigned i; 1060 int r; 1061 1062 if (ttm->state != tt_unpopulated) 1063 return 0; 1064 1065 dev_priv = nouveau_bdev(ttm->bdev); 1066 dev = dev_priv->dev; 1067 1068 #if __OS_HAS_AGP 1069 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) { 1070 return ttm_agp_tt_populate(ttm); 1071 } 1072 #endif 1073 1074 #ifdef CONFIG_SWIOTLB 1075 if (swiotlb_nr_tbl()) { 1076 return ttm_dma_populate((void *)ttm, dev->dev); 1077 } 1078 #endif 1079 1080 r = ttm_pool_populate(ttm); 1081 if (r) { 1082 return r; 1083 } 1084 1085 for (i = 0; i < ttm->num_pages; i++) { 1086 ttm_dma->dma_address[i] = pci_map_page(dev->pdev, ttm->pages[i], 1087 0, PAGE_SIZE, 1088 PCI_DMA_BIDIRECTIONAL); 1089 if (pci_dma_mapping_error(dev->pdev, ttm_dma->dma_address[i])) { 1090 while (--i) { 1091 pci_unmap_page(dev->pdev, ttm_dma->dma_address[i], 1092 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 1093 ttm_dma->dma_address[i] = 0; 1094 } 1095 ttm_pool_unpopulate(ttm); 1096 return -EFAULT; 1097 } 1098 } 1099 return 0; 1100 } 1101 1102 static void 1103 nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm) 1104 { 1105 struct ttm_dma_tt *ttm_dma = (void *)ttm; 1106 struct drm_nouveau_private *dev_priv; 1107 struct drm_device *dev; 1108 unsigned i; 1109 1110 dev_priv = nouveau_bdev(ttm->bdev); 1111 dev = dev_priv->dev; 1112 1113 #if __OS_HAS_AGP 1114 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) { 1115 ttm_agp_tt_unpopulate(ttm); 1116 return; 1117 } 1118 #endif 1119 1120 #ifdef CONFIG_SWIOTLB 1121 if (swiotlb_nr_tbl()) { 1122 ttm_dma_unpopulate((void *)ttm, dev->dev); 1123 return; 1124 } 1125 #endif 1126 1127 for (i = 0; i < ttm->num_pages; i++) { 1128 if (ttm_dma->dma_address[i]) { 1129 pci_unmap_page(dev->pdev, ttm_dma->dma_address[i], 1130 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 1131 } 1132 } 1133 1134 ttm_pool_unpopulate(ttm); 1135 } 1136 1137 struct ttm_bo_driver nouveau_bo_driver = { 1138 .ttm_tt_create = &nouveau_ttm_tt_create, 1139 .ttm_tt_populate = &nouveau_ttm_tt_populate, 1140 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate, 1141 .invalidate_caches = nouveau_bo_invalidate_caches, 1142 .init_mem_type = nouveau_bo_init_mem_type, 1143 .evict_flags = nouveau_bo_evict_flags, 1144 .move_notify = nouveau_bo_move_ntfy, 1145 .move = nouveau_bo_move, 1146 .verify_access = nouveau_bo_verify_access, 1147 .sync_obj_signaled = __nouveau_fence_signalled, 1148 .sync_obj_wait = __nouveau_fence_wait, 1149 .sync_obj_flush = __nouveau_fence_flush, 1150 .sync_obj_unref = __nouveau_fence_unref, 1151 .sync_obj_ref = __nouveau_fence_ref, 1152 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify, 1153 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1154 .io_mem_free = &nouveau_ttm_io_mem_free, 1155 }; 1156 1157 struct nouveau_vma * 1158 nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nouveau_vm *vm) 1159 { 1160 struct nouveau_vma *vma; 1161 list_for_each_entry(vma, &nvbo->vma_list, head) { 1162 if (vma->vm == vm) 1163 return vma; 1164 } 1165 1166 return NULL; 1167 } 1168 1169 int 1170 nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nouveau_vm *vm, 1171 struct nouveau_vma *vma) 1172 { 1173 const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 1174 struct nouveau_mem *node = nvbo->bo.mem.mm_node; 1175 int ret; 1176 1177 ret = nouveau_vm_get(vm, size, nvbo->page_shift, 1178 NV_MEM_ACCESS_RW, vma); 1179 if (ret) 1180 return ret; 1181 1182 if (nvbo->bo.mem.mem_type == TTM_PL_VRAM) 1183 nouveau_vm_map(vma, nvbo->bo.mem.mm_node); 1184 else 1185 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 1186 nouveau_vm_map_sg(vma, 0, size, node); 1187 1188 list_add_tail(&vma->head, &nvbo->vma_list); 1189 vma->refcount = 1; 1190 return 0; 1191 } 1192 1193 void 1194 nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 1195 { 1196 if (vma->node) { 1197 if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM) { 1198 spin_lock(&nvbo->bo.bdev->fence_lock); 1199 ttm_bo_wait(&nvbo->bo, false, false, false); 1200 spin_unlock(&nvbo->bo.bdev->fence_lock); 1201 nouveau_vm_unmap(vma); 1202 } 1203 1204 nouveau_vm_put(vma); 1205 list_del(&vma->head); 1206 } 1207 } 1208