1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 33 #include <linux/io.h> 34 #include <linux/list.h> 35 #include <linux/slab.h> 36 37 #include <drm/drm_cache.h> 38 #include <drm/drm_prime.h> 39 #include <drm/radeon_drm.h> 40 41 #include "radeon.h" 42 #include "radeon_trace.h" 43 44 int radeon_ttm_init(struct radeon_device *rdev); 45 void radeon_ttm_fini(struct radeon_device *rdev); 46 static void radeon_bo_clear_surface_reg(struct radeon_bo *bo); 47 48 /* 49 * To exclude mutual BO access we rely on bo_reserve exclusion, as all 50 * function are calling it. 51 */ 52 53 static void radeon_update_memory_usage(struct radeon_bo *bo, 54 unsigned mem_type, int sign) 55 { 56 struct radeon_device *rdev = bo->rdev; 57 u64 size = (u64)bo->tbo.num_pages << PAGE_SHIFT; 58 59 switch (mem_type) { 60 case TTM_PL_TT: 61 if (sign > 0) 62 atomic64_add(size, &rdev->gtt_usage); 63 else 64 atomic64_sub(size, &rdev->gtt_usage); 65 break; 66 case TTM_PL_VRAM: 67 if (sign > 0) 68 atomic64_add(size, &rdev->vram_usage); 69 else 70 atomic64_sub(size, &rdev->vram_usage); 71 break; 72 } 73 } 74 75 static void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo) 76 { 77 struct radeon_bo *bo; 78 79 bo = container_of(tbo, struct radeon_bo, tbo); 80 81 radeon_update_memory_usage(bo, bo->tbo.mem.mem_type, -1); 82 83 mutex_lock(&bo->rdev->gem.mutex); 84 list_del_init(&bo->list); 85 mutex_unlock(&bo->rdev->gem.mutex); 86 radeon_bo_clear_surface_reg(bo); 87 WARN_ON_ONCE(!list_empty(&bo->va)); 88 if (bo->tbo.base.import_attach) 89 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); 90 drm_gem_object_release(&bo->tbo.base); 91 kfree(bo); 92 } 93 94 bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo) 95 { 96 if (bo->destroy == &radeon_ttm_bo_destroy) 97 return true; 98 return false; 99 } 100 101 void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain) 102 { 103 u32 c = 0, i; 104 105 rbo->placement.placement = rbo->placements; 106 rbo->placement.busy_placement = rbo->placements; 107 if (domain & RADEON_GEM_DOMAIN_VRAM) { 108 /* Try placing BOs which don't need CPU access outside of the 109 * CPU accessible part of VRAM 110 */ 111 if ((rbo->flags & RADEON_GEM_NO_CPU_ACCESS) && 112 rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size) { 113 rbo->placements[c].fpfn = 114 rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; 115 rbo->placements[c].mem_type = TTM_PL_VRAM; 116 rbo->placements[c++].flags = 0; 117 } 118 119 rbo->placements[c].fpfn = 0; 120 rbo->placements[c].mem_type = TTM_PL_VRAM; 121 rbo->placements[c++].flags = 0; 122 } 123 124 if (domain & RADEON_GEM_DOMAIN_GTT) { 125 rbo->placements[c].fpfn = 0; 126 rbo->placements[c].mem_type = TTM_PL_TT; 127 rbo->placements[c++].flags = 0; 128 } 129 130 if (domain & RADEON_GEM_DOMAIN_CPU) { 131 rbo->placements[c].fpfn = 0; 132 rbo->placements[c].mem_type = TTM_PL_SYSTEM; 133 rbo->placements[c++].flags = 0; 134 } 135 if (!c) { 136 rbo->placements[c].fpfn = 0; 137 rbo->placements[c].mem_type = TTM_PL_SYSTEM; 138 rbo->placements[c++].flags = 0; 139 } 140 141 rbo->placement.num_placement = c; 142 rbo->placement.num_busy_placement = c; 143 144 for (i = 0; i < c; ++i) { 145 if ((rbo->flags & RADEON_GEM_CPU_ACCESS) && 146 (rbo->placements[i].mem_type == TTM_PL_VRAM) && 147 !rbo->placements[i].fpfn) 148 rbo->placements[i].lpfn = 149 rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; 150 else 151 rbo->placements[i].lpfn = 0; 152 } 153 } 154 155 int radeon_bo_create(struct radeon_device *rdev, 156 unsigned long size, int byte_align, bool kernel, 157 u32 domain, u32 flags, struct sg_table *sg, 158 struct dma_resv *resv, 159 struct radeon_bo **bo_ptr) 160 { 161 struct radeon_bo *bo; 162 enum ttm_bo_type type; 163 unsigned long page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT; 164 size_t acc_size; 165 int r; 166 167 size = ALIGN(size, PAGE_SIZE); 168 169 if (kernel) { 170 type = ttm_bo_type_kernel; 171 } else if (sg) { 172 type = ttm_bo_type_sg; 173 } else { 174 type = ttm_bo_type_device; 175 } 176 *bo_ptr = NULL; 177 178 acc_size = ttm_bo_dma_acc_size(&rdev->mman.bdev, size, 179 sizeof(struct radeon_bo)); 180 181 bo = kzalloc(sizeof(struct radeon_bo), GFP_KERNEL); 182 if (bo == NULL) 183 return -ENOMEM; 184 drm_gem_private_object_init(rdev->ddev, &bo->tbo.base, size); 185 bo->rdev = rdev; 186 bo->surface_reg = -1; 187 INIT_LIST_HEAD(&bo->list); 188 INIT_LIST_HEAD(&bo->va); 189 bo->initial_domain = domain & (RADEON_GEM_DOMAIN_VRAM | 190 RADEON_GEM_DOMAIN_GTT | 191 RADEON_GEM_DOMAIN_CPU); 192 193 bo->flags = flags; 194 /* PCI GART is always snooped */ 195 if (!(rdev->flags & RADEON_IS_PCIE)) 196 bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC); 197 198 /* Write-combined CPU mappings of GTT cause GPU hangs with RV6xx 199 * See https://bugs.freedesktop.org/show_bug.cgi?id=91268 200 */ 201 if (rdev->family >= CHIP_RV610 && rdev->family <= CHIP_RV635) 202 bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC); 203 204 #ifdef CONFIG_X86_32 205 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit 206 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 207 */ 208 bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC); 209 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) 210 /* Don't try to enable write-combining when it can't work, or things 211 * may be slow 212 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 213 */ 214 #ifndef CONFIG_COMPILE_TEST 215 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ 216 thanks to write-combining 217 #endif 218 219 if (bo->flags & RADEON_GEM_GTT_WC) 220 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " 221 "better performance thanks to write-combining\n"); 222 bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC); 223 #else 224 /* For architectures that don't support WC memory, 225 * mask out the WC flag from the BO 226 */ 227 if (!drm_arch_can_wc_memory()) 228 bo->flags &= ~RADEON_GEM_GTT_WC; 229 #endif 230 231 radeon_ttm_placement_from_domain(bo, domain); 232 /* Kernel allocation are uninterruptible */ 233 down_read(&rdev->pm.mclk_lock); 234 r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type, 235 &bo->placement, page_align, !kernel, acc_size, 236 sg, resv, &radeon_ttm_bo_destroy); 237 up_read(&rdev->pm.mclk_lock); 238 if (unlikely(r != 0)) { 239 return r; 240 } 241 *bo_ptr = bo; 242 243 trace_radeon_bo_create(bo); 244 245 return 0; 246 } 247 248 int radeon_bo_kmap(struct radeon_bo *bo, void **ptr) 249 { 250 bool is_iomem; 251 int r; 252 253 if (bo->kptr) { 254 if (ptr) { 255 *ptr = bo->kptr; 256 } 257 return 0; 258 } 259 r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap); 260 if (r) { 261 return r; 262 } 263 bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 264 if (ptr) { 265 *ptr = bo->kptr; 266 } 267 radeon_bo_check_tiling(bo, 0, 0); 268 return 0; 269 } 270 271 void radeon_bo_kunmap(struct radeon_bo *bo) 272 { 273 if (bo->kptr == NULL) 274 return; 275 bo->kptr = NULL; 276 radeon_bo_check_tiling(bo, 0, 0); 277 ttm_bo_kunmap(&bo->kmap); 278 } 279 280 struct radeon_bo *radeon_bo_ref(struct radeon_bo *bo) 281 { 282 if (bo == NULL) 283 return NULL; 284 285 ttm_bo_get(&bo->tbo); 286 return bo; 287 } 288 289 void radeon_bo_unref(struct radeon_bo **bo) 290 { 291 struct ttm_buffer_object *tbo; 292 293 if ((*bo) == NULL) 294 return; 295 tbo = &((*bo)->tbo); 296 ttm_bo_put(tbo); 297 *bo = NULL; 298 } 299 300 int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset, 301 u64 *gpu_addr) 302 { 303 struct ttm_operation_ctx ctx = { false, false }; 304 int r, i; 305 306 if (radeon_ttm_tt_has_userptr(bo->rdev, bo->tbo.ttm)) 307 return -EPERM; 308 309 if (bo->tbo.pin_count) { 310 ttm_bo_pin(&bo->tbo); 311 if (gpu_addr) 312 *gpu_addr = radeon_bo_gpu_offset(bo); 313 314 if (max_offset != 0) { 315 u64 domain_start; 316 317 if (domain == RADEON_GEM_DOMAIN_VRAM) 318 domain_start = bo->rdev->mc.vram_start; 319 else 320 domain_start = bo->rdev->mc.gtt_start; 321 WARN_ON_ONCE(max_offset < 322 (radeon_bo_gpu_offset(bo) - domain_start)); 323 } 324 325 return 0; 326 } 327 if (bo->prime_shared_count && domain == RADEON_GEM_DOMAIN_VRAM) { 328 /* A BO shared as a dma-buf cannot be sensibly migrated to VRAM */ 329 return -EINVAL; 330 } 331 332 radeon_ttm_placement_from_domain(bo, domain); 333 for (i = 0; i < bo->placement.num_placement; i++) { 334 /* force to pin into visible video ram */ 335 if ((bo->placements[i].mem_type == TTM_PL_VRAM) && 336 !(bo->flags & RADEON_GEM_NO_CPU_ACCESS) && 337 (!max_offset || max_offset > bo->rdev->mc.visible_vram_size)) 338 bo->placements[i].lpfn = 339 bo->rdev->mc.visible_vram_size >> PAGE_SHIFT; 340 else 341 bo->placements[i].lpfn = max_offset >> PAGE_SHIFT; 342 } 343 344 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 345 if (likely(r == 0)) { 346 ttm_bo_pin(&bo->tbo); 347 if (gpu_addr != NULL) 348 *gpu_addr = radeon_bo_gpu_offset(bo); 349 if (domain == RADEON_GEM_DOMAIN_VRAM) 350 bo->rdev->vram_pin_size += radeon_bo_size(bo); 351 else 352 bo->rdev->gart_pin_size += radeon_bo_size(bo); 353 } else { 354 dev_err(bo->rdev->dev, "%p pin failed\n", bo); 355 } 356 return r; 357 } 358 359 int radeon_bo_pin(struct radeon_bo *bo, u32 domain, u64 *gpu_addr) 360 { 361 return radeon_bo_pin_restricted(bo, domain, 0, gpu_addr); 362 } 363 364 void radeon_bo_unpin(struct radeon_bo *bo) 365 { 366 ttm_bo_unpin(&bo->tbo); 367 if (!bo->tbo.pin_count) { 368 if (bo->tbo.mem.mem_type == TTM_PL_VRAM) 369 bo->rdev->vram_pin_size -= radeon_bo_size(bo); 370 else 371 bo->rdev->gart_pin_size -= radeon_bo_size(bo); 372 } 373 } 374 375 int radeon_bo_evict_vram(struct radeon_device *rdev) 376 { 377 struct ttm_bo_device *bdev = &rdev->mman.bdev; 378 struct ttm_resource_manager *man; 379 380 /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */ 381 #ifndef CONFIG_HIBERNATION 382 if (rdev->flags & RADEON_IS_IGP) { 383 if (rdev->mc.igp_sideport_enabled == false) 384 /* Useless to evict on IGP chips */ 385 return 0; 386 } 387 #endif 388 man = ttm_manager_type(bdev, TTM_PL_VRAM); 389 return ttm_resource_manager_evict_all(bdev, man); 390 } 391 392 void radeon_bo_force_delete(struct radeon_device *rdev) 393 { 394 struct radeon_bo *bo, *n; 395 396 if (list_empty(&rdev->gem.objects)) { 397 return; 398 } 399 dev_err(rdev->dev, "Userspace still has active objects !\n"); 400 list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) { 401 dev_err(rdev->dev, "%p %p %lu %lu force free\n", 402 &bo->tbo.base, bo, (unsigned long)bo->tbo.base.size, 403 *((unsigned long *)&bo->tbo.base.refcount)); 404 mutex_lock(&bo->rdev->gem.mutex); 405 list_del_init(&bo->list); 406 mutex_unlock(&bo->rdev->gem.mutex); 407 /* this should unref the ttm bo */ 408 drm_gem_object_put(&bo->tbo.base); 409 } 410 } 411 412 int radeon_bo_init(struct radeon_device *rdev) 413 { 414 /* reserve PAT memory space to WC for VRAM */ 415 arch_io_reserve_memtype_wc(rdev->mc.aper_base, 416 rdev->mc.aper_size); 417 418 /* Add an MTRR for the VRAM */ 419 if (!rdev->fastfb_working) { 420 rdev->mc.vram_mtrr = arch_phys_wc_add(rdev->mc.aper_base, 421 rdev->mc.aper_size); 422 } 423 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", 424 rdev->mc.mc_vram_size >> 20, 425 (unsigned long long)rdev->mc.aper_size >> 20); 426 DRM_INFO("RAM width %dbits %cDR\n", 427 rdev->mc.vram_width, rdev->mc.vram_is_ddr ? 'D' : 'S'); 428 return radeon_ttm_init(rdev); 429 } 430 431 void radeon_bo_fini(struct radeon_device *rdev) 432 { 433 radeon_ttm_fini(rdev); 434 arch_phys_wc_del(rdev->mc.vram_mtrr); 435 arch_io_free_memtype_wc(rdev->mc.aper_base, rdev->mc.aper_size); 436 } 437 438 /* Returns how many bytes TTM can move per IB. 439 */ 440 static u64 radeon_bo_get_threshold_for_moves(struct radeon_device *rdev) 441 { 442 u64 real_vram_size = rdev->mc.real_vram_size; 443 u64 vram_usage = atomic64_read(&rdev->vram_usage); 444 445 /* This function is based on the current VRAM usage. 446 * 447 * - If all of VRAM is free, allow relocating the number of bytes that 448 * is equal to 1/4 of the size of VRAM for this IB. 449 450 * - If more than one half of VRAM is occupied, only allow relocating 451 * 1 MB of data for this IB. 452 * 453 * - From 0 to one half of used VRAM, the threshold decreases 454 * linearly. 455 * __________________ 456 * 1/4 of -|\ | 457 * VRAM | \ | 458 * | \ | 459 * | \ | 460 * | \ | 461 * | \ | 462 * | \ | 463 * | \________|1 MB 464 * |----------------| 465 * VRAM 0 % 100 % 466 * used used 467 * 468 * Note: It's a threshold, not a limit. The threshold must be crossed 469 * for buffer relocations to stop, so any buffer of an arbitrary size 470 * can be moved as long as the threshold isn't crossed before 471 * the relocation takes place. We don't want to disable buffer 472 * relocations completely. 473 * 474 * The idea is that buffers should be placed in VRAM at creation time 475 * and TTM should only do a minimum number of relocations during 476 * command submission. In practice, you need to submit at least 477 * a dozen IBs to move all buffers to VRAM if they are in GTT. 478 * 479 * Also, things can get pretty crazy under memory pressure and actual 480 * VRAM usage can change a lot, so playing safe even at 50% does 481 * consistently increase performance. 482 */ 483 484 u64 half_vram = real_vram_size >> 1; 485 u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage; 486 u64 bytes_moved_threshold = half_free_vram >> 1; 487 return max(bytes_moved_threshold, 1024*1024ull); 488 } 489 490 int radeon_bo_list_validate(struct radeon_device *rdev, 491 struct ww_acquire_ctx *ticket, 492 struct list_head *head, int ring) 493 { 494 struct ttm_operation_ctx ctx = { true, false }; 495 struct radeon_bo_list *lobj; 496 struct list_head duplicates; 497 int r; 498 u64 bytes_moved = 0, initial_bytes_moved; 499 u64 bytes_moved_threshold = radeon_bo_get_threshold_for_moves(rdev); 500 501 INIT_LIST_HEAD(&duplicates); 502 r = ttm_eu_reserve_buffers(ticket, head, true, &duplicates); 503 if (unlikely(r != 0)) { 504 return r; 505 } 506 507 list_for_each_entry(lobj, head, tv.head) { 508 struct radeon_bo *bo = lobj->robj; 509 if (!bo->tbo.pin_count) { 510 u32 domain = lobj->preferred_domains; 511 u32 allowed = lobj->allowed_domains; 512 u32 current_domain = 513 radeon_mem_type_to_domain(bo->tbo.mem.mem_type); 514 515 /* Check if this buffer will be moved and don't move it 516 * if we have moved too many buffers for this IB already. 517 * 518 * Note that this allows moving at least one buffer of 519 * any size, because it doesn't take the current "bo" 520 * into account. We don't want to disallow buffer moves 521 * completely. 522 */ 523 if ((allowed & current_domain) != 0 && 524 (domain & current_domain) == 0 && /* will be moved */ 525 bytes_moved > bytes_moved_threshold) { 526 /* don't move it */ 527 domain = current_domain; 528 } 529 530 retry: 531 radeon_ttm_placement_from_domain(bo, domain); 532 if (ring == R600_RING_TYPE_UVD_INDEX) 533 radeon_uvd_force_into_uvd_segment(bo, allowed); 534 535 initial_bytes_moved = atomic64_read(&rdev->num_bytes_moved); 536 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 537 bytes_moved += atomic64_read(&rdev->num_bytes_moved) - 538 initial_bytes_moved; 539 540 if (unlikely(r)) { 541 if (r != -ERESTARTSYS && 542 domain != lobj->allowed_domains) { 543 domain = lobj->allowed_domains; 544 goto retry; 545 } 546 ttm_eu_backoff_reservation(ticket, head); 547 return r; 548 } 549 } 550 lobj->gpu_offset = radeon_bo_gpu_offset(bo); 551 lobj->tiling_flags = bo->tiling_flags; 552 } 553 554 list_for_each_entry(lobj, &duplicates, tv.head) { 555 lobj->gpu_offset = radeon_bo_gpu_offset(lobj->robj); 556 lobj->tiling_flags = lobj->robj->tiling_flags; 557 } 558 559 return 0; 560 } 561 562 int radeon_bo_get_surface_reg(struct radeon_bo *bo) 563 { 564 struct radeon_device *rdev = bo->rdev; 565 struct radeon_surface_reg *reg; 566 struct radeon_bo *old_object; 567 int steal; 568 int i; 569 570 dma_resv_assert_held(bo->tbo.base.resv); 571 572 if (!bo->tiling_flags) 573 return 0; 574 575 if (bo->surface_reg >= 0) { 576 reg = &rdev->surface_regs[bo->surface_reg]; 577 i = bo->surface_reg; 578 goto out; 579 } 580 581 steal = -1; 582 for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) { 583 584 reg = &rdev->surface_regs[i]; 585 if (!reg->bo) 586 break; 587 588 old_object = reg->bo; 589 if (old_object->tbo.pin_count == 0) 590 steal = i; 591 } 592 593 /* if we are all out */ 594 if (i == RADEON_GEM_MAX_SURFACES) { 595 if (steal == -1) 596 return -ENOMEM; 597 /* find someone with a surface reg and nuke their BO */ 598 reg = &rdev->surface_regs[steal]; 599 old_object = reg->bo; 600 /* blow away the mapping */ 601 DRM_DEBUG("stealing surface reg %d from %p\n", steal, old_object); 602 ttm_bo_unmap_virtual(&old_object->tbo); 603 old_object->surface_reg = -1; 604 i = steal; 605 } 606 607 bo->surface_reg = i; 608 reg->bo = bo; 609 610 out: 611 radeon_set_surface_reg(rdev, i, bo->tiling_flags, bo->pitch, 612 bo->tbo.mem.start << PAGE_SHIFT, 613 bo->tbo.num_pages << PAGE_SHIFT); 614 return 0; 615 } 616 617 static void radeon_bo_clear_surface_reg(struct radeon_bo *bo) 618 { 619 struct radeon_device *rdev = bo->rdev; 620 struct radeon_surface_reg *reg; 621 622 if (bo->surface_reg == -1) 623 return; 624 625 reg = &rdev->surface_regs[bo->surface_reg]; 626 radeon_clear_surface_reg(rdev, bo->surface_reg); 627 628 reg->bo = NULL; 629 bo->surface_reg = -1; 630 } 631 632 int radeon_bo_set_tiling_flags(struct radeon_bo *bo, 633 uint32_t tiling_flags, uint32_t pitch) 634 { 635 struct radeon_device *rdev = bo->rdev; 636 int r; 637 638 if (rdev->family >= CHIP_CEDAR) { 639 unsigned bankw, bankh, mtaspect, tilesplit, stilesplit; 640 641 bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK; 642 bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK; 643 mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK; 644 tilesplit = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK; 645 stilesplit = (tiling_flags >> RADEON_TILING_EG_STENCIL_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_STENCIL_TILE_SPLIT_MASK; 646 switch (bankw) { 647 case 0: 648 case 1: 649 case 2: 650 case 4: 651 case 8: 652 break; 653 default: 654 return -EINVAL; 655 } 656 switch (bankh) { 657 case 0: 658 case 1: 659 case 2: 660 case 4: 661 case 8: 662 break; 663 default: 664 return -EINVAL; 665 } 666 switch (mtaspect) { 667 case 0: 668 case 1: 669 case 2: 670 case 4: 671 case 8: 672 break; 673 default: 674 return -EINVAL; 675 } 676 if (tilesplit > 6) { 677 return -EINVAL; 678 } 679 if (stilesplit > 6) { 680 return -EINVAL; 681 } 682 } 683 r = radeon_bo_reserve(bo, false); 684 if (unlikely(r != 0)) 685 return r; 686 bo->tiling_flags = tiling_flags; 687 bo->pitch = pitch; 688 radeon_bo_unreserve(bo); 689 return 0; 690 } 691 692 void radeon_bo_get_tiling_flags(struct radeon_bo *bo, 693 uint32_t *tiling_flags, 694 uint32_t *pitch) 695 { 696 dma_resv_assert_held(bo->tbo.base.resv); 697 698 if (tiling_flags) 699 *tiling_flags = bo->tiling_flags; 700 if (pitch) 701 *pitch = bo->pitch; 702 } 703 704 int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved, 705 bool force_drop) 706 { 707 if (!force_drop) 708 dma_resv_assert_held(bo->tbo.base.resv); 709 710 if (!(bo->tiling_flags & RADEON_TILING_SURFACE)) 711 return 0; 712 713 if (force_drop) { 714 radeon_bo_clear_surface_reg(bo); 715 return 0; 716 } 717 718 if (bo->tbo.mem.mem_type != TTM_PL_VRAM) { 719 if (!has_moved) 720 return 0; 721 722 if (bo->surface_reg >= 0) 723 radeon_bo_clear_surface_reg(bo); 724 return 0; 725 } 726 727 if ((bo->surface_reg >= 0) && !has_moved) 728 return 0; 729 730 return radeon_bo_get_surface_reg(bo); 731 } 732 733 void radeon_bo_move_notify(struct ttm_buffer_object *bo, 734 bool evict, 735 struct ttm_resource *new_mem) 736 { 737 struct radeon_bo *rbo; 738 739 if (!radeon_ttm_bo_is_radeon_bo(bo)) 740 return; 741 742 rbo = container_of(bo, struct radeon_bo, tbo); 743 radeon_bo_check_tiling(rbo, 0, 1); 744 radeon_vm_bo_invalidate(rbo->rdev, rbo); 745 746 /* update statistics */ 747 if (!new_mem) 748 return; 749 750 radeon_update_memory_usage(rbo, bo->mem.mem_type, -1); 751 radeon_update_memory_usage(rbo, new_mem->mem_type, 1); 752 } 753 754 vm_fault_t radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 755 { 756 struct ttm_operation_ctx ctx = { false, false }; 757 struct radeon_device *rdev; 758 struct radeon_bo *rbo; 759 unsigned long offset, size, lpfn; 760 int i, r; 761 762 if (!radeon_ttm_bo_is_radeon_bo(bo)) 763 return 0; 764 rbo = container_of(bo, struct radeon_bo, tbo); 765 radeon_bo_check_tiling(rbo, 0, 0); 766 rdev = rbo->rdev; 767 if (bo->mem.mem_type != TTM_PL_VRAM) 768 return 0; 769 770 size = bo->mem.num_pages << PAGE_SHIFT; 771 offset = bo->mem.start << PAGE_SHIFT; 772 if ((offset + size) <= rdev->mc.visible_vram_size) 773 return 0; 774 775 /* Can't move a pinned BO to visible VRAM */ 776 if (rbo->tbo.pin_count > 0) 777 return VM_FAULT_SIGBUS; 778 779 /* hurrah the memory is not visible ! */ 780 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM); 781 lpfn = rdev->mc.visible_vram_size >> PAGE_SHIFT; 782 for (i = 0; i < rbo->placement.num_placement; i++) { 783 /* Force into visible VRAM */ 784 if ((rbo->placements[i].mem_type == TTM_PL_VRAM) && 785 (!rbo->placements[i].lpfn || rbo->placements[i].lpfn > lpfn)) 786 rbo->placements[i].lpfn = lpfn; 787 } 788 r = ttm_bo_validate(bo, &rbo->placement, &ctx); 789 if (unlikely(r == -ENOMEM)) { 790 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); 791 r = ttm_bo_validate(bo, &rbo->placement, &ctx); 792 } else if (likely(!r)) { 793 offset = bo->mem.start << PAGE_SHIFT; 794 /* this should never happen */ 795 if ((offset + size) > rdev->mc.visible_vram_size) 796 return VM_FAULT_SIGBUS; 797 } 798 799 if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) 800 return VM_FAULT_NOPAGE; 801 else if (unlikely(r)) 802 return VM_FAULT_SIGBUS; 803 804 ttm_bo_move_to_lru_tail_unlocked(bo); 805 return 0; 806 } 807 808 /** 809 * radeon_bo_fence - add fence to buffer object 810 * 811 * @bo: buffer object in question 812 * @fence: fence to add 813 * @shared: true if fence should be added shared 814 * 815 */ 816 void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence, 817 bool shared) 818 { 819 struct dma_resv *resv = bo->tbo.base.resv; 820 821 if (shared) 822 dma_resv_add_shared_fence(resv, &fence->base); 823 else 824 dma_resv_add_excl_fence(resv, &fence->base); 825 } 826