1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #include <linux/dma-mapping.h> 26 #include <drm/ttm/ttm_range_manager.h> 27 28 #include "amdgpu.h" 29 #include "amdgpu_vm.h" 30 #include "amdgpu_res_cursor.h" 31 #include "amdgpu_atomfirmware.h" 32 #include "atom.h" 33 34 struct amdgpu_vram_reservation { 35 u64 start; 36 u64 size; 37 struct list_head allocated; 38 struct list_head blocks; 39 }; 40 41 static inline struct amdgpu_vram_mgr * 42 to_vram_mgr(struct ttm_resource_manager *man) 43 { 44 return container_of(man, struct amdgpu_vram_mgr, manager); 45 } 46 47 static inline struct amdgpu_device * 48 to_amdgpu_device(struct amdgpu_vram_mgr *mgr) 49 { 50 return container_of(mgr, struct amdgpu_device, mman.vram_mgr); 51 } 52 53 static inline struct drm_buddy_block * 54 amdgpu_vram_mgr_first_block(struct list_head *list) 55 { 56 return list_first_entry_or_null(list, struct drm_buddy_block, link); 57 } 58 59 static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head) 60 { 61 struct drm_buddy_block *block; 62 u64 start, size; 63 64 block = amdgpu_vram_mgr_first_block(head); 65 if (!block) 66 return false; 67 68 while (head != block->link.next) { 69 start = amdgpu_vram_mgr_block_start(block); 70 size = amdgpu_vram_mgr_block_size(block); 71 72 block = list_entry(block->link.next, struct drm_buddy_block, link); 73 if (start + size != amdgpu_vram_mgr_block_start(block)) 74 return false; 75 } 76 77 return true; 78 } 79 80 81 82 /** 83 * DOC: mem_info_vram_total 84 * 85 * The amdgpu driver provides a sysfs API for reporting current total VRAM 86 * available on the device 87 * The file mem_info_vram_total is used for this and returns the total 88 * amount of VRAM in bytes 89 */ 90 static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev, 91 struct device_attribute *attr, char *buf) 92 { 93 struct drm_device *ddev = dev_get_drvdata(dev); 94 struct amdgpu_device *adev = drm_to_adev(ddev); 95 96 return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size); 97 } 98 99 /** 100 * DOC: mem_info_vis_vram_total 101 * 102 * The amdgpu driver provides a sysfs API for reporting current total 103 * visible VRAM available on the device 104 * The file mem_info_vis_vram_total is used for this and returns the total 105 * amount of visible VRAM in bytes 106 */ 107 static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, 108 struct device_attribute *attr, char *buf) 109 { 110 struct drm_device *ddev = dev_get_drvdata(dev); 111 struct amdgpu_device *adev = drm_to_adev(ddev); 112 113 return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size); 114 } 115 116 /** 117 * DOC: mem_info_vram_used 118 * 119 * The amdgpu driver provides a sysfs API for reporting current total VRAM 120 * available on the device 121 * The file mem_info_vram_used is used for this and returns the total 122 * amount of currently used VRAM in bytes 123 */ 124 static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, 125 struct device_attribute *attr, 126 char *buf) 127 { 128 struct drm_device *ddev = dev_get_drvdata(dev); 129 struct amdgpu_device *adev = drm_to_adev(ddev); 130 struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager; 131 132 return sysfs_emit(buf, "%llu\n", ttm_resource_manager_usage(man)); 133 } 134 135 /** 136 * DOC: mem_info_vis_vram_used 137 * 138 * The amdgpu driver provides a sysfs API for reporting current total of 139 * used visible VRAM 140 * The file mem_info_vis_vram_used is used for this and returns the total 141 * amount of currently used visible VRAM in bytes 142 */ 143 static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev, 144 struct device_attribute *attr, 145 char *buf) 146 { 147 struct drm_device *ddev = dev_get_drvdata(dev); 148 struct amdgpu_device *adev = drm_to_adev(ddev); 149 150 return sysfs_emit(buf, "%llu\n", 151 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr)); 152 } 153 154 /** 155 * DOC: mem_info_vram_vendor 156 * 157 * The amdgpu driver provides a sysfs API for reporting the vendor of the 158 * installed VRAM 159 * The file mem_info_vram_vendor is used for this and returns the name of the 160 * vendor. 161 */ 162 static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev, 163 struct device_attribute *attr, 164 char *buf) 165 { 166 struct drm_device *ddev = dev_get_drvdata(dev); 167 struct amdgpu_device *adev = drm_to_adev(ddev); 168 169 switch (adev->gmc.vram_vendor) { 170 case SAMSUNG: 171 return sysfs_emit(buf, "samsung\n"); 172 case INFINEON: 173 return sysfs_emit(buf, "infineon\n"); 174 case ELPIDA: 175 return sysfs_emit(buf, "elpida\n"); 176 case ETRON: 177 return sysfs_emit(buf, "etron\n"); 178 case NANYA: 179 return sysfs_emit(buf, "nanya\n"); 180 case HYNIX: 181 return sysfs_emit(buf, "hynix\n"); 182 case MOSEL: 183 return sysfs_emit(buf, "mosel\n"); 184 case WINBOND: 185 return sysfs_emit(buf, "winbond\n"); 186 case ESMT: 187 return sysfs_emit(buf, "esmt\n"); 188 case MICRON: 189 return sysfs_emit(buf, "micron\n"); 190 default: 191 return sysfs_emit(buf, "unknown\n"); 192 } 193 } 194 195 static DEVICE_ATTR(mem_info_vram_total, S_IRUGO, 196 amdgpu_mem_info_vram_total_show, NULL); 197 static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO, 198 amdgpu_mem_info_vis_vram_total_show,NULL); 199 static DEVICE_ATTR(mem_info_vram_used, S_IRUGO, 200 amdgpu_mem_info_vram_used_show, NULL); 201 static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO, 202 amdgpu_mem_info_vis_vram_used_show, NULL); 203 static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO, 204 amdgpu_mem_info_vram_vendor, NULL); 205 206 static struct attribute *amdgpu_vram_mgr_attributes[] = { 207 &dev_attr_mem_info_vram_total.attr, 208 &dev_attr_mem_info_vis_vram_total.attr, 209 &dev_attr_mem_info_vram_used.attr, 210 &dev_attr_mem_info_vis_vram_used.attr, 211 &dev_attr_mem_info_vram_vendor.attr, 212 NULL 213 }; 214 215 const struct attribute_group amdgpu_vram_mgr_attr_group = { 216 .attrs = amdgpu_vram_mgr_attributes 217 }; 218 219 /** 220 * amdgpu_vram_mgr_vis_size - Calculate visible block size 221 * 222 * @adev: amdgpu_device pointer 223 * @block: DRM BUDDY block structure 224 * 225 * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM 226 */ 227 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, 228 struct drm_buddy_block *block) 229 { 230 u64 start = amdgpu_vram_mgr_block_start(block); 231 u64 end = start + amdgpu_vram_mgr_block_size(block); 232 233 if (start >= adev->gmc.visible_vram_size) 234 return 0; 235 236 return (end > adev->gmc.visible_vram_size ? 237 adev->gmc.visible_vram_size : end) - start; 238 } 239 240 /** 241 * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size 242 * 243 * @bo: &amdgpu_bo buffer object (must be in VRAM) 244 * 245 * Returns: 246 * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM. 247 */ 248 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) 249 { 250 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 251 struct ttm_resource *res = bo->tbo.resource; 252 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); 253 struct drm_buddy_block *block; 254 u64 usage = 0; 255 256 if (amdgpu_gmc_vram_full_visible(&adev->gmc)) 257 return amdgpu_bo_size(bo); 258 259 if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) 260 return 0; 261 262 list_for_each_entry(block, &vres->blocks, link) 263 usage += amdgpu_vram_mgr_vis_size(adev, block); 264 265 return usage; 266 } 267 268 /* Commit the reservation of VRAM pages */ 269 static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man) 270 { 271 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 272 struct amdgpu_device *adev = to_amdgpu_device(mgr); 273 struct drm_buddy *mm = &mgr->mm; 274 struct amdgpu_vram_reservation *rsv, *temp; 275 struct drm_buddy_block *block; 276 uint64_t vis_usage; 277 278 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) { 279 if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size, 280 rsv->size, mm->chunk_size, &rsv->allocated, 281 DRM_BUDDY_RANGE_ALLOCATION)) 282 continue; 283 284 block = amdgpu_vram_mgr_first_block(&rsv->allocated); 285 if (!block) 286 continue; 287 288 dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n", 289 rsv->start, rsv->size); 290 291 vis_usage = amdgpu_vram_mgr_vis_size(adev, block); 292 atomic64_add(vis_usage, &mgr->vis_usage); 293 spin_lock(&man->bdev->lru_lock); 294 man->usage += rsv->size; 295 spin_unlock(&man->bdev->lru_lock); 296 list_move(&rsv->blocks, &mgr->reserved_pages); 297 } 298 } 299 300 /** 301 * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM 302 * 303 * @mgr: amdgpu_vram_mgr pointer 304 * @start: start address of the range in VRAM 305 * @size: size of the range 306 * 307 * Reserve memory from start address with the specified size in VRAM 308 */ 309 int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr, 310 uint64_t start, uint64_t size) 311 { 312 struct amdgpu_vram_reservation *rsv; 313 314 rsv = kzalloc(sizeof(*rsv), GFP_KERNEL); 315 if (!rsv) 316 return -ENOMEM; 317 318 INIT_LIST_HEAD(&rsv->allocated); 319 INIT_LIST_HEAD(&rsv->blocks); 320 321 rsv->start = start; 322 rsv->size = size; 323 324 mutex_lock(&mgr->lock); 325 list_add_tail(&rsv->blocks, &mgr->reservations_pending); 326 amdgpu_vram_mgr_do_reserve(&mgr->manager); 327 mutex_unlock(&mgr->lock); 328 329 return 0; 330 } 331 332 /** 333 * amdgpu_vram_mgr_query_page_status - query the reservation status 334 * 335 * @mgr: amdgpu_vram_mgr pointer 336 * @start: start address of a page in VRAM 337 * 338 * Returns: 339 * -EBUSY: the page is still hold and in pending list 340 * 0: the page has been reserved 341 * -ENOENT: the input page is not a reservation 342 */ 343 int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr, 344 uint64_t start) 345 { 346 struct amdgpu_vram_reservation *rsv; 347 int ret; 348 349 mutex_lock(&mgr->lock); 350 351 list_for_each_entry(rsv, &mgr->reservations_pending, blocks) { 352 if (rsv->start <= start && 353 (start < (rsv->start + rsv->size))) { 354 ret = -EBUSY; 355 goto out; 356 } 357 } 358 359 list_for_each_entry(rsv, &mgr->reserved_pages, blocks) { 360 if (rsv->start <= start && 361 (start < (rsv->start + rsv->size))) { 362 ret = 0; 363 goto out; 364 } 365 } 366 367 ret = -ENOENT; 368 out: 369 mutex_unlock(&mgr->lock); 370 return ret; 371 } 372 373 static void amdgpu_dummy_vram_mgr_debug(struct ttm_resource_manager *man, 374 struct drm_printer *printer) 375 { 376 DRM_DEBUG_DRIVER("Dummy vram mgr debug\n"); 377 } 378 379 static bool amdgpu_dummy_vram_mgr_compatible(struct ttm_resource_manager *man, 380 struct ttm_resource *res, 381 const struct ttm_place *place, 382 size_t size) 383 { 384 DRM_DEBUG_DRIVER("Dummy vram mgr compatible\n"); 385 return false; 386 } 387 388 static bool amdgpu_dummy_vram_mgr_intersects(struct ttm_resource_manager *man, 389 struct ttm_resource *res, 390 const struct ttm_place *place, 391 size_t size) 392 { 393 DRM_DEBUG_DRIVER("Dummy vram mgr intersects\n"); 394 return true; 395 } 396 397 static void amdgpu_dummy_vram_mgr_del(struct ttm_resource_manager *man, 398 struct ttm_resource *res) 399 { 400 DRM_DEBUG_DRIVER("Dummy vram mgr deleted\n"); 401 } 402 403 static int amdgpu_dummy_vram_mgr_new(struct ttm_resource_manager *man, 404 struct ttm_buffer_object *tbo, 405 const struct ttm_place *place, 406 struct ttm_resource **res) 407 { 408 DRM_DEBUG_DRIVER("Dummy vram mgr new\n"); 409 return -ENOSPC; 410 } 411 412 /** 413 * amdgpu_vram_mgr_new - allocate new ranges 414 * 415 * @man: TTM memory type manager 416 * @tbo: TTM BO we need this range for 417 * @place: placement flags and restrictions 418 * @res: the resulting mem object 419 * 420 * Allocate VRAM for the given BO. 421 */ 422 static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, 423 struct ttm_buffer_object *tbo, 424 const struct ttm_place *place, 425 struct ttm_resource **res) 426 { 427 u64 vis_usage = 0, max_bytes, cur_size, min_block_size; 428 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 429 struct amdgpu_device *adev = to_amdgpu_device(mgr); 430 struct amdgpu_vram_mgr_resource *vres; 431 u64 size, remaining_size, lpfn, fpfn; 432 struct drm_buddy *mm = &mgr->mm; 433 struct drm_buddy_block *block; 434 unsigned long pages_per_block; 435 int r; 436 437 lpfn = (u64)place->lpfn << PAGE_SHIFT; 438 if (!lpfn) 439 lpfn = man->size; 440 441 fpfn = (u64)place->fpfn << PAGE_SHIFT; 442 443 max_bytes = adev->gmc.mc_vram_size; 444 if (tbo->type != ttm_bo_type_kernel) 445 max_bytes -= AMDGPU_VM_RESERVED_VRAM; 446 447 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 448 pages_per_block = ~0ul; 449 } else { 450 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 451 pages_per_block = HPAGE_PMD_NR; 452 #else 453 /* default to 2MB */ 454 pages_per_block = 2UL << (20UL - PAGE_SHIFT); 455 #endif 456 pages_per_block = max_t(uint32_t, pages_per_block, 457 tbo->page_alignment); 458 } 459 460 vres = kzalloc(sizeof(*vres), GFP_KERNEL); 461 if (!vres) 462 return -ENOMEM; 463 464 ttm_resource_init(tbo, place, &vres->base); 465 466 /* bail out quickly if there's likely not enough VRAM for this BO */ 467 if (ttm_resource_manager_usage(man) > max_bytes) { 468 r = -ENOSPC; 469 goto error_fini; 470 } 471 472 INIT_LIST_HEAD(&vres->blocks); 473 474 if (place->flags & TTM_PL_FLAG_TOPDOWN) 475 vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; 476 477 if (fpfn || lpfn != mgr->mm.size) 478 /* Allocate blocks in desired range */ 479 vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; 480 481 remaining_size = (u64)vres->base.size; 482 483 mutex_lock(&mgr->lock); 484 while (remaining_size) { 485 if (tbo->page_alignment) 486 min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT; 487 else 488 min_block_size = mgr->default_page_size; 489 490 BUG_ON(min_block_size < mm->chunk_size); 491 492 /* Limit maximum size to 2GiB due to SG table limitations */ 493 size = min(remaining_size, 2ULL << 30); 494 495 if ((size >= (u64)pages_per_block << PAGE_SHIFT) && 496 !(size & (((u64)pages_per_block << PAGE_SHIFT) - 1))) 497 min_block_size = (u64)pages_per_block << PAGE_SHIFT; 498 499 cur_size = size; 500 501 if (fpfn + size != (u64)place->lpfn << PAGE_SHIFT) { 502 /* 503 * Except for actual range allocation, modify the size and 504 * min_block_size conforming to continuous flag enablement 505 */ 506 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 507 size = roundup_pow_of_two(size); 508 min_block_size = size; 509 /* 510 * Modify the size value if size is not 511 * aligned with min_block_size 512 */ 513 } else if (!IS_ALIGNED(size, min_block_size)) { 514 size = round_up(size, min_block_size); 515 } 516 } 517 518 r = drm_buddy_alloc_blocks(mm, fpfn, 519 lpfn, 520 size, 521 min_block_size, 522 &vres->blocks, 523 vres->flags); 524 if (unlikely(r)) 525 goto error_free_blocks; 526 527 if (size > remaining_size) 528 remaining_size = 0; 529 else 530 remaining_size -= size; 531 } 532 mutex_unlock(&mgr->lock); 533 534 if (cur_size != size) { 535 struct drm_buddy_block *block; 536 struct list_head *trim_list; 537 u64 original_size; 538 LIST_HEAD(temp); 539 540 trim_list = &vres->blocks; 541 original_size = (u64)vres->base.size; 542 543 /* 544 * If size value is rounded up to min_block_size, trim the last 545 * block to the required size 546 */ 547 if (!list_is_singular(&vres->blocks)) { 548 block = list_last_entry(&vres->blocks, typeof(*block), link); 549 list_move_tail(&block->link, &temp); 550 trim_list = &temp; 551 /* 552 * Compute the original_size value by subtracting the 553 * last block size with (aligned size - original size) 554 */ 555 original_size = amdgpu_vram_mgr_block_size(block) - (size - cur_size); 556 } 557 558 mutex_lock(&mgr->lock); 559 drm_buddy_block_trim(mm, 560 original_size, 561 trim_list); 562 mutex_unlock(&mgr->lock); 563 564 if (!list_empty(&temp)) 565 list_splice_tail(trim_list, &vres->blocks); 566 } 567 568 vres->base.start = 0; 569 list_for_each_entry(block, &vres->blocks, link) { 570 unsigned long start; 571 572 start = amdgpu_vram_mgr_block_start(block) + 573 amdgpu_vram_mgr_block_size(block); 574 start >>= PAGE_SHIFT; 575 576 if (start > PFN_UP(vres->base.size)) 577 start -= PFN_UP(vres->base.size); 578 else 579 start = 0; 580 vres->base.start = max(vres->base.start, start); 581 582 vis_usage += amdgpu_vram_mgr_vis_size(adev, block); 583 } 584 585 if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks)) 586 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; 587 588 if (adev->gmc.xgmi.connected_to_cpu) 589 vres->base.bus.caching = ttm_cached; 590 else 591 vres->base.bus.caching = ttm_write_combined; 592 593 atomic64_add(vis_usage, &mgr->vis_usage); 594 *res = &vres->base; 595 return 0; 596 597 error_free_blocks: 598 drm_buddy_free_list(mm, &vres->blocks); 599 mutex_unlock(&mgr->lock); 600 error_fini: 601 ttm_resource_fini(man, &vres->base); 602 kfree(vres); 603 604 return r; 605 } 606 607 /** 608 * amdgpu_vram_mgr_del - free ranges 609 * 610 * @man: TTM memory type manager 611 * @res: TTM memory object 612 * 613 * Free the allocated VRAM again. 614 */ 615 static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, 616 struct ttm_resource *res) 617 { 618 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); 619 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 620 struct amdgpu_device *adev = to_amdgpu_device(mgr); 621 struct drm_buddy *mm = &mgr->mm; 622 struct drm_buddy_block *block; 623 uint64_t vis_usage = 0; 624 625 mutex_lock(&mgr->lock); 626 list_for_each_entry(block, &vres->blocks, link) 627 vis_usage += amdgpu_vram_mgr_vis_size(adev, block); 628 629 amdgpu_vram_mgr_do_reserve(man); 630 631 drm_buddy_free_list(mm, &vres->blocks); 632 mutex_unlock(&mgr->lock); 633 634 atomic64_sub(vis_usage, &mgr->vis_usage); 635 636 ttm_resource_fini(man, res); 637 kfree(vres); 638 } 639 640 /** 641 * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table 642 * 643 * @adev: amdgpu device pointer 644 * @res: TTM memory object 645 * @offset: byte offset from the base of VRAM BO 646 * @length: number of bytes to export in sg_table 647 * @dev: the other device 648 * @dir: dma direction 649 * @sgt: resulting sg table 650 * 651 * Allocate and fill a sg table from a VRAM allocation. 652 */ 653 int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, 654 struct ttm_resource *res, 655 u64 offset, u64 length, 656 struct device *dev, 657 enum dma_data_direction dir, 658 struct sg_table **sgt) 659 { 660 struct amdgpu_res_cursor cursor; 661 struct scatterlist *sg; 662 int num_entries = 0; 663 int i, r; 664 665 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL); 666 if (!*sgt) 667 return -ENOMEM; 668 669 /* Determine the number of DRM_BUDDY blocks to export */ 670 amdgpu_res_first(res, offset, length, &cursor); 671 while (cursor.remaining) { 672 num_entries++; 673 amdgpu_res_next(&cursor, cursor.size); 674 } 675 676 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); 677 if (r) 678 goto error_free; 679 680 /* Initialize scatterlist nodes of sg_table */ 681 for_each_sgtable_sg((*sgt), sg, i) 682 sg->length = 0; 683 684 /* 685 * Walk down DRM_BUDDY blocks to populate scatterlist nodes 686 * @note: Use iterator api to get first the DRM_BUDDY block 687 * and the number of bytes from it. Access the following 688 * DRM_BUDDY block(s) if more buffer needs to exported 689 */ 690 amdgpu_res_first(res, offset, length, &cursor); 691 for_each_sgtable_sg((*sgt), sg, i) { 692 phys_addr_t phys = cursor.start + adev->gmc.aper_base; 693 size_t size = cursor.size; 694 dma_addr_t addr; 695 696 addr = dma_map_resource(dev, phys, size, dir, 697 DMA_ATTR_SKIP_CPU_SYNC); 698 r = dma_mapping_error(dev, addr); 699 if (r) 700 goto error_unmap; 701 702 sg_set_page(sg, NULL, size, 0); 703 sg_dma_address(sg) = addr; 704 sg_dma_len(sg) = size; 705 706 amdgpu_res_next(&cursor, cursor.size); 707 } 708 709 return 0; 710 711 error_unmap: 712 for_each_sgtable_sg((*sgt), sg, i) { 713 if (!sg->length) 714 continue; 715 716 dma_unmap_resource(dev, sg->dma_address, 717 sg->length, dir, 718 DMA_ATTR_SKIP_CPU_SYNC); 719 } 720 sg_free_table(*sgt); 721 722 error_free: 723 kfree(*sgt); 724 return r; 725 } 726 727 /** 728 * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table 729 * 730 * @dev: device pointer 731 * @dir: data direction of resource to unmap 732 * @sgt: sg table to free 733 * 734 * Free a previously allocate sg table. 735 */ 736 void amdgpu_vram_mgr_free_sgt(struct device *dev, 737 enum dma_data_direction dir, 738 struct sg_table *sgt) 739 { 740 struct scatterlist *sg; 741 int i; 742 743 for_each_sgtable_sg(sgt, sg, i) 744 dma_unmap_resource(dev, sg->dma_address, 745 sg->length, dir, 746 DMA_ATTR_SKIP_CPU_SYNC); 747 sg_free_table(sgt); 748 kfree(sgt); 749 } 750 751 /** 752 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part 753 * 754 * @mgr: amdgpu_vram_mgr pointer 755 * 756 * Returns how many bytes are used in the visible part of VRAM 757 */ 758 uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr) 759 { 760 return atomic64_read(&mgr->vis_usage); 761 } 762 763 /** 764 * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection 765 * 766 * @man: TTM memory type manager 767 * @res: The resource to test 768 * @place: The place to test against 769 * @size: Size of the new allocation 770 * 771 * Test each drm buddy block for intersection for eviction decision. 772 */ 773 static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man, 774 struct ttm_resource *res, 775 const struct ttm_place *place, 776 size_t size) 777 { 778 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); 779 struct drm_buddy_block *block; 780 781 /* Check each drm buddy block individually */ 782 list_for_each_entry(block, &mgr->blocks, link) { 783 unsigned long fpfn = 784 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; 785 unsigned long lpfn = fpfn + 786 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT); 787 788 if (place->fpfn < lpfn && 789 (!place->lpfn || place->lpfn > fpfn)) 790 return true; 791 } 792 793 return false; 794 } 795 796 /** 797 * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility 798 * 799 * @man: TTM memory type manager 800 * @res: The resource to test 801 * @place: The place to test against 802 * @size: Size of the new allocation 803 * 804 * Test each drm buddy block for placement compatibility. 805 */ 806 static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man, 807 struct ttm_resource *res, 808 const struct ttm_place *place, 809 size_t size) 810 { 811 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); 812 struct drm_buddy_block *block; 813 814 /* Check each drm buddy block individually */ 815 list_for_each_entry(block, &mgr->blocks, link) { 816 unsigned long fpfn = 817 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; 818 unsigned long lpfn = fpfn + 819 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT); 820 821 if (fpfn < place->fpfn || 822 (place->lpfn && lpfn > place->lpfn)) 823 return false; 824 } 825 826 return true; 827 } 828 829 /** 830 * amdgpu_vram_mgr_debug - dump VRAM table 831 * 832 * @man: TTM memory type manager 833 * @printer: DRM printer to use 834 * 835 * Dump the table content using printk. 836 */ 837 static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, 838 struct drm_printer *printer) 839 { 840 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 841 struct drm_buddy *mm = &mgr->mm; 842 struct amdgpu_vram_reservation *rsv; 843 844 drm_printf(printer, " vis usage:%llu\n", 845 amdgpu_vram_mgr_vis_usage(mgr)); 846 847 mutex_lock(&mgr->lock); 848 drm_printf(printer, "default_page_size: %lluKiB\n", 849 mgr->default_page_size >> 10); 850 851 drm_buddy_print(mm, printer); 852 853 drm_printf(printer, "reserved:\n"); 854 list_for_each_entry(rsv, &mgr->reserved_pages, blocks) 855 drm_printf(printer, "%#018llx-%#018llx: %llu\n", 856 rsv->start, rsv->start + rsv->size, rsv->size); 857 mutex_unlock(&mgr->lock); 858 } 859 860 static const struct ttm_resource_manager_func amdgpu_dummy_vram_mgr_func = { 861 .alloc = amdgpu_dummy_vram_mgr_new, 862 .free = amdgpu_dummy_vram_mgr_del, 863 .intersects = amdgpu_dummy_vram_mgr_intersects, 864 .compatible = amdgpu_dummy_vram_mgr_compatible, 865 .debug = amdgpu_dummy_vram_mgr_debug 866 }; 867 868 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { 869 .alloc = amdgpu_vram_mgr_new, 870 .free = amdgpu_vram_mgr_del, 871 .intersects = amdgpu_vram_mgr_intersects, 872 .compatible = amdgpu_vram_mgr_compatible, 873 .debug = amdgpu_vram_mgr_debug 874 }; 875 876 /** 877 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM 878 * 879 * @adev: amdgpu_device pointer 880 * 881 * Allocate and initialize the VRAM manager. 882 */ 883 int amdgpu_vram_mgr_init(struct amdgpu_device *adev) 884 { 885 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 886 struct ttm_resource_manager *man = &mgr->manager; 887 int err; 888 889 ttm_resource_manager_init(man, &adev->mman.bdev, 890 adev->gmc.real_vram_size); 891 892 mutex_init(&mgr->lock); 893 INIT_LIST_HEAD(&mgr->reservations_pending); 894 INIT_LIST_HEAD(&mgr->reserved_pages); 895 mgr->default_page_size = PAGE_SIZE; 896 897 if (!adev->gmc.is_app_apu) { 898 man->func = &amdgpu_vram_mgr_func; 899 900 err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE); 901 if (err) 902 return err; 903 } else { 904 man->func = &amdgpu_dummy_vram_mgr_func; 905 DRM_INFO("Setup dummy vram mgr\n"); 906 } 907 908 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); 909 ttm_resource_manager_set_used(man, true); 910 return 0; 911 } 912 913 /** 914 * amdgpu_vram_mgr_fini - free and destroy VRAM manager 915 * 916 * @adev: amdgpu_device pointer 917 * 918 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still 919 * allocated inside it. 920 */ 921 void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) 922 { 923 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 924 struct ttm_resource_manager *man = &mgr->manager; 925 int ret; 926 struct amdgpu_vram_reservation *rsv, *temp; 927 928 ttm_resource_manager_set_used(man, false); 929 930 ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man); 931 if (ret) 932 return; 933 934 mutex_lock(&mgr->lock); 935 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) 936 kfree(rsv); 937 938 list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) { 939 drm_buddy_free_list(&mgr->mm, &rsv->allocated); 940 kfree(rsv); 941 } 942 if (!adev->gmc.is_app_apu) 943 drm_buddy_fini(&mgr->mm); 944 mutex_unlock(&mgr->lock); 945 946 ttm_resource_manager_cleanup(man); 947 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL); 948 } 949