1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #include <linux/dma-mapping.h> 26 #include <drm/ttm/ttm_range_manager.h> 27 28 #include "amdgpu.h" 29 #include "amdgpu_vm.h" 30 #include "amdgpu_res_cursor.h" 31 #include "amdgpu_atomfirmware.h" 32 #include "atom.h" 33 34 struct amdgpu_vram_reservation { 35 u64 start; 36 u64 size; 37 struct list_head allocated; 38 struct list_head blocks; 39 }; 40 41 static inline struct amdgpu_vram_mgr * 42 to_vram_mgr(struct ttm_resource_manager *man) 43 { 44 return container_of(man, struct amdgpu_vram_mgr, manager); 45 } 46 47 static inline struct amdgpu_device * 48 to_amdgpu_device(struct amdgpu_vram_mgr *mgr) 49 { 50 return container_of(mgr, struct amdgpu_device, mman.vram_mgr); 51 } 52 53 static inline struct drm_buddy_block * 54 amdgpu_vram_mgr_first_block(struct list_head *list) 55 { 56 return list_first_entry_or_null(list, struct drm_buddy_block, link); 57 } 58 59 static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head) 60 { 61 struct drm_buddy_block *block; 62 u64 start, size; 63 64 block = amdgpu_vram_mgr_first_block(head); 65 if (!block) 66 return false; 67 68 while (head != block->link.next) { 69 start = amdgpu_vram_mgr_block_start(block); 70 size = amdgpu_vram_mgr_block_size(block); 71 72 block = list_entry(block->link.next, struct drm_buddy_block, link); 73 if (start + size != amdgpu_vram_mgr_block_start(block)) 74 return false; 75 } 76 77 return true; 78 } 79 80 81 82 /** 83 * DOC: mem_info_vram_total 84 * 85 * The amdgpu driver provides a sysfs API for reporting current total VRAM 86 * available on the device 87 * The file mem_info_vram_total is used for this and returns the total 88 * amount of VRAM in bytes 89 */ 90 static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev, 91 struct device_attribute *attr, char *buf) 92 { 93 struct drm_device *ddev = dev_get_drvdata(dev); 94 struct amdgpu_device *adev = drm_to_adev(ddev); 95 96 return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size); 97 } 98 99 /** 100 * DOC: mem_info_vis_vram_total 101 * 102 * The amdgpu driver provides a sysfs API for reporting current total 103 * visible VRAM available on the device 104 * The file mem_info_vis_vram_total is used for this and returns the total 105 * amount of visible VRAM in bytes 106 */ 107 static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, 108 struct device_attribute *attr, char *buf) 109 { 110 struct drm_device *ddev = dev_get_drvdata(dev); 111 struct amdgpu_device *adev = drm_to_adev(ddev); 112 113 return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size); 114 } 115 116 /** 117 * DOC: mem_info_vram_used 118 * 119 * The amdgpu driver provides a sysfs API for reporting current total VRAM 120 * available on the device 121 * The file mem_info_vram_used is used for this and returns the total 122 * amount of currently used VRAM in bytes 123 */ 124 static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, 125 struct device_attribute *attr, 126 char *buf) 127 { 128 struct drm_device *ddev = dev_get_drvdata(dev); 129 struct amdgpu_device *adev = drm_to_adev(ddev); 130 struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager; 131 132 return sysfs_emit(buf, "%llu\n", ttm_resource_manager_usage(man)); 133 } 134 135 /** 136 * DOC: mem_info_vis_vram_used 137 * 138 * The amdgpu driver provides a sysfs API for reporting current total of 139 * used visible VRAM 140 * The file mem_info_vis_vram_used is used for this and returns the total 141 * amount of currently used visible VRAM in bytes 142 */ 143 static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev, 144 struct device_attribute *attr, 145 char *buf) 146 { 147 struct drm_device *ddev = dev_get_drvdata(dev); 148 struct amdgpu_device *adev = drm_to_adev(ddev); 149 150 return sysfs_emit(buf, "%llu\n", 151 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr)); 152 } 153 154 /** 155 * DOC: mem_info_vram_vendor 156 * 157 * The amdgpu driver provides a sysfs API for reporting the vendor of the 158 * installed VRAM 159 * The file mem_info_vram_vendor is used for this and returns the name of the 160 * vendor. 161 */ 162 static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev, 163 struct device_attribute *attr, 164 char *buf) 165 { 166 struct drm_device *ddev = dev_get_drvdata(dev); 167 struct amdgpu_device *adev = drm_to_adev(ddev); 168 169 switch (adev->gmc.vram_vendor) { 170 case SAMSUNG: 171 return sysfs_emit(buf, "samsung\n"); 172 case INFINEON: 173 return sysfs_emit(buf, "infineon\n"); 174 case ELPIDA: 175 return sysfs_emit(buf, "elpida\n"); 176 case ETRON: 177 return sysfs_emit(buf, "etron\n"); 178 case NANYA: 179 return sysfs_emit(buf, "nanya\n"); 180 case HYNIX: 181 return sysfs_emit(buf, "hynix\n"); 182 case MOSEL: 183 return sysfs_emit(buf, "mosel\n"); 184 case WINBOND: 185 return sysfs_emit(buf, "winbond\n"); 186 case ESMT: 187 return sysfs_emit(buf, "esmt\n"); 188 case MICRON: 189 return sysfs_emit(buf, "micron\n"); 190 default: 191 return sysfs_emit(buf, "unknown\n"); 192 } 193 } 194 195 static DEVICE_ATTR(mem_info_vram_total, S_IRUGO, 196 amdgpu_mem_info_vram_total_show, NULL); 197 static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO, 198 amdgpu_mem_info_vis_vram_total_show,NULL); 199 static DEVICE_ATTR(mem_info_vram_used, S_IRUGO, 200 amdgpu_mem_info_vram_used_show, NULL); 201 static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO, 202 amdgpu_mem_info_vis_vram_used_show, NULL); 203 static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO, 204 amdgpu_mem_info_vram_vendor, NULL); 205 206 static struct attribute *amdgpu_vram_mgr_attributes[] = { 207 &dev_attr_mem_info_vram_total.attr, 208 &dev_attr_mem_info_vis_vram_total.attr, 209 &dev_attr_mem_info_vram_used.attr, 210 &dev_attr_mem_info_vis_vram_used.attr, 211 &dev_attr_mem_info_vram_vendor.attr, 212 NULL 213 }; 214 215 const struct attribute_group amdgpu_vram_mgr_attr_group = { 216 .attrs = amdgpu_vram_mgr_attributes 217 }; 218 219 /** 220 * amdgpu_vram_mgr_vis_size - Calculate visible block size 221 * 222 * @adev: amdgpu_device pointer 223 * @block: DRM BUDDY block structure 224 * 225 * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM 226 */ 227 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, 228 struct drm_buddy_block *block) 229 { 230 u64 start = amdgpu_vram_mgr_block_start(block); 231 u64 end = start + amdgpu_vram_mgr_block_size(block); 232 233 if (start >= adev->gmc.visible_vram_size) 234 return 0; 235 236 return (end > adev->gmc.visible_vram_size ? 237 adev->gmc.visible_vram_size : end) - start; 238 } 239 240 /** 241 * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size 242 * 243 * @bo: &amdgpu_bo buffer object (must be in VRAM) 244 * 245 * Returns: 246 * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM. 247 */ 248 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) 249 { 250 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 251 struct ttm_resource *res = bo->tbo.resource; 252 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); 253 struct drm_buddy_block *block; 254 u64 usage = 0; 255 256 if (amdgpu_gmc_vram_full_visible(&adev->gmc)) 257 return amdgpu_bo_size(bo); 258 259 if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) 260 return 0; 261 262 list_for_each_entry(block, &vres->blocks, link) 263 usage += amdgpu_vram_mgr_vis_size(adev, block); 264 265 return usage; 266 } 267 268 /* Commit the reservation of VRAM pages */ 269 static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man) 270 { 271 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 272 struct amdgpu_device *adev = to_amdgpu_device(mgr); 273 struct drm_buddy *mm = &mgr->mm; 274 struct amdgpu_vram_reservation *rsv, *temp; 275 struct drm_buddy_block *block; 276 uint64_t vis_usage; 277 278 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) { 279 if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size, 280 rsv->size, mm->chunk_size, &rsv->allocated, 281 DRM_BUDDY_RANGE_ALLOCATION)) 282 continue; 283 284 block = amdgpu_vram_mgr_first_block(&rsv->allocated); 285 if (!block) 286 continue; 287 288 dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n", 289 rsv->start, rsv->size); 290 291 vis_usage = amdgpu_vram_mgr_vis_size(adev, block); 292 atomic64_add(vis_usage, &mgr->vis_usage); 293 spin_lock(&man->bdev->lru_lock); 294 man->usage += rsv->size; 295 spin_unlock(&man->bdev->lru_lock); 296 list_move(&rsv->blocks, &mgr->reserved_pages); 297 } 298 } 299 300 /** 301 * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM 302 * 303 * @mgr: amdgpu_vram_mgr pointer 304 * @start: start address of the range in VRAM 305 * @size: size of the range 306 * 307 * Reserve memory from start address with the specified size in VRAM 308 */ 309 int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr, 310 uint64_t start, uint64_t size) 311 { 312 struct amdgpu_vram_reservation *rsv; 313 314 rsv = kzalloc(sizeof(*rsv), GFP_KERNEL); 315 if (!rsv) 316 return -ENOMEM; 317 318 INIT_LIST_HEAD(&rsv->allocated); 319 INIT_LIST_HEAD(&rsv->blocks); 320 321 rsv->start = start; 322 rsv->size = size; 323 324 mutex_lock(&mgr->lock); 325 list_add_tail(&rsv->blocks, &mgr->reservations_pending); 326 amdgpu_vram_mgr_do_reserve(&mgr->manager); 327 mutex_unlock(&mgr->lock); 328 329 return 0; 330 } 331 332 /** 333 * amdgpu_vram_mgr_query_page_status - query the reservation status 334 * 335 * @mgr: amdgpu_vram_mgr pointer 336 * @start: start address of a page in VRAM 337 * 338 * Returns: 339 * -EBUSY: the page is still hold and in pending list 340 * 0: the page has been reserved 341 * -ENOENT: the input page is not a reservation 342 */ 343 int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr, 344 uint64_t start) 345 { 346 struct amdgpu_vram_reservation *rsv; 347 int ret; 348 349 mutex_lock(&mgr->lock); 350 351 list_for_each_entry(rsv, &mgr->reservations_pending, blocks) { 352 if (rsv->start <= start && 353 (start < (rsv->start + rsv->size))) { 354 ret = -EBUSY; 355 goto out; 356 } 357 } 358 359 list_for_each_entry(rsv, &mgr->reserved_pages, blocks) { 360 if (rsv->start <= start && 361 (start < (rsv->start + rsv->size))) { 362 ret = 0; 363 goto out; 364 } 365 } 366 367 ret = -ENOENT; 368 out: 369 mutex_unlock(&mgr->lock); 370 return ret; 371 } 372 373 /** 374 * amdgpu_vram_mgr_new - allocate new ranges 375 * 376 * @man: TTM memory type manager 377 * @tbo: TTM BO we need this range for 378 * @place: placement flags and restrictions 379 * @res: the resulting mem object 380 * 381 * Allocate VRAM for the given BO. 382 */ 383 static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, 384 struct ttm_buffer_object *tbo, 385 const struct ttm_place *place, 386 struct ttm_resource **res) 387 { 388 u64 vis_usage = 0, max_bytes, cur_size, min_block_size; 389 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 390 struct amdgpu_device *adev = to_amdgpu_device(mgr); 391 struct amdgpu_vram_mgr_resource *vres; 392 u64 size, remaining_size, lpfn, fpfn; 393 struct drm_buddy *mm = &mgr->mm; 394 struct drm_buddy_block *block; 395 unsigned long pages_per_block; 396 int r; 397 398 lpfn = (u64)place->lpfn << PAGE_SHIFT; 399 if (!lpfn) 400 lpfn = man->size; 401 402 fpfn = (u64)place->fpfn << PAGE_SHIFT; 403 404 max_bytes = adev->gmc.mc_vram_size; 405 if (tbo->type != ttm_bo_type_kernel) 406 max_bytes -= AMDGPU_VM_RESERVED_VRAM; 407 408 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 409 pages_per_block = ~0ul; 410 } else { 411 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 412 pages_per_block = HPAGE_PMD_NR; 413 #else 414 /* default to 2MB */ 415 pages_per_block = 2UL << (20UL - PAGE_SHIFT); 416 #endif 417 pages_per_block = max_t(uint32_t, pages_per_block, 418 tbo->page_alignment); 419 } 420 421 vres = kzalloc(sizeof(*vres), GFP_KERNEL); 422 if (!vres) 423 return -ENOMEM; 424 425 ttm_resource_init(tbo, place, &vres->base); 426 427 /* bail out quickly if there's likely not enough VRAM for this BO */ 428 if (ttm_resource_manager_usage(man) > max_bytes) { 429 r = -ENOSPC; 430 goto error_fini; 431 } 432 433 INIT_LIST_HEAD(&vres->blocks); 434 435 if (place->flags & TTM_PL_FLAG_TOPDOWN) 436 vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; 437 438 if (fpfn || lpfn != mgr->mm.size) 439 /* Allocate blocks in desired range */ 440 vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; 441 442 remaining_size = (u64)vres->base.size; 443 444 mutex_lock(&mgr->lock); 445 while (remaining_size) { 446 if (tbo->page_alignment) 447 min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT; 448 else 449 min_block_size = mgr->default_page_size; 450 451 BUG_ON(min_block_size < mm->chunk_size); 452 453 /* Limit maximum size to 2GiB due to SG table limitations */ 454 size = min(remaining_size, 2ULL << 30); 455 456 if (size >= (u64)pages_per_block << PAGE_SHIFT) 457 min_block_size = (u64)pages_per_block << PAGE_SHIFT; 458 459 cur_size = size; 460 461 if (fpfn + size != (u64)place->lpfn << PAGE_SHIFT) { 462 /* 463 * Except for actual range allocation, modify the size and 464 * min_block_size conforming to continuous flag enablement 465 */ 466 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 467 size = roundup_pow_of_two(size); 468 min_block_size = size; 469 /* 470 * Modify the size value if size is not 471 * aligned with min_block_size 472 */ 473 } else if (!IS_ALIGNED(size, min_block_size)) { 474 size = round_up(size, min_block_size); 475 } 476 } 477 478 r = drm_buddy_alloc_blocks(mm, fpfn, 479 lpfn, 480 size, 481 min_block_size, 482 &vres->blocks, 483 vres->flags); 484 if (unlikely(r)) 485 goto error_free_blocks; 486 487 if (size > remaining_size) 488 remaining_size = 0; 489 else 490 remaining_size -= size; 491 } 492 mutex_unlock(&mgr->lock); 493 494 if (cur_size != size) { 495 struct drm_buddy_block *block; 496 struct list_head *trim_list; 497 u64 original_size; 498 LIST_HEAD(temp); 499 500 trim_list = &vres->blocks; 501 original_size = (u64)vres->base.size; 502 503 /* 504 * If size value is rounded up to min_block_size, trim the last 505 * block to the required size 506 */ 507 if (!list_is_singular(&vres->blocks)) { 508 block = list_last_entry(&vres->blocks, typeof(*block), link); 509 list_move_tail(&block->link, &temp); 510 trim_list = &temp; 511 /* 512 * Compute the original_size value by subtracting the 513 * last block size with (aligned size - original size) 514 */ 515 original_size = amdgpu_vram_mgr_block_size(block) - (size - cur_size); 516 } 517 518 mutex_lock(&mgr->lock); 519 drm_buddy_block_trim(mm, 520 original_size, 521 trim_list); 522 mutex_unlock(&mgr->lock); 523 524 if (!list_empty(&temp)) 525 list_splice_tail(trim_list, &vres->blocks); 526 } 527 528 vres->base.start = 0; 529 list_for_each_entry(block, &vres->blocks, link) { 530 unsigned long start; 531 532 start = amdgpu_vram_mgr_block_start(block) + 533 amdgpu_vram_mgr_block_size(block); 534 start >>= PAGE_SHIFT; 535 536 if (start > PFN_UP(vres->base.size)) 537 start -= PFN_UP(vres->base.size); 538 else 539 start = 0; 540 vres->base.start = max(vres->base.start, start); 541 542 vis_usage += amdgpu_vram_mgr_vis_size(adev, block); 543 } 544 545 if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks)) 546 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; 547 548 if (adev->gmc.xgmi.connected_to_cpu) 549 vres->base.bus.caching = ttm_cached; 550 else 551 vres->base.bus.caching = ttm_write_combined; 552 553 atomic64_add(vis_usage, &mgr->vis_usage); 554 *res = &vres->base; 555 return 0; 556 557 error_free_blocks: 558 drm_buddy_free_list(mm, &vres->blocks); 559 mutex_unlock(&mgr->lock); 560 error_fini: 561 ttm_resource_fini(man, &vres->base); 562 kfree(vres); 563 564 return r; 565 } 566 567 /** 568 * amdgpu_vram_mgr_del - free ranges 569 * 570 * @man: TTM memory type manager 571 * @res: TTM memory object 572 * 573 * Free the allocated VRAM again. 574 */ 575 static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, 576 struct ttm_resource *res) 577 { 578 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); 579 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 580 struct amdgpu_device *adev = to_amdgpu_device(mgr); 581 struct drm_buddy *mm = &mgr->mm; 582 struct drm_buddy_block *block; 583 uint64_t vis_usage = 0; 584 585 mutex_lock(&mgr->lock); 586 list_for_each_entry(block, &vres->blocks, link) 587 vis_usage += amdgpu_vram_mgr_vis_size(adev, block); 588 589 amdgpu_vram_mgr_do_reserve(man); 590 591 drm_buddy_free_list(mm, &vres->blocks); 592 mutex_unlock(&mgr->lock); 593 594 atomic64_sub(vis_usage, &mgr->vis_usage); 595 596 ttm_resource_fini(man, res); 597 kfree(vres); 598 } 599 600 /** 601 * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table 602 * 603 * @adev: amdgpu device pointer 604 * @res: TTM memory object 605 * @offset: byte offset from the base of VRAM BO 606 * @length: number of bytes to export in sg_table 607 * @dev: the other device 608 * @dir: dma direction 609 * @sgt: resulting sg table 610 * 611 * Allocate and fill a sg table from a VRAM allocation. 612 */ 613 int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, 614 struct ttm_resource *res, 615 u64 offset, u64 length, 616 struct device *dev, 617 enum dma_data_direction dir, 618 struct sg_table **sgt) 619 { 620 struct amdgpu_res_cursor cursor; 621 struct scatterlist *sg; 622 int num_entries = 0; 623 int i, r; 624 625 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL); 626 if (!*sgt) 627 return -ENOMEM; 628 629 /* Determine the number of DRM_BUDDY blocks to export */ 630 amdgpu_res_first(res, offset, length, &cursor); 631 while (cursor.remaining) { 632 num_entries++; 633 amdgpu_res_next(&cursor, cursor.size); 634 } 635 636 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); 637 if (r) 638 goto error_free; 639 640 /* Initialize scatterlist nodes of sg_table */ 641 for_each_sgtable_sg((*sgt), sg, i) 642 sg->length = 0; 643 644 /* 645 * Walk down DRM_BUDDY blocks to populate scatterlist nodes 646 * @note: Use iterator api to get first the DRM_BUDDY block 647 * and the number of bytes from it. Access the following 648 * DRM_BUDDY block(s) if more buffer needs to exported 649 */ 650 amdgpu_res_first(res, offset, length, &cursor); 651 for_each_sgtable_sg((*sgt), sg, i) { 652 phys_addr_t phys = cursor.start + adev->gmc.aper_base; 653 size_t size = cursor.size; 654 dma_addr_t addr; 655 656 addr = dma_map_resource(dev, phys, size, dir, 657 DMA_ATTR_SKIP_CPU_SYNC); 658 r = dma_mapping_error(dev, addr); 659 if (r) 660 goto error_unmap; 661 662 sg_set_page(sg, NULL, size, 0); 663 sg_dma_address(sg) = addr; 664 sg_dma_len(sg) = size; 665 666 amdgpu_res_next(&cursor, cursor.size); 667 } 668 669 return 0; 670 671 error_unmap: 672 for_each_sgtable_sg((*sgt), sg, i) { 673 if (!sg->length) 674 continue; 675 676 dma_unmap_resource(dev, sg->dma_address, 677 sg->length, dir, 678 DMA_ATTR_SKIP_CPU_SYNC); 679 } 680 sg_free_table(*sgt); 681 682 error_free: 683 kfree(*sgt); 684 return r; 685 } 686 687 /** 688 * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table 689 * 690 * @dev: device pointer 691 * @dir: data direction of resource to unmap 692 * @sgt: sg table to free 693 * 694 * Free a previously allocate sg table. 695 */ 696 void amdgpu_vram_mgr_free_sgt(struct device *dev, 697 enum dma_data_direction dir, 698 struct sg_table *sgt) 699 { 700 struct scatterlist *sg; 701 int i; 702 703 for_each_sgtable_sg(sgt, sg, i) 704 dma_unmap_resource(dev, sg->dma_address, 705 sg->length, dir, 706 DMA_ATTR_SKIP_CPU_SYNC); 707 sg_free_table(sgt); 708 kfree(sgt); 709 } 710 711 /** 712 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part 713 * 714 * @mgr: amdgpu_vram_mgr pointer 715 * 716 * Returns how many bytes are used in the visible part of VRAM 717 */ 718 uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr) 719 { 720 return atomic64_read(&mgr->vis_usage); 721 } 722 723 /** 724 * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection 725 * 726 * @man: TTM memory type manager 727 * @res: The resource to test 728 * @place: The place to test against 729 * @size: Size of the new allocation 730 * 731 * Test each drm buddy block for intersection for eviction decision. 732 */ 733 static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man, 734 struct ttm_resource *res, 735 const struct ttm_place *place, 736 size_t size) 737 { 738 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); 739 struct drm_buddy_block *block; 740 741 /* Check each drm buddy block individually */ 742 list_for_each_entry(block, &mgr->blocks, link) { 743 unsigned long fpfn = 744 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; 745 unsigned long lpfn = fpfn + 746 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT); 747 748 if (place->fpfn < lpfn && 749 (!place->lpfn || place->lpfn > fpfn)) 750 return true; 751 } 752 753 return false; 754 } 755 756 /** 757 * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility 758 * 759 * @man: TTM memory type manager 760 * @res: The resource to test 761 * @place: The place to test against 762 * @size: Size of the new allocation 763 * 764 * Test each drm buddy block for placement compatibility. 765 */ 766 static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man, 767 struct ttm_resource *res, 768 const struct ttm_place *place, 769 size_t size) 770 { 771 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); 772 struct drm_buddy_block *block; 773 774 /* Check each drm buddy block individually */ 775 list_for_each_entry(block, &mgr->blocks, link) { 776 unsigned long fpfn = 777 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; 778 unsigned long lpfn = fpfn + 779 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT); 780 781 if (fpfn < place->fpfn || 782 (place->lpfn && lpfn > place->lpfn)) 783 return false; 784 } 785 786 return true; 787 } 788 789 /** 790 * amdgpu_vram_mgr_debug - dump VRAM table 791 * 792 * @man: TTM memory type manager 793 * @printer: DRM printer to use 794 * 795 * Dump the table content using printk. 796 */ 797 static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, 798 struct drm_printer *printer) 799 { 800 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 801 struct drm_buddy *mm = &mgr->mm; 802 struct drm_buddy_block *block; 803 804 drm_printf(printer, " vis usage:%llu\n", 805 amdgpu_vram_mgr_vis_usage(mgr)); 806 807 mutex_lock(&mgr->lock); 808 drm_printf(printer, "default_page_size: %lluKiB\n", 809 mgr->default_page_size >> 10); 810 811 drm_buddy_print(mm, printer); 812 813 drm_printf(printer, "reserved:\n"); 814 list_for_each_entry(block, &mgr->reserved_pages, link) 815 drm_buddy_block_print(mm, block, printer); 816 mutex_unlock(&mgr->lock); 817 } 818 819 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { 820 .alloc = amdgpu_vram_mgr_new, 821 .free = amdgpu_vram_mgr_del, 822 .intersects = amdgpu_vram_mgr_intersects, 823 .compatible = amdgpu_vram_mgr_compatible, 824 .debug = amdgpu_vram_mgr_debug 825 }; 826 827 /** 828 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM 829 * 830 * @adev: amdgpu_device pointer 831 * 832 * Allocate and initialize the VRAM manager. 833 */ 834 int amdgpu_vram_mgr_init(struct amdgpu_device *adev) 835 { 836 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 837 struct ttm_resource_manager *man = &mgr->manager; 838 int err; 839 840 ttm_resource_manager_init(man, &adev->mman.bdev, 841 adev->gmc.real_vram_size); 842 843 man->func = &amdgpu_vram_mgr_func; 844 845 err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE); 846 if (err) 847 return err; 848 849 mutex_init(&mgr->lock); 850 INIT_LIST_HEAD(&mgr->reservations_pending); 851 INIT_LIST_HEAD(&mgr->reserved_pages); 852 mgr->default_page_size = PAGE_SIZE; 853 854 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); 855 ttm_resource_manager_set_used(man, true); 856 return 0; 857 } 858 859 /** 860 * amdgpu_vram_mgr_fini - free and destroy VRAM manager 861 * 862 * @adev: amdgpu_device pointer 863 * 864 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still 865 * allocated inside it. 866 */ 867 void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) 868 { 869 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 870 struct ttm_resource_manager *man = &mgr->manager; 871 int ret; 872 struct amdgpu_vram_reservation *rsv, *temp; 873 874 ttm_resource_manager_set_used(man, false); 875 876 ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man); 877 if (ret) 878 return; 879 880 mutex_lock(&mgr->lock); 881 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) 882 kfree(rsv); 883 884 list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) { 885 drm_buddy_free_list(&mgr->mm, &rsv->blocks); 886 kfree(rsv); 887 } 888 drm_buddy_fini(&mgr->mm); 889 mutex_unlock(&mgr->lock); 890 891 ttm_resource_manager_cleanup(man); 892 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL); 893 } 894