1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #include <linux/dma-mapping.h> 26 #include "amdgpu.h" 27 #include "amdgpu_vm.h" 28 #include "amdgpu_atomfirmware.h" 29 #include "atom.h" 30 31 static inline struct amdgpu_vram_mgr *to_vram_mgr(struct ttm_resource_manager *man) 32 { 33 return container_of(man, struct amdgpu_vram_mgr, manager); 34 } 35 36 static inline struct amdgpu_device *to_amdgpu_device(struct amdgpu_vram_mgr *mgr) 37 { 38 return container_of(mgr, struct amdgpu_device, mman.vram_mgr); 39 } 40 41 /** 42 * DOC: mem_info_vram_total 43 * 44 * The amdgpu driver provides a sysfs API for reporting current total VRAM 45 * available on the device 46 * The file mem_info_vram_total is used for this and returns the total 47 * amount of VRAM in bytes 48 */ 49 static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev, 50 struct device_attribute *attr, char *buf) 51 { 52 struct drm_device *ddev = dev_get_drvdata(dev); 53 struct amdgpu_device *adev = drm_to_adev(ddev); 54 55 return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.real_vram_size); 56 } 57 58 /** 59 * DOC: mem_info_vis_vram_total 60 * 61 * The amdgpu driver provides a sysfs API for reporting current total 62 * visible VRAM available on the device 63 * The file mem_info_vis_vram_total is used for this and returns the total 64 * amount of visible VRAM in bytes 65 */ 66 static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, 67 struct device_attribute *attr, char *buf) 68 { 69 struct drm_device *ddev = dev_get_drvdata(dev); 70 struct amdgpu_device *adev = drm_to_adev(ddev); 71 72 return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.visible_vram_size); 73 } 74 75 /** 76 * DOC: mem_info_vram_used 77 * 78 * The amdgpu driver provides a sysfs API for reporting current total VRAM 79 * available on the device 80 * The file mem_info_vram_used is used for this and returns the total 81 * amount of currently used VRAM in bytes 82 */ 83 static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, 84 struct device_attribute *attr, char *buf) 85 { 86 struct drm_device *ddev = dev_get_drvdata(dev); 87 struct amdgpu_device *adev = drm_to_adev(ddev); 88 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 89 90 return snprintf(buf, PAGE_SIZE, "%llu\n", 91 amdgpu_vram_mgr_usage(man)); 92 } 93 94 /** 95 * DOC: mem_info_vis_vram_used 96 * 97 * The amdgpu driver provides a sysfs API for reporting current total of 98 * used visible VRAM 99 * The file mem_info_vis_vram_used is used for this and returns the total 100 * amount of currently used visible VRAM in bytes 101 */ 102 static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev, 103 struct device_attribute *attr, char *buf) 104 { 105 struct drm_device *ddev = dev_get_drvdata(dev); 106 struct amdgpu_device *adev = drm_to_adev(ddev); 107 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 108 109 return snprintf(buf, PAGE_SIZE, "%llu\n", 110 amdgpu_vram_mgr_vis_usage(man)); 111 } 112 113 static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev, 114 struct device_attribute *attr, 115 char *buf) 116 { 117 struct drm_device *ddev = dev_get_drvdata(dev); 118 struct amdgpu_device *adev = drm_to_adev(ddev); 119 120 switch (adev->gmc.vram_vendor) { 121 case SAMSUNG: 122 return snprintf(buf, PAGE_SIZE, "samsung\n"); 123 case INFINEON: 124 return snprintf(buf, PAGE_SIZE, "infineon\n"); 125 case ELPIDA: 126 return snprintf(buf, PAGE_SIZE, "elpida\n"); 127 case ETRON: 128 return snprintf(buf, PAGE_SIZE, "etron\n"); 129 case NANYA: 130 return snprintf(buf, PAGE_SIZE, "nanya\n"); 131 case HYNIX: 132 return snprintf(buf, PAGE_SIZE, "hynix\n"); 133 case MOSEL: 134 return snprintf(buf, PAGE_SIZE, "mosel\n"); 135 case WINBOND: 136 return snprintf(buf, PAGE_SIZE, "winbond\n"); 137 case ESMT: 138 return snprintf(buf, PAGE_SIZE, "esmt\n"); 139 case MICRON: 140 return snprintf(buf, PAGE_SIZE, "micron\n"); 141 default: 142 return snprintf(buf, PAGE_SIZE, "unknown\n"); 143 } 144 } 145 146 static DEVICE_ATTR(mem_info_vram_total, S_IRUGO, 147 amdgpu_mem_info_vram_total_show, NULL); 148 static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO, 149 amdgpu_mem_info_vis_vram_total_show,NULL); 150 static DEVICE_ATTR(mem_info_vram_used, S_IRUGO, 151 amdgpu_mem_info_vram_used_show, NULL); 152 static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO, 153 amdgpu_mem_info_vis_vram_used_show, NULL); 154 static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO, 155 amdgpu_mem_info_vram_vendor, NULL); 156 157 static const struct attribute *amdgpu_vram_mgr_attributes[] = { 158 &dev_attr_mem_info_vram_total.attr, 159 &dev_attr_mem_info_vis_vram_total.attr, 160 &dev_attr_mem_info_vram_used.attr, 161 &dev_attr_mem_info_vis_vram_used.attr, 162 &dev_attr_mem_info_vram_vendor.attr, 163 NULL 164 }; 165 166 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func; 167 168 /** 169 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM 170 * 171 * @man: TTM memory type manager 172 * @p_size: maximum size of VRAM 173 * 174 * Allocate and initialize the VRAM manager. 175 */ 176 int amdgpu_vram_mgr_init(struct amdgpu_device *adev) 177 { 178 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 179 struct ttm_resource_manager *man = &mgr->manager; 180 int ret; 181 182 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; 183 man->default_caching = TTM_PL_FLAG_WC; 184 185 ttm_resource_manager_init(man, adev->gmc.real_vram_size >> PAGE_SHIFT); 186 187 man->func = &amdgpu_vram_mgr_func; 188 189 drm_mm_init(&mgr->mm, 0, man->size); 190 spin_lock_init(&mgr->lock); 191 192 /* Add the two VRAM-related sysfs files */ 193 ret = sysfs_create_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes); 194 if (ret) 195 DRM_ERROR("Failed to register sysfs\n"); 196 197 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); 198 ttm_resource_manager_set_used(man, true); 199 return 0; 200 } 201 202 /** 203 * amdgpu_vram_mgr_fini - free and destroy VRAM manager 204 * 205 * @man: TTM memory type manager 206 * 207 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still 208 * allocated inside it. 209 */ 210 void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) 211 { 212 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 213 struct ttm_resource_manager *man = &mgr->manager; 214 int ret; 215 216 ttm_resource_manager_set_used(man, false); 217 218 ret = ttm_resource_manager_force_list_clean(&adev->mman.bdev, man); 219 if (ret) 220 return; 221 222 spin_lock(&mgr->lock); 223 drm_mm_takedown(&mgr->mm); 224 spin_unlock(&mgr->lock); 225 226 sysfs_remove_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes); 227 228 ttm_resource_manager_cleanup(man); 229 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL); 230 } 231 232 /** 233 * amdgpu_vram_mgr_vis_size - Calculate visible node size 234 * 235 * @adev: amdgpu device structure 236 * @node: MM node structure 237 * 238 * Calculate how many bytes of the MM node are inside visible VRAM 239 */ 240 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, 241 struct drm_mm_node *node) 242 { 243 uint64_t start = node->start << PAGE_SHIFT; 244 uint64_t end = (node->size + node->start) << PAGE_SHIFT; 245 246 if (start >= adev->gmc.visible_vram_size) 247 return 0; 248 249 return (end > adev->gmc.visible_vram_size ? 250 adev->gmc.visible_vram_size : end) - start; 251 } 252 253 /** 254 * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size 255 * 256 * @bo: &amdgpu_bo buffer object (must be in VRAM) 257 * 258 * Returns: 259 * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM. 260 */ 261 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) 262 { 263 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 264 struct ttm_resource *mem = &bo->tbo.mem; 265 struct drm_mm_node *nodes = mem->mm_node; 266 unsigned pages = mem->num_pages; 267 u64 usage; 268 269 if (amdgpu_gmc_vram_full_visible(&adev->gmc)) 270 return amdgpu_bo_size(bo); 271 272 if (mem->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) 273 return 0; 274 275 for (usage = 0; nodes && pages; pages -= nodes->size, nodes++) 276 usage += amdgpu_vram_mgr_vis_size(adev, nodes); 277 278 return usage; 279 } 280 281 /** 282 * amdgpu_vram_mgr_virt_start - update virtual start address 283 * 284 * @mem: ttm_resource to update 285 * @node: just allocated node 286 * 287 * Calculate a virtual BO start address to easily check if everything is CPU 288 * accessible. 289 */ 290 static void amdgpu_vram_mgr_virt_start(struct ttm_resource *mem, 291 struct drm_mm_node *node) 292 { 293 unsigned long start; 294 295 start = node->start + node->size; 296 if (start > mem->num_pages) 297 start -= mem->num_pages; 298 else 299 start = 0; 300 mem->start = max(mem->start, start); 301 } 302 303 /** 304 * amdgpu_vram_mgr_new - allocate new ranges 305 * 306 * @man: TTM memory type manager 307 * @tbo: TTM BO we need this range for 308 * @place: placement flags and restrictions 309 * @mem: the resulting mem object 310 * 311 * Allocate VRAM for the given BO. 312 */ 313 static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, 314 struct ttm_buffer_object *tbo, 315 const struct ttm_place *place, 316 struct ttm_resource *mem) 317 { 318 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 319 struct amdgpu_device *adev = to_amdgpu_device(mgr); 320 struct drm_mm *mm = &mgr->mm; 321 struct drm_mm_node *nodes; 322 enum drm_mm_insert_mode mode; 323 unsigned long lpfn, num_nodes, pages_per_node, pages_left; 324 uint64_t vis_usage = 0, mem_bytes, max_bytes; 325 unsigned i; 326 int r; 327 328 lpfn = place->lpfn; 329 if (!lpfn) 330 lpfn = man->size; 331 332 max_bytes = adev->gmc.mc_vram_size; 333 if (tbo->type != ttm_bo_type_kernel) 334 max_bytes -= AMDGPU_VM_RESERVED_VRAM; 335 336 /* bail out quickly if there's likely not enough VRAM for this BO */ 337 mem_bytes = (u64)mem->num_pages << PAGE_SHIFT; 338 if (atomic64_add_return(mem_bytes, &mgr->usage) > max_bytes) { 339 atomic64_sub(mem_bytes, &mgr->usage); 340 return -ENOSPC; 341 } 342 343 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 344 pages_per_node = ~0ul; 345 num_nodes = 1; 346 } else { 347 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 348 pages_per_node = HPAGE_PMD_NR; 349 #else 350 /* default to 2MB */ 351 pages_per_node = (2UL << (20UL - PAGE_SHIFT)); 352 #endif 353 pages_per_node = max((uint32_t)pages_per_node, mem->page_alignment); 354 num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node); 355 } 356 357 nodes = kvmalloc_array((uint32_t)num_nodes, sizeof(*nodes), 358 GFP_KERNEL | __GFP_ZERO); 359 if (!nodes) { 360 atomic64_sub(mem_bytes, &mgr->usage); 361 return -ENOMEM; 362 } 363 364 mode = DRM_MM_INSERT_BEST; 365 if (place->flags & TTM_PL_FLAG_TOPDOWN) 366 mode = DRM_MM_INSERT_HIGH; 367 368 mem->start = 0; 369 pages_left = mem->num_pages; 370 371 spin_lock(&mgr->lock); 372 for (i = 0; pages_left >= pages_per_node; ++i) { 373 unsigned long pages = rounddown_pow_of_two(pages_left); 374 375 r = drm_mm_insert_node_in_range(mm, &nodes[i], pages, 376 pages_per_node, 0, 377 place->fpfn, lpfn, 378 mode); 379 if (unlikely(r)) 380 break; 381 382 vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]); 383 amdgpu_vram_mgr_virt_start(mem, &nodes[i]); 384 pages_left -= pages; 385 } 386 387 for (; pages_left; ++i) { 388 unsigned long pages = min(pages_left, pages_per_node); 389 uint32_t alignment = mem->page_alignment; 390 391 if (pages == pages_per_node) 392 alignment = pages_per_node; 393 394 r = drm_mm_insert_node_in_range(mm, &nodes[i], 395 pages, alignment, 0, 396 place->fpfn, lpfn, 397 mode); 398 if (unlikely(r)) 399 goto error; 400 401 vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]); 402 amdgpu_vram_mgr_virt_start(mem, &nodes[i]); 403 pages_left -= pages; 404 } 405 spin_unlock(&mgr->lock); 406 407 atomic64_add(vis_usage, &mgr->vis_usage); 408 409 mem->mm_node = nodes; 410 411 return 0; 412 413 error: 414 while (i--) 415 drm_mm_remove_node(&nodes[i]); 416 spin_unlock(&mgr->lock); 417 atomic64_sub(mem->num_pages << PAGE_SHIFT, &mgr->usage); 418 419 kvfree(nodes); 420 return r; 421 } 422 423 /** 424 * amdgpu_vram_mgr_del - free ranges 425 * 426 * @man: TTM memory type manager 427 * @mem: TTM memory object 428 * 429 * Free the allocated VRAM again. 430 */ 431 static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, 432 struct ttm_resource *mem) 433 { 434 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 435 struct amdgpu_device *adev = to_amdgpu_device(mgr); 436 struct drm_mm_node *nodes = mem->mm_node; 437 uint64_t usage = 0, vis_usage = 0; 438 unsigned pages = mem->num_pages; 439 440 if (!mem->mm_node) 441 return; 442 443 spin_lock(&mgr->lock); 444 while (pages) { 445 pages -= nodes->size; 446 drm_mm_remove_node(nodes); 447 usage += nodes->size << PAGE_SHIFT; 448 vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes); 449 ++nodes; 450 } 451 spin_unlock(&mgr->lock); 452 453 atomic64_sub(usage, &mgr->usage); 454 atomic64_sub(vis_usage, &mgr->vis_usage); 455 456 kvfree(mem->mm_node); 457 mem->mm_node = NULL; 458 } 459 460 /** 461 * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table 462 * 463 * @adev: amdgpu device pointer 464 * @mem: TTM memory object 465 * @dev: the other device 466 * @dir: dma direction 467 * @sgt: resulting sg table 468 * 469 * Allocate and fill a sg table from a VRAM allocation. 470 */ 471 int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, 472 struct ttm_resource *mem, 473 struct device *dev, 474 enum dma_data_direction dir, 475 struct sg_table **sgt) 476 { 477 struct drm_mm_node *node; 478 struct scatterlist *sg; 479 int num_entries = 0; 480 unsigned int pages; 481 int i, r; 482 483 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL); 484 if (!*sgt) 485 return -ENOMEM; 486 487 for (pages = mem->num_pages, node = mem->mm_node; 488 pages; pages -= node->size, ++node) 489 ++num_entries; 490 491 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); 492 if (r) 493 goto error_free; 494 495 for_each_sgtable_sg((*sgt), sg, i) 496 sg->length = 0; 497 498 node = mem->mm_node; 499 for_each_sgtable_sg((*sgt), sg, i) { 500 phys_addr_t phys = (node->start << PAGE_SHIFT) + 501 adev->gmc.aper_base; 502 size_t size = node->size << PAGE_SHIFT; 503 dma_addr_t addr; 504 505 ++node; 506 addr = dma_map_resource(dev, phys, size, dir, 507 DMA_ATTR_SKIP_CPU_SYNC); 508 r = dma_mapping_error(dev, addr); 509 if (r) 510 goto error_unmap; 511 512 sg_set_page(sg, NULL, size, 0); 513 sg_dma_address(sg) = addr; 514 sg_dma_len(sg) = size; 515 } 516 return 0; 517 518 error_unmap: 519 for_each_sgtable_sg((*sgt), sg, i) { 520 if (!sg->length) 521 continue; 522 523 dma_unmap_resource(dev, sg->dma_address, 524 sg->length, dir, 525 DMA_ATTR_SKIP_CPU_SYNC); 526 } 527 sg_free_table(*sgt); 528 529 error_free: 530 kfree(*sgt); 531 return r; 532 } 533 534 /** 535 * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table 536 * 537 * @adev: amdgpu device pointer 538 * @sgt: sg table to free 539 * 540 * Free a previously allocate sg table. 541 */ 542 void amdgpu_vram_mgr_free_sgt(struct amdgpu_device *adev, 543 struct device *dev, 544 enum dma_data_direction dir, 545 struct sg_table *sgt) 546 { 547 struct scatterlist *sg; 548 int i; 549 550 for_each_sgtable_sg(sgt, sg, i) 551 dma_unmap_resource(dev, sg->dma_address, 552 sg->length, dir, 553 DMA_ATTR_SKIP_CPU_SYNC); 554 sg_free_table(sgt); 555 kfree(sgt); 556 } 557 558 /** 559 * amdgpu_vram_mgr_usage - how many bytes are used in this domain 560 * 561 * @man: TTM memory type manager 562 * 563 * Returns how many bytes are used in this domain. 564 */ 565 uint64_t amdgpu_vram_mgr_usage(struct ttm_resource_manager *man) 566 { 567 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 568 569 return atomic64_read(&mgr->usage); 570 } 571 572 /** 573 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part 574 * 575 * @man: TTM memory type manager 576 * 577 * Returns how many bytes are used in the visible part of VRAM 578 */ 579 uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_resource_manager *man) 580 { 581 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 582 583 return atomic64_read(&mgr->vis_usage); 584 } 585 586 /** 587 * amdgpu_vram_mgr_debug - dump VRAM table 588 * 589 * @man: TTM memory type manager 590 * @printer: DRM printer to use 591 * 592 * Dump the table content using printk. 593 */ 594 static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, 595 struct drm_printer *printer) 596 { 597 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 598 599 spin_lock(&mgr->lock); 600 drm_mm_print(&mgr->mm, printer); 601 spin_unlock(&mgr->lock); 602 603 drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n", 604 man->size, amdgpu_vram_mgr_usage(man) >> 20, 605 amdgpu_vram_mgr_vis_usage(man) >> 20); 606 } 607 608 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { 609 .alloc = amdgpu_vram_mgr_new, 610 .free = amdgpu_vram_mgr_del, 611 .debug = amdgpu_vram_mgr_debug 612 }; 613