16a7f76e7SChristian König /* 26a7f76e7SChristian König * Copyright 2016 Advanced Micro Devices, Inc. 36a7f76e7SChristian König * 46a7f76e7SChristian König * Permission is hereby granted, free of charge, to any person obtaining a 56a7f76e7SChristian König * copy of this software and associated documentation files (the "Software"), 66a7f76e7SChristian König * to deal in the Software without restriction, including without limitation 76a7f76e7SChristian König * the rights to use, copy, modify, merge, publish, distribute, sublicense, 86a7f76e7SChristian König * and/or sell copies of the Software, and to permit persons to whom the 96a7f76e7SChristian König * Software is furnished to do so, subject to the following conditions: 106a7f76e7SChristian König * 116a7f76e7SChristian König * The above copyright notice and this permission notice shall be included in 126a7f76e7SChristian König * all copies or substantial portions of the Software. 136a7f76e7SChristian König * 146a7f76e7SChristian König * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 156a7f76e7SChristian König * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 166a7f76e7SChristian König * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 176a7f76e7SChristian König * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 186a7f76e7SChristian König * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 196a7f76e7SChristian König * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 206a7f76e7SChristian König * OTHER DEALINGS IN THE SOFTWARE. 216a7f76e7SChristian König * 226a7f76e7SChristian König * Authors: Christian König 236a7f76e7SChristian König */ 246a7f76e7SChristian König 25f44ffd67SChristian König #include <linux/dma-mapping.h> 266a7f76e7SChristian König #include "amdgpu.h" 279d1b3c78SChristian König #include "amdgpu_vm.h" 28ba5b662cSRamesh Errabolu #include "amdgpu_res_cursor.h" 29ad02e08eSOri Messinger #include "amdgpu_atomfirmware.h" 30ad02e08eSOri Messinger #include "atom.h" 316a7f76e7SChristian König 32*3b5d86fcSChristian König struct amdgpu_vram_reservation { 33*3b5d86fcSChristian König struct list_head node; 34*3b5d86fcSChristian König struct drm_mm_node mm_node; 35*3b5d86fcSChristian König }; 36*3b5d86fcSChristian König 37589939d4SChristian König static inline struct amdgpu_vram_mgr * 38589939d4SChristian König to_vram_mgr(struct ttm_resource_manager *man) 390af135b8SDave Airlie { 400af135b8SDave Airlie return container_of(man, struct amdgpu_vram_mgr, manager); 410af135b8SDave Airlie } 420af135b8SDave Airlie 43589939d4SChristian König static inline struct amdgpu_device * 44589939d4SChristian König to_amdgpu_device(struct amdgpu_vram_mgr *mgr) 4537362793SDave Airlie { 4637362793SDave Airlie return container_of(mgr, struct amdgpu_device, mman.vram_mgr); 4737362793SDave Airlie } 486a7f76e7SChristian König 496a7f76e7SChristian König /** 5055c374e9SKent Russell * DOC: mem_info_vram_total 5155c374e9SKent Russell * 5255c374e9SKent Russell * The amdgpu driver provides a sysfs API for reporting current total VRAM 5355c374e9SKent Russell * available on the device 5455c374e9SKent Russell * The file mem_info_vram_total is used for this and returns the total 5555c374e9SKent Russell * amount of VRAM in bytes 5655c374e9SKent Russell */ 5755c374e9SKent Russell static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev, 5855c374e9SKent Russell struct device_attribute *attr, char *buf) 5955c374e9SKent Russell { 6055c374e9SKent Russell struct drm_device *ddev = dev_get_drvdata(dev); 611348969aSLuben Tuikov struct amdgpu_device *adev = drm_to_adev(ddev); 6255c374e9SKent Russell 6336000c7aSTian Tao return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size); 6455c374e9SKent Russell } 6555c374e9SKent Russell 6655c374e9SKent Russell /** 6755c374e9SKent Russell * DOC: mem_info_vis_vram_total 6855c374e9SKent Russell * 6955c374e9SKent Russell * The amdgpu driver provides a sysfs API for reporting current total 7055c374e9SKent Russell * visible VRAM available on the device 7155c374e9SKent Russell * The file mem_info_vis_vram_total is used for this and returns the total 7255c374e9SKent Russell * amount of visible VRAM in bytes 7355c374e9SKent Russell */ 7455c374e9SKent Russell static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, 7555c374e9SKent Russell struct device_attribute *attr, char *buf) 7655c374e9SKent Russell { 7755c374e9SKent Russell struct drm_device *ddev = dev_get_drvdata(dev); 781348969aSLuben Tuikov struct amdgpu_device *adev = drm_to_adev(ddev); 7955c374e9SKent Russell 8036000c7aSTian Tao return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size); 8155c374e9SKent Russell } 8255c374e9SKent Russell 8355c374e9SKent Russell /** 8455c374e9SKent Russell * DOC: mem_info_vram_used 8555c374e9SKent Russell * 8655c374e9SKent Russell * The amdgpu driver provides a sysfs API for reporting current total VRAM 8755c374e9SKent Russell * available on the device 8855c374e9SKent Russell * The file mem_info_vram_used is used for this and returns the total 8955c374e9SKent Russell * amount of currently used VRAM in bytes 9055c374e9SKent Russell */ 9155c374e9SKent Russell static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, 92589939d4SChristian König struct device_attribute *attr, 93589939d4SChristian König char *buf) 9455c374e9SKent Russell { 9555c374e9SKent Russell struct drm_device *ddev = dev_get_drvdata(dev); 961348969aSLuben Tuikov struct amdgpu_device *adev = drm_to_adev(ddev); 97589939d4SChristian König struct ttm_resource_manager *man; 9855c374e9SKent Russell 99589939d4SChristian König man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 10036000c7aSTian Tao return sysfs_emit(buf, "%llu\n", amdgpu_vram_mgr_usage(man)); 10155c374e9SKent Russell } 10255c374e9SKent Russell 10355c374e9SKent Russell /** 10455c374e9SKent Russell * DOC: mem_info_vis_vram_used 10555c374e9SKent Russell * 10655c374e9SKent Russell * The amdgpu driver provides a sysfs API for reporting current total of 10755c374e9SKent Russell * used visible VRAM 10855c374e9SKent Russell * The file mem_info_vis_vram_used is used for this and returns the total 10955c374e9SKent Russell * amount of currently used visible VRAM in bytes 11055c374e9SKent Russell */ 11155c374e9SKent Russell static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev, 112589939d4SChristian König struct device_attribute *attr, 113589939d4SChristian König char *buf) 11455c374e9SKent Russell { 11555c374e9SKent Russell struct drm_device *ddev = dev_get_drvdata(dev); 1161348969aSLuben Tuikov struct amdgpu_device *adev = drm_to_adev(ddev); 117589939d4SChristian König struct ttm_resource_manager *man; 11855c374e9SKent Russell 119589939d4SChristian König man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 12036000c7aSTian Tao return sysfs_emit(buf, "%llu\n", amdgpu_vram_mgr_vis_usage(man)); 12155c374e9SKent Russell } 12255c374e9SKent Russell 123589939d4SChristian König /** 124589939d4SChristian König * DOC: mem_info_vram_vendor 125589939d4SChristian König * 126589939d4SChristian König * The amdgpu driver provides a sysfs API for reporting the vendor of the 127589939d4SChristian König * installed VRAM 128589939d4SChristian König * The file mem_info_vram_vendor is used for this and returns the name of the 129589939d4SChristian König * vendor. 130589939d4SChristian König */ 131ad02e08eSOri Messinger static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev, 132ad02e08eSOri Messinger struct device_attribute *attr, 133ad02e08eSOri Messinger char *buf) 134ad02e08eSOri Messinger { 135ad02e08eSOri Messinger struct drm_device *ddev = dev_get_drvdata(dev); 1361348969aSLuben Tuikov struct amdgpu_device *adev = drm_to_adev(ddev); 137ad02e08eSOri Messinger 138ad02e08eSOri Messinger switch (adev->gmc.vram_vendor) { 139ad02e08eSOri Messinger case SAMSUNG: 14036000c7aSTian Tao return sysfs_emit(buf, "samsung\n"); 141ad02e08eSOri Messinger case INFINEON: 14236000c7aSTian Tao return sysfs_emit(buf, "infineon\n"); 143ad02e08eSOri Messinger case ELPIDA: 14436000c7aSTian Tao return sysfs_emit(buf, "elpida\n"); 145ad02e08eSOri Messinger case ETRON: 14636000c7aSTian Tao return sysfs_emit(buf, "etron\n"); 147ad02e08eSOri Messinger case NANYA: 14836000c7aSTian Tao return sysfs_emit(buf, "nanya\n"); 149ad02e08eSOri Messinger case HYNIX: 15036000c7aSTian Tao return sysfs_emit(buf, "hynix\n"); 151ad02e08eSOri Messinger case MOSEL: 15236000c7aSTian Tao return sysfs_emit(buf, "mosel\n"); 153ad02e08eSOri Messinger case WINBOND: 15436000c7aSTian Tao return sysfs_emit(buf, "winbond\n"); 155ad02e08eSOri Messinger case ESMT: 15636000c7aSTian Tao return sysfs_emit(buf, "esmt\n"); 157ad02e08eSOri Messinger case MICRON: 15836000c7aSTian Tao return sysfs_emit(buf, "micron\n"); 159ad02e08eSOri Messinger default: 16036000c7aSTian Tao return sysfs_emit(buf, "unknown\n"); 161ad02e08eSOri Messinger } 162ad02e08eSOri Messinger } 163ad02e08eSOri Messinger 16455c374e9SKent Russell static DEVICE_ATTR(mem_info_vram_total, S_IRUGO, 16555c374e9SKent Russell amdgpu_mem_info_vram_total_show, NULL); 16655c374e9SKent Russell static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO, 16755c374e9SKent Russell amdgpu_mem_info_vis_vram_total_show,NULL); 16855c374e9SKent Russell static DEVICE_ATTR(mem_info_vram_used, S_IRUGO, 16955c374e9SKent Russell amdgpu_mem_info_vram_used_show, NULL); 17055c374e9SKent Russell static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO, 17155c374e9SKent Russell amdgpu_mem_info_vis_vram_used_show, NULL); 172ad02e08eSOri Messinger static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO, 173ad02e08eSOri Messinger amdgpu_mem_info_vram_vendor, NULL); 17455c374e9SKent Russell 175ecc8c2e1SBernard Zhao static const struct attribute *amdgpu_vram_mgr_attributes[] = { 176ecc8c2e1SBernard Zhao &dev_attr_mem_info_vram_total.attr, 177ecc8c2e1SBernard Zhao &dev_attr_mem_info_vis_vram_total.attr, 178ecc8c2e1SBernard Zhao &dev_attr_mem_info_vram_used.attr, 179ecc8c2e1SBernard Zhao &dev_attr_mem_info_vis_vram_used.attr, 180ecc8c2e1SBernard Zhao &dev_attr_mem_info_vram_vendor.attr, 181ecc8c2e1SBernard Zhao NULL 182ecc8c2e1SBernard Zhao }; 183ecc8c2e1SBernard Zhao 1846a7f76e7SChristian König /** 1853c848bb3SChristian König * amdgpu_vram_mgr_vis_size - Calculate visible node size 1863c848bb3SChristian König * 187982a820bSMauro Carvalho Chehab * @adev: amdgpu_device pointer 1883c848bb3SChristian König * @node: MM node structure 1893c848bb3SChristian König * 1903c848bb3SChristian König * Calculate how many bytes of the MM node are inside visible VRAM 1913c848bb3SChristian König */ 1923c848bb3SChristian König static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, 1933c848bb3SChristian König struct drm_mm_node *node) 1943c848bb3SChristian König { 1953c848bb3SChristian König uint64_t start = node->start << PAGE_SHIFT; 1963c848bb3SChristian König uint64_t end = (node->size + node->start) << PAGE_SHIFT; 1973c848bb3SChristian König 198770d13b1SChristian König if (start >= adev->gmc.visible_vram_size) 1993c848bb3SChristian König return 0; 2003c848bb3SChristian König 201770d13b1SChristian König return (end > adev->gmc.visible_vram_size ? 202770d13b1SChristian König adev->gmc.visible_vram_size : end) - start; 2033c848bb3SChristian König } 2043c848bb3SChristian König 2053c848bb3SChristian König /** 206ddc21af4SMichel Dänzer * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size 2075e9244ffSMichel Dänzer * 2085e9244ffSMichel Dänzer * @bo: &amdgpu_bo buffer object (must be in VRAM) 2095e9244ffSMichel Dänzer * 2105e9244ffSMichel Dänzer * Returns: 211ddc21af4SMichel Dänzer * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM. 2125e9244ffSMichel Dänzer */ 213ddc21af4SMichel Dänzer u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) 2145e9244ffSMichel Dänzer { 2157303b39eSMichel Dänzer struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 2162966141aSDave Airlie struct ttm_resource *mem = &bo->tbo.mem; 2177303b39eSMichel Dänzer struct drm_mm_node *nodes = mem->mm_node; 2187303b39eSMichel Dänzer unsigned pages = mem->num_pages; 219ddc21af4SMichel Dänzer u64 usage; 2207303b39eSMichel Dänzer 2219735bf19SMichel Dänzer if (amdgpu_gmc_vram_full_visible(&adev->gmc)) 2225e9244ffSMichel Dänzer return amdgpu_bo_size(bo); 2235e9244ffSMichel Dänzer 224ddc21af4SMichel Dänzer if (mem->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) 225ddc21af4SMichel Dänzer return 0; 226ddc21af4SMichel Dänzer 227ddc21af4SMichel Dänzer for (usage = 0; nodes && pages; pages -= nodes->size, nodes++) 228ddc21af4SMichel Dänzer usage += amdgpu_vram_mgr_vis_size(adev, nodes); 2297303b39eSMichel Dänzer 2307303b39eSMichel Dänzer return usage; 2315e9244ffSMichel Dänzer } 2325e9244ffSMichel Dänzer 233589939d4SChristian König /* Commit the reservation of VRAM pages */ 234676deb38SDennis Li static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man) 235676deb38SDennis Li { 236676deb38SDennis Li struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 237676deb38SDennis Li struct amdgpu_device *adev = to_amdgpu_device(mgr); 238676deb38SDennis Li struct drm_mm *mm = &mgr->mm; 239676deb38SDennis Li struct amdgpu_vram_reservation *rsv, *temp; 240676deb38SDennis Li uint64_t vis_usage; 241676deb38SDennis Li 242676deb38SDennis Li list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node) { 243676deb38SDennis Li if (drm_mm_reserve_node(mm, &rsv->mm_node)) 244676deb38SDennis Li continue; 245676deb38SDennis Li 2467dee4d51SColin Ian King dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n", 247e5c04edfSChristian König rsv->mm_node.start, rsv->mm_node.size); 248676deb38SDennis Li 249676deb38SDennis Li vis_usage = amdgpu_vram_mgr_vis_size(adev, &rsv->mm_node); 250676deb38SDennis Li atomic64_add(vis_usage, &mgr->vis_usage); 251676deb38SDennis Li atomic64_add(rsv->mm_node.size << PAGE_SHIFT, &mgr->usage); 252676deb38SDennis Li list_move(&rsv->node, &mgr->reserved_pages); 253676deb38SDennis Li } 254676deb38SDennis Li } 255676deb38SDennis Li 256676deb38SDennis Li /** 257676deb38SDennis Li * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM 258676deb38SDennis Li * 259676deb38SDennis Li * @man: TTM memory type manager 260676deb38SDennis Li * @start: start address of the range in VRAM 261676deb38SDennis Li * @size: size of the range 262676deb38SDennis Li * 263676deb38SDennis Li * Reserve memory from start addess with the specified size in VRAM 264676deb38SDennis Li */ 265676deb38SDennis Li int amdgpu_vram_mgr_reserve_range(struct ttm_resource_manager *man, 266676deb38SDennis Li uint64_t start, uint64_t size) 267676deb38SDennis Li { 268676deb38SDennis Li struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 269676deb38SDennis Li struct amdgpu_vram_reservation *rsv; 270676deb38SDennis Li 271676deb38SDennis Li rsv = kzalloc(sizeof(*rsv), GFP_KERNEL); 272676deb38SDennis Li if (!rsv) 273676deb38SDennis Li return -ENOMEM; 274676deb38SDennis Li 275676deb38SDennis Li INIT_LIST_HEAD(&rsv->node); 276676deb38SDennis Li rsv->mm_node.start = start >> PAGE_SHIFT; 277676deb38SDennis Li rsv->mm_node.size = size >> PAGE_SHIFT; 278676deb38SDennis Li 279676deb38SDennis Li spin_lock(&mgr->lock); 280e5c04edfSChristian König list_add_tail(&mgr->reservations_pending, &rsv->node); 281676deb38SDennis Li amdgpu_vram_mgr_do_reserve(man); 282676deb38SDennis Li spin_unlock(&mgr->lock); 283676deb38SDennis Li 284676deb38SDennis Li return 0; 285676deb38SDennis Li } 286676deb38SDennis Li 287676deb38SDennis Li /** 288676deb38SDennis Li * amdgpu_vram_mgr_query_page_status - query the reservation status 289676deb38SDennis Li * 290676deb38SDennis Li * @man: TTM memory type manager 291676deb38SDennis Li * @start: start address of a page in VRAM 292676deb38SDennis Li * 293676deb38SDennis Li * Returns: 294676deb38SDennis Li * -EBUSY: the page is still hold and in pending list 295676deb38SDennis Li * 0: the page has been reserved 296676deb38SDennis Li * -ENOENT: the input page is not a reservation 297676deb38SDennis Li */ 298676deb38SDennis Li int amdgpu_vram_mgr_query_page_status(struct ttm_resource_manager *man, 299676deb38SDennis Li uint64_t start) 300676deb38SDennis Li { 301676deb38SDennis Li struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 302676deb38SDennis Li struct amdgpu_vram_reservation *rsv; 303676deb38SDennis Li int ret; 304676deb38SDennis Li 305676deb38SDennis Li spin_lock(&mgr->lock); 306676deb38SDennis Li 307676deb38SDennis Li list_for_each_entry(rsv, &mgr->reservations_pending, node) { 308676deb38SDennis Li if ((rsv->mm_node.start <= start) && 309676deb38SDennis Li (start < (rsv->mm_node.start + rsv->mm_node.size))) { 310676deb38SDennis Li ret = -EBUSY; 311676deb38SDennis Li goto out; 312676deb38SDennis Li } 313676deb38SDennis Li } 314676deb38SDennis Li 315676deb38SDennis Li list_for_each_entry(rsv, &mgr->reserved_pages, node) { 316676deb38SDennis Li if ((rsv->mm_node.start <= start) && 317676deb38SDennis Li (start < (rsv->mm_node.start + rsv->mm_node.size))) { 318676deb38SDennis Li ret = 0; 319676deb38SDennis Li goto out; 320676deb38SDennis Li } 321676deb38SDennis Li } 322676deb38SDennis Li 323676deb38SDennis Li ret = -ENOENT; 324676deb38SDennis Li out: 325676deb38SDennis Li spin_unlock(&mgr->lock); 326676deb38SDennis Li return ret; 327676deb38SDennis Li } 328676deb38SDennis Li 3295e9244ffSMichel Dänzer /** 330433ca054SChristian König * amdgpu_vram_mgr_virt_start - update virtual start address 331433ca054SChristian König * 3322966141aSDave Airlie * @mem: ttm_resource to update 333433ca054SChristian König * @node: just allocated node 334433ca054SChristian König * 335433ca054SChristian König * Calculate a virtual BO start address to easily check if everything is CPU 336433ca054SChristian König * accessible. 337433ca054SChristian König */ 3382966141aSDave Airlie static void amdgpu_vram_mgr_virt_start(struct ttm_resource *mem, 339433ca054SChristian König struct drm_mm_node *node) 340433ca054SChristian König { 341433ca054SChristian König unsigned long start; 342433ca054SChristian König 343433ca054SChristian König start = node->start + node->size; 344433ca054SChristian König if (start > mem->num_pages) 345433ca054SChristian König start -= mem->num_pages; 346433ca054SChristian König else 347433ca054SChristian König start = 0; 348433ca054SChristian König mem->start = max(mem->start, start); 349433ca054SChristian König } 350433ca054SChristian König 351433ca054SChristian König /** 3526a7f76e7SChristian König * amdgpu_vram_mgr_new - allocate new ranges 3536a7f76e7SChristian König * 3546a7f76e7SChristian König * @man: TTM memory type manager 3556a7f76e7SChristian König * @tbo: TTM BO we need this range for 3566a7f76e7SChristian König * @place: placement flags and restrictions 3576a7f76e7SChristian König * @mem: the resulting mem object 3586a7f76e7SChristian König * 3596a7f76e7SChristian König * Allocate VRAM for the given BO. 3606a7f76e7SChristian König */ 3619de59bc2SDave Airlie static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, 3626a7f76e7SChristian König struct ttm_buffer_object *tbo, 3636a7f76e7SChristian König const struct ttm_place *place, 3642966141aSDave Airlie struct ttm_resource *mem) 3656a7f76e7SChristian König { 366dd03daecSChristian König unsigned long lpfn, num_nodes, pages_per_node, pages_left, pages; 3670af135b8SDave Airlie struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 36837362793SDave Airlie struct amdgpu_device *adev = to_amdgpu_device(mgr); 3699d1b3c78SChristian König uint64_t vis_usage = 0, mem_bytes, max_bytes; 370dd03daecSChristian König struct drm_mm *mm = &mgr->mm; 371dd03daecSChristian König enum drm_mm_insert_mode mode; 372dd03daecSChristian König struct drm_mm_node *nodes; 3736a7f76e7SChristian König unsigned i; 3746a7f76e7SChristian König int r; 3756a7f76e7SChristian König 3766a7f76e7SChristian König lpfn = place->lpfn; 3776a7f76e7SChristian König if (!lpfn) 3786a7f76e7SChristian König lpfn = man->size; 3796a7f76e7SChristian König 3809d1b3c78SChristian König max_bytes = adev->gmc.mc_vram_size; 3819d1b3c78SChristian König if (tbo->type != ttm_bo_type_kernel) 3829d1b3c78SChristian König max_bytes -= AMDGPU_VM_RESERVED_VRAM; 3839d1b3c78SChristian König 384c5e4c6bbSTom St Denis /* bail out quickly if there's likely not enough VRAM for this BO */ 385c1d827d6STom St Denis mem_bytes = (u64)mem->num_pages << PAGE_SHIFT; 3869d1b3c78SChristian König if (atomic64_add_return(mem_bytes, &mgr->usage) > max_bytes) { 387c1d827d6STom St Denis atomic64_sub(mem_bytes, &mgr->usage); 38858e4d686SChristian König return -ENOSPC; 389c5e4c6bbSTom St Denis } 390c5e4c6bbSTom St Denis 391b4559a16STom St Denis if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 3926a7f76e7SChristian König pages_per_node = ~0ul; 3936a7f76e7SChristian König num_nodes = 1; 3946a7f76e7SChristian König } else { 395b4559a16STom St Denis #ifdef CONFIG_TRANSPARENT_HUGEPAGE 396b4559a16STom St Denis pages_per_node = HPAGE_PMD_NR; 397b4559a16STom St Denis #else 398b4559a16STom St Denis /* default to 2MB */ 399dd03daecSChristian König pages_per_node = 2UL << (20UL - PAGE_SHIFT); 400b4559a16STom St Denis #endif 401dd03daecSChristian König pages_per_node = max_t(uint32_t, pages_per_node, 402dd03daecSChristian König mem->page_alignment); 4036a7f76e7SChristian König num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node); 4046a7f76e7SChristian König } 4056a7f76e7SChristian König 406b4559a16STom St Denis nodes = kvmalloc_array((uint32_t)num_nodes, sizeof(*nodes), 4076fa39bc1SMichel Dänzer GFP_KERNEL | __GFP_ZERO); 408c5e4c6bbSTom St Denis if (!nodes) { 409c1d827d6STom St Denis atomic64_sub(mem_bytes, &mgr->usage); 4106a7f76e7SChristian König return -ENOMEM; 411c5e4c6bbSTom St Denis } 4126a7f76e7SChristian König 4134e64e553SChris Wilson mode = DRM_MM_INSERT_BEST; 4144e64e553SChris Wilson if (place->flags & TTM_PL_FLAG_TOPDOWN) 4154e64e553SChris Wilson mode = DRM_MM_INSERT_HIGH; 4166a7f76e7SChristian König 41789bb5752SChristian König mem->start = 0; 4186a7f76e7SChristian König pages_left = mem->num_pages; 4196a7f76e7SChristian König 420b131c363SRamesh Errabolu /* Limit maximum size to 2GB due to SG table limitations */ 421dd03daecSChristian König pages = min(pages_left, 2UL << (30 - PAGE_SHIFT)); 422b131c363SRamesh Errabolu 423dd03daecSChristian König i = 0; 424dd03daecSChristian König spin_lock(&mgr->lock); 425dd03daecSChristian König while (pages_left) { 4266a7f76e7SChristian König uint32_t alignment = mem->page_alignment; 4276a7f76e7SChristian König 428dd03daecSChristian König if (pages >= pages_per_node) 4296a7f76e7SChristian König alignment = pages_per_node; 4306a7f76e7SChristian König 431dd03daecSChristian König r = drm_mm_insert_node_in_range(mm, &nodes[i], pages, alignment, 432dd03daecSChristian König 0, place->fpfn, lpfn, mode); 433dd03daecSChristian König if (unlikely(r)) { 434dd03daecSChristian König if (pages > pages_per_node) { 435dd03daecSChristian König if (is_power_of_2(pages)) 436dd03daecSChristian König pages = pages / 2; 437dd03daecSChristian König else 438dd03daecSChristian König pages = rounddown_pow_of_two(pages); 439dd03daecSChristian König continue; 440dd03daecSChristian König } 4416a7f76e7SChristian König goto error; 442dd03daecSChristian König } 4436a7f76e7SChristian König 4443c848bb3SChristian König vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]); 445433ca054SChristian König amdgpu_vram_mgr_virt_start(mem, &nodes[i]); 4466a7f76e7SChristian König pages_left -= pages; 447dd03daecSChristian König ++i; 448dd03daecSChristian König 449dd03daecSChristian König if (pages > pages_left) 450dd03daecSChristian König pages = pages_left; 4516a7f76e7SChristian König } 4526a7f76e7SChristian König spin_unlock(&mgr->lock); 4536a7f76e7SChristian König 454abf91e0dSChristian König if (i == 1) 455abf91e0dSChristian König mem->placement |= TTM_PL_FLAG_CONTIGUOUS; 456abf91e0dSChristian König 4573c848bb3SChristian König atomic64_add(vis_usage, &mgr->vis_usage); 4586a7f76e7SChristian König mem->mm_node = nodes; 4596a7f76e7SChristian König return 0; 4606a7f76e7SChristian König 4616a7f76e7SChristian König error: 4626a7f76e7SChristian König while (i--) 4636a7f76e7SChristian König drm_mm_remove_node(&nodes[i]); 4646a7f76e7SChristian König spin_unlock(&mgr->lock); 465c5e4c6bbSTom St Denis atomic64_sub(mem->num_pages << PAGE_SHIFT, &mgr->usage); 4666a7f76e7SChristian König 4676fa39bc1SMichel Dänzer kvfree(nodes); 46858e4d686SChristian König return r; 4696a7f76e7SChristian König } 4706a7f76e7SChristian König 4716a7f76e7SChristian König /** 4726a7f76e7SChristian König * amdgpu_vram_mgr_del - free ranges 4736a7f76e7SChristian König * 4746a7f76e7SChristian König * @man: TTM memory type manager 4756a7f76e7SChristian König * @mem: TTM memory object 4766a7f76e7SChristian König * 4776a7f76e7SChristian König * Free the allocated VRAM again. 4786a7f76e7SChristian König */ 4799de59bc2SDave Airlie static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, 4802966141aSDave Airlie struct ttm_resource *mem) 4816a7f76e7SChristian König { 4820af135b8SDave Airlie struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 48337362793SDave Airlie struct amdgpu_device *adev = to_amdgpu_device(mgr); 4846a7f76e7SChristian König struct drm_mm_node *nodes = mem->mm_node; 4853c848bb3SChristian König uint64_t usage = 0, vis_usage = 0; 4866a7f76e7SChristian König unsigned pages = mem->num_pages; 4876a7f76e7SChristian König 4886a7f76e7SChristian König if (!mem->mm_node) 4896a7f76e7SChristian König return; 4906a7f76e7SChristian König 4916a7f76e7SChristian König spin_lock(&mgr->lock); 4926a7f76e7SChristian König while (pages) { 4936a7f76e7SChristian König pages -= nodes->size; 4946a7f76e7SChristian König drm_mm_remove_node(nodes); 4953c848bb3SChristian König usage += nodes->size << PAGE_SHIFT; 4963c848bb3SChristian König vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes); 4976a7f76e7SChristian König ++nodes; 4986a7f76e7SChristian König } 499676deb38SDennis Li amdgpu_vram_mgr_do_reserve(man); 5006a7f76e7SChristian König spin_unlock(&mgr->lock); 5016a7f76e7SChristian König 5023c848bb3SChristian König atomic64_sub(usage, &mgr->usage); 5033c848bb3SChristian König atomic64_sub(vis_usage, &mgr->vis_usage); 5043c848bb3SChristian König 5056fa39bc1SMichel Dänzer kvfree(mem->mm_node); 5066a7f76e7SChristian König mem->mm_node = NULL; 5076a7f76e7SChristian König } 5086a7f76e7SChristian König 5096a7f76e7SChristian König /** 510f44ffd67SChristian König * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table 511f44ffd67SChristian König * 512f44ffd67SChristian König * @adev: amdgpu device pointer 513f44ffd67SChristian König * @mem: TTM memory object 514ba5b662cSRamesh Errabolu * @offset: byte offset from the base of VRAM BO 515ba5b662cSRamesh Errabolu * @length: number of bytes to export in sg_table 516f44ffd67SChristian König * @dev: the other device 517f44ffd67SChristian König * @dir: dma direction 518f44ffd67SChristian König * @sgt: resulting sg table 519f44ffd67SChristian König * 520f44ffd67SChristian König * Allocate and fill a sg table from a VRAM allocation. 521f44ffd67SChristian König */ 522f44ffd67SChristian König int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, 5232966141aSDave Airlie struct ttm_resource *mem, 524ba5b662cSRamesh Errabolu u64 offset, u64 length, 525f44ffd67SChristian König struct device *dev, 526f44ffd67SChristian König enum dma_data_direction dir, 527f44ffd67SChristian König struct sg_table **sgt) 528f44ffd67SChristian König { 529ba5b662cSRamesh Errabolu struct amdgpu_res_cursor cursor; 530f44ffd67SChristian König struct scatterlist *sg; 531f44ffd67SChristian König int num_entries = 0; 532f44ffd67SChristian König int i, r; 533f44ffd67SChristian König 53478484d7cSChristophe JAILLET *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL); 535f44ffd67SChristian König if (!*sgt) 536f44ffd67SChristian König return -ENOMEM; 537f44ffd67SChristian König 538ba5b662cSRamesh Errabolu /* Determine the number of DRM_MM nodes to export */ 539ba5b662cSRamesh Errabolu amdgpu_res_first(mem, offset, length, &cursor); 540ba5b662cSRamesh Errabolu while (cursor.remaining) { 541ba5b662cSRamesh Errabolu num_entries++; 542ba5b662cSRamesh Errabolu amdgpu_res_next(&cursor, cursor.size); 543ba5b662cSRamesh Errabolu } 544f44ffd67SChristian König 545f44ffd67SChristian König r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); 546f44ffd67SChristian König if (r) 547f44ffd67SChristian König goto error_free; 548f44ffd67SChristian König 549ba5b662cSRamesh Errabolu /* Initialize scatterlist nodes of sg_table */ 55039913934SMarek Szyprowski for_each_sgtable_sg((*sgt), sg, i) 551f44ffd67SChristian König sg->length = 0; 552f44ffd67SChristian König 553ba5b662cSRamesh Errabolu /* 554ba5b662cSRamesh Errabolu * Walk down DRM_MM nodes to populate scatterlist nodes 555ba5b662cSRamesh Errabolu * @note: Use iterator api to get first the DRM_MM node 556ba5b662cSRamesh Errabolu * and the number of bytes from it. Access the following 557ba5b662cSRamesh Errabolu * DRM_MM node(s) if more buffer needs to exported 558ba5b662cSRamesh Errabolu */ 559ba5b662cSRamesh Errabolu amdgpu_res_first(mem, offset, length, &cursor); 56039913934SMarek Szyprowski for_each_sgtable_sg((*sgt), sg, i) { 561ba5b662cSRamesh Errabolu phys_addr_t phys = cursor.start + adev->gmc.aper_base; 562ba5b662cSRamesh Errabolu size_t size = cursor.size; 563f44ffd67SChristian König dma_addr_t addr; 564f44ffd67SChristian König 565f44ffd67SChristian König addr = dma_map_resource(dev, phys, size, dir, 566f44ffd67SChristian König DMA_ATTR_SKIP_CPU_SYNC); 567f44ffd67SChristian König r = dma_mapping_error(dev, addr); 568f44ffd67SChristian König if (r) 569f44ffd67SChristian König goto error_unmap; 570f44ffd67SChristian König 571f44ffd67SChristian König sg_set_page(sg, NULL, size, 0); 572f44ffd67SChristian König sg_dma_address(sg) = addr; 573f44ffd67SChristian König sg_dma_len(sg) = size; 574ba5b662cSRamesh Errabolu 575ba5b662cSRamesh Errabolu amdgpu_res_next(&cursor, cursor.size); 576f44ffd67SChristian König } 577ba5b662cSRamesh Errabolu 578f44ffd67SChristian König return 0; 579f44ffd67SChristian König 580f44ffd67SChristian König error_unmap: 58139913934SMarek Szyprowski for_each_sgtable_sg((*sgt), sg, i) { 582f44ffd67SChristian König if (!sg->length) 583f44ffd67SChristian König continue; 584f44ffd67SChristian König 585f44ffd67SChristian König dma_unmap_resource(dev, sg->dma_address, 586f44ffd67SChristian König sg->length, dir, 587f44ffd67SChristian König DMA_ATTR_SKIP_CPU_SYNC); 588f44ffd67SChristian König } 589f44ffd67SChristian König sg_free_table(*sgt); 590f44ffd67SChristian König 591f44ffd67SChristian König error_free: 592f44ffd67SChristian König kfree(*sgt); 593f44ffd67SChristian König return r; 594f44ffd67SChristian König } 595f44ffd67SChristian König 596f44ffd67SChristian König /** 597c45dd3bdSMauro Carvalho Chehab * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table 598f44ffd67SChristian König * 5992c8645b7SLee Jones * @dev: device pointer 6002c8645b7SLee Jones * @dir: data direction of resource to unmap 601f44ffd67SChristian König * @sgt: sg table to free 602f44ffd67SChristian König * 603f44ffd67SChristian König * Free a previously allocate sg table. 604f44ffd67SChristian König */ 6055392b2afSRamesh Errabolu void amdgpu_vram_mgr_free_sgt(struct device *dev, 606f44ffd67SChristian König enum dma_data_direction dir, 607f44ffd67SChristian König struct sg_table *sgt) 608f44ffd67SChristian König { 609f44ffd67SChristian König struct scatterlist *sg; 610f44ffd67SChristian König int i; 611f44ffd67SChristian König 61239913934SMarek Szyprowski for_each_sgtable_sg(sgt, sg, i) 613f44ffd67SChristian König dma_unmap_resource(dev, sg->dma_address, 614f44ffd67SChristian König sg->length, dir, 615f44ffd67SChristian König DMA_ATTR_SKIP_CPU_SYNC); 616f44ffd67SChristian König sg_free_table(sgt); 617f44ffd67SChristian König kfree(sgt); 618f44ffd67SChristian König } 619f44ffd67SChristian König 620f44ffd67SChristian König /** 6213c848bb3SChristian König * amdgpu_vram_mgr_usage - how many bytes are used in this domain 6223c848bb3SChristian König * 6233c848bb3SChristian König * @man: TTM memory type manager 6243c848bb3SChristian König * 6253c848bb3SChristian König * Returns how many bytes are used in this domain. 6263c848bb3SChristian König */ 6279de59bc2SDave Airlie uint64_t amdgpu_vram_mgr_usage(struct ttm_resource_manager *man) 6283c848bb3SChristian König { 6290af135b8SDave Airlie struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 6303c848bb3SChristian König 6313c848bb3SChristian König return atomic64_read(&mgr->usage); 6323c848bb3SChristian König } 6333c848bb3SChristian König 6343c848bb3SChristian König /** 6353c848bb3SChristian König * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part 6363c848bb3SChristian König * 6373c848bb3SChristian König * @man: TTM memory type manager 6383c848bb3SChristian König * 6393c848bb3SChristian König * Returns how many bytes are used in the visible part of VRAM 6403c848bb3SChristian König */ 6419de59bc2SDave Airlie uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_resource_manager *man) 6423c848bb3SChristian König { 6430af135b8SDave Airlie struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 6443c848bb3SChristian König 6453c848bb3SChristian König return atomic64_read(&mgr->vis_usage); 6463c848bb3SChristian König } 6473c848bb3SChristian König 6483c848bb3SChristian König /** 6496a7f76e7SChristian König * amdgpu_vram_mgr_debug - dump VRAM table 6506a7f76e7SChristian König * 6516a7f76e7SChristian König * @man: TTM memory type manager 652373533f8SChristian König * @printer: DRM printer to use 6536a7f76e7SChristian König * 6546a7f76e7SChristian König * Dump the table content using printk. 6556a7f76e7SChristian König */ 6569de59bc2SDave Airlie static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, 657373533f8SChristian König struct drm_printer *printer) 6586a7f76e7SChristian König { 6590af135b8SDave Airlie struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 6606a7f76e7SChristian König 6616a7f76e7SChristian König spin_lock(&mgr->lock); 662373533f8SChristian König drm_mm_print(&mgr->mm, printer); 6636a7f76e7SChristian König spin_unlock(&mgr->lock); 66497cbb284SChristian König 66597cbb284SChristian König drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n", 6663c848bb3SChristian König man->size, amdgpu_vram_mgr_usage(man) >> 20, 6673c848bb3SChristian König amdgpu_vram_mgr_vis_usage(man) >> 20); 6686a7f76e7SChristian König } 6696a7f76e7SChristian König 6709de59bc2SDave Airlie static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { 671e92ae67dSChristian König .alloc = amdgpu_vram_mgr_new, 672e92ae67dSChristian König .free = amdgpu_vram_mgr_del, 6732a9d6d26SKees Cook .debug = amdgpu_vram_mgr_debug 6746a7f76e7SChristian König }; 675589939d4SChristian König 676589939d4SChristian König /** 677589939d4SChristian König * amdgpu_vram_mgr_init - init VRAM manager and DRM MM 678589939d4SChristian König * 679589939d4SChristian König * @adev: amdgpu_device pointer 680589939d4SChristian König * 681589939d4SChristian König * Allocate and initialize the VRAM manager. 682589939d4SChristian König */ 683589939d4SChristian König int amdgpu_vram_mgr_init(struct amdgpu_device *adev) 684589939d4SChristian König { 685589939d4SChristian König struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 686589939d4SChristian König struct ttm_resource_manager *man = &mgr->manager; 687589939d4SChristian König int ret; 688589939d4SChristian König 689589939d4SChristian König ttm_resource_manager_init(man, adev->gmc.real_vram_size >> PAGE_SHIFT); 690589939d4SChristian König 691589939d4SChristian König man->func = &amdgpu_vram_mgr_func; 692589939d4SChristian König 693589939d4SChristian König drm_mm_init(&mgr->mm, 0, man->size); 694589939d4SChristian König spin_lock_init(&mgr->lock); 695589939d4SChristian König INIT_LIST_HEAD(&mgr->reservations_pending); 696589939d4SChristian König INIT_LIST_HEAD(&mgr->reserved_pages); 697589939d4SChristian König 698589939d4SChristian König /* Add the two VRAM-related sysfs files */ 699589939d4SChristian König ret = sysfs_create_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes); 700589939d4SChristian König if (ret) 701589939d4SChristian König DRM_ERROR("Failed to register sysfs\n"); 702589939d4SChristian König 703589939d4SChristian König ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); 704589939d4SChristian König ttm_resource_manager_set_used(man, true); 705589939d4SChristian König return 0; 706589939d4SChristian König } 707589939d4SChristian König 708589939d4SChristian König /** 709589939d4SChristian König * amdgpu_vram_mgr_fini - free and destroy VRAM manager 710589939d4SChristian König * 711589939d4SChristian König * @adev: amdgpu_device pointer 712589939d4SChristian König * 713589939d4SChristian König * Destroy and free the VRAM manager, returns -EBUSY if ranges are still 714589939d4SChristian König * allocated inside it. 715589939d4SChristian König */ 716589939d4SChristian König void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) 717589939d4SChristian König { 718589939d4SChristian König struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 719589939d4SChristian König struct ttm_resource_manager *man = &mgr->manager; 720589939d4SChristian König int ret; 721589939d4SChristian König struct amdgpu_vram_reservation *rsv, *temp; 722589939d4SChristian König 723589939d4SChristian König ttm_resource_manager_set_used(man, false); 724589939d4SChristian König 725589939d4SChristian König ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man); 726589939d4SChristian König if (ret) 727589939d4SChristian König return; 728589939d4SChristian König 729589939d4SChristian König spin_lock(&mgr->lock); 730589939d4SChristian König list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node) 731589939d4SChristian König kfree(rsv); 732589939d4SChristian König 733589939d4SChristian König list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) { 734589939d4SChristian König drm_mm_remove_node(&rsv->mm_node); 735589939d4SChristian König kfree(rsv); 736589939d4SChristian König } 737589939d4SChristian König drm_mm_takedown(&mgr->mm); 738589939d4SChristian König spin_unlock(&mgr->lock); 739589939d4SChristian König 740589939d4SChristian König sysfs_remove_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes); 741589939d4SChristian König 742589939d4SChristian König ttm_resource_manager_cleanup(man); 743589939d4SChristian König ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL); 744589939d4SChristian König } 745