1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2667ce33eSRob Clark /* 3667ce33eSRob Clark * Copyright (C) 2016 Red Hat 4667ce33eSRob Clark * Author: Rob Clark <robdclark@gmail.com> 5667ce33eSRob Clark */ 6667ce33eSRob Clark 7667ce33eSRob Clark #include "msm_drv.h" 895d1deb0SRob Clark #include "msm_fence.h" 9667ce33eSRob Clark #include "msm_gem.h" 10667ce33eSRob Clark #include "msm_mmu.h" 11667ce33eSRob Clark 12ee546cd3SJordan Crouse static void 13ee546cd3SJordan Crouse msm_gem_address_space_destroy(struct kref *kref) 14ee546cd3SJordan Crouse { 15ee546cd3SJordan Crouse struct msm_gem_address_space *aspace = container_of(kref, 16ee546cd3SJordan Crouse struct msm_gem_address_space, kref); 17ee546cd3SJordan Crouse 18ee546cd3SJordan Crouse drm_mm_takedown(&aspace->mm); 19ee546cd3SJordan Crouse if (aspace->mmu) 20ee546cd3SJordan Crouse aspace->mmu->funcs->destroy(aspace->mmu); 2125faf2f2SRob Clark put_pid(aspace->pid); 22ee546cd3SJordan Crouse kfree(aspace); 23ee546cd3SJordan Crouse } 24ee546cd3SJordan Crouse 25ee546cd3SJordan Crouse 26ee546cd3SJordan Crouse void msm_gem_address_space_put(struct msm_gem_address_space *aspace) 27ee546cd3SJordan Crouse { 28ee546cd3SJordan Crouse if (aspace) 29ee546cd3SJordan Crouse kref_put(&aspace->kref, msm_gem_address_space_destroy); 30ee546cd3SJordan Crouse } 31ee546cd3SJordan Crouse 32933415e2SJordan Crouse struct msm_gem_address_space * 33933415e2SJordan Crouse msm_gem_address_space_get(struct msm_gem_address_space *aspace) 34933415e2SJordan Crouse { 35933415e2SJordan Crouse if (!IS_ERR_OR_NULL(aspace)) 36933415e2SJordan Crouse kref_get(&aspace->kref); 37933415e2SJordan Crouse 38933415e2SJordan Crouse return aspace; 39933415e2SJordan Crouse } 40933415e2SJordan Crouse 41ca35ab2aSRob Clark bool msm_gem_vma_inuse(struct msm_gem_vma *vma) 42ca35ab2aSRob Clark { 4395d1deb0SRob Clark if (vma->inuse > 0) 4495d1deb0SRob Clark return true; 4595d1deb0SRob Clark 4695d1deb0SRob Clark while (vma->fence_mask) { 4795d1deb0SRob Clark unsigned idx = ffs(vma->fence_mask) - 1; 4895d1deb0SRob Clark 4995d1deb0SRob Clark if (!msm_fence_completed(vma->fctx[idx], vma->fence[idx])) 5095d1deb0SRob Clark return true; 5195d1deb0SRob Clark 5295d1deb0SRob Clark vma->fence_mask &= ~BIT(idx); 5395d1deb0SRob Clark } 5495d1deb0SRob Clark 5595d1deb0SRob Clark return false; 56ca35ab2aSRob Clark } 57ca35ab2aSRob Clark 587ad0e8cfSJordan Crouse /* Actually unmap memory for the vma */ 597ad0e8cfSJordan Crouse void msm_gem_purge_vma(struct msm_gem_address_space *aspace, 6070dc51b4SJordan Crouse struct msm_gem_vma *vma) 61667ce33eSRob Clark { 622ee4b5d2SRob Clark unsigned size = vma->node.size; 637ad0e8cfSJordan Crouse 647ad0e8cfSJordan Crouse /* Print a message if we try to purge a vma in use */ 65ca35ab2aSRob Clark if (GEM_WARN_ON(msm_gem_vma_inuse(vma))) 66667ce33eSRob Clark return; 67667ce33eSRob Clark 687ad0e8cfSJordan Crouse /* Don't do anything if the memory isn't mapped */ 697ad0e8cfSJordan Crouse if (!vma->mapped) 707ad0e8cfSJordan Crouse return; 717ad0e8cfSJordan Crouse 727ad0e8cfSJordan Crouse if (aspace->mmu) 7370dc51b4SJordan Crouse aspace->mmu->funcs->unmap(aspace->mmu, vma->iova, size); 747ad0e8cfSJordan Crouse 757ad0e8cfSJordan Crouse vma->mapped = false; 76667ce33eSRob Clark } 77667ce33eSRob Clark 787ad0e8cfSJordan Crouse /* Remove reference counts for the mapping */ 7995d1deb0SRob Clark void msm_gem_unpin_vma(struct msm_gem_vma *vma) 807ad0e8cfSJordan Crouse { 8195d1deb0SRob Clark if (GEM_WARN_ON(!vma->inuse)) 8295d1deb0SRob Clark return; 838e30fa32SRob Clark if (!GEM_WARN_ON(!vma->iova)) 847ad0e8cfSJordan Crouse vma->inuse--; 85667ce33eSRob Clark } 86667ce33eSRob Clark 8795d1deb0SRob Clark /* Replace pin reference with fence: */ 8895d1deb0SRob Clark void msm_gem_unpin_vma_fenced(struct msm_gem_vma *vma, struct msm_fence_context *fctx) 8995d1deb0SRob Clark { 9095d1deb0SRob Clark vma->fctx[fctx->index] = fctx; 9195d1deb0SRob Clark vma->fence[fctx->index] = fctx->last_fence; 9295d1deb0SRob Clark vma->fence_mask |= BIT(fctx->index); 9395d1deb0SRob Clark msm_gem_unpin_vma(vma); 9495d1deb0SRob Clark } 9595d1deb0SRob Clark 9695d1deb0SRob Clark /* Map and pin vma: */ 97667ce33eSRob Clark int 98667ce33eSRob Clark msm_gem_map_vma(struct msm_gem_address_space *aspace, 99bbc2cd07SRob Clark struct msm_gem_vma *vma, int prot, 1002ee4b5d2SRob Clark struct sg_table *sgt, int size) 101667ce33eSRob Clark { 102c0ee9794SJordan Crouse int ret = 0; 103667ce33eSRob Clark 1048e30fa32SRob Clark if (GEM_WARN_ON(!vma->iova)) 105c0ee9794SJordan Crouse return -EINVAL; 106c0ee9794SJordan Crouse 1077ad0e8cfSJordan Crouse /* Increase the usage counter */ 1087ad0e8cfSJordan Crouse vma->inuse++; 1097ad0e8cfSJordan Crouse 110c0ee9794SJordan Crouse if (vma->mapped) 111667ce33eSRob Clark return 0; 112c0ee9794SJordan Crouse 113c0ee9794SJordan Crouse vma->mapped = true; 114c0ee9794SJordan Crouse 1157603df38SLuca Weiss if (aspace && aspace->mmu) 116c0ee9794SJordan Crouse ret = aspace->mmu->funcs->map(aspace->mmu, vma->iova, sgt, 117bbc2cd07SRob Clark size, prot); 118c0ee9794SJordan Crouse 11963ca94adSAkhil P Oommen if (ret) { 120c0ee9794SJordan Crouse vma->mapped = false; 12163ca94adSAkhil P Oommen vma->inuse--; 12263ca94adSAkhil P Oommen } 123c0ee9794SJordan Crouse 124c0ee9794SJordan Crouse return ret; 1250e08270aSSushmita Susheelendra } 126667ce33eSRob Clark 1277ad0e8cfSJordan Crouse /* Close an iova. Warn if it is still in use */ 1287ad0e8cfSJordan Crouse void msm_gem_close_vma(struct msm_gem_address_space *aspace, 1297ad0e8cfSJordan Crouse struct msm_gem_vma *vma) 1307ad0e8cfSJordan Crouse { 131ca35ab2aSRob Clark if (GEM_WARN_ON(msm_gem_vma_inuse(vma) || vma->mapped)) 1327ad0e8cfSJordan Crouse return; 1337ad0e8cfSJordan Crouse 1347ad0e8cfSJordan Crouse spin_lock(&aspace->lock); 1357ad0e8cfSJordan Crouse if (vma->iova) 1367ad0e8cfSJordan Crouse drm_mm_remove_node(&vma->node); 1377ad0e8cfSJordan Crouse spin_unlock(&aspace->lock); 1387ad0e8cfSJordan Crouse 1397ad0e8cfSJordan Crouse vma->iova = 0; 1407ad0e8cfSJordan Crouse 1417ad0e8cfSJordan Crouse msm_gem_address_space_put(aspace); 1427ad0e8cfSJordan Crouse } 1437ad0e8cfSJordan Crouse 144c0ee9794SJordan Crouse /* Initialize a new vma and allocate an iova for it */ 145c0ee9794SJordan Crouse int msm_gem_init_vma(struct msm_gem_address_space *aspace, 1462ee4b5d2SRob Clark struct msm_gem_vma *vma, int size, 147d3b8877eSJonathan Marek u64 range_start, u64 range_end) 148c0ee9794SJordan Crouse { 149c0ee9794SJordan Crouse int ret; 150c0ee9794SJordan Crouse 1518e30fa32SRob Clark if (GEM_WARN_ON(vma->iova)) 152c0ee9794SJordan Crouse return -EBUSY; 153c0ee9794SJordan Crouse 154c0ee9794SJordan Crouse spin_lock(&aspace->lock); 1552ee4b5d2SRob Clark ret = drm_mm_insert_node_in_range(&aspace->mm, &vma->node, 1562ee4b5d2SRob Clark size, PAGE_SIZE, 0, 1572ee4b5d2SRob Clark range_start, range_end, 0); 1580e08270aSSushmita Susheelendra spin_unlock(&aspace->lock); 1590e08270aSSushmita Susheelendra 160667ce33eSRob Clark if (ret) 161667ce33eSRob Clark return ret; 162667ce33eSRob Clark 1632ee4b5d2SRob Clark vma->iova = vma->node.start; 164c0ee9794SJordan Crouse vma->mapped = false; 165667ce33eSRob Clark 166ee546cd3SJordan Crouse kref_get(&aspace->kref); 167667ce33eSRob Clark 168c0ee9794SJordan Crouse return 0; 169667ce33eSRob Clark } 170667ce33eSRob Clark 171667ce33eSRob Clark struct msm_gem_address_space * 172ccac7ce3SJordan Crouse msm_gem_address_space_create(struct msm_mmu *mmu, const char *name, 173ccac7ce3SJordan Crouse u64 va_start, u64 size) 174667ce33eSRob Clark { 175667ce33eSRob Clark struct msm_gem_address_space *aspace; 176ccac7ce3SJordan Crouse 177ccac7ce3SJordan Crouse if (IS_ERR(mmu)) 178ccac7ce3SJordan Crouse return ERR_CAST(mmu); 179667ce33eSRob Clark 180667ce33eSRob Clark aspace = kzalloc(sizeof(*aspace), GFP_KERNEL); 181667ce33eSRob Clark if (!aspace) 182667ce33eSRob Clark return ERR_PTR(-ENOMEM); 183667ce33eSRob Clark 1840e08270aSSushmita Susheelendra spin_lock_init(&aspace->lock); 185667ce33eSRob Clark aspace->name = name; 186ccac7ce3SJordan Crouse aspace->mmu = mmu; 187*a636a0ffSRob Clark aspace->va_start = va_start; 188*a636a0ffSRob Clark aspace->va_size = size; 189667ce33eSRob Clark 1902ee4b5d2SRob Clark drm_mm_init(&aspace->mm, va_start, size); 191c2052a4eSJonathan Marek 192c2052a4eSJonathan Marek kref_init(&aspace->kref); 193c2052a4eSJonathan Marek 194c2052a4eSJonathan Marek return aspace; 195c2052a4eSJonathan Marek } 196