1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Christian König <christian.koenig@amd.com> 29 */ 30 31 /** 32 * DOC: MMU Notifier 33 * 34 * For coherent userptr handling registers an MMU notifier to inform the driver 35 * about updates on the page tables of a process. 36 * 37 * When somebody tries to invalidate the page tables we block the update until 38 * all operations on the pages in question are completed, then those pages are 39 * marked as accessed and also dirty if it wasn't a read only access. 40 * 41 * New command submissions using the userptrs in question are delayed until all 42 * page table invalidation are completed and we once more see a coherent process 43 * address space. 44 */ 45 46 #include <linux/firmware.h> 47 #include <linux/module.h> 48 #include <drm/drm.h> 49 50 #include "amdgpu.h" 51 #include "amdgpu_amdkfd.h" 52 #include "amdgpu_hmm.h" 53 54 /** 55 * amdgpu_hmm_invalidate_gfx - callback to notify about mm change 56 * 57 * @mni: the range (mm) is about to update 58 * @range: details on the invalidation 59 * @cur_seq: Value to pass to mmu_interval_set_seq() 60 * 61 * Block for operations on BOs to finish and mark pages as accessed and 62 * potentially dirty. 63 */ 64 static bool amdgpu_hmm_invalidate_gfx(struct mmu_interval_notifier *mni, 65 const struct mmu_notifier_range *range, 66 unsigned long cur_seq) 67 { 68 struct amdgpu_bo *bo = container_of(mni, struct amdgpu_bo, notifier); 69 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 70 long r; 71 72 if (!mmu_notifier_range_blockable(range)) 73 return false; 74 75 mutex_lock(&adev->notifier_lock); 76 77 mmu_interval_set_seq(mni, cur_seq); 78 79 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP, 80 false, MAX_SCHEDULE_TIMEOUT); 81 mutex_unlock(&adev->notifier_lock); 82 if (r <= 0) 83 DRM_ERROR("(%ld) failed to wait for user bo\n", r); 84 return true; 85 } 86 87 static const struct mmu_interval_notifier_ops amdgpu_hmm_gfx_ops = { 88 .invalidate = amdgpu_hmm_invalidate_gfx, 89 }; 90 91 /** 92 * amdgpu_hmm_invalidate_hsa - callback to notify about mm change 93 * 94 * @mni: the range (mm) is about to update 95 * @range: details on the invalidation 96 * @cur_seq: Value to pass to mmu_interval_set_seq() 97 * 98 * We temporarily evict the BO attached to this range. This necessitates 99 * evicting all user-mode queues of the process. 100 */ 101 static bool amdgpu_hmm_invalidate_hsa(struct mmu_interval_notifier *mni, 102 const struct mmu_notifier_range *range, 103 unsigned long cur_seq) 104 { 105 struct amdgpu_bo *bo = container_of(mni, struct amdgpu_bo, notifier); 106 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 107 108 if (!mmu_notifier_range_blockable(range)) 109 return false; 110 111 mutex_lock(&adev->notifier_lock); 112 113 mmu_interval_set_seq(mni, cur_seq); 114 115 amdgpu_amdkfd_evict_userptr(bo->kfd_bo, bo->notifier.mm); 116 mutex_unlock(&adev->notifier_lock); 117 118 return true; 119 } 120 121 static const struct mmu_interval_notifier_ops amdgpu_hmm_hsa_ops = { 122 .invalidate = amdgpu_hmm_invalidate_hsa, 123 }; 124 125 /** 126 * amdgpu_hmm_register - register a BO for notifier updates 127 * 128 * @bo: amdgpu buffer object 129 * @addr: userptr addr we should monitor 130 * 131 * Registers a mmu_notifier for the given BO at the specified address. 132 * Returns 0 on success, -ERRNO if anything goes wrong. 133 */ 134 int amdgpu_hmm_register(struct amdgpu_bo *bo, unsigned long addr) 135 { 136 if (bo->kfd_bo) 137 return mmu_interval_notifier_insert(&bo->notifier, current->mm, 138 addr, amdgpu_bo_size(bo), 139 &amdgpu_hmm_hsa_ops); 140 return mmu_interval_notifier_insert(&bo->notifier, current->mm, addr, 141 amdgpu_bo_size(bo), 142 &amdgpu_hmm_gfx_ops); 143 } 144 145 /** 146 * amdgpu_hmm_unregister - unregister a BO for notifier updates 147 * 148 * @bo: amdgpu buffer object 149 * 150 * Remove any registration of mmu notifier updates from the buffer object. 151 */ 152 void amdgpu_hmm_unregister(struct amdgpu_bo *bo) 153 { 154 if (!bo->notifier.mm) 155 return; 156 mmu_interval_notifier_remove(&bo->notifier); 157 bo->notifier.mm = NULL; 158 } 159 160 int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, 161 uint64_t start, uint64_t npages, bool readonly, 162 void *owner, struct page **pages, 163 struct hmm_range **phmm_range) 164 { 165 struct hmm_range *hmm_range; 166 unsigned long timeout; 167 unsigned long i; 168 unsigned long *pfns; 169 int r = 0; 170 171 hmm_range = kzalloc(sizeof(*hmm_range), GFP_KERNEL); 172 if (unlikely(!hmm_range)) 173 return -ENOMEM; 174 175 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); 176 if (unlikely(!pfns)) { 177 r = -ENOMEM; 178 goto out_free_range; 179 } 180 181 hmm_range->notifier = notifier; 182 hmm_range->default_flags = HMM_PFN_REQ_FAULT; 183 if (!readonly) 184 hmm_range->default_flags |= HMM_PFN_REQ_WRITE; 185 hmm_range->hmm_pfns = pfns; 186 hmm_range->start = start; 187 hmm_range->end = start + npages * PAGE_SIZE; 188 hmm_range->dev_private_owner = owner; 189 190 /* Assuming 512MB takes maxmium 1 second to fault page address */ 191 timeout = max(npages >> 17, 1ULL) * HMM_RANGE_DEFAULT_TIMEOUT; 192 timeout = jiffies + msecs_to_jiffies(timeout); 193 194 retry: 195 hmm_range->notifier_seq = mmu_interval_read_begin(notifier); 196 r = hmm_range_fault(hmm_range); 197 if (unlikely(r)) { 198 /* 199 * FIXME: This timeout should encompass the retry from 200 * mmu_interval_read_retry() as well. 201 */ 202 if (r == -EBUSY && !time_after(jiffies, timeout)) 203 goto retry; 204 goto out_free_pfns; 205 } 206 207 /* 208 * Due to default_flags, all pages are HMM_PFN_VALID or 209 * hmm_range_fault() fails. FIXME: The pages cannot be touched outside 210 * the notifier_lock, and mmu_interval_read_retry() must be done first. 211 */ 212 for (i = 0; pages && i < npages; i++) 213 pages[i] = hmm_pfn_to_page(pfns[i]); 214 215 *phmm_range = hmm_range; 216 217 return 0; 218 219 out_free_pfns: 220 kvfree(pfns); 221 out_free_range: 222 kfree(hmm_range); 223 224 return r; 225 } 226 227 int amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range) 228 { 229 int r; 230 231 r = mmu_interval_read_retry(hmm_range->notifier, 232 hmm_range->notifier_seq); 233 kvfree(hmm_range->hmm_pfns); 234 kfree(hmm_range); 235 236 return r; 237 } 238