1d38ceaf9SAlex Deucher /*
2d38ceaf9SAlex Deucher * Copyright 2008 Advanced Micro Devices, Inc.
3d38ceaf9SAlex Deucher * Copyright 2008 Red Hat Inc.
4d38ceaf9SAlex Deucher * Copyright 2009 Jerome Glisse.
5d38ceaf9SAlex Deucher *
6d38ceaf9SAlex Deucher * Permission is hereby granted, free of charge, to any person obtaining a
7d38ceaf9SAlex Deucher * copy of this software and associated documentation files (the "Software"),
8d38ceaf9SAlex Deucher * to deal in the Software without restriction, including without limitation
9d38ceaf9SAlex Deucher * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10d38ceaf9SAlex Deucher * and/or sell copies of the Software, and to permit persons to whom the
11d38ceaf9SAlex Deucher * Software is furnished to do so, subject to the following conditions:
12d38ceaf9SAlex Deucher *
13d38ceaf9SAlex Deucher * The above copyright notice and this permission notice shall be included in
14d38ceaf9SAlex Deucher * all copies or substantial portions of the Software.
15d38ceaf9SAlex Deucher *
16d38ceaf9SAlex Deucher * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17d38ceaf9SAlex Deucher * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18d38ceaf9SAlex Deucher * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19d38ceaf9SAlex Deucher * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20d38ceaf9SAlex Deucher * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21d38ceaf9SAlex Deucher * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22d38ceaf9SAlex Deucher * OTHER DEALINGS IN THE SOFTWARE.
23d38ceaf9SAlex Deucher *
24d38ceaf9SAlex Deucher * Authors: Dave Airlie
25d38ceaf9SAlex Deucher * Alex Deucher
26d38ceaf9SAlex Deucher * Jerome Glisse
27d38ceaf9SAlex Deucher */
2887444254SRoy Sun
29f54d1867SChris Wilson #include <linux/dma-fence-array.h>
30a9f87f64SChristian König #include <linux/interval_tree_generic.h>
3102208441SFelix Kuehling #include <linux/idr.h>
320cf0ee98SArunpravin #include <linux/dma-buf.h>
33fdf2f6c5SSam Ravnborg
34d38ceaf9SAlex Deucher #include <drm/amdgpu_drm.h>
35f89f8c6bSAndrey Grodzovsky #include <drm/drm_drv.h>
36a3185f91SChristian König #include <drm/ttm/ttm_tt.h>
378abc1eb2SChristian König #include <drm/drm_exec.h>
38d38ceaf9SAlex Deucher #include "amdgpu.h"
39d38ceaf9SAlex Deucher #include "amdgpu_trace.h"
40ede0dd86SFelix Kuehling #include "amdgpu_amdkfd.h"
41c8c5e569SAndrey Grodzovsky #include "amdgpu_gmc.h"
42df399b06Sshaoyunl #include "amdgpu_xgmi.h"
430cf0ee98SArunpravin #include "amdgpu_dma_buf.h"
440ccc3ccfSChristian König #include "amdgpu_res_cursor.h"
45ea53af8aSAlex Sierra #include "kfd_svm.h"
46d38ceaf9SAlex Deucher
477fc48e59SAndrey Grodzovsky /**
487fc48e59SAndrey Grodzovsky * DOC: GPUVM
497fc48e59SAndrey Grodzovsky *
504670ac70SAlex Deucher * GPUVM is the MMU functionality provided on the GPU.
514670ac70SAlex Deucher * GPUVM is similar to the legacy GART on older asics, however
524670ac70SAlex Deucher * rather than there being a single global GART table
534670ac70SAlex Deucher * for the entire GPU, there can be multiple GPUVM page tables active
544670ac70SAlex Deucher * at any given time. The GPUVM page tables can contain a mix
554670ac70SAlex Deucher * VRAM pages and system pages (both memory and MMIO) and system pages
56d38ceaf9SAlex Deucher * can be mapped as snooped (cached system pages) or unsnooped
57d38ceaf9SAlex Deucher * (uncached system pages).
584670ac70SAlex Deucher *
594670ac70SAlex Deucher * Each active GPUVM has an ID associated with it and there is a page table
604670ac70SAlex Deucher * linked with each VMID. When executing a command buffer,
614670ac70SAlex Deucher * the kernel tells the engine what VMID to use for that command
62d38ceaf9SAlex Deucher * buffer. VMIDs are allocated dynamically as commands are submitted.
63d38ceaf9SAlex Deucher * The userspace drivers maintain their own address space and the kernel
64d38ceaf9SAlex Deucher * sets up their pages tables accordingly when they submit their
65d38ceaf9SAlex Deucher * command buffers and a VMID is assigned.
664670ac70SAlex Deucher * The hardware supports up to 16 active GPUVMs at any given time.
674670ac70SAlex Deucher *
684670ac70SAlex Deucher * Each GPUVM is represented by a 1-2 or 1-5 level page table, depending
694670ac70SAlex Deucher * on the ASIC family. GPUVM supports RWX attributes on each page as well
704670ac70SAlex Deucher * as other features such as encryption and caching attributes.
714670ac70SAlex Deucher *
724670ac70SAlex Deucher * VMID 0 is special. It is the GPUVM used for the kernel driver. In
734670ac70SAlex Deucher * addition to an aperture managed by a page table, VMID 0 also has
744670ac70SAlex Deucher * several other apertures. There is an aperture for direct access to VRAM
754670ac70SAlex Deucher * and there is a legacy AGP aperture which just forwards accesses directly
764670ac70SAlex Deucher * to the matching system physical addresses (or IOVAs when an IOMMU is
774670ac70SAlex Deucher * present). These apertures provide direct access to these memories without
784670ac70SAlex Deucher * incurring the overhead of a page table. VMID 0 is used by the kernel
794670ac70SAlex Deucher * driver for tasks like memory management.
804670ac70SAlex Deucher *
814670ac70SAlex Deucher * GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory.
824670ac70SAlex Deucher * For user applications, each application can have their own unique GPUVM
834670ac70SAlex Deucher * address space. The application manages the address space and the kernel
844670ac70SAlex Deucher * driver manages the GPUVM page tables for each process. If an GPU client
854670ac70SAlex Deucher * accesses an invalid page, it will generate a GPU page fault, similar to
864670ac70SAlex Deucher * accessing an invalid page on a CPU.
87d38ceaf9SAlex Deucher */
88d38ceaf9SAlex Deucher
89a9f87f64SChristian König #define START(node) ((node)->start)
90a9f87f64SChristian König #define LAST(node) ((node)->last)
91a9f87f64SChristian König
92a9f87f64SChristian König INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
93a9f87f64SChristian König START, LAST, static, amdgpu_vm_it)
94a9f87f64SChristian König
95a9f87f64SChristian König #undef START
96a9f87f64SChristian König #undef LAST
97a9f87f64SChristian König
987fc48e59SAndrey Grodzovsky /**
997fc48e59SAndrey Grodzovsky * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
1007fc48e59SAndrey Grodzovsky */
101284710faSChristian König struct amdgpu_prt_cb {
1027fc48e59SAndrey Grodzovsky
1037fc48e59SAndrey Grodzovsky /**
1047fc48e59SAndrey Grodzovsky * @adev: amdgpu device
1057fc48e59SAndrey Grodzovsky */
106284710faSChristian König struct amdgpu_device *adev;
1077fc48e59SAndrey Grodzovsky
1087fc48e59SAndrey Grodzovsky /**
1097fc48e59SAndrey Grodzovsky * @cb: callback
1107fc48e59SAndrey Grodzovsky */
111284710faSChristian König struct dma_fence_cb cb;
112284710faSChristian König };
113284710faSChristian König
114dcb388edSNirmoy Das /**
11552b82609SLuben Tuikov * struct amdgpu_vm_tlb_seq_struct - Helper to increment the TLB flush sequence
1165255e146SChristian König */
11752b82609SLuben Tuikov struct amdgpu_vm_tlb_seq_struct {
1185255e146SChristian König /**
1195255e146SChristian König * @vm: pointer to the amdgpu_vm structure to set the fence sequence on
1205255e146SChristian König */
1215255e146SChristian König struct amdgpu_vm *vm;
1225255e146SChristian König
1235255e146SChristian König /**
1245255e146SChristian König * @cb: callback
1255255e146SChristian König */
1265255e146SChristian König struct dma_fence_cb cb;
1275255e146SChristian König };
1285255e146SChristian König
1295255e146SChristian König /**
130dcb388edSNirmoy Das * amdgpu_vm_set_pasid - manage pasid and vm ptr mapping
131dcb388edSNirmoy Das *
132dcb388edSNirmoy Das * @adev: amdgpu_device pointer
133dcb388edSNirmoy Das * @vm: amdgpu_vm pointer
134dcb388edSNirmoy Das * @pasid: the pasid the VM is using on this GPU
135dcb388edSNirmoy Das *
136dcb388edSNirmoy Das * Set the pasid this VM is using on this GPU, can also be used to remove the
137dcb388edSNirmoy Das * pasid by passing in zero.
138dcb388edSNirmoy Das *
139dcb388edSNirmoy Das */
amdgpu_vm_set_pasid(struct amdgpu_device * adev,struct amdgpu_vm * vm,u32 pasid)140dcb388edSNirmoy Das int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm,
141dcb388edSNirmoy Das u32 pasid)
142dcb388edSNirmoy Das {
143dcb388edSNirmoy Das int r;
144dcb388edSNirmoy Das
145dcb388edSNirmoy Das if (vm->pasid == pasid)
146dcb388edSNirmoy Das return 0;
147dcb388edSNirmoy Das
148dcb388edSNirmoy Das if (vm->pasid) {
149dcb388edSNirmoy Das r = xa_err(xa_erase_irq(&adev->vm_manager.pasids, vm->pasid));
150dcb388edSNirmoy Das if (r < 0)
151dcb388edSNirmoy Das return r;
152dcb388edSNirmoy Das
153dcb388edSNirmoy Das vm->pasid = 0;
154dcb388edSNirmoy Das }
155dcb388edSNirmoy Das
156dcb388edSNirmoy Das if (pasid) {
157dcb388edSNirmoy Das r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm,
158dcb388edSNirmoy Das GFP_KERNEL));
159dcb388edSNirmoy Das if (r < 0)
160dcb388edSNirmoy Das return r;
161dcb388edSNirmoy Das
162dcb388edSNirmoy Das vm->pasid = pasid;
163dcb388edSNirmoy Das }
164dcb388edSNirmoy Das
165dcb388edSNirmoy Das
166dcb388edSNirmoy Das return 0;
167dcb388edSNirmoy Das }
168dcb388edSNirmoy Das
169a269e449SAlex Sierra /**
170bcdc9fd6SChristian König * amdgpu_vm_bo_evicted - vm_bo is evicted
171bcdc9fd6SChristian König *
172bcdc9fd6SChristian König * @vm_bo: vm_bo which is evicted
173bcdc9fd6SChristian König *
174bcdc9fd6SChristian König * State for PDs/PTs and per VM BOs which are not at the location they should
175bcdc9fd6SChristian König * be.
176bcdc9fd6SChristian König */
amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base * vm_bo)177bcdc9fd6SChristian König static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
178bcdc9fd6SChristian König {
179bcdc9fd6SChristian König struct amdgpu_vm *vm = vm_bo->vm;
180bcdc9fd6SChristian König struct amdgpu_bo *bo = vm_bo->bo;
181bcdc9fd6SChristian König
182bcdc9fd6SChristian König vm_bo->moved = true;
183757eb2beSPhilip Yang spin_lock(&vm_bo->vm->status_lock);
184bcdc9fd6SChristian König if (bo->tbo.type == ttm_bo_type_kernel)
185bcdc9fd6SChristian König list_move(&vm_bo->vm_status, &vm->evicted);
186bcdc9fd6SChristian König else
187bcdc9fd6SChristian König list_move_tail(&vm_bo->vm_status, &vm->evicted);
188757eb2beSPhilip Yang spin_unlock(&vm_bo->vm->status_lock);
189bcdc9fd6SChristian König }
190bcdc9fd6SChristian König /**
191bcdc9fd6SChristian König * amdgpu_vm_bo_moved - vm_bo is moved
192bcdc9fd6SChristian König *
193bcdc9fd6SChristian König * @vm_bo: vm_bo which is moved
194bcdc9fd6SChristian König *
195bcdc9fd6SChristian König * State for per VM BOs which are moved, but that change is not yet reflected
196bcdc9fd6SChristian König * in the page tables.
197bcdc9fd6SChristian König */
amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base * vm_bo)198bcdc9fd6SChristian König static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
199bcdc9fd6SChristian König {
200998debbdSPhilip Yang spin_lock(&vm_bo->vm->status_lock);
201bcdc9fd6SChristian König list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
202998debbdSPhilip Yang spin_unlock(&vm_bo->vm->status_lock);
203bcdc9fd6SChristian König }
204bcdc9fd6SChristian König
205bcdc9fd6SChristian König /**
206bcdc9fd6SChristian König * amdgpu_vm_bo_idle - vm_bo is idle
207bcdc9fd6SChristian König *
208bcdc9fd6SChristian König * @vm_bo: vm_bo which is now idle
209bcdc9fd6SChristian König *
210bcdc9fd6SChristian König * State for PDs/PTs and per VM BOs which have gone through the state machine
211bcdc9fd6SChristian König * and are now idle.
212bcdc9fd6SChristian König */
amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base * vm_bo)213bcdc9fd6SChristian König static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
214bcdc9fd6SChristian König {
215c1806d78SPhilip Yang spin_lock(&vm_bo->vm->status_lock);
216bcdc9fd6SChristian König list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
217c1806d78SPhilip Yang spin_unlock(&vm_bo->vm->status_lock);
218bcdc9fd6SChristian König vm_bo->moved = false;
219bcdc9fd6SChristian König }
220bcdc9fd6SChristian König
221bcdc9fd6SChristian König /**
222bcdc9fd6SChristian König * amdgpu_vm_bo_invalidated - vm_bo is invalidated
223bcdc9fd6SChristian König *
224bcdc9fd6SChristian König * @vm_bo: vm_bo which is now invalidated
225bcdc9fd6SChristian König *
226bcdc9fd6SChristian König * State for normal BOs which are invalidated and that change not yet reflected
227bcdc9fd6SChristian König * in the PTs.
228bcdc9fd6SChristian König */
amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base * vm_bo)229bcdc9fd6SChristian König static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
230bcdc9fd6SChristian König {
2310479956cSPhilip Yang spin_lock(&vm_bo->vm->status_lock);
232bcdc9fd6SChristian König list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
2330479956cSPhilip Yang spin_unlock(&vm_bo->vm->status_lock);
234bcdc9fd6SChristian König }
235bcdc9fd6SChristian König
236bcdc9fd6SChristian König /**
237a6605c43Sxinhui pan * amdgpu_vm_bo_relocated - vm_bo is reloacted
238a6605c43Sxinhui pan *
239a6605c43Sxinhui pan * @vm_bo: vm_bo which is relocated
240a6605c43Sxinhui pan *
241a6605c43Sxinhui pan * State for PDs/PTs which needs to update their parent PD.
242a6605c43Sxinhui pan * For the root PD, just move to idle state.
243a6605c43Sxinhui pan */
amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base * vm_bo)244a6605c43Sxinhui pan static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
245a6605c43Sxinhui pan {
246b38e77cbSPhilip Yang if (vm_bo->bo->parent) {
247b38e77cbSPhilip Yang spin_lock(&vm_bo->vm->status_lock);
248a6605c43Sxinhui pan list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
249b38e77cbSPhilip Yang spin_unlock(&vm_bo->vm->status_lock);
250b38e77cbSPhilip Yang } else {
251a6605c43Sxinhui pan amdgpu_vm_bo_idle(vm_bo);
252a6605c43Sxinhui pan }
253b38e77cbSPhilip Yang }
254a6605c43Sxinhui pan
255a6605c43Sxinhui pan /**
256bcdc9fd6SChristian König * amdgpu_vm_bo_done - vm_bo is done
257bcdc9fd6SChristian König *
258bcdc9fd6SChristian König * @vm_bo: vm_bo which is now done
259bcdc9fd6SChristian König *
260bcdc9fd6SChristian König * State for normal BOs which are invalidated and that change has been updated
261bcdc9fd6SChristian König * in the PTs.
262bcdc9fd6SChristian König */
amdgpu_vm_bo_done(struct amdgpu_vm_bo_base * vm_bo)263bcdc9fd6SChristian König static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
264bcdc9fd6SChristian König {
2650479956cSPhilip Yang spin_lock(&vm_bo->vm->status_lock);
2660e601a04SMihir Bhogilal Patel list_move(&vm_bo->vm_status, &vm_bo->vm->done);
2670479956cSPhilip Yang spin_unlock(&vm_bo->vm->status_lock);
268bcdc9fd6SChristian König }
269bcdc9fd6SChristian König
270bcdc9fd6SChristian König /**
27155bf196fSChristian König * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
27255bf196fSChristian König * @vm: the VM which state machine to reset
27355bf196fSChristian König *
27455bf196fSChristian König * Move all vm_bo object in the VM into a state where they will be updated
27555bf196fSChristian König * again during validation.
27655bf196fSChristian König */
amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm * vm)27755bf196fSChristian König static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
27855bf196fSChristian König {
27955bf196fSChristian König struct amdgpu_vm_bo_base *vm_bo, *tmp;
28055bf196fSChristian König
28155bf196fSChristian König spin_lock(&vm->status_lock);
28255bf196fSChristian König list_splice_init(&vm->done, &vm->invalidated);
28355bf196fSChristian König list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
28455bf196fSChristian König vm_bo->moved = true;
28555bf196fSChristian König list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
28655bf196fSChristian König struct amdgpu_bo *bo = vm_bo->bo;
28755bf196fSChristian König
28863835e3fSZhenGuo Yin vm_bo->moved = true;
28955bf196fSChristian König if (!bo || bo->tbo.type != ttm_bo_type_kernel)
29055bf196fSChristian König list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
29155bf196fSChristian König else if (bo->parent)
29255bf196fSChristian König list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
29355bf196fSChristian König }
29455bf196fSChristian König spin_unlock(&vm->status_lock);
29555bf196fSChristian König }
29655bf196fSChristian König
29755bf196fSChristian König /**
298c460f8a6SChristian König * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
299c460f8a6SChristian König *
300c460f8a6SChristian König * @base: base structure for tracking BO usage in a VM
301c460f8a6SChristian König * @vm: vm to which bo is to be added
302c460f8a6SChristian König * @bo: amdgpu buffer object
303c460f8a6SChristian König *
304c460f8a6SChristian König * Initialize a bo_va_base structure and add it to the appropriate lists
305c460f8a6SChristian König *
306c460f8a6SChristian König */
amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base * base,struct amdgpu_vm * vm,struct amdgpu_bo * bo)307184a69caSChristian König void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
308184a69caSChristian König struct amdgpu_vm *vm, struct amdgpu_bo *bo)
309c460f8a6SChristian König {
310c460f8a6SChristian König base->vm = vm;
311c460f8a6SChristian König base->bo = bo;
312646b9025SChristian König base->next = NULL;
313c460f8a6SChristian König INIT_LIST_HEAD(&base->vm_status);
314c460f8a6SChristian König
315c460f8a6SChristian König if (!bo)
316c460f8a6SChristian König return;
317646b9025SChristian König base->next = bo->vm_bo;
318646b9025SChristian König bo->vm_bo = base;
319c460f8a6SChristian König
320391629bdSNirmoy Das if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv)
321c460f8a6SChristian König return;
322c460f8a6SChristian König
323d7d7ddc1SChristian König dma_resv_assert_held(vm->root.bo->tbo.base.resv);
324d7d7ddc1SChristian König
325fee2ede1SChristian König ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
326fda43ab6SChristian König if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
327bcdc9fd6SChristian König amdgpu_vm_bo_relocated(base);
328c460f8a6SChristian König else
329bcdc9fd6SChristian König amdgpu_vm_bo_idle(base);
330c460f8a6SChristian König
331c460f8a6SChristian König if (bo->preferred_domains &
332d3116756SChristian König amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
333c460f8a6SChristian König return;
334c460f8a6SChristian König
335c460f8a6SChristian König /*
336c460f8a6SChristian König * we checked all the prerequisites, but it looks like this per vm bo
337c460f8a6SChristian König * is currently evicted. add the bo to the evicted list to make sure it
338c460f8a6SChristian König * is validated on next vm use to avoid fault.
339c460f8a6SChristian König * */
340bcdc9fd6SChristian König amdgpu_vm_bo_evicted(base);
341c460f8a6SChristian König }
342c460f8a6SChristian König
343c460f8a6SChristian König /**
3448abc1eb2SChristian König * amdgpu_vm_lock_pd - lock PD in drm_exec
345d38ceaf9SAlex Deucher *
346d38ceaf9SAlex Deucher * @vm: vm providing the BOs
3478abc1eb2SChristian König * @exec: drm execution context
3488abc1eb2SChristian König * @num_fences: number of extra fences to reserve
349d38ceaf9SAlex Deucher *
3508abc1eb2SChristian König * Lock the VM root PD in the DRM execution context.
351d38ceaf9SAlex Deucher */
amdgpu_vm_lock_pd(struct amdgpu_vm * vm,struct drm_exec * exec,unsigned int num_fences)3528abc1eb2SChristian König int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
3538abc1eb2SChristian König unsigned int num_fences)
354d38ceaf9SAlex Deucher {
3558abc1eb2SChristian König /* We need at least two fences for the VM PD/PT updates */
3568abc1eb2SChristian König return drm_exec_prepare_obj(exec, &vm->root.bo->tbo.base,
3578abc1eb2SChristian König 2 + num_fences);
3583d5a08c1Smonk.liu }
359d38ceaf9SAlex Deucher
360fc39d903SChristian König /**
361f921661bSHuang Rui * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
362f921661bSHuang Rui *
363f921661bSHuang Rui * @adev: amdgpu device pointer
364f921661bSHuang Rui * @vm: vm providing the BOs
365f921661bSHuang Rui *
366f921661bSHuang Rui * Move all BOs to the end of LRU and remember their positions to put them
367f921661bSHuang Rui * together.
368f921661bSHuang Rui */
amdgpu_vm_move_to_lru_tail(struct amdgpu_device * adev,struct amdgpu_vm * vm)369f921661bSHuang Rui void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
370f921661bSHuang Rui struct amdgpu_vm *vm)
371f921661bSHuang Rui {
372a1f091f8SChristian König spin_lock(&adev->mman.bdev.lru_lock);
3736a9b0289SChristian König ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
374a1f091f8SChristian König spin_unlock(&adev->mman.bdev.lru_lock);
375f921661bSHuang Rui }
376f921661bSHuang Rui
37755bf196fSChristian König /* Create scheduler entities for page table updates */
amdgpu_vm_init_entities(struct amdgpu_device * adev,struct amdgpu_vm * vm)37855bf196fSChristian König static int amdgpu_vm_init_entities(struct amdgpu_device *adev,
37955bf196fSChristian König struct amdgpu_vm *vm)
38055bf196fSChristian König {
38155bf196fSChristian König int r;
38255bf196fSChristian König
38355bf196fSChristian König r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL,
38455bf196fSChristian König adev->vm_manager.vm_pte_scheds,
38555bf196fSChristian König adev->vm_manager.vm_pte_num_scheds, NULL);
38655bf196fSChristian König if (r)
38755bf196fSChristian König goto error;
38855bf196fSChristian König
38955bf196fSChristian König return drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL,
39055bf196fSChristian König adev->vm_manager.vm_pte_scheds,
39155bf196fSChristian König adev->vm_manager.vm_pte_num_scheds, NULL);
39255bf196fSChristian König
39355bf196fSChristian König error:
39455bf196fSChristian König drm_sched_entity_destroy(&vm->immediate);
39555bf196fSChristian König return r;
39655bf196fSChristian König }
39755bf196fSChristian König
39855bf196fSChristian König /* Destroy the entities for page table updates again */
amdgpu_vm_fini_entities(struct amdgpu_vm * vm)39955bf196fSChristian König static void amdgpu_vm_fini_entities(struct amdgpu_vm *vm)
40055bf196fSChristian König {
40155bf196fSChristian König drm_sched_entity_destroy(&vm->immediate);
40255bf196fSChristian König drm_sched_entity_destroy(&vm->delayed);
40355bf196fSChristian König }
40455bf196fSChristian König
405f921661bSHuang Rui /**
406f88e295eSChristian König * amdgpu_vm_generation - return the page table re-generation counter
407f88e295eSChristian König * @adev: the amdgpu_device
408f88e295eSChristian König * @vm: optional VM to check, might be NULL
409f88e295eSChristian König *
410f88e295eSChristian König * Returns a page table re-generation token to allow checking if submissions
411f88e295eSChristian König * are still valid to use this VM. The VM parameter might be NULL in which case
412f88e295eSChristian König * just the VRAM lost counter will be used.
413f88e295eSChristian König */
amdgpu_vm_generation(struct amdgpu_device * adev,struct amdgpu_vm * vm)414f88e295eSChristian König uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm)
415f88e295eSChristian König {
416f88e295eSChristian König uint64_t result = (u64)atomic_read(&adev->vram_lost_counter) << 32;
417f88e295eSChristian König
418f88e295eSChristian König if (!vm)
419f88e295eSChristian König return result;
420f88e295eSChristian König
421f88e295eSChristian König result += vm->generation;
422f88e295eSChristian König /* Add one if the page tables will be re-generated on next CS */
423f88e295eSChristian König if (drm_sched_entity_error(&vm->delayed))
424f88e295eSChristian König ++result;
425f88e295eSChristian König
426f88e295eSChristian König return result;
427f88e295eSChristian König }
428f88e295eSChristian König
429f88e295eSChristian König /**
430f7da30d9SChristian König * amdgpu_vm_validate_pt_bos - validate the page table BOs
43156467ebfSChristian König *
4325a712a87SChristian König * @adev: amdgpu device pointer
43356467ebfSChristian König * @vm: vm providing the BOs
434f7da30d9SChristian König * @validate: callback to do the validation
435f7da30d9SChristian König * @param: parameter for the validation callback
436d38ceaf9SAlex Deucher *
437f7da30d9SChristian König * Validate the page table BOs on command submission if neccessary.
4387fc48e59SAndrey Grodzovsky *
4397fc48e59SAndrey Grodzovsky * Returns:
4407fc48e59SAndrey Grodzovsky * Validation result.
441d38ceaf9SAlex Deucher */
amdgpu_vm_validate_pt_bos(struct amdgpu_device * adev,struct amdgpu_vm * vm,int (* validate)(void * p,struct amdgpu_bo * bo),void * param)442f7da30d9SChristian König int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
443f7da30d9SChristian König int (*validate)(void *p, struct amdgpu_bo *bo),
444f7da30d9SChristian König void *param)
445d38ceaf9SAlex Deucher {
446757eb2beSPhilip Yang struct amdgpu_vm_bo_base *bo_base;
447757eb2beSPhilip Yang struct amdgpu_bo *shadow;
448757eb2beSPhilip Yang struct amdgpu_bo *bo;
449b4ff0f8aSChristian König int r;
450d38ceaf9SAlex Deucher
45155bf196fSChristian König if (drm_sched_entity_error(&vm->delayed)) {
452f88e295eSChristian König ++vm->generation;
45355bf196fSChristian König amdgpu_vm_bo_reset_state_machine(vm);
45455bf196fSChristian König amdgpu_vm_fini_entities(vm);
45555bf196fSChristian König r = amdgpu_vm_init_entities(adev, vm);
45655bf196fSChristian König if (r)
45755bf196fSChristian König return r;
45855bf196fSChristian König }
45955bf196fSChristian König
460757eb2beSPhilip Yang spin_lock(&vm->status_lock);
461757eb2beSPhilip Yang while (!list_empty(&vm->evicted)) {
462757eb2beSPhilip Yang bo_base = list_first_entry(&vm->evicted,
463757eb2beSPhilip Yang struct amdgpu_vm_bo_base,
464757eb2beSPhilip Yang vm_status);
465757eb2beSPhilip Yang spin_unlock(&vm->status_lock);
466757eb2beSPhilip Yang
467757eb2beSPhilip Yang bo = bo_base->bo;
468757eb2beSPhilip Yang shadow = amdgpu_bo_shadowed(bo);
4695a712a87SChristian König
4703f3333f8SChristian König r = validate(param, bo);
4713f3333f8SChristian König if (r)
472b4ff0f8aSChristian König return r;
47359276f05SNirmoy Das if (shadow) {
47459276f05SNirmoy Das r = validate(param, shadow);
4752a675640SNirmoy Das if (r)
4762a675640SNirmoy Das return r;
4772a675640SNirmoy Das }
4783f3333f8SChristian König
479af4c0f65SChristian König if (bo->tbo.type != ttm_bo_type_kernel) {
480bcdc9fd6SChristian König amdgpu_vm_bo_moved(bo_base);
481af4c0f65SChristian König } else {
48259276f05SNirmoy Das vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
483bcdc9fd6SChristian König amdgpu_vm_bo_relocated(bo_base);
4843f3333f8SChristian König }
485757eb2beSPhilip Yang spin_lock(&vm->status_lock);
486af4c0f65SChristian König }
487757eb2beSPhilip Yang spin_unlock(&vm->status_lock);
48834d7be5dSChristian König
489a269e449SAlex Sierra amdgpu_vm_eviction_lock(vm);
490b4ff0f8aSChristian König vm->evicting = false;
491a269e449SAlex Sierra amdgpu_vm_eviction_unlock(vm);
492b4ff0f8aSChristian König
493b4ff0f8aSChristian König return 0;
49434d7be5dSChristian König }
495d38ceaf9SAlex Deucher
49634d7be5dSChristian König /**
49734d7be5dSChristian König * amdgpu_vm_ready - check VM is ready for updates
49834d7be5dSChristian König *
49934d7be5dSChristian König * @vm: VM to check
50034d7be5dSChristian König *
50134d7be5dSChristian König * Check if all VM PDs/PTs are ready for updates
5027fc48e59SAndrey Grodzovsky *
5037fc48e59SAndrey Grodzovsky * Returns:
504c1a66c3bSQiang Yu * True if VM is not evicting.
50534d7be5dSChristian König */
amdgpu_vm_ready(struct amdgpu_vm * vm)5063f3333f8SChristian König bool amdgpu_vm_ready(struct amdgpu_vm *vm)
50734d7be5dSChristian König {
508757eb2beSPhilip Yang bool empty;
509c1a66c3bSQiang Yu bool ret;
510c1a66c3bSQiang Yu
511c1a66c3bSQiang Yu amdgpu_vm_eviction_lock(vm);
512c1a66c3bSQiang Yu ret = !vm->evicting;
513c1a66c3bSQiang Yu amdgpu_vm_eviction_unlock(vm);
514b6901d93SQiang Yu
515757eb2beSPhilip Yang spin_lock(&vm->status_lock);
516757eb2beSPhilip Yang empty = list_empty(&vm->evicted);
517757eb2beSPhilip Yang spin_unlock(&vm->status_lock);
518757eb2beSPhilip Yang
519757eb2beSPhilip Yang return ret && empty;
520d38ceaf9SAlex Deucher }
521d38ceaf9SAlex Deucher
522663e4577SChristian König /**
523e59c0205SAlex Xie * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
524e59c0205SAlex Xie *
525e59c0205SAlex Xie * @adev: amdgpu_device pointer
526e59c0205SAlex Xie */
amdgpu_vm_check_compute_bug(struct amdgpu_device * adev)527e59c0205SAlex Xie void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
528e59c0205SAlex Xie {
529e59c0205SAlex Xie const struct amdgpu_ip_block *ip_block;
530e59c0205SAlex Xie bool has_compute_vm_bug;
531e59c0205SAlex Xie struct amdgpu_ring *ring;
532e59c0205SAlex Xie int i;
533e59c0205SAlex Xie
534e59c0205SAlex Xie has_compute_vm_bug = false;
53593dcc37dSAlex Deucher
5362990a1fcSAlex Deucher ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
537e59c0205SAlex Xie if (ip_block) {
538e59c0205SAlex Xie /* Compute has a VM bug for GFX version < 7.
539e59c0205SAlex Xie Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
540e59c0205SAlex Xie if (ip_block->version->major <= 7)
541e59c0205SAlex Xie has_compute_vm_bug = true;
542e59c0205SAlex Xie else if (ip_block->version->major == 8)
543e59c0205SAlex Xie if (adev->gfx.mec_fw_version < 673)
544e59c0205SAlex Xie has_compute_vm_bug = true;
545e59c0205SAlex Xie }
54693dcc37dSAlex Deucher
547e59c0205SAlex Xie for (i = 0; i < adev->num_rings; i++) {
548e59c0205SAlex Xie ring = adev->rings[i];
549e59c0205SAlex Xie if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
550e59c0205SAlex Xie /* only compute rings */
551e59c0205SAlex Xie ring->has_compute_vm_bug = has_compute_vm_bug;
55293dcc37dSAlex Deucher else
553e59c0205SAlex Xie ring->has_compute_vm_bug = false;
55493dcc37dSAlex Deucher }
55593dcc37dSAlex Deucher }
55693dcc37dSAlex Deucher
5577fc48e59SAndrey Grodzovsky /**
5587fc48e59SAndrey Grodzovsky * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
5597fc48e59SAndrey Grodzovsky *
5607fc48e59SAndrey Grodzovsky * @ring: ring on which the job will be submitted
5617fc48e59SAndrey Grodzovsky * @job: job to submit
5627fc48e59SAndrey Grodzovsky *
5637fc48e59SAndrey Grodzovsky * Returns:
5647fc48e59SAndrey Grodzovsky * True if sync is needed.
5657fc48e59SAndrey Grodzovsky */
amdgpu_vm_need_pipeline_sync(struct amdgpu_ring * ring,struct amdgpu_job * job)566b9bf33d5SChunming Zhou bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
567b9bf33d5SChunming Zhou struct amdgpu_job *job)
568e60f8db5SAlex Xie {
569b9bf33d5SChunming Zhou struct amdgpu_device *adev = ring->adev;
5700530553bSLe Ma unsigned vmhub = ring->vm_hub;
571620f774fSChristian König struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
572e60f8db5SAlex Xie
573c4f46f22SChristian König if (job->vmid == 0)
574b9bf33d5SChunming Zhou return false;
575e60f8db5SAlex Xie
57656b0989eSChristian König if (job->vm_needs_flush || ring->has_compute_vm_bug)
577b9bf33d5SChunming Zhou return true;
578bb37b67dSAlex Xie
57956b0989eSChristian König if (ring->funcs->emit_gds_switch && job->gds_switch_needed)
58056b0989eSChristian König return true;
58156b0989eSChristian König
58256b0989eSChristian König if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid]))
58356b0989eSChristian König return true;
58456b0989eSChristian König
58556b0989eSChristian König return false;
586b9bf33d5SChunming Zhou }
587b9bf33d5SChunming Zhou
5887fc48e59SAndrey Grodzovsky /**
589d38ceaf9SAlex Deucher * amdgpu_vm_flush - hardware flush the vm
590d38ceaf9SAlex Deucher *
591d38ceaf9SAlex Deucher * @ring: ring to use for flush
59200553cf8SAndrey Grodzovsky * @job: related job
5937fc48e59SAndrey Grodzovsky * @need_pipe_sync: is pipe sync needed
594d38ceaf9SAlex Deucher *
5954ff37a83SChristian König * Emit a VM flush when it is necessary.
5967fc48e59SAndrey Grodzovsky *
5977fc48e59SAndrey Grodzovsky * Returns:
5987fc48e59SAndrey Grodzovsky * 0 on success, errno otherwise.
599d38ceaf9SAlex Deucher */
amdgpu_vm_flush(struct amdgpu_ring * ring,struct amdgpu_job * job,bool need_pipe_sync)600fc39d903SChristian König int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
601fc39d903SChristian König bool need_pipe_sync)
602d38ceaf9SAlex Deucher {
603971fe9a9SChristian König struct amdgpu_device *adev = ring->adev;
6040530553bSLe Ma unsigned vmhub = ring->vm_hub;
605620f774fSChristian König struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
606c4f46f22SChristian König struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
6075f3c40e9SChristian König bool spm_update_needed = job->spm_update_needed;
60856b0989eSChristian König bool gds_switch_needed = ring->funcs->emit_gds_switch &&
60956b0989eSChristian König job->gds_switch_needed;
610de37e68aSFlora Cui bool vm_flush_needed = job->vm_needs_flush;
611b3cd285fSChristian König struct dma_fence *fence = NULL;
61217cf678aSColin Ian King bool pasid_mapping_needed = false;
613c0e51931SChristian König unsigned patch_offset = 0;
61441d9eb2cSChristian König int r;
615971fe9a9SChristian König
616620f774fSChristian König if (amdgpu_vmid_had_gpu_reset(adev, id)) {
617f7d015b9SChristian König gds_switch_needed = true;
618f7d015b9SChristian König vm_flush_needed = true;
619b3cd285fSChristian König pasid_mapping_needed = true;
6205f3c40e9SChristian König spm_update_needed = true;
621f7d015b9SChristian König }
622c0e51931SChristian König
6236817bf28SChristian König mutex_lock(&id_mgr->lock);
6246817bf28SChristian König if (id->pasid != job->pasid || !id->pasid_mapping ||
6256817bf28SChristian König !dma_fence_is_signaled(id->pasid_mapping))
6266817bf28SChristian König pasid_mapping_needed = true;
6276817bf28SChristian König mutex_unlock(&id_mgr->lock);
6286817bf28SChristian König
629b3cd285fSChristian König gds_switch_needed &= !!ring->funcs->emit_gds_switch;
630d8de8260SAndrey Grodzovsky vm_flush_needed &= !!ring->funcs->emit_vm_flush &&
631d8de8260SAndrey Grodzovsky job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
632b3cd285fSChristian König pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
633b3cd285fSChristian König ring->funcs->emit_wreg;
634b3cd285fSChristian König
6358fdf074fSMonk Liu if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
636f7d015b9SChristian König return 0;
637e9d672b2SMonk Liu
6383f4c175dSJiadong.Zhu amdgpu_ring_ib_begin(ring);
639e9d672b2SMonk Liu if (ring->funcs->init_cond_exec)
640e9d672b2SMonk Liu patch_offset = amdgpu_ring_init_cond_exec(ring);
641e9d672b2SMonk Liu
6428fdf074fSMonk Liu if (need_pipe_sync)
6438fdf074fSMonk Liu amdgpu_ring_emit_pipeline_sync(ring);
6448fdf074fSMonk Liu
645b3cd285fSChristian König if (vm_flush_needed) {
646c4f46f22SChristian König trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
647c633c00bSChristian König amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
648b3cd285fSChristian König }
64941d9eb2cSChristian König
650b3cd285fSChristian König if (pasid_mapping_needed)
651b3cd285fSChristian König amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
652b3cd285fSChristian König
6535f3c40e9SChristian König if (spm_update_needed && adev->gfx.rlc.funcs->update_spm_vmid)
6545f3c40e9SChristian König adev->gfx.rlc.funcs->update_spm_vmid(adev, job->vmid);
6555f3c40e9SChristian König
65656b0989eSChristian König if (!ring->is_mes_queue && ring->funcs->emit_gds_switch &&
65756b0989eSChristian König gds_switch_needed) {
65856b0989eSChristian König amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
65956b0989eSChristian König job->gds_size, job->gws_base,
66056b0989eSChristian König job->gws_size, job->oa_base,
66156b0989eSChristian König job->oa_size);
66256b0989eSChristian König }
66356b0989eSChristian König
664b3cd285fSChristian König if (vm_flush_needed || pasid_mapping_needed) {
665c530b02fSJack Zhang r = amdgpu_fence_emit(ring, &fence, NULL, 0);
6663dab83beSChristian König if (r)
66768befebeSChunming Zhou return r;
668b3cd285fSChristian König }
6693dab83beSChristian König
670b3cd285fSChristian König if (vm_flush_needed) {
6717645670dSChristian König mutex_lock(&id_mgr->lock);
672f54d1867SChris Wilson dma_fence_put(id->last_flush);
673b3cd285fSChristian König id->last_flush = dma_fence_get(fence);
674b3cd285fSChristian König id->current_gpu_reset_count =
675b3cd285fSChristian König atomic_read(&adev->gpu_reset_counter);
6767645670dSChristian König mutex_unlock(&id_mgr->lock);
677d564a06eSChristian König }
678d564a06eSChristian König
679b3cd285fSChristian König if (pasid_mapping_needed) {
6806817bf28SChristian König mutex_lock(&id_mgr->lock);
681b3cd285fSChristian König id->pasid = job->pasid;
682b3cd285fSChristian König dma_fence_put(id->pasid_mapping);
683b3cd285fSChristian König id->pasid_mapping = dma_fence_get(fence);
6846817bf28SChristian König mutex_unlock(&id_mgr->lock);
685b3cd285fSChristian König }
686b3cd285fSChristian König dma_fence_put(fence);
687b3cd285fSChristian König
688e9d672b2SMonk Liu if (ring->funcs->patch_cond_exec)
689e9d672b2SMonk Liu amdgpu_ring_patch_cond_exec(ring, patch_offset);
690e9d672b2SMonk Liu
691e9d672b2SMonk Liu /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
692e9d672b2SMonk Liu if (ring->funcs->emit_switch_buffer) {
693e9d672b2SMonk Liu amdgpu_ring_emit_switch_buffer(ring);
694e9d672b2SMonk Liu amdgpu_ring_emit_switch_buffer(ring);
695e9d672b2SMonk Liu }
6963f4c175dSJiadong.Zhu amdgpu_ring_ib_end(ring);
69741d9eb2cSChristian König return 0;
698971fe9a9SChristian König }
699971fe9a9SChristian König
700971fe9a9SChristian König /**
701d38ceaf9SAlex Deucher * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
702d38ceaf9SAlex Deucher *
703d38ceaf9SAlex Deucher * @vm: requested vm
704d38ceaf9SAlex Deucher * @bo: requested buffer object
705d38ceaf9SAlex Deucher *
7068843dbbbSChristian König * Find @bo inside the requested vm.
707d38ceaf9SAlex Deucher * Search inside the @bos vm list for the requested vm
708d38ceaf9SAlex Deucher * Returns the found bo_va or NULL if none is found
709d38ceaf9SAlex Deucher *
710d38ceaf9SAlex Deucher * Object has to be reserved!
7117fc48e59SAndrey Grodzovsky *
7127fc48e59SAndrey Grodzovsky * Returns:
7137fc48e59SAndrey Grodzovsky * Found bo_va or NULL.
714d38ceaf9SAlex Deucher */
amdgpu_vm_bo_find(struct amdgpu_vm * vm,struct amdgpu_bo * bo)715d38ceaf9SAlex Deucher struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
716d38ceaf9SAlex Deucher struct amdgpu_bo *bo)
717d38ceaf9SAlex Deucher {
718646b9025SChristian König struct amdgpu_vm_bo_base *base;
719d38ceaf9SAlex Deucher
720646b9025SChristian König for (base = bo->vm_bo; base; base = base->next) {
721646b9025SChristian König if (base->vm != vm)
722646b9025SChristian König continue;
723646b9025SChristian König
724646b9025SChristian König return container_of(base, struct amdgpu_bo_va, base);
725d38ceaf9SAlex Deucher }
726d38ceaf9SAlex Deucher return NULL;
727d38ceaf9SAlex Deucher }
728d38ceaf9SAlex Deucher
729d38ceaf9SAlex Deucher /**
730b07c9d2aSChristian König * amdgpu_vm_map_gart - Resolve gart mapping of addr
731d38ceaf9SAlex Deucher *
732b07c9d2aSChristian König * @pages_addr: optional DMA address to use for lookup
733d38ceaf9SAlex Deucher * @addr: the unmapped addr
734d38ceaf9SAlex Deucher *
735d38ceaf9SAlex Deucher * Look up the physical address of the page that the pte resolves
7367fc48e59SAndrey Grodzovsky * to.
7377fc48e59SAndrey Grodzovsky *
7387fc48e59SAndrey Grodzovsky * Returns:
7397fc48e59SAndrey Grodzovsky * The pointer for the page table entry.
740d38ceaf9SAlex Deucher */
amdgpu_vm_map_gart(const dma_addr_t * pages_addr,uint64_t addr)7416dd09027SChristian König uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
742d38ceaf9SAlex Deucher {
743d38ceaf9SAlex Deucher uint64_t result;
744d38ceaf9SAlex Deucher
745d38ceaf9SAlex Deucher /* page table offset */
746b07c9d2aSChristian König result = pages_addr[addr >> PAGE_SHIFT];
747d38ceaf9SAlex Deucher
748d38ceaf9SAlex Deucher /* in case cpu page size != gpu page size*/
749d38ceaf9SAlex Deucher result |= addr & (~PAGE_MASK);
750d38ceaf9SAlex Deucher
751b07c9d2aSChristian König result &= 0xFFFFFFFFFFFFF000ULL;
752b07c9d2aSChristian König
753d38ceaf9SAlex Deucher return result;
754d38ceaf9SAlex Deucher }
755d38ceaf9SAlex Deucher
7561d614dedSAlex Deucher /**
757807e2994SChristian König * amdgpu_vm_update_pdes - make sure that all directories are valid
758194d2161SChristian König *
759194d2161SChristian König * @adev: amdgpu_device pointer
760194d2161SChristian König * @vm: requested vm
761eaad0c3aSChristian König * @immediate: submit immediately to the paging queue
762194d2161SChristian König *
763194d2161SChristian König * Makes sure all directories are up to date.
7647fc48e59SAndrey Grodzovsky *
7657fc48e59SAndrey Grodzovsky * Returns:
7667fc48e59SAndrey Grodzovsky * 0 for success, error for failure.
767194d2161SChristian König */
amdgpu_vm_update_pdes(struct amdgpu_device * adev,struct amdgpu_vm * vm,bool immediate)768807e2994SChristian König int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
769eaad0c3aSChristian König struct amdgpu_vm *vm, bool immediate)
770194d2161SChristian König {
771d1e29462SChristian König struct amdgpu_vm_update_params params;
7726e97c2f9SChristian König struct amdgpu_vm_bo_base *entry;
7734d1e5f12SPhilip Yang bool flush_tlb_needed = false;
774b38e77cbSPhilip Yang LIST_HEAD(relocated);
775b2fe31cfSxinhui pan int r, idx;
77692456b93SChristian König
777b38e77cbSPhilip Yang spin_lock(&vm->status_lock);
778b38e77cbSPhilip Yang list_splice_init(&vm->relocated, &relocated);
779b38e77cbSPhilip Yang spin_unlock(&vm->status_lock);
780b38e77cbSPhilip Yang
781b38e77cbSPhilip Yang if (list_empty(&relocated))
7826989f246SChristian König return 0;
7836989f246SChristian König
784c58a863bSGuchun Chen if (!drm_dev_enter(adev_to_drm(adev), &idx))
785b2fe31cfSxinhui pan return -ENODEV;
786b2fe31cfSxinhui pan
7876989f246SChristian König memset(¶ms, 0, sizeof(params));
7886989f246SChristian König params.adev = adev;
789e6899d55SChristian König params.vm = vm;
790eaad0c3aSChristian König params.immediate = immediate;
7916989f246SChristian König
7929f3cc18dSChristian König r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT);
7936989f246SChristian König if (r)
7946e97c2f9SChristian König goto error;
7956989f246SChristian König
796b38e77cbSPhilip Yang list_for_each_entry(entry, &relocated, vm_status) {
7974d1e5f12SPhilip Yang /* vm_flush_needed after updating moved PDEs */
7984d1e5f12SPhilip Yang flush_tlb_needed |= entry->moved;
7994d1e5f12SPhilip Yang
800184a69caSChristian König r = amdgpu_vm_pde_update(¶ms, entry);
8016989f246SChristian König if (r)
8026989f246SChristian König goto error;
80368c62306SChristian König }
80468c62306SChristian König
805e6899d55SChristian König r = vm->update_funcs->commit(¶ms, &vm->last_update);
806e6899d55SChristian König if (r)
807e6899d55SChristian König goto error;
8086e97c2f9SChristian König
8094d1e5f12SPhilip Yang if (flush_tlb_needed)
8105be32356SPhilip Yang atomic64_inc(&vm->tlb_seq);
8115be32356SPhilip Yang
812b38e77cbSPhilip Yang while (!list_empty(&relocated)) {
813b38e77cbSPhilip Yang entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
8146e97c2f9SChristian König vm_status);
8156e97c2f9SChristian König amdgpu_vm_bo_idle(entry);
8166e97c2f9SChristian König }
8176989f246SChristian König
8186989f246SChristian König error:
819b2fe31cfSxinhui pan drm_dev_exit(idx);
82092456b93SChristian König return r;
821194d2161SChristian König }
822194d2161SChristian König
823d38ceaf9SAlex Deucher /**
8245255e146SChristian König * amdgpu_vm_tlb_seq_cb - make sure to increment tlb sequence
8255255e146SChristian König * @fence: unused
8265255e146SChristian König * @cb: the callback structure
827d38ceaf9SAlex Deucher *
8285255e146SChristian König * Increments the tlb sequence to make sure that future CS execute a VM flush.
829d38ceaf9SAlex Deucher */
amdgpu_vm_tlb_seq_cb(struct dma_fence * fence,struct dma_fence_cb * cb)8305255e146SChristian König static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence,
8315255e146SChristian König struct dma_fence_cb *cb)
832d38ceaf9SAlex Deucher {
83352b82609SLuben Tuikov struct amdgpu_vm_tlb_seq_struct *tlb_cb;
834d38ceaf9SAlex Deucher
8355255e146SChristian König tlb_cb = container_of(cb, typeof(*tlb_cb), cb);
8365255e146SChristian König atomic64_inc(&tlb_cb->vm->tlb_seq);
8375255e146SChristian König kfree(tlb_cb);
838d38ceaf9SAlex Deucher }
839d38ceaf9SAlex Deucher
840d38ceaf9SAlex Deucher /**
84130671b44SChristian König * amdgpu_vm_update_range - update a range in the vm page table
842d38ceaf9SAlex Deucher *
84330671b44SChristian König * @adev: amdgpu_device pointer to use for commands
84430671b44SChristian König * @vm: the VM to update the range
845eaad0c3aSChristian König * @immediate: immediate submission in a page fault
8469c466bcbSChristian König * @unlocked: unlocked invalidation during MM callback
84730671b44SChristian König * @flush_tlb: trigger tlb invalidation after update completed
8489f3cc18dSChristian König * @resv: fences we need to sync to
849a14faa65SChristian König * @start: start of mapped range
850a14faa65SChristian König * @last: last mapped entry
851a14faa65SChristian König * @flags: flags for the entries
852a39f2a8dSChristian König * @offset: offset into nodes and pages_addr
85330671b44SChristian König * @vram_base: base for vram mappings
8540ccc3ccfSChristian König * @res: ttm_resource to map
855acb476f5SChristian König * @pages_addr: DMA addresses to use for mapping
856d38ceaf9SAlex Deucher * @fence: optional resulting fence
857d38ceaf9SAlex Deucher *
858a14faa65SChristian König * Fill in the page table entries between @start and @last.
8597fc48e59SAndrey Grodzovsky *
8607fc48e59SAndrey Grodzovsky * Returns:
86130671b44SChristian König * 0 for success, negative erro code for failure.
862d38ceaf9SAlex Deucher */
amdgpu_vm_update_range(struct amdgpu_device * adev,struct amdgpu_vm * vm,bool immediate,bool unlocked,bool flush_tlb,struct dma_resv * resv,uint64_t start,uint64_t last,uint64_t flags,uint64_t offset,uint64_t vram_base,struct ttm_resource * res,dma_addr_t * pages_addr,struct dma_fence ** fence)86330671b44SChristian König int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
86430671b44SChristian König bool immediate, bool unlocked, bool flush_tlb,
86530671b44SChristian König struct dma_resv *resv, uint64_t start, uint64_t last,
86630671b44SChristian König uint64_t flags, uint64_t offset, uint64_t vram_base,
86730671b44SChristian König struct ttm_resource *res, dma_addr_t *pages_addr,
8688f8cc3fbSChristian König struct dma_fence **fence)
869d38ceaf9SAlex Deucher {
870d1e29462SChristian König struct amdgpu_vm_update_params params;
87152b82609SLuben Tuikov struct amdgpu_vm_tlb_seq_struct *tlb_cb;
8720ccc3ccfSChristian König struct amdgpu_res_cursor cursor;
8739f3cc18dSChristian König enum amdgpu_sync_mode sync_mode;
874f89f8c6bSAndrey Grodzovsky int r, idx;
875f89f8c6bSAndrey Grodzovsky
876c58a863bSGuchun Chen if (!drm_dev_enter(adev_to_drm(adev), &idx))
877f89f8c6bSAndrey Grodzovsky return -ENODEV;
878d38ceaf9SAlex Deucher
8795255e146SChristian König tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
8805255e146SChristian König if (!tlb_cb) {
8815255e146SChristian König r = -ENOMEM;
8825255e146SChristian König goto error_unlock;
8835255e146SChristian König }
8845255e146SChristian König
8850f12a22fSPhilip Yang /* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache,
8860f12a22fSPhilip Yang * heavy-weight flush TLB unconditionally.
8870f12a22fSPhilip Yang */
8880f12a22fSPhilip Yang flush_tlb |= adev->gmc.xgmi.num_physical_nodes &&
8890f12a22fSPhilip Yang adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0);
8900f12a22fSPhilip Yang
89164f6516eSChristian König /*
89264f6516eSChristian König * On GFX8 and older any 8 PTE block with a valid bit set enters the TLB
89364f6516eSChristian König */
89464f6516eSChristian König flush_tlb |= adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 0, 0);
89564f6516eSChristian König
896afef8b8fSChristian König memset(¶ms, 0, sizeof(params));
897afef8b8fSChristian König params.adev = adev;
89849ac8a24SChristian König params.vm = vm;
899eaad0c3aSChristian König params.immediate = immediate;
900072b7a0bSChristian König params.pages_addr = pages_addr;
9015654b897SAlex Sierra params.unlocked = unlocked;
902afef8b8fSChristian König
9039f3cc18dSChristian König /* Implicitly sync to command submissions in the same VM before
9049f3cc18dSChristian König * unmapping. Sync to moving fences before mapping.
9059f3cc18dSChristian König */
906a33cab7aSChristian König if (!(flags & AMDGPU_PTE_VALID))
9079f3cc18dSChristian König sync_mode = AMDGPU_SYNC_EQ_OWNER;
9089f3cc18dSChristian König else
9099f3cc18dSChristian König sync_mode = AMDGPU_SYNC_EXPLICIT;
910a33cab7aSChristian König
911a269e449SAlex Sierra amdgpu_vm_eviction_lock(vm);
912b4ff0f8aSChristian König if (vm->evicting) {
913b4ff0f8aSChristian König r = -EBUSY;
9145255e146SChristian König goto error_free;
915b4ff0f8aSChristian König }
916b4ff0f8aSChristian König
9179c466bcbSChristian König if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
9189c466bcbSChristian König struct dma_fence *tmp = dma_fence_get_stub();
91942e5fee6SChristian König
920391629bdSNirmoy Das amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
9219c466bcbSChristian König swap(vm->last_unlocked, tmp);
9229c466bcbSChristian König dma_fence_put(tmp);
92342e5fee6SChristian König }
92442e5fee6SChristian König
9259f3cc18dSChristian König r = vm->update_funcs->prepare(¶ms, resv, sync_mode);
926d71518b5SChristian König if (r)
9275255e146SChristian König goto error_free;
928d71518b5SChristian König
9290ac8f587SChristian König amdgpu_res_first(pages_addr ? NULL : res, offset,
9300ac8f587SChristian König (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor);
9310ccc3ccfSChristian König while (cursor.remaining) {
932a39f2a8dSChristian König uint64_t tmp, num_entries, addr;
93363e0ba40SChristian König
9340ccc3ccfSChristian König num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;
9358358dceeSChristian König if (pages_addr) {
936a39f2a8dSChristian König bool contiguous = true;
937a39f2a8dSChristian König
938a39f2a8dSChristian König if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) {
9390ccc3ccfSChristian König uint64_t pfn = cursor.start >> PAGE_SHIFT;
9409fc8fc70SChristian König uint64_t count;
9419fc8fc70SChristian König
942a39f2a8dSChristian König contiguous = pages_addr[pfn + 1] ==
943a39f2a8dSChristian König pages_addr[pfn] + PAGE_SIZE;
944a39f2a8dSChristian König
945a39f2a8dSChristian König tmp = num_entries /
946a39f2a8dSChristian König AMDGPU_GPU_PAGES_IN_CPU_PAGE;
947a39f2a8dSChristian König for (count = 2; count < tmp; ++count) {
9489fc8fc70SChristian König uint64_t idx = pfn + count;
9499fc8fc70SChristian König
950a39f2a8dSChristian König if (contiguous != (pages_addr[idx] ==
951a39f2a8dSChristian König pages_addr[idx - 1] + PAGE_SIZE))
9529fc8fc70SChristian König break;
9539fc8fc70SChristian König }
95413e3a038SFelix Kuehling if (!contiguous)
95513e3a038SFelix Kuehling count--;
956a39f2a8dSChristian König num_entries = count *
957a39f2a8dSChristian König AMDGPU_GPU_PAGES_IN_CPU_PAGE;
958a39f2a8dSChristian König }
9599fc8fc70SChristian König
960a39f2a8dSChristian König if (!contiguous) {
9610ccc3ccfSChristian König addr = cursor.start;
962a39f2a8dSChristian König params.pages_addr = pages_addr;
9639fc8fc70SChristian König } else {
9640ccc3ccfSChristian König addr = pages_addr[cursor.start >> PAGE_SHIFT];
965a39f2a8dSChristian König params.pages_addr = NULL;
9669fc8fc70SChristian König }
9679fc8fc70SChristian König
96831d0271dSYintian Tao } else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) {
96930671b44SChristian König addr = vram_base + cursor.start;
9700ccc3ccfSChristian König } else {
9710ccc3ccfSChristian König addr = 0;
9729fc8fc70SChristian König }
973a14faa65SChristian König
974a39f2a8dSChristian König tmp = start + num_entries;
975184a69caSChristian König r = amdgpu_vm_ptes_update(¶ms, start, tmp, addr, flags);
976a14faa65SChristian König if (r)
9775255e146SChristian König goto error_free;
978a14faa65SChristian König
9790ccc3ccfSChristian König amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE);
980a39f2a8dSChristian König start = tmp;
98148b03309SWan Jiabing }
982a14faa65SChristian König
983a39f2a8dSChristian König r = vm->update_funcs->commit(¶ms, fence);
984a39f2a8dSChristian König
98530671b44SChristian König if (flush_tlb || params.table_freed) {
9865255e146SChristian König tlb_cb->vm = vm;
9877c703a7dSxinhui pan if (fence && *fence &&
9887c703a7dSxinhui pan !dma_fence_add_callback(*fence, &tlb_cb->cb,
9897c703a7dSxinhui pan amdgpu_vm_tlb_seq_cb)) {
9907c703a7dSxinhui pan dma_fence_put(vm->last_tlb_flush);
9917c703a7dSxinhui pan vm->last_tlb_flush = dma_fence_get(*fence);
9927c703a7dSxinhui pan } else {
9935255e146SChristian König amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);
9947c703a7dSxinhui pan }
9955255e146SChristian König tlb_cb = NULL;
9965255e146SChristian König }
9975255e146SChristian König
9985255e146SChristian König error_free:
9995255e146SChristian König kfree(tlb_cb);
1000bf546940SPhilip Yang
1001a39f2a8dSChristian König error_unlock:
1002a39f2a8dSChristian König amdgpu_vm_eviction_unlock(vm);
1003f89f8c6bSAndrey Grodzovsky drm_dev_exit(idx);
1004a39f2a8dSChristian König return r;
1005a14faa65SChristian König }
1006a14faa65SChristian König
amdgpu_vm_bo_get_memory(struct amdgpu_bo_va * bo_va,struct amdgpu_mem_stats * stats)1007e2ad8e2dSChristian König static void amdgpu_vm_bo_get_memory(struct amdgpu_bo_va *bo_va,
1008e2ad8e2dSChristian König struct amdgpu_mem_stats *stats)
1009e2ad8e2dSChristian König {
1010e2ad8e2dSChristian König struct amdgpu_vm *vm = bo_va->base.vm;
1011e2ad8e2dSChristian König struct amdgpu_bo *bo = bo_va->base.bo;
1012e2ad8e2dSChristian König
1013e2ad8e2dSChristian König if (!bo)
1014e2ad8e2dSChristian König return;
1015e2ad8e2dSChristian König
1016e2ad8e2dSChristian König /*
1017e2ad8e2dSChristian König * For now ignore BOs which are currently locked and potentially
1018e2ad8e2dSChristian König * changing their location.
1019e2ad8e2dSChristian König */
1020e2ad8e2dSChristian König if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv &&
1021e2ad8e2dSChristian König !dma_resv_trylock(bo->tbo.base.resv))
1022e2ad8e2dSChristian König return;
1023e2ad8e2dSChristian König
1024e2ad8e2dSChristian König amdgpu_bo_get_memory(bo, stats);
1025e2ad8e2dSChristian König if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv)
1026e2ad8e2dSChristian König dma_resv_unlock(bo->tbo.base.resv);
1027e2ad8e2dSChristian König }
1028e2ad8e2dSChristian König
amdgpu_vm_get_memory(struct amdgpu_vm * vm,struct amdgpu_mem_stats * stats)1029d6530c33SMarek Olšák void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
1030d6530c33SMarek Olšák struct amdgpu_mem_stats *stats)
103187444254SRoy Sun {
103287444254SRoy Sun struct amdgpu_bo_va *bo_va, *tmp;
103387444254SRoy Sun
1034b38e77cbSPhilip Yang spin_lock(&vm->status_lock);
1035e2ad8e2dSChristian König list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status)
1036e2ad8e2dSChristian König amdgpu_vm_bo_get_memory(bo_va, stats);
1037e2ad8e2dSChristian König
1038e2ad8e2dSChristian König list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status)
1039e2ad8e2dSChristian König amdgpu_vm_bo_get_memory(bo_va, stats);
1040e2ad8e2dSChristian König
1041e2ad8e2dSChristian König list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status)
1042e2ad8e2dSChristian König amdgpu_vm_bo_get_memory(bo_va, stats);
1043e2ad8e2dSChristian König
1044e2ad8e2dSChristian König list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status)
1045e2ad8e2dSChristian König amdgpu_vm_bo_get_memory(bo_va, stats);
1046e2ad8e2dSChristian König
1047e2ad8e2dSChristian König list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status)
1048e2ad8e2dSChristian König amdgpu_vm_bo_get_memory(bo_va, stats);
1049e2ad8e2dSChristian König
1050e2ad8e2dSChristian König list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status)
1051e2ad8e2dSChristian König amdgpu_vm_bo_get_memory(bo_va, stats);
10520479956cSPhilip Yang spin_unlock(&vm->status_lock);
105387444254SRoy Sun }
1054d6530c33SMarek Olšák
1055a14faa65SChristian König /**
1056d38ceaf9SAlex Deucher * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1057d38ceaf9SAlex Deucher *
1058d38ceaf9SAlex Deucher * @adev: amdgpu_device pointer
1059d38ceaf9SAlex Deucher * @bo_va: requested BO and VM object
106099e124f4SChristian König * @clear: if true clear the entries
1061d38ceaf9SAlex Deucher *
1062d38ceaf9SAlex Deucher * Fill in the page table entries for @bo_va.
10637fc48e59SAndrey Grodzovsky *
10647fc48e59SAndrey Grodzovsky * Returns:
10657fc48e59SAndrey Grodzovsky * 0 for success, -EINVAL for failure.
1066d38ceaf9SAlex Deucher */
amdgpu_vm_bo_update(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va,bool clear)1067fc39d903SChristian König int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
10688f8cc3fbSChristian König bool clear)
1069d38ceaf9SAlex Deucher {
1070ec681545SChristian König struct amdgpu_bo *bo = bo_va->base.bo;
1071ec681545SChristian König struct amdgpu_vm *vm = bo_va->base.vm;
1072d38ceaf9SAlex Deucher struct amdgpu_bo_va_mapping *mapping;
10738358dceeSChristian König dma_addr_t *pages_addr = NULL;
10742966141aSDave Airlie struct ttm_resource *mem;
10759f3cc18dSChristian König struct dma_fence **last_update;
107630671b44SChristian König bool flush_tlb = clear;
10779f3cc18dSChristian König struct dma_resv *resv;
107830671b44SChristian König uint64_t vram_base;
1079457e0feeSChristian König uint64_t flags;
1080d38ceaf9SAlex Deucher int r;
1081d38ceaf9SAlex Deucher
10827eb80427SHuang Rui if (clear || !bo) {
108399e124f4SChristian König mem = NULL;
1084391629bdSNirmoy Das resv = vm->root.bo->tbo.base.resv;
108599e124f4SChristian König } else {
10860cf0ee98SArunpravin struct drm_gem_object *obj = &bo->tbo.base;
10878358dceeSChristian König
10880cf0ee98SArunpravin resv = bo->tbo.base.resv;
10890cf0ee98SArunpravin if (obj->import_attach && bo_va->is_xgmi) {
10900cf0ee98SArunpravin struct dma_buf *dma_buf = obj->import_attach->dmabuf;
10910cf0ee98SArunpravin struct drm_gem_object *gobj = dma_buf->priv;
10920cf0ee98SArunpravin struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
10930cf0ee98SArunpravin
109451b79f33SFelix Kuehling if (abo->tbo.resource &&
109551b79f33SFelix Kuehling abo->tbo.resource->mem_type == TTM_PL_VRAM)
10960cf0ee98SArunpravin bo = gem_to_amdgpu_bo(gobj);
10970cf0ee98SArunpravin }
1098d3116756SChristian König mem = bo->tbo.resource;
10995e4d9fdaSFelix Kuehling if (mem && (mem->mem_type == TTM_PL_TT ||
11005e4d9fdaSFelix Kuehling mem->mem_type == AMDGPU_PL_PREEMPT))
1101e34b8feeSChristian König pages_addr = bo->tbo.ttm->dma_address;
1102d38ceaf9SAlex Deucher }
1103d38ceaf9SAlex Deucher
1104a690aa0fSshaoyunl if (bo) {
110530671b44SChristian König struct amdgpu_device *bo_adev;
110630671b44SChristian König
1107ec681545SChristian König flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
11084cd24494SAlex Deucher
11094cd24494SAlex Deucher if (amdgpu_bo_encrypted(bo))
11104cd24494SAlex Deucher flags |= AMDGPU_PTE_TMZ;
11114cd24494SAlex Deucher
1112a690aa0fSshaoyunl bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
111330671b44SChristian König vram_base = bo_adev->vm_manager.vram_base_offset;
1114a690aa0fSshaoyunl } else {
1115a5f6b5b1SChristian König flags = 0x0;
111630671b44SChristian König vram_base = 0;
1117a690aa0fSshaoyunl }
1118d38ceaf9SAlex Deucher
11199f3cc18dSChristian König if (clear || (bo && bo->tbo.base.resv ==
1120391629bdSNirmoy Das vm->root.bo->tbo.base.resv))
11214e55eb38SChristian König last_update = &vm->last_update;
11224e55eb38SChristian König else
11234e55eb38SChristian König last_update = &bo_va->last_pt_update;
11244e55eb38SChristian König
11253d7d4d3aSChristian König if (!clear && bo_va->base.moved) {
112630671b44SChristian König flush_tlb = true;
11277fc11959SChristian König list_splice_init(&bo_va->valids, &bo_va->invalids);
11283d7d4d3aSChristian König
1129cb7b6ec2SChristian König } else if (bo_va->cleared != clear) {
11307fc11959SChristian König list_splice_init(&bo_va->valids, &bo_va->invalids);
11313d7d4d3aSChristian König }
11327fc11959SChristian König
11337fc11959SChristian König list_for_each_entry(mapping, &bo_va->invalids, list) {
1134a39f2a8dSChristian König uint64_t update_flags = flags;
1135a39f2a8dSChristian König
1136a39f2a8dSChristian König /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1137a39f2a8dSChristian König * but in case of something, we filter the flags in first place
1138a39f2a8dSChristian König */
1139a39f2a8dSChristian König if (!(mapping->flags & AMDGPU_PTE_READABLE))
1140a39f2a8dSChristian König update_flags &= ~AMDGPU_PTE_READABLE;
1141a39f2a8dSChristian König if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1142a39f2a8dSChristian König update_flags &= ~AMDGPU_PTE_WRITEABLE;
1143a39f2a8dSChristian König
1144a39f2a8dSChristian König /* Apply ASIC specific mapping flags */
1145a39f2a8dSChristian König amdgpu_gmc_get_vm_pte(adev, mapping, &update_flags);
1146a39f2a8dSChristian König
1147a39f2a8dSChristian König trace_amdgpu_vm_bo_update(mapping);
1148a39f2a8dSChristian König
114930671b44SChristian König r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb,
115030671b44SChristian König resv, mapping->start, mapping->last,
115130671b44SChristian König update_flags, mapping->offset,
115230671b44SChristian König vram_base, mem, pages_addr,
115330671b44SChristian König last_update);
1154d38ceaf9SAlex Deucher if (r)
1155d38ceaf9SAlex Deucher return r;
1156d38ceaf9SAlex Deucher }
1157d38ceaf9SAlex Deucher
115836188364SChristian König /* If the BO is not in its preferred location add it back to
115936188364SChristian König * the evicted list so that it gets validated again on the
116036188364SChristian König * next command submission.
116136188364SChristian König */
1162391629bdSNirmoy Das if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
1163d3116756SChristian König uint32_t mem_type = bo->tbo.resource->mem_type;
1164806f043fSChristian König
1165fc39d903SChristian König if (!(bo->preferred_domains &
1166fc39d903SChristian König amdgpu_mem_type_to_domain(mem_type)))
1167bcdc9fd6SChristian König amdgpu_vm_bo_evicted(&bo_va->base);
1168806f043fSChristian König else
1169bcdc9fd6SChristian König amdgpu_vm_bo_idle(&bo_va->base);
1170c12a2ee5SChristian König } else {
1171bcdc9fd6SChristian König amdgpu_vm_bo_done(&bo_va->base);
1172806f043fSChristian König }
1173cb7b6ec2SChristian König
1174cb7b6ec2SChristian König list_splice_init(&bo_va->invalids, &bo_va->valids);
1175cb7b6ec2SChristian König bo_va->cleared = clear;
117630671b44SChristian König bo_va->base.moved = false;
1177cb7b6ec2SChristian König
1178cb7b6ec2SChristian König if (trace_amdgpu_vm_bo_mapping_enabled()) {
1179cb7b6ec2SChristian König list_for_each_entry(mapping, &bo_va->valids, list)
1180cb7b6ec2SChristian König trace_amdgpu_vm_bo_mapping(mapping);
1181cb7b6ec2SChristian König }
1182cb7b6ec2SChristian König
1183d38ceaf9SAlex Deucher return 0;
1184d38ceaf9SAlex Deucher }
1185d38ceaf9SAlex Deucher
1186d38ceaf9SAlex Deucher /**
1187284710faSChristian König * amdgpu_vm_update_prt_state - update the global PRT state
11887fc48e59SAndrey Grodzovsky *
11897fc48e59SAndrey Grodzovsky * @adev: amdgpu_device pointer
1190284710faSChristian König */
amdgpu_vm_update_prt_state(struct amdgpu_device * adev)1191284710faSChristian König static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1192284710faSChristian König {
1193284710faSChristian König unsigned long flags;
1194284710faSChristian König bool enable;
1195284710faSChristian König
1196284710faSChristian König spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1197451bc8ebSChristian König enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1198132f34e4SChristian König adev->gmc.gmc_funcs->set_prt(adev, enable);
1199284710faSChristian König spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1200284710faSChristian König }
1201284710faSChristian König
1202284710faSChristian König /**
12034388fc2aSChristian König * amdgpu_vm_prt_get - add a PRT user
12047fc48e59SAndrey Grodzovsky *
12057fc48e59SAndrey Grodzovsky * @adev: amdgpu_device pointer
1206451bc8ebSChristian König */
amdgpu_vm_prt_get(struct amdgpu_device * adev)1207451bc8ebSChristian König static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1208451bc8ebSChristian König {
1209132f34e4SChristian König if (!adev->gmc.gmc_funcs->set_prt)
12104388fc2aSChristian König return;
12114388fc2aSChristian König
1212451bc8ebSChristian König if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1213451bc8ebSChristian König amdgpu_vm_update_prt_state(adev);
1214451bc8ebSChristian König }
1215451bc8ebSChristian König
1216451bc8ebSChristian König /**
12170b15f2fcSChristian König * amdgpu_vm_prt_put - drop a PRT user
12187fc48e59SAndrey Grodzovsky *
12197fc48e59SAndrey Grodzovsky * @adev: amdgpu_device pointer
12200b15f2fcSChristian König */
amdgpu_vm_prt_put(struct amdgpu_device * adev)12210b15f2fcSChristian König static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
12220b15f2fcSChristian König {
1223451bc8ebSChristian König if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
12240b15f2fcSChristian König amdgpu_vm_update_prt_state(adev);
12250b15f2fcSChristian König }
12260b15f2fcSChristian König
12270b15f2fcSChristian König /**
1228451bc8ebSChristian König * amdgpu_vm_prt_cb - callback for updating the PRT status
12297fc48e59SAndrey Grodzovsky *
12307fc48e59SAndrey Grodzovsky * @fence: fence for the callback
123100553cf8SAndrey Grodzovsky * @_cb: the callback function
1232284710faSChristian König */
amdgpu_vm_prt_cb(struct dma_fence * fence,struct dma_fence_cb * _cb)1233284710faSChristian König static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1234284710faSChristian König {
1235284710faSChristian König struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1236284710faSChristian König
12370b15f2fcSChristian König amdgpu_vm_prt_put(cb->adev);
1238284710faSChristian König kfree(cb);
1239284710faSChristian König }
1240284710faSChristian König
1241284710faSChristian König /**
1242451bc8ebSChristian König * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
12437fc48e59SAndrey Grodzovsky *
12447fc48e59SAndrey Grodzovsky * @adev: amdgpu_device pointer
12457fc48e59SAndrey Grodzovsky * @fence: fence for the callback
1246451bc8ebSChristian König */
amdgpu_vm_add_prt_cb(struct amdgpu_device * adev,struct dma_fence * fence)1247451bc8ebSChristian König static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1248451bc8ebSChristian König struct dma_fence *fence)
1249451bc8ebSChristian König {
12504388fc2aSChristian König struct amdgpu_prt_cb *cb;
1251451bc8ebSChristian König
1252132f34e4SChristian König if (!adev->gmc.gmc_funcs->set_prt)
12534388fc2aSChristian König return;
12544388fc2aSChristian König
12554388fc2aSChristian König cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1256451bc8ebSChristian König if (!cb) {
1257451bc8ebSChristian König /* Last resort when we are OOM */
1258451bc8ebSChristian König if (fence)
1259451bc8ebSChristian König dma_fence_wait(fence, false);
1260451bc8ebSChristian König
1261486a68f5SDan Carpenter amdgpu_vm_prt_put(adev);
1262451bc8ebSChristian König } else {
1263451bc8ebSChristian König cb->adev = adev;
1264451bc8ebSChristian König if (!fence || dma_fence_add_callback(fence, &cb->cb,
1265451bc8ebSChristian König amdgpu_vm_prt_cb))
1266451bc8ebSChristian König amdgpu_vm_prt_cb(fence, &cb->cb);
1267451bc8ebSChristian König }
1268451bc8ebSChristian König }
1269451bc8ebSChristian König
1270451bc8ebSChristian König /**
1271284710faSChristian König * amdgpu_vm_free_mapping - free a mapping
1272284710faSChristian König *
1273284710faSChristian König * @adev: amdgpu_device pointer
1274284710faSChristian König * @vm: requested vm
1275284710faSChristian König * @mapping: mapping to be freed
1276284710faSChristian König * @fence: fence of the unmap operation
1277284710faSChristian König *
1278284710faSChristian König * Free a mapping and make sure we decrease the PRT usage count if applicable.
1279284710faSChristian König */
amdgpu_vm_free_mapping(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo_va_mapping * mapping,struct dma_fence * fence)1280284710faSChristian König static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1281284710faSChristian König struct amdgpu_vm *vm,
1282284710faSChristian König struct amdgpu_bo_va_mapping *mapping,
1283284710faSChristian König struct dma_fence *fence)
1284284710faSChristian König {
1285451bc8ebSChristian König if (mapping->flags & AMDGPU_PTE_PRT)
1286451bc8ebSChristian König amdgpu_vm_add_prt_cb(adev, fence);
1287284710faSChristian König kfree(mapping);
1288284710faSChristian König }
1289284710faSChristian König
1290284710faSChristian König /**
1291451bc8ebSChristian König * amdgpu_vm_prt_fini - finish all prt mappings
1292451bc8ebSChristian König *
1293451bc8ebSChristian König * @adev: amdgpu_device pointer
1294451bc8ebSChristian König * @vm: requested vm
1295451bc8ebSChristian König *
1296451bc8ebSChristian König * Register a cleanup callback to disable PRT support after VM dies.
1297451bc8ebSChristian König */
amdgpu_vm_prt_fini(struct amdgpu_device * adev,struct amdgpu_vm * vm)1298451bc8ebSChristian König static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1299451bc8ebSChristian König {
1300391629bdSNirmoy Das struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1301a0a8e759SChristian König struct dma_resv_iter cursor;
1302a0a8e759SChristian König struct dma_fence *fence;
1303451bc8ebSChristian König
13040cc848a7SChristian König dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {
1305451bc8ebSChristian König /* Add a callback for each fence in the reservation object */
1306451bc8ebSChristian König amdgpu_vm_prt_get(adev);
1307a0a8e759SChristian König amdgpu_vm_add_prt_cb(adev, fence);
1308451bc8ebSChristian König }
1309451bc8ebSChristian König }
1310451bc8ebSChristian König
1311451bc8ebSChristian König /**
1312d38ceaf9SAlex Deucher * amdgpu_vm_clear_freed - clear freed BOs in the PT
1313d38ceaf9SAlex Deucher *
1314d38ceaf9SAlex Deucher * @adev: amdgpu_device pointer
1315d38ceaf9SAlex Deucher * @vm: requested vm
1316f3467818SNicolai Hähnle * @fence: optional resulting fence (unchanged if no work needed to be done
1317f3467818SNicolai Hähnle * or if an error occurred)
1318d38ceaf9SAlex Deucher *
1319d38ceaf9SAlex Deucher * Make sure all freed BOs are cleared in the PT.
1320d38ceaf9SAlex Deucher * PTs have to be reserved and mutex must be locked!
13217fc48e59SAndrey Grodzovsky *
13227fc48e59SAndrey Grodzovsky * Returns:
13237fc48e59SAndrey Grodzovsky * 0 for success.
13247fc48e59SAndrey Grodzovsky *
1325d38ceaf9SAlex Deucher */
amdgpu_vm_clear_freed(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct dma_fence ** fence)1326d38ceaf9SAlex Deucher int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1327f3467818SNicolai Hähnle struct amdgpu_vm *vm,
1328f3467818SNicolai Hähnle struct dma_fence **fence)
1329d38ceaf9SAlex Deucher {
1330391629bdSNirmoy Das struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1331d38ceaf9SAlex Deucher struct amdgpu_bo_va_mapping *mapping;
13324584312dSChristian König uint64_t init_pte_value = 0;
1333f3467818SNicolai Hähnle struct dma_fence *f = NULL;
1334d38ceaf9SAlex Deucher int r;
1335d38ceaf9SAlex Deucher
1336d38ceaf9SAlex Deucher while (!list_empty(&vm->freed)) {
1337d38ceaf9SAlex Deucher mapping = list_first_entry(&vm->freed,
1338d38ceaf9SAlex Deucher struct amdgpu_bo_va_mapping, list);
1339d38ceaf9SAlex Deucher list_del(&mapping->list);
1340e17841b9SChristian König
1341ad9a5b78SChristian König if (vm->pte_support_ats &&
1342ad9a5b78SChristian König mapping->start < AMDGPU_GMC_HOLE_START)
13436d16dac8SYong Zhao init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
134451ac7eecSYong Zhao
134530671b44SChristian König r = amdgpu_vm_update_range(adev, vm, false, false, true, resv,
134630671b44SChristian König mapping->start, mapping->last,
134730671b44SChristian König init_pte_value, 0, 0, NULL, NULL,
134830671b44SChristian König &f);
1349f3467818SNicolai Hähnle amdgpu_vm_free_mapping(adev, vm, mapping, f);
1350284710faSChristian König if (r) {
1351f3467818SNicolai Hähnle dma_fence_put(f);
1352d38ceaf9SAlex Deucher return r;
1353284710faSChristian König }
1354d38ceaf9SAlex Deucher }
1355f3467818SNicolai Hähnle
1356f3467818SNicolai Hähnle if (fence && f) {
1357f3467818SNicolai Hähnle dma_fence_put(*fence);
1358f3467818SNicolai Hähnle *fence = f;
1359f3467818SNicolai Hähnle } else {
1360f3467818SNicolai Hähnle dma_fence_put(f);
1361f3467818SNicolai Hähnle }
1362f3467818SNicolai Hähnle
1363d38ceaf9SAlex Deucher return 0;
1364d38ceaf9SAlex Deucher
1365d38ceaf9SAlex Deucher }
1366d38ceaf9SAlex Deucher
1367d38ceaf9SAlex Deucher /**
136873fb16e7SChristian König * amdgpu_vm_handle_moved - handle moved BOs in the PT
1369d38ceaf9SAlex Deucher *
1370d38ceaf9SAlex Deucher * @adev: amdgpu_device pointer
1371d38ceaf9SAlex Deucher * @vm: requested vm
1372d38ceaf9SAlex Deucher *
137373fb16e7SChristian König * Make sure all BOs which are moved are updated in the PTs.
13747fc48e59SAndrey Grodzovsky *
13757fc48e59SAndrey Grodzovsky * Returns:
13767fc48e59SAndrey Grodzovsky * 0 for success.
1377d38ceaf9SAlex Deucher *
137873fb16e7SChristian König * PTs have to be reserved!
1379d38ceaf9SAlex Deucher */
amdgpu_vm_handle_moved(struct amdgpu_device * adev,struct amdgpu_vm * vm)138073fb16e7SChristian König int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
13814e55eb38SChristian König struct amdgpu_vm *vm)
1382d38ceaf9SAlex Deucher {
1383998debbdSPhilip Yang struct amdgpu_bo_va *bo_va;
138452791eeeSChristian König struct dma_resv *resv;
138573fb16e7SChristian König bool clear;
1386789f3317SChristian König int r;
1387d38ceaf9SAlex Deucher
1388998debbdSPhilip Yang spin_lock(&vm->status_lock);
1389998debbdSPhilip Yang while (!list_empty(&vm->moved)) {
1390998debbdSPhilip Yang bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
1391998debbdSPhilip Yang base.vm_status);
1392998debbdSPhilip Yang spin_unlock(&vm->status_lock);
1393998debbdSPhilip Yang
139473fb16e7SChristian König /* Per VM BOs never need to bo cleared in the page tables */
13958f8cc3fbSChristian König r = amdgpu_vm_bo_update(adev, bo_va, false);
1396c12a2ee5SChristian König if (r)
1397c12a2ee5SChristian König return r;
1398998debbdSPhilip Yang spin_lock(&vm->status_lock);
1399c12a2ee5SChristian König }
1400c12a2ee5SChristian König
1401c12a2ee5SChristian König while (!list_empty(&vm->invalidated)) {
1402c12a2ee5SChristian König bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
1403c12a2ee5SChristian König base.vm_status);
14045a5011a7SGerd Hoffmann resv = bo_va->base.bo->tbo.base.resv;
14050479956cSPhilip Yang spin_unlock(&vm->status_lock);
1406c12a2ee5SChristian König
1407ec363e0dSChristian König /* Try to reserve the BO to avoid clearing its ptes */
140852791eeeSChristian König if (!amdgpu_vm_debug && dma_resv_trylock(resv))
1409ec363e0dSChristian König clear = false;
1410ec363e0dSChristian König /* Somebody else is using the BO right now */
1411ec363e0dSChristian König else
1412ec363e0dSChristian König clear = true;
141373fb16e7SChristian König
14148f8cc3fbSChristian König r = amdgpu_vm_bo_update(adev, bo_va, clear);
1415c12a2ee5SChristian König if (r)
1416d38ceaf9SAlex Deucher return r;
1417d38ceaf9SAlex Deucher
1418c12a2ee5SChristian König if (!clear)
141952791eeeSChristian König dma_resv_unlock(resv);
14200479956cSPhilip Yang spin_lock(&vm->status_lock);
1421d38ceaf9SAlex Deucher }
14220479956cSPhilip Yang spin_unlock(&vm->status_lock);
1423d38ceaf9SAlex Deucher
1424789f3317SChristian König return 0;
1425d38ceaf9SAlex Deucher }
1426d38ceaf9SAlex Deucher
1427d38ceaf9SAlex Deucher /**
1428d38ceaf9SAlex Deucher * amdgpu_vm_bo_add - add a bo to a specific vm
1429d38ceaf9SAlex Deucher *
1430d38ceaf9SAlex Deucher * @adev: amdgpu_device pointer
1431d38ceaf9SAlex Deucher * @vm: requested vm
1432d38ceaf9SAlex Deucher * @bo: amdgpu buffer object
1433d38ceaf9SAlex Deucher *
14348843dbbbSChristian König * Add @bo into the requested vm.
1435d38ceaf9SAlex Deucher * Add @bo to the list of bos associated with the vm
14367fc48e59SAndrey Grodzovsky *
14377fc48e59SAndrey Grodzovsky * Returns:
14387fc48e59SAndrey Grodzovsky * Newly added bo_va or NULL for failure
1439d38ceaf9SAlex Deucher *
1440d38ceaf9SAlex Deucher * Object has to be reserved!
1441d38ceaf9SAlex Deucher */
amdgpu_vm_bo_add(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo * bo)1442d38ceaf9SAlex Deucher struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1443d38ceaf9SAlex Deucher struct amdgpu_vm *vm,
1444d38ceaf9SAlex Deucher struct amdgpu_bo *bo)
1445d38ceaf9SAlex Deucher {
1446d38ceaf9SAlex Deucher struct amdgpu_bo_va *bo_va;
1447d38ceaf9SAlex Deucher
1448d38ceaf9SAlex Deucher bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1449d38ceaf9SAlex Deucher if (bo_va == NULL) {
1450d38ceaf9SAlex Deucher return NULL;
1451d38ceaf9SAlex Deucher }
14523f4299beSChunming Zhou amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
1453ec681545SChristian König
1454d38ceaf9SAlex Deucher bo_va->ref_count = 1;
1455187916e6SLang Yu bo_va->last_pt_update = dma_fence_get_stub();
14567fc11959SChristian König INIT_LIST_HEAD(&bo_va->valids);
14577fc11959SChristian König INIT_LIST_HEAD(&bo_va->invalids);
145832b41ac2SChristian König
14590cf0ee98SArunpravin if (!bo)
14600cf0ee98SArunpravin return bo_va;
14610cf0ee98SArunpravin
14622d022081SChristian König dma_resv_assert_held(bo->tbo.base.resv);
14630cf0ee98SArunpravin if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) {
1464df399b06Sshaoyunl bo_va->is_xgmi = true;
1465df399b06Sshaoyunl /* Power up XGMI if it can be potentially used */
1466d84a430dSJonathan Kim amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20);
1467df399b06Sshaoyunl }
1468df399b06Sshaoyunl
1469d38ceaf9SAlex Deucher return bo_va;
1470d38ceaf9SAlex Deucher }
1471d38ceaf9SAlex Deucher
147273fb16e7SChristian König
147373fb16e7SChristian König /**
1474c45dd3bdSMauro Carvalho Chehab * amdgpu_vm_bo_insert_map - insert a new mapping
147573fb16e7SChristian König *
147673fb16e7SChristian König * @adev: amdgpu_device pointer
147773fb16e7SChristian König * @bo_va: bo_va to store the address
147873fb16e7SChristian König * @mapping: the mapping to insert
147973fb16e7SChristian König *
148073fb16e7SChristian König * Insert a new mapping into all structures.
148173fb16e7SChristian König */
amdgpu_vm_bo_insert_map(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va,struct amdgpu_bo_va_mapping * mapping)148273fb16e7SChristian König static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
148373fb16e7SChristian König struct amdgpu_bo_va *bo_va,
148473fb16e7SChristian König struct amdgpu_bo_va_mapping *mapping)
148573fb16e7SChristian König {
148673fb16e7SChristian König struct amdgpu_vm *vm = bo_va->base.vm;
148773fb16e7SChristian König struct amdgpu_bo *bo = bo_va->base.bo;
148873fb16e7SChristian König
1489aebc5e6fSChristian König mapping->bo_va = bo_va;
149073fb16e7SChristian König list_add(&mapping->list, &bo_va->invalids);
149173fb16e7SChristian König amdgpu_vm_it_insert(mapping, &vm->va);
149273fb16e7SChristian König
149373fb16e7SChristian König if (mapping->flags & AMDGPU_PTE_PRT)
149473fb16e7SChristian König amdgpu_vm_prt_get(adev);
149573fb16e7SChristian König
1496391629bdSNirmoy Das if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv &&
1497862b8c57SChristian König !bo_va->base.moved) {
1498998debbdSPhilip Yang amdgpu_vm_bo_moved(&bo_va->base);
149973fb16e7SChristian König }
150073fb16e7SChristian König trace_amdgpu_vm_bo_map(bo_va, mapping);
150173fb16e7SChristian König }
150273fb16e7SChristian König
1503ef13eecaSxinhui pan /* Validate operation parameters to prevent potential abuse */
amdgpu_vm_verify_parameters(struct amdgpu_device * adev,struct amdgpu_bo * bo,uint64_t saddr,uint64_t offset,uint64_t size)1504ef13eecaSxinhui pan static int amdgpu_vm_verify_parameters(struct amdgpu_device *adev,
1505ef13eecaSxinhui pan struct amdgpu_bo *bo,
1506ef13eecaSxinhui pan uint64_t saddr,
1507ef13eecaSxinhui pan uint64_t offset,
1508ef13eecaSxinhui pan uint64_t size)
1509ef13eecaSxinhui pan {
1510ef13eecaSxinhui pan uint64_t tmp, lpfn;
1511ef13eecaSxinhui pan
1512ef13eecaSxinhui pan if (saddr & AMDGPU_GPU_PAGE_MASK
1513ef13eecaSxinhui pan || offset & AMDGPU_GPU_PAGE_MASK
1514ef13eecaSxinhui pan || size & AMDGPU_GPU_PAGE_MASK)
1515ef13eecaSxinhui pan return -EINVAL;
1516ef13eecaSxinhui pan
1517ef13eecaSxinhui pan if (check_add_overflow(saddr, size, &tmp)
1518ef13eecaSxinhui pan || check_add_overflow(offset, size, &tmp)
1519ef13eecaSxinhui pan || size == 0 /* which also leads to end < begin */)
1520ef13eecaSxinhui pan return -EINVAL;
1521ef13eecaSxinhui pan
1522ef13eecaSxinhui pan /* make sure object fit at this offset */
1523ef13eecaSxinhui pan if (bo && offset + size > amdgpu_bo_size(bo))
1524ef13eecaSxinhui pan return -EINVAL;
1525ef13eecaSxinhui pan
1526ef13eecaSxinhui pan /* Ensure last pfn not exceed max_pfn */
1527ef13eecaSxinhui pan lpfn = (saddr + size - 1) >> AMDGPU_GPU_PAGE_SHIFT;
1528ef13eecaSxinhui pan if (lpfn >= adev->vm_manager.max_pfn)
1529ef13eecaSxinhui pan return -EINVAL;
1530ef13eecaSxinhui pan
1531ef13eecaSxinhui pan return 0;
1532ef13eecaSxinhui pan }
1533ef13eecaSxinhui pan
1534d38ceaf9SAlex Deucher /**
1535d38ceaf9SAlex Deucher * amdgpu_vm_bo_map - map bo inside a vm
1536d38ceaf9SAlex Deucher *
1537d38ceaf9SAlex Deucher * @adev: amdgpu_device pointer
1538d38ceaf9SAlex Deucher * @bo_va: bo_va to store the address
1539d38ceaf9SAlex Deucher * @saddr: where to map the BO
1540d38ceaf9SAlex Deucher * @offset: requested offset in the BO
154100553cf8SAndrey Grodzovsky * @size: BO size in bytes
1542d38ceaf9SAlex Deucher * @flags: attributes of pages (read/write/valid/etc.)
1543d38ceaf9SAlex Deucher *
1544d38ceaf9SAlex Deucher * Add a mapping of the BO at the specefied addr into the VM.
15457fc48e59SAndrey Grodzovsky *
15467fc48e59SAndrey Grodzovsky * Returns:
15477fc48e59SAndrey Grodzovsky * 0 for success, error for failure.
1548d38ceaf9SAlex Deucher *
154949b02b18SChunming Zhou * Object has to be reserved and unreserved outside!
1550d38ceaf9SAlex Deucher */
amdgpu_vm_bo_map(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va,uint64_t saddr,uint64_t offset,uint64_t size,uint64_t flags)1551d38ceaf9SAlex Deucher int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1552d38ceaf9SAlex Deucher struct amdgpu_bo_va *bo_va,
1553d38ceaf9SAlex Deucher uint64_t saddr, uint64_t offset,
1554268c3001SChristian König uint64_t size, uint64_t flags)
1555d38ceaf9SAlex Deucher {
1556a9f87f64SChristian König struct amdgpu_bo_va_mapping *mapping, *tmp;
1557ec681545SChristian König struct amdgpu_bo *bo = bo_va->base.bo;
1558ec681545SChristian König struct amdgpu_vm *vm = bo_va->base.vm;
1559d38ceaf9SAlex Deucher uint64_t eaddr;
1560ef13eecaSxinhui pan int r;
1561d38ceaf9SAlex Deucher
1562ef13eecaSxinhui pan r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);
1563ef13eecaSxinhui pan if (r)
1564ef13eecaSxinhui pan return r;
1565d38ceaf9SAlex Deucher
1566d38ceaf9SAlex Deucher saddr /= AMDGPU_GPU_PAGE_SIZE;
1567ef13eecaSxinhui pan eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
1568d38ceaf9SAlex Deucher
1569a9f87f64SChristian König tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1570a9f87f64SChristian König if (tmp) {
1571d38ceaf9SAlex Deucher /* bo and tmp overlap, invalid addr */
1572d38ceaf9SAlex Deucher dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1573ec681545SChristian König "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
1574a9f87f64SChristian König tmp->start, tmp->last + 1);
1575663e4577SChristian König return -EINVAL;
1576d38ceaf9SAlex Deucher }
1577d38ceaf9SAlex Deucher
1578d38ceaf9SAlex Deucher mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1579663e4577SChristian König if (!mapping)
1580663e4577SChristian König return -ENOMEM;
1581d38ceaf9SAlex Deucher
1582a9f87f64SChristian König mapping->start = saddr;
1583a9f87f64SChristian König mapping->last = eaddr;
1584d38ceaf9SAlex Deucher mapping->offset = offset;
1585d38ceaf9SAlex Deucher mapping->flags = flags;
1586d38ceaf9SAlex Deucher
158773fb16e7SChristian König amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
15884388fc2aSChristian König
1589d38ceaf9SAlex Deucher return 0;
1590d38ceaf9SAlex Deucher }
1591d38ceaf9SAlex Deucher
1592d38ceaf9SAlex Deucher /**
159380f95c57SChristian König * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
159480f95c57SChristian König *
159580f95c57SChristian König * @adev: amdgpu_device pointer
159680f95c57SChristian König * @bo_va: bo_va to store the address
159780f95c57SChristian König * @saddr: where to map the BO
159880f95c57SChristian König * @offset: requested offset in the BO
159900553cf8SAndrey Grodzovsky * @size: BO size in bytes
160080f95c57SChristian König * @flags: attributes of pages (read/write/valid/etc.)
160180f95c57SChristian König *
160280f95c57SChristian König * Add a mapping of the BO at the specefied addr into the VM. Replace existing
160380f95c57SChristian König * mappings as we do so.
16047fc48e59SAndrey Grodzovsky *
16057fc48e59SAndrey Grodzovsky * Returns:
16067fc48e59SAndrey Grodzovsky * 0 for success, error for failure.
160780f95c57SChristian König *
160880f95c57SChristian König * Object has to be reserved and unreserved outside!
160980f95c57SChristian König */
amdgpu_vm_bo_replace_map(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va,uint64_t saddr,uint64_t offset,uint64_t size,uint64_t flags)161080f95c57SChristian König int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
161180f95c57SChristian König struct amdgpu_bo_va *bo_va,
161280f95c57SChristian König uint64_t saddr, uint64_t offset,
161380f95c57SChristian König uint64_t size, uint64_t flags)
161480f95c57SChristian König {
161580f95c57SChristian König struct amdgpu_bo_va_mapping *mapping;
1616ec681545SChristian König struct amdgpu_bo *bo = bo_va->base.bo;
161780f95c57SChristian König uint64_t eaddr;
161880f95c57SChristian König int r;
161980f95c57SChristian König
1620ef13eecaSxinhui pan r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);
1621ef13eecaSxinhui pan if (r)
1622ef13eecaSxinhui pan return r;
162380f95c57SChristian König
162480f95c57SChristian König /* Allocate all the needed memory */
162580f95c57SChristian König mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
162680f95c57SChristian König if (!mapping)
162780f95c57SChristian König return -ENOMEM;
162880f95c57SChristian König
1629ec681545SChristian König r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
163080f95c57SChristian König if (r) {
163180f95c57SChristian König kfree(mapping);
163280f95c57SChristian König return r;
163380f95c57SChristian König }
163480f95c57SChristian König
163580f95c57SChristian König saddr /= AMDGPU_GPU_PAGE_SIZE;
1636ef13eecaSxinhui pan eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
163780f95c57SChristian König
1638a9f87f64SChristian König mapping->start = saddr;
1639a9f87f64SChristian König mapping->last = eaddr;
164080f95c57SChristian König mapping->offset = offset;
164180f95c57SChristian König mapping->flags = flags;
164280f95c57SChristian König
164373fb16e7SChristian König amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
164480f95c57SChristian König
164580f95c57SChristian König return 0;
164680f95c57SChristian König }
164780f95c57SChristian König
164880f95c57SChristian König /**
1649d38ceaf9SAlex Deucher * amdgpu_vm_bo_unmap - remove bo mapping from vm
1650d38ceaf9SAlex Deucher *
1651d38ceaf9SAlex Deucher * @adev: amdgpu_device pointer
1652d38ceaf9SAlex Deucher * @bo_va: bo_va to remove the address from
1653d38ceaf9SAlex Deucher * @saddr: where to the BO is mapped
1654d38ceaf9SAlex Deucher *
1655d38ceaf9SAlex Deucher * Remove a mapping of the BO at the specefied addr from the VM.
16567fc48e59SAndrey Grodzovsky *
16577fc48e59SAndrey Grodzovsky * Returns:
16587fc48e59SAndrey Grodzovsky * 0 for success, error for failure.
1659d38ceaf9SAlex Deucher *
166049b02b18SChunming Zhou * Object has to be reserved and unreserved outside!
1661d38ceaf9SAlex Deucher */
amdgpu_vm_bo_unmap(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va,uint64_t saddr)1662d38ceaf9SAlex Deucher int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1663d38ceaf9SAlex Deucher struct amdgpu_bo_va *bo_va,
1664d38ceaf9SAlex Deucher uint64_t saddr)
1665d38ceaf9SAlex Deucher {
1666d38ceaf9SAlex Deucher struct amdgpu_bo_va_mapping *mapping;
1667ec681545SChristian König struct amdgpu_vm *vm = bo_va->base.vm;
16687fc11959SChristian König bool valid = true;
1669d38ceaf9SAlex Deucher
16706c7fc503SChristian König saddr /= AMDGPU_GPU_PAGE_SIZE;
167132b41ac2SChristian König
16727fc11959SChristian König list_for_each_entry(mapping, &bo_va->valids, list) {
1673a9f87f64SChristian König if (mapping->start == saddr)
1674d38ceaf9SAlex Deucher break;
1675d38ceaf9SAlex Deucher }
1676d38ceaf9SAlex Deucher
16777fc11959SChristian König if (&mapping->list == &bo_va->valids) {
16787fc11959SChristian König valid = false;
16797fc11959SChristian König
16807fc11959SChristian König list_for_each_entry(mapping, &bo_va->invalids, list) {
1681a9f87f64SChristian König if (mapping->start == saddr)
16827fc11959SChristian König break;
16837fc11959SChristian König }
16847fc11959SChristian König
168532b41ac2SChristian König if (&mapping->list == &bo_va->invalids)
1686d38ceaf9SAlex Deucher return -ENOENT;
1687d38ceaf9SAlex Deucher }
168832b41ac2SChristian König
1689d38ceaf9SAlex Deucher list_del(&mapping->list);
1690a9f87f64SChristian König amdgpu_vm_it_remove(mapping, &vm->va);
1691aebc5e6fSChristian König mapping->bo_va = NULL;
169293e3e438SChristian König trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1693d38ceaf9SAlex Deucher
1694e17841b9SChristian König if (valid)
1695d38ceaf9SAlex Deucher list_add(&mapping->list, &vm->freed);
1696e17841b9SChristian König else
1697284710faSChristian König amdgpu_vm_free_mapping(adev, vm, mapping,
1698284710faSChristian König bo_va->last_pt_update);
1699d38ceaf9SAlex Deucher
1700d38ceaf9SAlex Deucher return 0;
1701d38ceaf9SAlex Deucher }
1702d38ceaf9SAlex Deucher
1703d38ceaf9SAlex Deucher /**
1704dc54d3d1SChristian König * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
1705dc54d3d1SChristian König *
1706dc54d3d1SChristian König * @adev: amdgpu_device pointer
1707dc54d3d1SChristian König * @vm: VM structure to use
1708dc54d3d1SChristian König * @saddr: start of the range
1709dc54d3d1SChristian König * @size: size of the range
1710dc54d3d1SChristian König *
1711dc54d3d1SChristian König * Remove all mappings in a range, split them as appropriate.
17127fc48e59SAndrey Grodzovsky *
17137fc48e59SAndrey Grodzovsky * Returns:
17147fc48e59SAndrey Grodzovsky * 0 for success, error for failure.
1715dc54d3d1SChristian König */
amdgpu_vm_bo_clear_mappings(struct amdgpu_device * adev,struct amdgpu_vm * vm,uint64_t saddr,uint64_t size)1716dc54d3d1SChristian König int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
1717dc54d3d1SChristian König struct amdgpu_vm *vm,
1718dc54d3d1SChristian König uint64_t saddr, uint64_t size)
1719dc54d3d1SChristian König {
1720dc54d3d1SChristian König struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
1721dc54d3d1SChristian König LIST_HEAD(removed);
1722dc54d3d1SChristian König uint64_t eaddr;
1723ef13eecaSxinhui pan int r;
1724dc54d3d1SChristian König
1725ef13eecaSxinhui pan r = amdgpu_vm_verify_parameters(adev, NULL, saddr, 0, size);
1726ef13eecaSxinhui pan if (r)
1727ef13eecaSxinhui pan return r;
1728ef13eecaSxinhui pan
1729dc54d3d1SChristian König saddr /= AMDGPU_GPU_PAGE_SIZE;
1730ef13eecaSxinhui pan eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
1731dc54d3d1SChristian König
1732dc54d3d1SChristian König /* Allocate all the needed memory */
1733dc54d3d1SChristian König before = kzalloc(sizeof(*before), GFP_KERNEL);
1734dc54d3d1SChristian König if (!before)
1735dc54d3d1SChristian König return -ENOMEM;
173627f6d610SJunwei Zhang INIT_LIST_HEAD(&before->list);
1737dc54d3d1SChristian König
1738dc54d3d1SChristian König after = kzalloc(sizeof(*after), GFP_KERNEL);
1739dc54d3d1SChristian König if (!after) {
1740dc54d3d1SChristian König kfree(before);
1741dc54d3d1SChristian König return -ENOMEM;
1742dc54d3d1SChristian König }
174327f6d610SJunwei Zhang INIT_LIST_HEAD(&after->list);
1744dc54d3d1SChristian König
1745dc54d3d1SChristian König /* Now gather all removed mappings */
1746a9f87f64SChristian König tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1747a9f87f64SChristian König while (tmp) {
1748dc54d3d1SChristian König /* Remember mapping split at the start */
1749a9f87f64SChristian König if (tmp->start < saddr) {
1750a9f87f64SChristian König before->start = tmp->start;
1751a9f87f64SChristian König before->last = saddr - 1;
1752dc54d3d1SChristian König before->offset = tmp->offset;
1753dc54d3d1SChristian König before->flags = tmp->flags;
1754387f49e5SJunwei Zhang before->bo_va = tmp->bo_va;
1755387f49e5SJunwei Zhang list_add(&before->list, &tmp->bo_va->invalids);
1756dc54d3d1SChristian König }
1757dc54d3d1SChristian König
1758dc54d3d1SChristian König /* Remember mapping split at the end */
1759a9f87f64SChristian König if (tmp->last > eaddr) {
1760a9f87f64SChristian König after->start = eaddr + 1;
1761a9f87f64SChristian König after->last = tmp->last;
1762dc54d3d1SChristian König after->offset = tmp->offset;
176384e070f5SNirmoy Das after->offset += (after->start - tmp->start) << PAGE_SHIFT;
1764dc54d3d1SChristian König after->flags = tmp->flags;
1765387f49e5SJunwei Zhang after->bo_va = tmp->bo_va;
1766387f49e5SJunwei Zhang list_add(&after->list, &tmp->bo_va->invalids);
1767dc54d3d1SChristian König }
1768dc54d3d1SChristian König
1769dc54d3d1SChristian König list_del(&tmp->list);
1770dc54d3d1SChristian König list_add(&tmp->list, &removed);
1771a9f87f64SChristian König
1772a9f87f64SChristian König tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
1773dc54d3d1SChristian König }
1774dc54d3d1SChristian König
1775dc54d3d1SChristian König /* And free them up */
1776dc54d3d1SChristian König list_for_each_entry_safe(tmp, next, &removed, list) {
1777a9f87f64SChristian König amdgpu_vm_it_remove(tmp, &vm->va);
1778dc54d3d1SChristian König list_del(&tmp->list);
1779dc54d3d1SChristian König
1780a9f87f64SChristian König if (tmp->start < saddr)
1781a9f87f64SChristian König tmp->start = saddr;
1782a9f87f64SChristian König if (tmp->last > eaddr)
1783a9f87f64SChristian König tmp->last = eaddr;
1784dc54d3d1SChristian König
1785aebc5e6fSChristian König tmp->bo_va = NULL;
1786dc54d3d1SChristian König list_add(&tmp->list, &vm->freed);
1787dc54d3d1SChristian König trace_amdgpu_vm_bo_unmap(NULL, tmp);
1788dc54d3d1SChristian König }
1789dc54d3d1SChristian König
1790dc54d3d1SChristian König /* Insert partial mapping before the range */
179127f6d610SJunwei Zhang if (!list_empty(&before->list)) {
1792ea2c3c08SSamuel Pitoiset struct amdgpu_bo *bo = before->bo_va->base.bo;
1793ea2c3c08SSamuel Pitoiset
1794a9f87f64SChristian König amdgpu_vm_it_insert(before, &vm->va);
1795dc54d3d1SChristian König if (before->flags & AMDGPU_PTE_PRT)
1796dc54d3d1SChristian König amdgpu_vm_prt_get(adev);
1797ea2c3c08SSamuel Pitoiset
1798ea2c3c08SSamuel Pitoiset if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv &&
1799ea2c3c08SSamuel Pitoiset !before->bo_va->base.moved)
1800ea2c3c08SSamuel Pitoiset amdgpu_vm_bo_moved(&before->bo_va->base);
1801dc54d3d1SChristian König } else {
1802dc54d3d1SChristian König kfree(before);
1803dc54d3d1SChristian König }
1804dc54d3d1SChristian König
1805dc54d3d1SChristian König /* Insert partial mapping after the range */
180627f6d610SJunwei Zhang if (!list_empty(&after->list)) {
1807ea2c3c08SSamuel Pitoiset struct amdgpu_bo *bo = after->bo_va->base.bo;
1808ea2c3c08SSamuel Pitoiset
1809a9f87f64SChristian König amdgpu_vm_it_insert(after, &vm->va);
1810dc54d3d1SChristian König if (after->flags & AMDGPU_PTE_PRT)
1811dc54d3d1SChristian König amdgpu_vm_prt_get(adev);
1812ea2c3c08SSamuel Pitoiset
1813ea2c3c08SSamuel Pitoiset if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv &&
1814ea2c3c08SSamuel Pitoiset !after->bo_va->base.moved)
1815ea2c3c08SSamuel Pitoiset amdgpu_vm_bo_moved(&after->bo_va->base);
1816dc54d3d1SChristian König } else {
1817dc54d3d1SChristian König kfree(after);
1818dc54d3d1SChristian König }
1819dc54d3d1SChristian König
1820dc54d3d1SChristian König return 0;
1821dc54d3d1SChristian König }
1822dc54d3d1SChristian König
1823dc54d3d1SChristian König /**
1824aebc5e6fSChristian König * amdgpu_vm_bo_lookup_mapping - find mapping by address
1825aebc5e6fSChristian König *
1826aebc5e6fSChristian König * @vm: the requested VM
182700553cf8SAndrey Grodzovsky * @addr: the address
1828aebc5e6fSChristian König *
1829aebc5e6fSChristian König * Find a mapping by it's address.
18307fc48e59SAndrey Grodzovsky *
18317fc48e59SAndrey Grodzovsky * Returns:
18327fc48e59SAndrey Grodzovsky * The amdgpu_bo_va_mapping matching for addr or NULL
18337fc48e59SAndrey Grodzovsky *
1834aebc5e6fSChristian König */
amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm * vm,uint64_t addr)1835aebc5e6fSChristian König struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
1836aebc5e6fSChristian König uint64_t addr)
1837aebc5e6fSChristian König {
1838aebc5e6fSChristian König return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
1839aebc5e6fSChristian König }
1840aebc5e6fSChristian König
1841aebc5e6fSChristian König /**
18428ab19ea6SChristian König * amdgpu_vm_bo_trace_cs - trace all reserved mappings
18438ab19ea6SChristian König *
18448ab19ea6SChristian König * @vm: the requested vm
18458ab19ea6SChristian König * @ticket: CS ticket
18468ab19ea6SChristian König *
18478ab19ea6SChristian König * Trace all mappings of BOs reserved during a command submission.
18488ab19ea6SChristian König */
amdgpu_vm_bo_trace_cs(struct amdgpu_vm * vm,struct ww_acquire_ctx * ticket)18498ab19ea6SChristian König void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
18508ab19ea6SChristian König {
18518ab19ea6SChristian König struct amdgpu_bo_va_mapping *mapping;
18528ab19ea6SChristian König
18538ab19ea6SChristian König if (!trace_amdgpu_vm_bo_cs_enabled())
18548ab19ea6SChristian König return;
18558ab19ea6SChristian König
18568ab19ea6SChristian König for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
18578ab19ea6SChristian König mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
18588ab19ea6SChristian König if (mapping->bo_va && mapping->bo_va->base.bo) {
18598ab19ea6SChristian König struct amdgpu_bo *bo;
18608ab19ea6SChristian König
18618ab19ea6SChristian König bo = mapping->bo_va->base.bo;
186252791eeeSChristian König if (dma_resv_locking_ctx(bo->tbo.base.resv) !=
18630dbd555aSChristian König ticket)
18648ab19ea6SChristian König continue;
18658ab19ea6SChristian König }
18668ab19ea6SChristian König
18678ab19ea6SChristian König trace_amdgpu_vm_bo_cs(mapping);
18688ab19ea6SChristian König }
18698ab19ea6SChristian König }
18708ab19ea6SChristian König
18718ab19ea6SChristian König /**
1872e56694f7SChristian König * amdgpu_vm_bo_del - remove a bo from a specific vm
1873d38ceaf9SAlex Deucher *
1874d38ceaf9SAlex Deucher * @adev: amdgpu_device pointer
1875d38ceaf9SAlex Deucher * @bo_va: requested bo_va
1876d38ceaf9SAlex Deucher *
18778843dbbbSChristian König * Remove @bo_va->bo from the requested vm.
1878d38ceaf9SAlex Deucher *
1879d38ceaf9SAlex Deucher * Object have to be reserved!
1880d38ceaf9SAlex Deucher */
amdgpu_vm_bo_del(struct amdgpu_device * adev,struct amdgpu_bo_va * bo_va)1881e56694f7SChristian König void amdgpu_vm_bo_del(struct amdgpu_device *adev,
1882d38ceaf9SAlex Deucher struct amdgpu_bo_va *bo_va)
1883d38ceaf9SAlex Deucher {
1884d38ceaf9SAlex Deucher struct amdgpu_bo_va_mapping *mapping, *next;
1885fbbf794cSChristian König struct amdgpu_bo *bo = bo_va->base.bo;
1886ec681545SChristian König struct amdgpu_vm *vm = bo_va->base.vm;
1887646b9025SChristian König struct amdgpu_vm_bo_base **base;
1888d38ceaf9SAlex Deucher
18892d022081SChristian König dma_resv_assert_held(vm->root.bo->tbo.base.resv);
18902d022081SChristian König
1891646b9025SChristian König if (bo) {
18922d022081SChristian König dma_resv_assert_held(bo->tbo.base.resv);
1893391629bdSNirmoy Das if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
1894fee2ede1SChristian König ttm_bo_set_bulk_move(&bo->tbo, NULL);
1895fbbf794cSChristian König
1896646b9025SChristian König for (base = &bo_va->base.bo->vm_bo; *base;
1897646b9025SChristian König base = &(*base)->next) {
1898646b9025SChristian König if (*base != &bo_va->base)
1899646b9025SChristian König continue;
1900646b9025SChristian König
1901646b9025SChristian König *base = bo_va->base.next;
1902646b9025SChristian König break;
1903646b9025SChristian König }
1904646b9025SChristian König }
1905d38ceaf9SAlex Deucher
19060479956cSPhilip Yang spin_lock(&vm->status_lock);
1907ec681545SChristian König list_del(&bo_va->base.vm_status);
19080479956cSPhilip Yang spin_unlock(&vm->status_lock);
1909d38ceaf9SAlex Deucher
19107fc11959SChristian König list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
1911d38ceaf9SAlex Deucher list_del(&mapping->list);
1912a9f87f64SChristian König amdgpu_vm_it_remove(mapping, &vm->va);
1913aebc5e6fSChristian König mapping->bo_va = NULL;
191493e3e438SChristian König trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1915d38ceaf9SAlex Deucher list_add(&mapping->list, &vm->freed);
19167fc11959SChristian König }
19177fc11959SChristian König list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
19187fc11959SChristian König list_del(&mapping->list);
1919a9f87f64SChristian König amdgpu_vm_it_remove(mapping, &vm->va);
1920284710faSChristian König amdgpu_vm_free_mapping(adev, vm, mapping,
1921284710faSChristian König bo_va->last_pt_update);
1922d38ceaf9SAlex Deucher }
192332b41ac2SChristian König
1924f54d1867SChris Wilson dma_fence_put(bo_va->last_pt_update);
1925df399b06Sshaoyunl
1926d84a430dSJonathan Kim if (bo && bo_va->is_xgmi)
1927d84a430dSJonathan Kim amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN);
1928df399b06Sshaoyunl
1929d38ceaf9SAlex Deucher kfree(bo_va);
1930d38ceaf9SAlex Deucher }
1931d38ceaf9SAlex Deucher
1932d38ceaf9SAlex Deucher /**
19336ceeb144SChristian König * amdgpu_vm_evictable - check if we can evict a VM
19346ceeb144SChristian König *
19356ceeb144SChristian König * @bo: A page table of the VM.
19366ceeb144SChristian König *
19376ceeb144SChristian König * Check if it is possible to evict a VM.
19386ceeb144SChristian König */
amdgpu_vm_evictable(struct amdgpu_bo * bo)19396ceeb144SChristian König bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
19406ceeb144SChristian König {
19416ceeb144SChristian König struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
19426ceeb144SChristian König
19436ceeb144SChristian König /* Page tables of a destroyed VM can go away immediately */
19446ceeb144SChristian König if (!bo_base || !bo_base->vm)
19456ceeb144SChristian König return true;
19466ceeb144SChristian König
19476ceeb144SChristian König /* Don't evict VM page tables while they are busy */
19480cc848a7SChristian König if (!dma_resv_test_signaled(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP))
19496ceeb144SChristian König return false;
19506ceeb144SChristian König
1951b4ff0f8aSChristian König /* Try to block ongoing updates */
1952a269e449SAlex Sierra if (!amdgpu_vm_eviction_trylock(bo_base->vm))
195390b69cdcSChristian König return false;
195490b69cdcSChristian König
1955b4ff0f8aSChristian König /* Don't evict VM page tables while they are updated */
19569c466bcbSChristian König if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
1957a269e449SAlex Sierra amdgpu_vm_eviction_unlock(bo_base->vm);
1958b4ff0f8aSChristian König return false;
1959b4ff0f8aSChristian König }
1960b4ff0f8aSChristian König
1961b4ff0f8aSChristian König bo_base->vm->evicting = true;
1962a269e449SAlex Sierra amdgpu_vm_eviction_unlock(bo_base->vm);
19636ceeb144SChristian König return true;
19646ceeb144SChristian König }
19656ceeb144SChristian König
19666ceeb144SChristian König /**
1967d38ceaf9SAlex Deucher * amdgpu_vm_bo_invalidate - mark the bo as invalid
1968d38ceaf9SAlex Deucher *
1969d38ceaf9SAlex Deucher * @adev: amdgpu_device pointer
1970d38ceaf9SAlex Deucher * @bo: amdgpu buffer object
197100553cf8SAndrey Grodzovsky * @evicted: is the BO evicted
1972d38ceaf9SAlex Deucher *
19738843dbbbSChristian König * Mark @bo as invalid.
1974d38ceaf9SAlex Deucher */
amdgpu_vm_bo_invalidate(struct amdgpu_device * adev,struct amdgpu_bo * bo,bool evicted)1975d38ceaf9SAlex Deucher void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
19763f3333f8SChristian König struct amdgpu_bo *bo, bool evicted)
1977d38ceaf9SAlex Deucher {
1978ec681545SChristian König struct amdgpu_vm_bo_base *bo_base;
1979d38ceaf9SAlex Deucher
19804bebcceeSChunming Zhou /* shadow bo doesn't have bo base, its validation needs its parent */
198159276f05SNirmoy Das if (bo->parent && (amdgpu_bo_shadowed(bo->parent) == bo))
19824bebcceeSChunming Zhou bo = bo->parent;
19834bebcceeSChunming Zhou
1984646b9025SChristian König for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
19853f3333f8SChristian König struct amdgpu_vm *vm = bo_base->vm;
19863f3333f8SChristian König
1987391629bdSNirmoy Das if (evicted && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
1988bcdc9fd6SChristian König amdgpu_vm_bo_evicted(bo_base);
1989bcdc9fd6SChristian König continue;
1990bcdc9fd6SChristian König }
1991bcdc9fd6SChristian König
1992bcdc9fd6SChristian König if (bo_base->moved)
1993bcdc9fd6SChristian König continue;
1994bcdc9fd6SChristian König bo_base->moved = true;
1995bcdc9fd6SChristian König
199673fb16e7SChristian König if (bo->tbo.type == ttm_bo_type_kernel)
1997bcdc9fd6SChristian König amdgpu_vm_bo_relocated(bo_base);
1998391629bdSNirmoy Das else if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
1999bcdc9fd6SChristian König amdgpu_vm_bo_moved(bo_base);
200073fb16e7SChristian König else
2001bcdc9fd6SChristian König amdgpu_vm_bo_invalidated(bo_base);
2002d38ceaf9SAlex Deucher }
2003862b8c57SChristian König }
2004d38ceaf9SAlex Deucher
20057fc48e59SAndrey Grodzovsky /**
20067fc48e59SAndrey Grodzovsky * amdgpu_vm_get_block_size - calculate VM page table size as power of two
20077fc48e59SAndrey Grodzovsky *
20087fc48e59SAndrey Grodzovsky * @vm_size: VM size
20097fc48e59SAndrey Grodzovsky *
20107fc48e59SAndrey Grodzovsky * Returns:
20117fc48e59SAndrey Grodzovsky * VM page table as power of two
20127fc48e59SAndrey Grodzovsky */
amdgpu_vm_get_block_size(uint64_t vm_size)2013bab4fee7SJunwei Zhang static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2014bab4fee7SJunwei Zhang {
2015bab4fee7SJunwei Zhang /* Total bits covered by PD + PTs */
2016bab4fee7SJunwei Zhang unsigned bits = ilog2(vm_size) + 18;
2017bab4fee7SJunwei Zhang
2018bab4fee7SJunwei Zhang /* Make sure the PD is 4K in size up to 8GB address space.
2019bab4fee7SJunwei Zhang Above that split equal between PD and PTs */
2020bab4fee7SJunwei Zhang if (vm_size <= 8)
2021bab4fee7SJunwei Zhang return (bits - 9);
2022bab4fee7SJunwei Zhang else
2023bab4fee7SJunwei Zhang return ((bits + 3) / 2);
2024bab4fee7SJunwei Zhang }
2025bab4fee7SJunwei Zhang
2026bab4fee7SJunwei Zhang /**
2027d07f14beSRoger He * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2028bab4fee7SJunwei Zhang *
2029bab4fee7SJunwei Zhang * @adev: amdgpu_device pointer
203043370c4cSFelix Kuehling * @min_vm_size: the minimum vm size in GB if it's set auto
203100553cf8SAndrey Grodzovsky * @fragment_size_default: Default PTE fragment size
203200553cf8SAndrey Grodzovsky * @max_level: max VMPT level
203300553cf8SAndrey Grodzovsky * @max_bits: max address space size in bits
203400553cf8SAndrey Grodzovsky *
2035bab4fee7SJunwei Zhang */
amdgpu_vm_adjust_size(struct amdgpu_device * adev,uint32_t min_vm_size,uint32_t fragment_size_default,unsigned max_level,unsigned max_bits)203643370c4cSFelix Kuehling void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
2037f3368128SChristian König uint32_t fragment_size_default, unsigned max_level,
2038f3368128SChristian König unsigned max_bits)
2039bab4fee7SJunwei Zhang {
204043370c4cSFelix Kuehling unsigned int max_size = 1 << (max_bits - 30);
204143370c4cSFelix Kuehling unsigned int vm_size;
204236539dceSChristian König uint64_t tmp;
204336539dceSChristian König
204436539dceSChristian König /* adjust vm size first */
2045f3368128SChristian König if (amdgpu_vm_size != -1) {
2046fdd5faaaSChristian König vm_size = amdgpu_vm_size;
2047f3368128SChristian König if (vm_size > max_size) {
2048f3368128SChristian König dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2049f3368128SChristian König amdgpu_vm_size, max_size);
2050f3368128SChristian König vm_size = max_size;
2051f3368128SChristian König }
205243370c4cSFelix Kuehling } else {
205343370c4cSFelix Kuehling struct sysinfo si;
205443370c4cSFelix Kuehling unsigned int phys_ram_gb;
205543370c4cSFelix Kuehling
205643370c4cSFelix Kuehling /* Optimal VM size depends on the amount of physical
205743370c4cSFelix Kuehling * RAM available. Underlying requirements and
205843370c4cSFelix Kuehling * assumptions:
205943370c4cSFelix Kuehling *
206043370c4cSFelix Kuehling * - Need to map system memory and VRAM from all GPUs
206143370c4cSFelix Kuehling * - VRAM from other GPUs not known here
206243370c4cSFelix Kuehling * - Assume VRAM <= system memory
206343370c4cSFelix Kuehling * - On GFX8 and older, VM space can be segmented for
206443370c4cSFelix Kuehling * different MTYPEs
206543370c4cSFelix Kuehling * - Need to allow room for fragmentation, guard pages etc.
206643370c4cSFelix Kuehling *
206743370c4cSFelix Kuehling * This adds up to a rough guess of system memory x3.
206843370c4cSFelix Kuehling * Round up to power of two to maximize the available
206943370c4cSFelix Kuehling * VM size with the given page table size.
207043370c4cSFelix Kuehling */
207143370c4cSFelix Kuehling si_meminfo(&si);
207243370c4cSFelix Kuehling phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
207343370c4cSFelix Kuehling (1 << 30) - 1) >> 30;
207443370c4cSFelix Kuehling vm_size = roundup_pow_of_two(
207543370c4cSFelix Kuehling min(max(phys_ram_gb * 3, min_vm_size), max_size));
2076f3368128SChristian König }
2077fdd5faaaSChristian König
2078fdd5faaaSChristian König adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
207936539dceSChristian König
208036539dceSChristian König tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
208197489129SChristian König if (amdgpu_vm_block_size != -1)
208297489129SChristian König tmp >>= amdgpu_vm_block_size - 9;
208336539dceSChristian König tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
208436539dceSChristian König adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
2085196f7489SChunming Zhou switch (adev->vm_manager.num_level) {
2086196f7489SChunming Zhou case 3:
2087196f7489SChunming Zhou adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2088196f7489SChunming Zhou break;
2089196f7489SChunming Zhou case 2:
2090196f7489SChunming Zhou adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2091196f7489SChunming Zhou break;
2092196f7489SChunming Zhou case 1:
2093196f7489SChunming Zhou adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2094196f7489SChunming Zhou break;
2095196f7489SChunming Zhou default:
2096196f7489SChunming Zhou dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2097196f7489SChunming Zhou }
2098b38f41ebSChristian König /* block size depends on vm size and hw setup*/
209997489129SChristian König if (amdgpu_vm_block_size != -1)
2100bab4fee7SJunwei Zhang adev->vm_manager.block_size =
210197489129SChristian König min((unsigned)amdgpu_vm_block_size, max_bits
210297489129SChristian König - AMDGPU_GPU_PAGE_SHIFT
210397489129SChristian König - 9 * adev->vm_manager.num_level);
210497489129SChristian König else if (adev->vm_manager.num_level > 1)
210597489129SChristian König adev->vm_manager.block_size = 9;
2106bab4fee7SJunwei Zhang else
210797489129SChristian König adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2108bab4fee7SJunwei Zhang
2109b38f41ebSChristian König if (amdgpu_vm_fragment_size == -1)
2110b38f41ebSChristian König adev->vm_manager.fragment_size = fragment_size_default;
2111b38f41ebSChristian König else
2112b38f41ebSChristian König adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2113d07f14beSRoger He
211436539dceSChristian König DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
211536539dceSChristian König vm_size, adev->vm_manager.num_level + 1,
211636539dceSChristian König adev->vm_manager.block_size,
2117d07f14beSRoger He adev->vm_manager.fragment_size);
2118bab4fee7SJunwei Zhang }
2119bab4fee7SJunwei Zhang
2120d38ceaf9SAlex Deucher /**
212156753e73SChristian König * amdgpu_vm_wait_idle - wait for the VM to become idle
212256753e73SChristian König *
212356753e73SChristian König * @vm: VM object to wait for
212456753e73SChristian König * @timeout: timeout to wait for VM to become idle
212556753e73SChristian König */
amdgpu_vm_wait_idle(struct amdgpu_vm * vm,long timeout)212656753e73SChristian König long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
2127d38ceaf9SAlex Deucher {
21287bc80a54SChristian König timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv,
21290cc848a7SChristian König DMA_RESV_USAGE_BOOKKEEP,
2130d3fae3b3SChristian König true, timeout);
213190b69cdcSChristian König if (timeout <= 0)
213290b69cdcSChristian König return timeout;
213390b69cdcSChristian König
21349c466bcbSChristian König return dma_fence_wait_timeout(vm->last_unlocked, true, timeout);
21352bd9ccfaSChristian König }
2136f566ceb1SChristian König
21372bd9ccfaSChristian König /**
2138a24960f3SChristian König * amdgpu_vm_init - initialize a vm instance
213905906decSBas Nieuwenhuizen *
2140f566ceb1SChristian König * @adev: amdgpu_device pointer
2141857d913dSAlex Deucher * @vm: requested vm
21425003ca63SGuchun Chen * @xcp_id: GPU partition selection id
2143d38ceaf9SAlex Deucher *
2144d38ceaf9SAlex Deucher * Init @vm fields.
21457fc48e59SAndrey Grodzovsky *
21467fc48e59SAndrey Grodzovsky * Returns:
21477fc48e59SAndrey Grodzovsky * 0 for success, error for failure.
2148d38ceaf9SAlex Deucher */
amdgpu_vm_init(struct amdgpu_device * adev,struct amdgpu_vm * vm,int32_t xcp_id)21495e94f18dSChristian König int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
21505e94f18dSChristian König int32_t xcp_id)
2151d38ceaf9SAlex Deucher {
215259276f05SNirmoy Das struct amdgpu_bo *root_bo;
215359276f05SNirmoy Das struct amdgpu_bo_vm *root;
215436bbf3bfSChunming Zhou int r, i;
2155d38ceaf9SAlex Deucher
2156f808c13fSDavidlohr Bueso vm->va = RB_ROOT_CACHED;
215736bbf3bfSChunming Zhou for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
215836bbf3bfSChunming Zhou vm->reserved_vmid[i] = NULL;
21593f3333f8SChristian König INIT_LIST_HEAD(&vm->evicted);
2160ea09729cSChristian König INIT_LIST_HEAD(&vm->relocated);
216127c7b9aeSChristian König INIT_LIST_HEAD(&vm->moved);
2162806f043fSChristian König INIT_LIST_HEAD(&vm->idle);
2163c12a2ee5SChristian König INIT_LIST_HEAD(&vm->invalidated);
21640479956cSPhilip Yang spin_lock_init(&vm->status_lock);
2165d38ceaf9SAlex Deucher INIT_LIST_HEAD(&vm->freed);
21660e601a04SMihir Bhogilal Patel INIT_LIST_HEAD(&vm->done);
21673e43b760SPhilip Yang INIT_LIST_HEAD(&vm->pt_freed);
21683e43b760SPhilip Yang INIT_WORK(&vm->pt_free_work, amdgpu_vm_pt_free_work);
21695e94f18dSChristian König INIT_KFIFO(vm->faults);
2170b3ac1766SNirmoy Das
217155bf196fSChristian König r = amdgpu_vm_init_entities(adev, vm);
2172d38ceaf9SAlex Deucher if (r)
2173d38ceaf9SAlex Deucher return r;
2174d38ceaf9SAlex Deucher
217551ac7eecSYong Zhao vm->pte_support_ats = false;
2176f43ef951SAlex Sierra vm->is_compute_context = false;
217751ac7eecSYong Zhao
21789a4b7d4cSHarish Kasiviswanathan vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
21799a4b7d4cSHarish Kasiviswanathan AMDGPU_VM_USE_CPU_FOR_GFX);
2180a35455d0SNirmoy Das
21819a4b7d4cSHarish Kasiviswanathan DRM_DEBUG_DRIVER("VM update mode is %s\n",
21829a4b7d4cSHarish Kasiviswanathan vm->use_cpu_for_update ? "CPU" : "SDMA");
2183fc39d903SChristian König WARN_ONCE((vm->use_cpu_for_update &&
2184fc39d903SChristian König !amdgpu_gmc_vram_full_visible(&adev->gmc)),
21859a4b7d4cSHarish Kasiviswanathan "CPU update of VM recommended only for large BAR system\n");
21866dd09027SChristian König
21876dd09027SChristian König if (vm->use_cpu_for_update)
21886dd09027SChristian König vm->update_funcs = &amdgpu_vm_cpu_funcs;
21896dd09027SChristian König else
21906dd09027SChristian König vm->update_funcs = &amdgpu_vm_sdma_funcs;
2191187916e6SLang Yu
2192187916e6SLang Yu vm->last_update = dma_fence_get_stub();
21939c466bcbSChristian König vm->last_unlocked = dma_fence_get_stub();
21947c703a7dSxinhui pan vm->last_tlb_flush = dma_fence_get_stub();
2195f88e295eSChristian König vm->generation = 0;
2196d38ceaf9SAlex Deucher
2197b4ff0f8aSChristian König mutex_init(&vm->eviction_lock);
2198b4ff0f8aSChristian König vm->evicting = false;
2199b4ff0f8aSChristian König
2200adf6f5c5SNirmoy Das r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
22015003ca63SGuchun Chen false, &root, xcp_id);
2202d38ceaf9SAlex Deucher if (r)
2203a2cf3247SChristian König goto error_free_delayed;
22045e94f18dSChristian König
22055e94f18dSChristian König root_bo = amdgpu_bo_ref(&root->bo);
220659276f05SNirmoy Das r = amdgpu_bo_reserve(root_bo, true);
22075e94f18dSChristian König if (r) {
22085e94f18dSChristian König amdgpu_bo_unref(&root->shadow);
22095e94f18dSChristian König amdgpu_bo_unref(&root_bo);
22105e94f18dSChristian König goto error_free_delayed;
22115e94f18dSChristian König }
22125e94f18dSChristian König
22135e94f18dSChristian König amdgpu_vm_bo_base_init(&vm->root, vm, root_bo);
22145e94f18dSChristian König r = dma_resv_reserve_fences(root_bo->tbo.base.resv, 1);
2215d3aab672SChristian König if (r)
2216d3aab672SChristian König goto error_free_root;
2217d3aab672SChristian König
2218184a69caSChristian König r = amdgpu_vm_pt_clear(adev, vm, root, false);
221913307f7eSChristian König if (r)
22205e94f18dSChristian König goto error_free_root;
222113307f7eSChristian König
2222391629bdSNirmoy Das amdgpu_bo_unreserve(vm->root.bo);
22235e94f18dSChristian König amdgpu_bo_unref(&root_bo);
2224d38ceaf9SAlex Deucher
2225d38ceaf9SAlex Deucher return 0;
22262bd9ccfaSChristian König
222767003a15SChristian König error_free_root:
22285e94f18dSChristian König amdgpu_vm_pt_free_root(adev, vm);
22295e94f18dSChristian König amdgpu_bo_unreserve(vm->root.bo);
223059276f05SNirmoy Das amdgpu_bo_unref(&root_bo);
22312bd9ccfaSChristian König
2232a2cf3247SChristian König error_free_delayed:
22337c703a7dSxinhui pan dma_fence_put(vm->last_tlb_flush);
22349c466bcbSChristian König dma_fence_put(vm->last_unlocked);
223555bf196fSChristian König amdgpu_vm_fini_entities(vm);
22362bd9ccfaSChristian König
22372bd9ccfaSChristian König return r;
2238d38ceaf9SAlex Deucher }
2239d38ceaf9SAlex Deucher
2240d38ceaf9SAlex Deucher /**
2241b236fa1dSFelix Kuehling * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2242b236fa1dSFelix Kuehling *
22437fc48e59SAndrey Grodzovsky * @adev: amdgpu_device pointer
22447fc48e59SAndrey Grodzovsky * @vm: requested vm
22457fc48e59SAndrey Grodzovsky *
2246b236fa1dSFelix Kuehling * This only works on GFX VMs that don't have any BOs added and no
2247b236fa1dSFelix Kuehling * page tables allocated yet.
2248b236fa1dSFelix Kuehling *
2249b236fa1dSFelix Kuehling * Changes the following VM parameters:
2250b236fa1dSFelix Kuehling * - use_cpu_for_update
2251b236fa1dSFelix Kuehling * - pte_supports_ats
2252b236fa1dSFelix Kuehling *
2253b236fa1dSFelix Kuehling * Reinitializes the page directory to reflect the changed ATS
2254b5d21aacSShaoyun Liu * setting.
2255b236fa1dSFelix Kuehling *
22567fc48e59SAndrey Grodzovsky * Returns:
22577fc48e59SAndrey Grodzovsky * 0 for success, -errno for errors.
2258b236fa1dSFelix Kuehling */
amdgpu_vm_make_compute(struct amdgpu_device * adev,struct amdgpu_vm * vm)225988f7f881SNirmoy Das int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2260b236fa1dSFelix Kuehling {
2261741deadeSAlex Deucher bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
2262b236fa1dSFelix Kuehling int r;
2263b236fa1dSFelix Kuehling
2264391629bdSNirmoy Das r = amdgpu_bo_reserve(vm->root.bo, true);
2265b236fa1dSFelix Kuehling if (r)
2266b236fa1dSFelix Kuehling return r;
2267b236fa1dSFelix Kuehling
22681d7776ccSXiaogang Chen /* Check if PD needs to be reinitialized and do it before
22691d7776ccSXiaogang Chen * changing any other state, in case it fails.
22701d7776ccSXiaogang Chen */
22711d7776ccSXiaogang Chen if (pte_support_ats != vm->pte_support_ats) {
2272b236fa1dSFelix Kuehling /* Sanity checks */
2273184a69caSChristian König if (!amdgpu_vm_pt_is_root_clean(adev, vm)) {
2274184a69caSChristian König r = -EINVAL;
22751685b01aSOak Zeng goto unreserve_bo;
2276184a69caSChristian König }
22771685b01aSOak Zeng
2278780637cbSChristian König vm->pte_support_ats = pte_support_ats;
2279184a69caSChristian König r = amdgpu_vm_pt_clear(adev, vm, to_amdgpu_bo_vm(vm->root.bo),
228059276f05SNirmoy Das false);
2281b236fa1dSFelix Kuehling if (r)
228288f7f881SNirmoy Das goto unreserve_bo;
2283b236fa1dSFelix Kuehling }
2284b236fa1dSFelix Kuehling
2285b236fa1dSFelix Kuehling /* Update VM state */
2286b236fa1dSFelix Kuehling vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2287b236fa1dSFelix Kuehling AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2288b236fa1dSFelix Kuehling DRM_DEBUG_DRIVER("VM update mode is %s\n",
2289b236fa1dSFelix Kuehling vm->use_cpu_for_update ? "CPU" : "SDMA");
2290fc39d903SChristian König WARN_ONCE((vm->use_cpu_for_update &&
2291fc39d903SChristian König !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2292b236fa1dSFelix Kuehling "CPU update of VM recommended only for large BAR system\n");
2293b236fa1dSFelix Kuehling
229490ca78deSFelix Kuehling if (vm->use_cpu_for_update) {
229590ca78deSFelix Kuehling /* Sync with last SDMA update/clear before switching to CPU */
2296391629bdSNirmoy Das r = amdgpu_bo_sync_wait(vm->root.bo,
229790ca78deSFelix Kuehling AMDGPU_FENCE_OWNER_UNDEFINED, true);
229890ca78deSFelix Kuehling if (r)
229988f7f881SNirmoy Das goto unreserve_bo;
230090ca78deSFelix Kuehling
2301108b4d92SGang Ba vm->update_funcs = &amdgpu_vm_cpu_funcs;
2302eb58ad14SXiaogang Chen r = amdgpu_vm_pt_map_tables(adev, vm);
2303eb58ad14SXiaogang Chen if (r)
2304eb58ad14SXiaogang Chen goto unreserve_bo;
2305eb58ad14SXiaogang Chen
230690ca78deSFelix Kuehling } else {
2307108b4d92SGang Ba vm->update_funcs = &amdgpu_vm_sdma_funcs;
230890ca78deSFelix Kuehling }
23094bdb9d65SLang Yu
2310108b4d92SGang Ba dma_fence_put(vm->last_update);
2311187916e6SLang Yu vm->last_update = dma_fence_get_stub();
2312f43ef951SAlex Sierra vm->is_compute_context = true;
2313108b4d92SGang Ba
2314b5d21aacSShaoyun Liu /* Free the shadow bo for compute VM */
2315391629bdSNirmoy Das amdgpu_bo_unref(&to_amdgpu_bo_vm(vm->root.bo)->shadow);
2316b5d21aacSShaoyun Liu
23171685b01aSOak Zeng goto unreserve_bo;
23181685b01aSOak Zeng
23191685b01aSOak Zeng unreserve_bo:
2320391629bdSNirmoy Das amdgpu_bo_unreserve(vm->root.bo);
2321b236fa1dSFelix Kuehling return r;
2322b236fa1dSFelix Kuehling }
2323b236fa1dSFelix Kuehling
2324b236fa1dSFelix Kuehling /**
2325bf47afbaSOak Zeng * amdgpu_vm_release_compute - release a compute vm
2326bf47afbaSOak Zeng * @adev: amdgpu_device pointer
2327bf47afbaSOak Zeng * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
2328bf47afbaSOak Zeng *
2329bf47afbaSOak Zeng * This is a correspondant of amdgpu_vm_make_compute. It decouples compute
2330bf47afbaSOak Zeng * pasid from vm. Compute should stop use of vm after this call.
2331bf47afbaSOak Zeng */
amdgpu_vm_release_compute(struct amdgpu_device * adev,struct amdgpu_vm * vm)2332bf47afbaSOak Zeng void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2333bf47afbaSOak Zeng {
2334dcb388edSNirmoy Das amdgpu_vm_set_pasid(adev, vm, 0);
2335f43ef951SAlex Sierra vm->is_compute_context = false;
2336bf47afbaSOak Zeng }
2337bf47afbaSOak Zeng
2338bf47afbaSOak Zeng /**
2339d38ceaf9SAlex Deucher * amdgpu_vm_fini - tear down a vm instance
2340d38ceaf9SAlex Deucher *
2341d38ceaf9SAlex Deucher * @adev: amdgpu_device pointer
2342d38ceaf9SAlex Deucher * @vm: requested vm
2343d38ceaf9SAlex Deucher *
23448843dbbbSChristian König * Tear down @vm.
2345d38ceaf9SAlex Deucher * Unbind the VM and remove all bos from the vm bo list
2346d38ceaf9SAlex Deucher */
amdgpu_vm_fini(struct amdgpu_device * adev,struct amdgpu_vm * vm)2347d38ceaf9SAlex Deucher void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2348d38ceaf9SAlex Deucher {
2349d38ceaf9SAlex Deucher struct amdgpu_bo_va_mapping *mapping, *tmp;
2350132f34e4SChristian König bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
23512642cf11SChristian König struct amdgpu_bo *root;
23527c703a7dSxinhui pan unsigned long flags;
2353b65709a9SChristian König int i;
2354d38ceaf9SAlex Deucher
2355ede0dd86SFelix Kuehling amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2356ede0dd86SFelix Kuehling
23573e43b760SPhilip Yang flush_work(&vm->pt_free_work);
23583e43b760SPhilip Yang
2359391629bdSNirmoy Das root = amdgpu_bo_ref(vm->root.bo);
2360b65709a9SChristian König amdgpu_bo_reserve(root, true);
2361dcb388edSNirmoy Das amdgpu_vm_set_pasid(adev, vm, 0);
23629c466bcbSChristian König dma_fence_wait(vm->last_unlocked, false);
23639c466bcbSChristian König dma_fence_put(vm->last_unlocked);
23647c703a7dSxinhui pan dma_fence_wait(vm->last_tlb_flush, false);
23657c703a7dSxinhui pan /* Make sure that all fence callbacks have completed */
23667c703a7dSxinhui pan spin_lock_irqsave(vm->last_tlb_flush->lock, flags);
23677c703a7dSxinhui pan spin_unlock_irqrestore(vm->last_tlb_flush->lock, flags);
23687c703a7dSxinhui pan dma_fence_put(vm->last_tlb_flush);
236990b69cdcSChristian König
2370ee8bcc23SPelloux-prayer, Pierre-eric list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2371ee8bcc23SPelloux-prayer, Pierre-eric if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2372ee8bcc23SPelloux-prayer, Pierre-eric amdgpu_vm_prt_fini(adev, vm);
2373ee8bcc23SPelloux-prayer, Pierre-eric prt_fini_needed = false;
2374ee8bcc23SPelloux-prayer, Pierre-eric }
2375ee8bcc23SPelloux-prayer, Pierre-eric
2376ee8bcc23SPelloux-prayer, Pierre-eric list_del(&mapping->list);
2377ee8bcc23SPelloux-prayer, Pierre-eric amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2378ee8bcc23SPelloux-prayer, Pierre-eric }
2379ee8bcc23SPelloux-prayer, Pierre-eric
2380184a69caSChristian König amdgpu_vm_pt_free_root(adev, vm);
2381b65709a9SChristian König amdgpu_bo_unreserve(root);
2382b65709a9SChristian König amdgpu_bo_unref(&root);
2383391629bdSNirmoy Das WARN_ON(vm->root.bo);
2384b65709a9SChristian König
238555bf196fSChristian König amdgpu_vm_fini_entities(vm);
23862bd9ccfaSChristian König
2387f808c13fSDavidlohr Bueso if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2388d38ceaf9SAlex Deucher dev_err(adev->dev, "still active bo inside vm\n");
2389d38ceaf9SAlex Deucher }
2390f808c13fSDavidlohr Bueso rbtree_postorder_for_each_entry_safe(mapping, tmp,
2391f808c13fSDavidlohr Bueso &vm->va.rb_root, rb) {
23920af5c656SChristian König /* Don't remove the mapping here, we don't want to trigger a
23930af5c656SChristian König * rebalance and the tree is about to be destroyed anyway.
23940af5c656SChristian König */
2395d38ceaf9SAlex Deucher list_del(&mapping->list);
2396d38ceaf9SAlex Deucher kfree(mapping);
2397d38ceaf9SAlex Deucher }
2398d38ceaf9SAlex Deucher
2399d5884513SChristian König dma_fence_put(vm->last_update);
240080e709eeSChong Li
240180e709eeSChong Li for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) {
240280e709eeSChong Li if (vm->reserved_vmid[i]) {
240380e709eeSChong Li amdgpu_vmid_free_reserved(adev, i);
240480e709eeSChong Li vm->reserved_vmid[i] = false;
240580e709eeSChong Li }
240680e709eeSChong Li }
240780e709eeSChong Li
2408444066b9SChunming Zhou }
2409ea89f8c9SChristian König
2410ea89f8c9SChristian König /**
2411a9a78b32SChristian König * amdgpu_vm_manager_init - init the VM manager
2412a9a78b32SChristian König *
2413a9a78b32SChristian König * @adev: amdgpu_device pointer
2414a9a78b32SChristian König *
2415a9a78b32SChristian König * Initialize the VM manager structures
2416a9a78b32SChristian König */
amdgpu_vm_manager_init(struct amdgpu_device * adev)2417a9a78b32SChristian König void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2418a9a78b32SChristian König {
2419620f774fSChristian König unsigned i;
2420a9a78b32SChristian König
242120a5f5a9SChristian König /* Concurrent flushes are only possible starting with Vega10 and
242220a5f5a9SChristian König * are broken on Navi10 and Navi14.
242320a5f5a9SChristian König */
242420a5f5a9SChristian König adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 ||
242520a5f5a9SChristian König adev->asic_type == CHIP_NAVI10 ||
242620a5f5a9SChristian König adev->asic_type == CHIP_NAVI14);
2427620f774fSChristian König amdgpu_vmid_mgr_init(adev);
24282d55e45aSChristian König
2429f54d1867SChris Wilson adev->vm_manager.fence_context =
2430f54d1867SChris Wilson dma_fence_context_alloc(AMDGPU_MAX_RINGS);
24311fbb2e92SChristian König for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
24321fbb2e92SChristian König adev->vm_manager.seqno[i] = 0;
24331fbb2e92SChristian König
2434284710faSChristian König spin_lock_init(&adev->vm_manager.prt_lock);
2435451bc8ebSChristian König atomic_set(&adev->vm_manager.num_prt_users, 0);
24369a4b7d4cSHarish Kasiviswanathan
24379a4b7d4cSHarish Kasiviswanathan /* If not overridden by the user, by default, only in large BAR systems
24389a4b7d4cSHarish Kasiviswanathan * Compute VM tables will be updated by CPU
24399a4b7d4cSHarish Kasiviswanathan */
24409a4b7d4cSHarish Kasiviswanathan #ifdef CONFIG_X86_64
24419a4b7d4cSHarish Kasiviswanathan if (amdgpu_vm_update_mode == -1) {
244265f8682bSDanijel Slivka /* For asic with VF MMIO access protection
244365f8682bSDanijel Slivka * avoid using CPU for VM table updates
244465f8682bSDanijel Slivka */
244565f8682bSDanijel Slivka if (amdgpu_gmc_vram_full_visible(&adev->gmc) &&
244665f8682bSDanijel Slivka !amdgpu_sriov_vf_mmio_access_protection(adev))
24479a4b7d4cSHarish Kasiviswanathan adev->vm_manager.vm_update_mode =
24489a4b7d4cSHarish Kasiviswanathan AMDGPU_VM_USE_CPU_FOR_COMPUTE;
24499a4b7d4cSHarish Kasiviswanathan else
24509a4b7d4cSHarish Kasiviswanathan adev->vm_manager.vm_update_mode = 0;
24519a4b7d4cSHarish Kasiviswanathan } else
24529a4b7d4cSHarish Kasiviswanathan adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
24539a4b7d4cSHarish Kasiviswanathan #else
24549a4b7d4cSHarish Kasiviswanathan adev->vm_manager.vm_update_mode = 0;
24559a4b7d4cSHarish Kasiviswanathan #endif
24569a4b7d4cSHarish Kasiviswanathan
2457dcb388edSNirmoy Das xa_init_flags(&adev->vm_manager.pasids, XA_FLAGS_LOCK_IRQ);
2458a9a78b32SChristian König }
2459a9a78b32SChristian König
2460a9a78b32SChristian König /**
2461ea89f8c9SChristian König * amdgpu_vm_manager_fini - cleanup VM manager
2462ea89f8c9SChristian König *
2463ea89f8c9SChristian König * @adev: amdgpu_device pointer
2464ea89f8c9SChristian König *
2465ea89f8c9SChristian König * Cleanup the VM manager and free resources.
2466ea89f8c9SChristian König */
amdgpu_vm_manager_fini(struct amdgpu_device * adev)2467ea89f8c9SChristian König void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2468ea89f8c9SChristian König {
2469dcb388edSNirmoy Das WARN_ON(!xa_empty(&adev->vm_manager.pasids));
2470dcb388edSNirmoy Das xa_destroy(&adev->vm_manager.pasids);
247102208441SFelix Kuehling
2472620f774fSChristian König amdgpu_vmid_mgr_fini(adev);
24737645670dSChristian König }
2474cfbcacf4SChunming Zhou
24757fc48e59SAndrey Grodzovsky /**
24767fc48e59SAndrey Grodzovsky * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
24777fc48e59SAndrey Grodzovsky *
24787fc48e59SAndrey Grodzovsky * @dev: drm device pointer
24797fc48e59SAndrey Grodzovsky * @data: drm_amdgpu_vm
24807fc48e59SAndrey Grodzovsky * @filp: drm file pointer
24817fc48e59SAndrey Grodzovsky *
24827fc48e59SAndrey Grodzovsky * Returns:
24837fc48e59SAndrey Grodzovsky * 0 for success, -errno for errors.
24847fc48e59SAndrey Grodzovsky */
amdgpu_vm_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)2485cfbcacf4SChunming Zhou int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2486cfbcacf4SChunming Zhou {
2487cfbcacf4SChunming Zhou union drm_amdgpu_vm *args = data;
24881348969aSLuben Tuikov struct amdgpu_device *adev = drm_to_adev(dev);
24891e9ef26fSChunming Zhou struct amdgpu_fpriv *fpriv = filp->driver_priv;
2490cfbcacf4SChunming Zhou
2491a2b30804SBas Nieuwenhuizen /* No valid flags defined yet */
2492a2b30804SBas Nieuwenhuizen if (args->in.flags)
2493a2b30804SBas Nieuwenhuizen return -EINVAL;
2494a2b30804SBas Nieuwenhuizen
2495cfbcacf4SChunming Zhou switch (args->in.op) {
2496cfbcacf4SChunming Zhou case AMDGPU_VM_OP_RESERVE_VMID:
2497fc39d903SChristian König /* We only have requirement to reserve vmid from gfxhub */
249880e709eeSChong Li if (!fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)]) {
249980e709eeSChong Li amdgpu_vmid_alloc_reserved(adev, AMDGPU_GFXHUB(0));
250080e709eeSChong Li fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)] = true;
250180e709eeSChong Li }
250280e709eeSChong Li
25031e9ef26fSChunming Zhou break;
2504cfbcacf4SChunming Zhou case AMDGPU_VM_OP_UNRESERVE_VMID:
250580e709eeSChong Li if (fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)]) {
250680e709eeSChong Li amdgpu_vmid_free_reserved(adev, AMDGPU_GFXHUB(0));
250780e709eeSChong Li fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)] = false;
250880e709eeSChong Li }
2509cfbcacf4SChunming Zhou break;
2510cfbcacf4SChunming Zhou default:
2511cfbcacf4SChunming Zhou return -EINVAL;
2512cfbcacf4SChunming Zhou }
2513cfbcacf4SChunming Zhou
2514cfbcacf4SChunming Zhou return 0;
2515cfbcacf4SChunming Zhou }
25162aa37bf5SAndrey Grodzovsky
25172aa37bf5SAndrey Grodzovsky /**
25182aa37bf5SAndrey Grodzovsky * amdgpu_vm_get_task_info - Extracts task info for a PASID.
25192aa37bf5SAndrey Grodzovsky *
2520989edc69SMasanari Iida * @adev: drm device pointer
25212aa37bf5SAndrey Grodzovsky * @pasid: PASID identifier for VM
25222aa37bf5SAndrey Grodzovsky * @task_info: task_info to fill.
25232aa37bf5SAndrey Grodzovsky */
amdgpu_vm_get_task_info(struct amdgpu_device * adev,u32 pasid,struct amdgpu_task_info * task_info)2524c7b6bac9SFenghua Yu void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid,
25252aa37bf5SAndrey Grodzovsky struct amdgpu_task_info *task_info)
25262aa37bf5SAndrey Grodzovsky {
25272aa37bf5SAndrey Grodzovsky struct amdgpu_vm *vm;
25280a5f49cbSPhilip Yang unsigned long flags;
25292aa37bf5SAndrey Grodzovsky
2530dcb388edSNirmoy Das xa_lock_irqsave(&adev->vm_manager.pasids, flags);
25312aa37bf5SAndrey Grodzovsky
2532dcb388edSNirmoy Das vm = xa_load(&adev->vm_manager.pasids, pasid);
25332aa37bf5SAndrey Grodzovsky if (vm)
25342aa37bf5SAndrey Grodzovsky *task_info = vm->task_info;
25352aa37bf5SAndrey Grodzovsky
2536dcb388edSNirmoy Das xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
25372aa37bf5SAndrey Grodzovsky }
25382aa37bf5SAndrey Grodzovsky
25392aa37bf5SAndrey Grodzovsky /**
25402aa37bf5SAndrey Grodzovsky * amdgpu_vm_set_task_info - Sets VMs task info.
25412aa37bf5SAndrey Grodzovsky *
25422aa37bf5SAndrey Grodzovsky * @vm: vm for which to set the info
25432aa37bf5SAndrey Grodzovsky */
amdgpu_vm_set_task_info(struct amdgpu_vm * vm)25442aa37bf5SAndrey Grodzovsky void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
25452aa37bf5SAndrey Grodzovsky {
2546fc39d903SChristian König if (vm->task_info.pid)
2547fc39d903SChristian König return;
2548fc39d903SChristian König
25492aa37bf5SAndrey Grodzovsky vm->task_info.pid = current->pid;
25502aa37bf5SAndrey Grodzovsky get_task_comm(vm->task_info.task_name, current);
25512aa37bf5SAndrey Grodzovsky
2552fc39d903SChristian König if (current->group_leader->mm != current->mm)
2553fc39d903SChristian König return;
2554fc39d903SChristian König
25552aa37bf5SAndrey Grodzovsky vm->task_info.tgid = current->group_leader->pid;
25562aa37bf5SAndrey Grodzovsky get_task_comm(vm->task_info.process_name, current->group_leader);
25572aa37bf5SAndrey Grodzovsky }
2558ec671737SChristian König
2559ec671737SChristian König /**
2560ec671737SChristian König * amdgpu_vm_handle_fault - graceful handling of VM faults.
2561ec671737SChristian König * @adev: amdgpu device pointer
2562ec671737SChristian König * @pasid: PASID of the VM
2563f5fe7edfSMukul Joshi * @vmid: VMID, only used for GFX 9.4.3.
2564f5fe7edfSMukul Joshi * @node_id: Node_id received in IH cookie. Only applicable for
2565f5fe7edfSMukul Joshi * GFX 9.4.3.
2566ec671737SChristian König * @addr: Address of the fault
2567ff891a2eSPhilip Yang * @write_fault: true is write fault, false is read fault
2568ec671737SChristian König *
2569ec671737SChristian König * Try to gracefully handle a VM fault. Return true if the fault was handled and
2570ec671737SChristian König * shouldn't be reported any more.
2571ec671737SChristian König */
amdgpu_vm_handle_fault(struct amdgpu_device * adev,u32 pasid,u32 vmid,u32 node_id,uint64_t addr,bool write_fault)2572c7b6bac9SFenghua Yu bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
2573f5fe7edfSMukul Joshi u32 vmid, u32 node_id, uint64_t addr,
25745fb34bd9SAlex Sierra bool write_fault)
2575ec671737SChristian König {
2576ea53af8aSAlex Sierra bool is_compute_context = false;
2577ec671737SChristian König struct amdgpu_bo *root;
2578d760895dSFelix Kuehling unsigned long irqflags;
2579ec671737SChristian König uint64_t value, flags;
2580ec671737SChristian König struct amdgpu_vm *vm;
25812b665c37SPhilip Yang int r;
2582ec671737SChristian König
2583dcb388edSNirmoy Das xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2584dcb388edSNirmoy Das vm = xa_load(&adev->vm_manager.pasids, pasid);
2585ea53af8aSAlex Sierra if (vm) {
2586391629bdSNirmoy Das root = amdgpu_bo_ref(vm->root.bo);
2587ea53af8aSAlex Sierra is_compute_context = vm->is_compute_context;
2588ea53af8aSAlex Sierra } else {
2589ec671737SChristian König root = NULL;
2590ea53af8aSAlex Sierra }
2591dcb388edSNirmoy Das xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2592ec671737SChristian König
2593ec671737SChristian König if (!root)
2594ec671737SChristian König return false;
2595ec671737SChristian König
2596ea53af8aSAlex Sierra addr /= AMDGPU_GPU_PAGE_SIZE;
2597ea53af8aSAlex Sierra
2598f5fe7edfSMukul Joshi if (is_compute_context && !svm_range_restore_pages(adev, pasid, vmid,
25995fb34bd9SAlex Sierra node_id, addr, write_fault)) {
2600ea53af8aSAlex Sierra amdgpu_bo_unref(&root);
2601ea53af8aSAlex Sierra return true;
2602ea53af8aSAlex Sierra }
2603ea53af8aSAlex Sierra
2604ec671737SChristian König r = amdgpu_bo_reserve(root, true);
2605ec671737SChristian König if (r)
2606ec671737SChristian König goto error_unref;
2607ec671737SChristian König
2608ec671737SChristian König /* Double check that the VM still exists */
2609dcb388edSNirmoy Das xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2610dcb388edSNirmoy Das vm = xa_load(&adev->vm_manager.pasids, pasid);
2611391629bdSNirmoy Das if (vm && vm->root.bo != root)
2612ec671737SChristian König vm = NULL;
2613dcb388edSNirmoy Das xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2614ec671737SChristian König if (!vm)
2615ec671737SChristian König goto error_unlock;
2616ec671737SChristian König
2617ec671737SChristian König flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
2618ec671737SChristian König AMDGPU_PTE_SYSTEM;
2619ec671737SChristian König
2620ea53af8aSAlex Sierra if (is_compute_context) {
2621b4672c8aSAlex Sierra /* Intentionally setting invalid PTE flag
2622b4672c8aSAlex Sierra * combination to force a no-retry-fault
2623b4672c8aSAlex Sierra */
2624e77673d1SMukul Joshi flags = AMDGPU_VM_NORETRY_FLAGS;
2625b4672c8aSAlex Sierra value = 0;
2626b4672c8aSAlex Sierra } else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
2627ec671737SChristian König /* Redirect the access to the dummy page */
2628ec671737SChristian König value = adev->dummy_page_addr;
2629ec671737SChristian König flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
2630ec671737SChristian König AMDGPU_PTE_WRITEABLE;
2631b4672c8aSAlex Sierra
2632ec671737SChristian König } else {
2633ec671737SChristian König /* Let the hw retry silently on the PTE */
2634ec671737SChristian König value = 0;
2635ec671737SChristian König }
2636ec671737SChristian König
2637c8d4c18bSChristian König r = dma_resv_reserve_fences(root->tbo.base.resv, 1);
26382b665c37SPhilip Yang if (r) {
26392b665c37SPhilip Yang pr_debug("failed %d to reserve fence slot\n", r);
26402b665c37SPhilip Yang goto error_unlock;
26412b665c37SPhilip Yang }
26422b665c37SPhilip Yang
264330671b44SChristian König r = amdgpu_vm_update_range(adev, vm, true, false, false, NULL, addr,
264430671b44SChristian König addr, flags, value, 0, NULL, NULL, NULL);
2645ec671737SChristian König if (r)
2646ec671737SChristian König goto error_unlock;
2647ec671737SChristian König
2648ec671737SChristian König r = amdgpu_vm_update_pdes(adev, vm, true);
2649ec671737SChristian König
2650ec671737SChristian König error_unlock:
2651ec671737SChristian König amdgpu_bo_unreserve(root);
2652ec671737SChristian König if (r < 0)
26532b665c37SPhilip Yang DRM_ERROR("Can't handle page fault (%d)\n", r);
2654ec671737SChristian König
2655ec671737SChristian König error_unref:
2656ec671737SChristian König amdgpu_bo_unref(&root);
2657ec671737SChristian König
2658ec671737SChristian König return false;
2659ec671737SChristian König }
2660ff72bc40SMihir Bhogilal Patel
2661ff72bc40SMihir Bhogilal Patel #if defined(CONFIG_DEBUG_FS)
2662ff72bc40SMihir Bhogilal Patel /**
2663ff72bc40SMihir Bhogilal Patel * amdgpu_debugfs_vm_bo_info - print BO info for the VM
2664ff72bc40SMihir Bhogilal Patel *
2665ff72bc40SMihir Bhogilal Patel * @vm: Requested VM for printing BO info
2666ff72bc40SMihir Bhogilal Patel * @m: debugfs file
2667ff72bc40SMihir Bhogilal Patel *
2668ff72bc40SMihir Bhogilal Patel * Print BO information in debugfs file for the VM
2669ff72bc40SMihir Bhogilal Patel */
amdgpu_debugfs_vm_bo_info(struct amdgpu_vm * vm,struct seq_file * m)2670ff72bc40SMihir Bhogilal Patel void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
2671ff72bc40SMihir Bhogilal Patel {
2672ff72bc40SMihir Bhogilal Patel struct amdgpu_bo_va *bo_va, *tmp;
2673ff72bc40SMihir Bhogilal Patel u64 total_idle = 0;
2674ff72bc40SMihir Bhogilal Patel u64 total_evicted = 0;
2675ff72bc40SMihir Bhogilal Patel u64 total_relocated = 0;
2676ff72bc40SMihir Bhogilal Patel u64 total_moved = 0;
2677ff72bc40SMihir Bhogilal Patel u64 total_invalidated = 0;
26780e601a04SMihir Bhogilal Patel u64 total_done = 0;
2679ff72bc40SMihir Bhogilal Patel unsigned int total_idle_objs = 0;
2680ff72bc40SMihir Bhogilal Patel unsigned int total_evicted_objs = 0;
2681ff72bc40SMihir Bhogilal Patel unsigned int total_relocated_objs = 0;
2682ff72bc40SMihir Bhogilal Patel unsigned int total_moved_objs = 0;
2683ff72bc40SMihir Bhogilal Patel unsigned int total_invalidated_objs = 0;
26840e601a04SMihir Bhogilal Patel unsigned int total_done_objs = 0;
2685ff72bc40SMihir Bhogilal Patel unsigned int id = 0;
2686ff72bc40SMihir Bhogilal Patel
2687c1806d78SPhilip Yang spin_lock(&vm->status_lock);
2688ff72bc40SMihir Bhogilal Patel seq_puts(m, "\tIdle BOs:\n");
2689ff72bc40SMihir Bhogilal Patel list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
2690ff72bc40SMihir Bhogilal Patel if (!bo_va->base.bo)
2691ff72bc40SMihir Bhogilal Patel continue;
2692ff72bc40SMihir Bhogilal Patel total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2693ff72bc40SMihir Bhogilal Patel }
2694ff72bc40SMihir Bhogilal Patel total_idle_objs = id;
2695ff72bc40SMihir Bhogilal Patel id = 0;
2696ff72bc40SMihir Bhogilal Patel
2697ff72bc40SMihir Bhogilal Patel seq_puts(m, "\tEvicted BOs:\n");
2698ff72bc40SMihir Bhogilal Patel list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
2699ff72bc40SMihir Bhogilal Patel if (!bo_va->base.bo)
2700ff72bc40SMihir Bhogilal Patel continue;
2701ff72bc40SMihir Bhogilal Patel total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2702ff72bc40SMihir Bhogilal Patel }
2703ff72bc40SMihir Bhogilal Patel total_evicted_objs = id;
2704ff72bc40SMihir Bhogilal Patel id = 0;
2705ff72bc40SMihir Bhogilal Patel
2706ff72bc40SMihir Bhogilal Patel seq_puts(m, "\tRelocated BOs:\n");
2707ff72bc40SMihir Bhogilal Patel list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
2708ff72bc40SMihir Bhogilal Patel if (!bo_va->base.bo)
2709ff72bc40SMihir Bhogilal Patel continue;
2710ff72bc40SMihir Bhogilal Patel total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2711ff72bc40SMihir Bhogilal Patel }
2712ff72bc40SMihir Bhogilal Patel total_relocated_objs = id;
2713ff72bc40SMihir Bhogilal Patel id = 0;
2714ff72bc40SMihir Bhogilal Patel
2715ff72bc40SMihir Bhogilal Patel seq_puts(m, "\tMoved BOs:\n");
2716ff72bc40SMihir Bhogilal Patel list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
2717ff72bc40SMihir Bhogilal Patel if (!bo_va->base.bo)
2718ff72bc40SMihir Bhogilal Patel continue;
2719ff72bc40SMihir Bhogilal Patel total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2720ff72bc40SMihir Bhogilal Patel }
2721ff72bc40SMihir Bhogilal Patel total_moved_objs = id;
2722ff72bc40SMihir Bhogilal Patel id = 0;
2723ff72bc40SMihir Bhogilal Patel
2724ff72bc40SMihir Bhogilal Patel seq_puts(m, "\tInvalidated BOs:\n");
2725ff72bc40SMihir Bhogilal Patel list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
2726ff72bc40SMihir Bhogilal Patel if (!bo_va->base.bo)
2727ff72bc40SMihir Bhogilal Patel continue;
2728ff72bc40SMihir Bhogilal Patel total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2729ff72bc40SMihir Bhogilal Patel }
2730ff72bc40SMihir Bhogilal Patel total_invalidated_objs = id;
27310e601a04SMihir Bhogilal Patel id = 0;
27320e601a04SMihir Bhogilal Patel
27330e601a04SMihir Bhogilal Patel seq_puts(m, "\tDone BOs:\n");
27340e601a04SMihir Bhogilal Patel list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
27350e601a04SMihir Bhogilal Patel if (!bo_va->base.bo)
27360e601a04SMihir Bhogilal Patel continue;
27370e601a04SMihir Bhogilal Patel total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
27380e601a04SMihir Bhogilal Patel }
27390479956cSPhilip Yang spin_unlock(&vm->status_lock);
27400e601a04SMihir Bhogilal Patel total_done_objs = id;
2741ff72bc40SMihir Bhogilal Patel
2742ff72bc40SMihir Bhogilal Patel seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle,
2743ff72bc40SMihir Bhogilal Patel total_idle_objs);
2744ff72bc40SMihir Bhogilal Patel seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n", total_evicted,
2745ff72bc40SMihir Bhogilal Patel total_evicted_objs);
2746ff72bc40SMihir Bhogilal Patel seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n", total_relocated,
2747ff72bc40SMihir Bhogilal Patel total_relocated_objs);
2748ff72bc40SMihir Bhogilal Patel seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n", total_moved,
2749ff72bc40SMihir Bhogilal Patel total_moved_objs);
2750ff72bc40SMihir Bhogilal Patel seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
2751ff72bc40SMihir Bhogilal Patel total_invalidated_objs);
27520e601a04SMihir Bhogilal Patel seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n", total_done,
27530e601a04SMihir Bhogilal Patel total_done_objs);
2754ff72bc40SMihir Bhogilal Patel }
2755ff72bc40SMihir Bhogilal Patel #endif
2756