1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 
29 #include <linux/dma-fence-array.h>
30 #include <linux/interval_tree_generic.h>
31 #include <linux/idr.h>
32 #include <linux/dma-buf.h>
33 
34 #include <drm/amdgpu_drm.h>
35 #include <drm/drm_drv.h>
36 #include <drm/ttm/ttm_tt.h>
37 #include "amdgpu.h"
38 #include "amdgpu_trace.h"
39 #include "amdgpu_amdkfd.h"
40 #include "amdgpu_gmc.h"
41 #include "amdgpu_xgmi.h"
42 #include "amdgpu_dma_buf.h"
43 #include "amdgpu_res_cursor.h"
44 #include "kfd_svm.h"
45 
46 /**
47  * DOC: GPUVM
48  *
49  * GPUVM is the MMU functionality provided on the GPU.
50  * GPUVM is similar to the legacy GART on older asics, however
51  * rather than there being a single global GART table
52  * for the entire GPU, there can be multiple GPUVM page tables active
53  * at any given time.  The GPUVM page tables can contain a mix
54  * VRAM pages and system pages (both memory and MMIO) and system pages
55  * can be mapped as snooped (cached system pages) or unsnooped
56  * (uncached system pages).
57  *
58  * Each active GPUVM has an ID associated with it and there is a page table
59  * linked with each VMID.  When executing a command buffer,
60  * the kernel tells the engine what VMID to use for that command
61  * buffer.  VMIDs are allocated dynamically as commands are submitted.
62  * The userspace drivers maintain their own address space and the kernel
63  * sets up their pages tables accordingly when they submit their
64  * command buffers and a VMID is assigned.
65  * The hardware supports up to 16 active GPUVMs at any given time.
66  *
67  * Each GPUVM is represented by a 1-2 or 1-5 level page table, depending
68  * on the ASIC family.  GPUVM supports RWX attributes on each page as well
69  * as other features such as encryption and caching attributes.
70  *
71  * VMID 0 is special.  It is the GPUVM used for the kernel driver.  In
72  * addition to an aperture managed by a page table, VMID 0 also has
73  * several other apertures.  There is an aperture for direct access to VRAM
74  * and there is a legacy AGP aperture which just forwards accesses directly
75  * to the matching system physical addresses (or IOVAs when an IOMMU is
76  * present).  These apertures provide direct access to these memories without
77  * incurring the overhead of a page table.  VMID 0 is used by the kernel
78  * driver for tasks like memory management.
79  *
80  * GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory.
81  * For user applications, each application can have their own unique GPUVM
82  * address space.  The application manages the address space and the kernel
83  * driver manages the GPUVM page tables for each process.  If an GPU client
84  * accesses an invalid page, it will generate a GPU page fault, similar to
85  * accessing an invalid page on a CPU.
86  */
87 
88 #define START(node) ((node)->start)
89 #define LAST(node) ((node)->last)
90 
91 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
92 		     START, LAST, static, amdgpu_vm_it)
93 
94 #undef START
95 #undef LAST
96 
97 /**
98  * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
99  */
100 struct amdgpu_prt_cb {
101 
102 	/**
103 	 * @adev: amdgpu device
104 	 */
105 	struct amdgpu_device *adev;
106 
107 	/**
108 	 * @cb: callback
109 	 */
110 	struct dma_fence_cb cb;
111 };
112 
113 /**
114  * struct amdgpu_vm_tlb_seq_cb - Helper to increment the TLB flush sequence
115  */
116 struct amdgpu_vm_tlb_seq_cb {
117 	/**
118 	 * @vm: pointer to the amdgpu_vm structure to set the fence sequence on
119 	 */
120 	struct amdgpu_vm *vm;
121 
122 	/**
123 	 * @cb: callback
124 	 */
125 	struct dma_fence_cb cb;
126 };
127 
128 /**
129  * amdgpu_vm_set_pasid - manage pasid and vm ptr mapping
130  *
131  * @adev: amdgpu_device pointer
132  * @vm: amdgpu_vm pointer
133  * @pasid: the pasid the VM is using on this GPU
134  *
135  * Set the pasid this VM is using on this GPU, can also be used to remove the
136  * pasid by passing in zero.
137  *
138  */
139 int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm,
140 			u32 pasid)
141 {
142 	int r;
143 
144 	if (vm->pasid == pasid)
145 		return 0;
146 
147 	if (vm->pasid) {
148 		r = xa_err(xa_erase_irq(&adev->vm_manager.pasids, vm->pasid));
149 		if (r < 0)
150 			return r;
151 
152 		vm->pasid = 0;
153 	}
154 
155 	if (pasid) {
156 		r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm,
157 					GFP_KERNEL));
158 		if (r < 0)
159 			return r;
160 
161 		vm->pasid = pasid;
162 	}
163 
164 
165 	return 0;
166 }
167 
168 /**
169  * amdgpu_vm_bo_evicted - vm_bo is evicted
170  *
171  * @vm_bo: vm_bo which is evicted
172  *
173  * State for PDs/PTs and per VM BOs which are not at the location they should
174  * be.
175  */
176 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
177 {
178 	struct amdgpu_vm *vm = vm_bo->vm;
179 	struct amdgpu_bo *bo = vm_bo->bo;
180 
181 	vm_bo->moved = true;
182 	spin_lock(&vm_bo->vm->status_lock);
183 	if (bo->tbo.type == ttm_bo_type_kernel)
184 		list_move(&vm_bo->vm_status, &vm->evicted);
185 	else
186 		list_move_tail(&vm_bo->vm_status, &vm->evicted);
187 	spin_unlock(&vm_bo->vm->status_lock);
188 }
189 /**
190  * amdgpu_vm_bo_moved - vm_bo is moved
191  *
192  * @vm_bo: vm_bo which is moved
193  *
194  * State for per VM BOs which are moved, but that change is not yet reflected
195  * in the page tables.
196  */
197 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
198 {
199 	spin_lock(&vm_bo->vm->status_lock);
200 	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
201 	spin_unlock(&vm_bo->vm->status_lock);
202 }
203 
204 /**
205  * amdgpu_vm_bo_idle - vm_bo is idle
206  *
207  * @vm_bo: vm_bo which is now idle
208  *
209  * State for PDs/PTs and per VM BOs which have gone through the state machine
210  * and are now idle.
211  */
212 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
213 {
214 	spin_lock(&vm_bo->vm->status_lock);
215 	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
216 	spin_unlock(&vm_bo->vm->status_lock);
217 	vm_bo->moved = false;
218 }
219 
220 /**
221  * amdgpu_vm_bo_invalidated - vm_bo is invalidated
222  *
223  * @vm_bo: vm_bo which is now invalidated
224  *
225  * State for normal BOs which are invalidated and that change not yet reflected
226  * in the PTs.
227  */
228 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
229 {
230 	spin_lock(&vm_bo->vm->status_lock);
231 	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
232 	spin_unlock(&vm_bo->vm->status_lock);
233 }
234 
235 /**
236  * amdgpu_vm_bo_relocated - vm_bo is reloacted
237  *
238  * @vm_bo: vm_bo which is relocated
239  *
240  * State for PDs/PTs which needs to update their parent PD.
241  * For the root PD, just move to idle state.
242  */
243 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
244 {
245 	if (vm_bo->bo->parent) {
246 		spin_lock(&vm_bo->vm->status_lock);
247 		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
248 		spin_unlock(&vm_bo->vm->status_lock);
249 	} else {
250 		amdgpu_vm_bo_idle(vm_bo);
251 	}
252 }
253 
254 /**
255  * amdgpu_vm_bo_done - vm_bo is done
256  *
257  * @vm_bo: vm_bo which is now done
258  *
259  * State for normal BOs which are invalidated and that change has been updated
260  * in the PTs.
261  */
262 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
263 {
264 	spin_lock(&vm_bo->vm->status_lock);
265 	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
266 	spin_unlock(&vm_bo->vm->status_lock);
267 }
268 
269 /**
270  * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
271  *
272  * @base: base structure for tracking BO usage in a VM
273  * @vm: vm to which bo is to be added
274  * @bo: amdgpu buffer object
275  *
276  * Initialize a bo_va_base structure and add it to the appropriate lists
277  *
278  */
279 void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
280 			    struct amdgpu_vm *vm, struct amdgpu_bo *bo)
281 {
282 	base->vm = vm;
283 	base->bo = bo;
284 	base->next = NULL;
285 	INIT_LIST_HEAD(&base->vm_status);
286 
287 	if (!bo)
288 		return;
289 	base->next = bo->vm_bo;
290 	bo->vm_bo = base;
291 
292 	if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv)
293 		return;
294 
295 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
296 
297 	ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
298 	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
299 		amdgpu_vm_bo_relocated(base);
300 	else
301 		amdgpu_vm_bo_idle(base);
302 
303 	if (bo->preferred_domains &
304 	    amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
305 		return;
306 
307 	/*
308 	 * we checked all the prerequisites, but it looks like this per vm bo
309 	 * is currently evicted. add the bo to the evicted list to make sure it
310 	 * is validated on next vm use to avoid fault.
311 	 * */
312 	amdgpu_vm_bo_evicted(base);
313 }
314 
315 /**
316  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
317  *
318  * @vm: vm providing the BOs
319  * @validated: head of validation list
320  * @entry: entry to add
321  *
322  * Add the page directory to the list of BOs to
323  * validate for command submission.
324  */
325 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
326 			 struct list_head *validated,
327 			 struct amdgpu_bo_list_entry *entry)
328 {
329 	entry->priority = 0;
330 	entry->tv.bo = &vm->root.bo->tbo;
331 	/* Two for VM updates, one for TTM and one for the CS job */
332 	entry->tv.num_shared = 4;
333 	entry->user_pages = NULL;
334 	list_add(&entry->tv.head, validated);
335 }
336 
337 /**
338  * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
339  *
340  * @adev: amdgpu device pointer
341  * @vm: vm providing the BOs
342  *
343  * Move all BOs to the end of LRU and remember their positions to put them
344  * together.
345  */
346 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
347 				struct amdgpu_vm *vm)
348 {
349 	spin_lock(&adev->mman.bdev.lru_lock);
350 	ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
351 	spin_unlock(&adev->mman.bdev.lru_lock);
352 }
353 
354 /**
355  * amdgpu_vm_validate_pt_bos - validate the page table BOs
356  *
357  * @adev: amdgpu device pointer
358  * @vm: vm providing the BOs
359  * @validate: callback to do the validation
360  * @param: parameter for the validation callback
361  *
362  * Validate the page table BOs on command submission if neccessary.
363  *
364  * Returns:
365  * Validation result.
366  */
367 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
368 			      int (*validate)(void *p, struct amdgpu_bo *bo),
369 			      void *param)
370 {
371 	struct amdgpu_vm_bo_base *bo_base;
372 	struct amdgpu_bo *shadow;
373 	struct amdgpu_bo *bo;
374 	int r;
375 
376 	spin_lock(&vm->status_lock);
377 	while (!list_empty(&vm->evicted)) {
378 		bo_base = list_first_entry(&vm->evicted,
379 					   struct amdgpu_vm_bo_base,
380 					   vm_status);
381 		spin_unlock(&vm->status_lock);
382 
383 		bo = bo_base->bo;
384 		shadow = amdgpu_bo_shadowed(bo);
385 
386 		r = validate(param, bo);
387 		if (r)
388 			return r;
389 		if (shadow) {
390 			r = validate(param, shadow);
391 			if (r)
392 				return r;
393 		}
394 
395 		if (bo->tbo.type != ttm_bo_type_kernel) {
396 			amdgpu_vm_bo_moved(bo_base);
397 		} else {
398 			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
399 			amdgpu_vm_bo_relocated(bo_base);
400 		}
401 		spin_lock(&vm->status_lock);
402 	}
403 	spin_unlock(&vm->status_lock);
404 
405 	amdgpu_vm_eviction_lock(vm);
406 	vm->evicting = false;
407 	amdgpu_vm_eviction_unlock(vm);
408 
409 	return 0;
410 }
411 
412 /**
413  * amdgpu_vm_ready - check VM is ready for updates
414  *
415  * @vm: VM to check
416  *
417  * Check if all VM PDs/PTs are ready for updates
418  *
419  * Returns:
420  * True if VM is not evicting.
421  */
422 bool amdgpu_vm_ready(struct amdgpu_vm *vm)
423 {
424 	bool empty;
425 	bool ret;
426 
427 	amdgpu_vm_eviction_lock(vm);
428 	ret = !vm->evicting;
429 	amdgpu_vm_eviction_unlock(vm);
430 
431 	spin_lock(&vm->status_lock);
432 	empty = list_empty(&vm->evicted);
433 	spin_unlock(&vm->status_lock);
434 
435 	return ret && empty;
436 }
437 
438 /**
439  * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
440  *
441  * @adev: amdgpu_device pointer
442  */
443 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
444 {
445 	const struct amdgpu_ip_block *ip_block;
446 	bool has_compute_vm_bug;
447 	struct amdgpu_ring *ring;
448 	int i;
449 
450 	has_compute_vm_bug = false;
451 
452 	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
453 	if (ip_block) {
454 		/* Compute has a VM bug for GFX version < 7.
455 		   Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
456 		if (ip_block->version->major <= 7)
457 			has_compute_vm_bug = true;
458 		else if (ip_block->version->major == 8)
459 			if (adev->gfx.mec_fw_version < 673)
460 				has_compute_vm_bug = true;
461 	}
462 
463 	for (i = 0; i < adev->num_rings; i++) {
464 		ring = adev->rings[i];
465 		if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
466 			/* only compute rings */
467 			ring->has_compute_vm_bug = has_compute_vm_bug;
468 		else
469 			ring->has_compute_vm_bug = false;
470 	}
471 }
472 
473 /**
474  * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
475  *
476  * @ring: ring on which the job will be submitted
477  * @job: job to submit
478  *
479  * Returns:
480  * True if sync is needed.
481  */
482 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
483 				  struct amdgpu_job *job)
484 {
485 	struct amdgpu_device *adev = ring->adev;
486 	unsigned vmhub = ring->vm_hub;
487 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
488 
489 	if (job->vmid == 0)
490 		return false;
491 
492 	if (job->vm_needs_flush || ring->has_compute_vm_bug)
493 		return true;
494 
495 	if (ring->funcs->emit_gds_switch && job->gds_switch_needed)
496 		return true;
497 
498 	if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid]))
499 		return true;
500 
501 	return false;
502 }
503 
504 /**
505  * amdgpu_vm_flush - hardware flush the vm
506  *
507  * @ring: ring to use for flush
508  * @job:  related job
509  * @need_pipe_sync: is pipe sync needed
510  *
511  * Emit a VM flush when it is necessary.
512  *
513  * Returns:
514  * 0 on success, errno otherwise.
515  */
516 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
517 		    bool need_pipe_sync)
518 {
519 	struct amdgpu_device *adev = ring->adev;
520 	unsigned vmhub = ring->vm_hub;
521 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
522 	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
523 	bool spm_update_needed = job->spm_update_needed;
524 	bool gds_switch_needed = ring->funcs->emit_gds_switch &&
525 		job->gds_switch_needed;
526 	bool vm_flush_needed = job->vm_needs_flush;
527 	struct dma_fence *fence = NULL;
528 	bool pasid_mapping_needed = false;
529 	unsigned patch_offset = 0;
530 	int r;
531 
532 	if (amdgpu_vmid_had_gpu_reset(adev, id)) {
533 		gds_switch_needed = true;
534 		vm_flush_needed = true;
535 		pasid_mapping_needed = true;
536 		spm_update_needed = true;
537 	}
538 
539 	mutex_lock(&id_mgr->lock);
540 	if (id->pasid != job->pasid || !id->pasid_mapping ||
541 	    !dma_fence_is_signaled(id->pasid_mapping))
542 		pasid_mapping_needed = true;
543 	mutex_unlock(&id_mgr->lock);
544 
545 	gds_switch_needed &= !!ring->funcs->emit_gds_switch;
546 	vm_flush_needed &= !!ring->funcs->emit_vm_flush  &&
547 			job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
548 	pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
549 		ring->funcs->emit_wreg;
550 
551 	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
552 		return 0;
553 
554 	amdgpu_ring_ib_begin(ring);
555 	if (ring->funcs->init_cond_exec)
556 		patch_offset = amdgpu_ring_init_cond_exec(ring);
557 
558 	if (need_pipe_sync)
559 		amdgpu_ring_emit_pipeline_sync(ring);
560 
561 	if (vm_flush_needed) {
562 		trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
563 		amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
564 	}
565 
566 	if (pasid_mapping_needed)
567 		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
568 
569 	if (spm_update_needed && adev->gfx.rlc.funcs->update_spm_vmid)
570 		adev->gfx.rlc.funcs->update_spm_vmid(adev, job->vmid);
571 
572 	if (!ring->is_mes_queue && ring->funcs->emit_gds_switch &&
573 	    gds_switch_needed) {
574 		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
575 					    job->gds_size, job->gws_base,
576 					    job->gws_size, job->oa_base,
577 					    job->oa_size);
578 	}
579 
580 	if (vm_flush_needed || pasid_mapping_needed) {
581 		r = amdgpu_fence_emit(ring, &fence, NULL, 0);
582 		if (r)
583 			return r;
584 	}
585 
586 	if (vm_flush_needed) {
587 		mutex_lock(&id_mgr->lock);
588 		dma_fence_put(id->last_flush);
589 		id->last_flush = dma_fence_get(fence);
590 		id->current_gpu_reset_count =
591 			atomic_read(&adev->gpu_reset_counter);
592 		mutex_unlock(&id_mgr->lock);
593 	}
594 
595 	if (pasid_mapping_needed) {
596 		mutex_lock(&id_mgr->lock);
597 		id->pasid = job->pasid;
598 		dma_fence_put(id->pasid_mapping);
599 		id->pasid_mapping = dma_fence_get(fence);
600 		mutex_unlock(&id_mgr->lock);
601 	}
602 	dma_fence_put(fence);
603 
604 	if (ring->funcs->patch_cond_exec)
605 		amdgpu_ring_patch_cond_exec(ring, patch_offset);
606 
607 	/* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
608 	if (ring->funcs->emit_switch_buffer) {
609 		amdgpu_ring_emit_switch_buffer(ring);
610 		amdgpu_ring_emit_switch_buffer(ring);
611 	}
612 	amdgpu_ring_ib_end(ring);
613 	return 0;
614 }
615 
616 /**
617  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
618  *
619  * @vm: requested vm
620  * @bo: requested buffer object
621  *
622  * Find @bo inside the requested vm.
623  * Search inside the @bos vm list for the requested vm
624  * Returns the found bo_va or NULL if none is found
625  *
626  * Object has to be reserved!
627  *
628  * Returns:
629  * Found bo_va or NULL.
630  */
631 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
632 				       struct amdgpu_bo *bo)
633 {
634 	struct amdgpu_vm_bo_base *base;
635 
636 	for (base = bo->vm_bo; base; base = base->next) {
637 		if (base->vm != vm)
638 			continue;
639 
640 		return container_of(base, struct amdgpu_bo_va, base);
641 	}
642 	return NULL;
643 }
644 
645 /**
646  * amdgpu_vm_map_gart - Resolve gart mapping of addr
647  *
648  * @pages_addr: optional DMA address to use for lookup
649  * @addr: the unmapped addr
650  *
651  * Look up the physical address of the page that the pte resolves
652  * to.
653  *
654  * Returns:
655  * The pointer for the page table entry.
656  */
657 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
658 {
659 	uint64_t result;
660 
661 	/* page table offset */
662 	result = pages_addr[addr >> PAGE_SHIFT];
663 
664 	/* in case cpu page size != gpu page size*/
665 	result |= addr & (~PAGE_MASK);
666 
667 	result &= 0xFFFFFFFFFFFFF000ULL;
668 
669 	return result;
670 }
671 
672 /**
673  * amdgpu_vm_update_pdes - make sure that all directories are valid
674  *
675  * @adev: amdgpu_device pointer
676  * @vm: requested vm
677  * @immediate: submit immediately to the paging queue
678  *
679  * Makes sure all directories are up to date.
680  *
681  * Returns:
682  * 0 for success, error for failure.
683  */
684 int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
685 			  struct amdgpu_vm *vm, bool immediate)
686 {
687 	struct amdgpu_vm_update_params params;
688 	struct amdgpu_vm_bo_base *entry;
689 	bool flush_tlb_needed = false;
690 	LIST_HEAD(relocated);
691 	int r, idx;
692 
693 	spin_lock(&vm->status_lock);
694 	list_splice_init(&vm->relocated, &relocated);
695 	spin_unlock(&vm->status_lock);
696 
697 	if (list_empty(&relocated))
698 		return 0;
699 
700 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
701 		return -ENODEV;
702 
703 	memset(&params, 0, sizeof(params));
704 	params.adev = adev;
705 	params.vm = vm;
706 	params.immediate = immediate;
707 
708 	r = vm->update_funcs->prepare(&params, NULL, AMDGPU_SYNC_EXPLICIT);
709 	if (r)
710 		goto error;
711 
712 	list_for_each_entry(entry, &relocated, vm_status) {
713 		/* vm_flush_needed after updating moved PDEs */
714 		flush_tlb_needed |= entry->moved;
715 
716 		r = amdgpu_vm_pde_update(&params, entry);
717 		if (r)
718 			goto error;
719 	}
720 
721 	r = vm->update_funcs->commit(&params, &vm->last_update);
722 	if (r)
723 		goto error;
724 
725 	if (flush_tlb_needed)
726 		atomic64_inc(&vm->tlb_seq);
727 
728 	while (!list_empty(&relocated)) {
729 		entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
730 					 vm_status);
731 		amdgpu_vm_bo_idle(entry);
732 	}
733 
734 error:
735 	drm_dev_exit(idx);
736 	return r;
737 }
738 
739 /**
740  * amdgpu_vm_tlb_seq_cb - make sure to increment tlb sequence
741  * @fence: unused
742  * @cb: the callback structure
743  *
744  * Increments the tlb sequence to make sure that future CS execute a VM flush.
745  */
746 static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence,
747 				 struct dma_fence_cb *cb)
748 {
749 	struct amdgpu_vm_tlb_seq_cb *tlb_cb;
750 
751 	tlb_cb = container_of(cb, typeof(*tlb_cb), cb);
752 	atomic64_inc(&tlb_cb->vm->tlb_seq);
753 	kfree(tlb_cb);
754 }
755 
756 /**
757  * amdgpu_vm_update_range - update a range in the vm page table
758  *
759  * @adev: amdgpu_device pointer to use for commands
760  * @vm: the VM to update the range
761  * @immediate: immediate submission in a page fault
762  * @unlocked: unlocked invalidation during MM callback
763  * @flush_tlb: trigger tlb invalidation after update completed
764  * @resv: fences we need to sync to
765  * @start: start of mapped range
766  * @last: last mapped entry
767  * @flags: flags for the entries
768  * @offset: offset into nodes and pages_addr
769  * @vram_base: base for vram mappings
770  * @res: ttm_resource to map
771  * @pages_addr: DMA addresses to use for mapping
772  * @fence: optional resulting fence
773  *
774  * Fill in the page table entries between @start and @last.
775  *
776  * Returns:
777  * 0 for success, negative erro code for failure.
778  */
779 int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
780 			   bool immediate, bool unlocked, bool flush_tlb,
781 			   struct dma_resv *resv, uint64_t start, uint64_t last,
782 			   uint64_t flags, uint64_t offset, uint64_t vram_base,
783 			   struct ttm_resource *res, dma_addr_t *pages_addr,
784 			   struct dma_fence **fence)
785 {
786 	struct amdgpu_vm_update_params params;
787 	struct amdgpu_vm_tlb_seq_cb *tlb_cb;
788 	struct amdgpu_res_cursor cursor;
789 	enum amdgpu_sync_mode sync_mode;
790 	int r, idx;
791 
792 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
793 		return -ENODEV;
794 
795 	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
796 	if (!tlb_cb) {
797 		r = -ENOMEM;
798 		goto error_unlock;
799 	}
800 
801 	/* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache,
802 	 * heavy-weight flush TLB unconditionally.
803 	 */
804 	flush_tlb |= adev->gmc.xgmi.num_physical_nodes &&
805 		     adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0);
806 
807 	/*
808 	 * On GFX8 and older any 8 PTE block with a valid bit set enters the TLB
809 	 */
810 	flush_tlb |= adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 0, 0);
811 
812 	memset(&params, 0, sizeof(params));
813 	params.adev = adev;
814 	params.vm = vm;
815 	params.immediate = immediate;
816 	params.pages_addr = pages_addr;
817 	params.unlocked = unlocked;
818 
819 	/* Implicitly sync to command submissions in the same VM before
820 	 * unmapping. Sync to moving fences before mapping.
821 	 */
822 	if (!(flags & AMDGPU_PTE_VALID))
823 		sync_mode = AMDGPU_SYNC_EQ_OWNER;
824 	else
825 		sync_mode = AMDGPU_SYNC_EXPLICIT;
826 
827 	amdgpu_vm_eviction_lock(vm);
828 	if (vm->evicting) {
829 		r = -EBUSY;
830 		goto error_free;
831 	}
832 
833 	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
834 		struct dma_fence *tmp = dma_fence_get_stub();
835 
836 		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
837 		swap(vm->last_unlocked, tmp);
838 		dma_fence_put(tmp);
839 	}
840 
841 	r = vm->update_funcs->prepare(&params, resv, sync_mode);
842 	if (r)
843 		goto error_free;
844 
845 	amdgpu_res_first(pages_addr ? NULL : res, offset,
846 			 (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor);
847 	while (cursor.remaining) {
848 		uint64_t tmp, num_entries, addr;
849 
850 		num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;
851 		if (pages_addr) {
852 			bool contiguous = true;
853 
854 			if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) {
855 				uint64_t pfn = cursor.start >> PAGE_SHIFT;
856 				uint64_t count;
857 
858 				contiguous = pages_addr[pfn + 1] ==
859 					pages_addr[pfn] + PAGE_SIZE;
860 
861 				tmp = num_entries /
862 					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
863 				for (count = 2; count < tmp; ++count) {
864 					uint64_t idx = pfn + count;
865 
866 					if (contiguous != (pages_addr[idx] ==
867 					    pages_addr[idx - 1] + PAGE_SIZE))
868 						break;
869 				}
870 				if (!contiguous)
871 					count--;
872 				num_entries = count *
873 					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
874 			}
875 
876 			if (!contiguous) {
877 				addr = cursor.start;
878 				params.pages_addr = pages_addr;
879 			} else {
880 				addr = pages_addr[cursor.start >> PAGE_SHIFT];
881 				params.pages_addr = NULL;
882 			}
883 
884 		} else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) {
885 			addr = vram_base + cursor.start;
886 		} else {
887 			addr = 0;
888 		}
889 
890 		tmp = start + num_entries;
891 		r = amdgpu_vm_ptes_update(&params, start, tmp, addr, flags);
892 		if (r)
893 			goto error_free;
894 
895 		amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE);
896 		start = tmp;
897 	}
898 
899 	r = vm->update_funcs->commit(&params, fence);
900 
901 	if (flush_tlb || params.table_freed) {
902 		tlb_cb->vm = vm;
903 		if (fence && *fence &&
904 		    !dma_fence_add_callback(*fence, &tlb_cb->cb,
905 					   amdgpu_vm_tlb_seq_cb)) {
906 			dma_fence_put(vm->last_tlb_flush);
907 			vm->last_tlb_flush = dma_fence_get(*fence);
908 		} else {
909 			amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);
910 		}
911 		tlb_cb = NULL;
912 	}
913 
914 error_free:
915 	kfree(tlb_cb);
916 
917 error_unlock:
918 	amdgpu_vm_eviction_unlock(vm);
919 	drm_dev_exit(idx);
920 	return r;
921 }
922 
923 void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
924 			  struct amdgpu_mem_stats *stats)
925 {
926 	struct amdgpu_bo_va *bo_va, *tmp;
927 
928 	spin_lock(&vm->status_lock);
929 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
930 		if (!bo_va->base.bo)
931 			continue;
932 		amdgpu_bo_get_memory(bo_va->base.bo, stats);
933 	}
934 	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
935 		if (!bo_va->base.bo)
936 			continue;
937 		amdgpu_bo_get_memory(bo_va->base.bo, stats);
938 	}
939 	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
940 		if (!bo_va->base.bo)
941 			continue;
942 		amdgpu_bo_get_memory(bo_va->base.bo, stats);
943 	}
944 	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
945 		if (!bo_va->base.bo)
946 			continue;
947 		amdgpu_bo_get_memory(bo_va->base.bo, stats);
948 	}
949 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
950 		if (!bo_va->base.bo)
951 			continue;
952 		amdgpu_bo_get_memory(bo_va->base.bo, stats);
953 	}
954 	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
955 		if (!bo_va->base.bo)
956 			continue;
957 		amdgpu_bo_get_memory(bo_va->base.bo, stats);
958 	}
959 	spin_unlock(&vm->status_lock);
960 }
961 
962 /**
963  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
964  *
965  * @adev: amdgpu_device pointer
966  * @bo_va: requested BO and VM object
967  * @clear: if true clear the entries
968  *
969  * Fill in the page table entries for @bo_va.
970  *
971  * Returns:
972  * 0 for success, -EINVAL for failure.
973  */
974 int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
975 			bool clear)
976 {
977 	struct amdgpu_bo *bo = bo_va->base.bo;
978 	struct amdgpu_vm *vm = bo_va->base.vm;
979 	struct amdgpu_bo_va_mapping *mapping;
980 	dma_addr_t *pages_addr = NULL;
981 	struct ttm_resource *mem;
982 	struct dma_fence **last_update;
983 	bool flush_tlb = clear;
984 	struct dma_resv *resv;
985 	uint64_t vram_base;
986 	uint64_t flags;
987 	int r;
988 
989 	if (clear || !bo) {
990 		mem = NULL;
991 		resv = vm->root.bo->tbo.base.resv;
992 	} else {
993 		struct drm_gem_object *obj = &bo->tbo.base;
994 
995 		resv = bo->tbo.base.resv;
996 		if (obj->import_attach && bo_va->is_xgmi) {
997 			struct dma_buf *dma_buf = obj->import_attach->dmabuf;
998 			struct drm_gem_object *gobj = dma_buf->priv;
999 			struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
1000 
1001 			if (abo->tbo.resource->mem_type == TTM_PL_VRAM)
1002 				bo = gem_to_amdgpu_bo(gobj);
1003 		}
1004 		mem = bo->tbo.resource;
1005 		if (mem->mem_type == TTM_PL_TT ||
1006 		    mem->mem_type == AMDGPU_PL_PREEMPT)
1007 			pages_addr = bo->tbo.ttm->dma_address;
1008 	}
1009 
1010 	if (bo) {
1011 		struct amdgpu_device *bo_adev;
1012 
1013 		flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1014 
1015 		if (amdgpu_bo_encrypted(bo))
1016 			flags |= AMDGPU_PTE_TMZ;
1017 
1018 		bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1019 		vram_base = bo_adev->vm_manager.vram_base_offset;
1020 	} else {
1021 		flags = 0x0;
1022 		vram_base = 0;
1023 	}
1024 
1025 	if (clear || (bo && bo->tbo.base.resv ==
1026 		      vm->root.bo->tbo.base.resv))
1027 		last_update = &vm->last_update;
1028 	else
1029 		last_update = &bo_va->last_pt_update;
1030 
1031 	if (!clear && bo_va->base.moved) {
1032 		flush_tlb = true;
1033 		list_splice_init(&bo_va->valids, &bo_va->invalids);
1034 
1035 	} else if (bo_va->cleared != clear) {
1036 		list_splice_init(&bo_va->valids, &bo_va->invalids);
1037 	}
1038 
1039 	list_for_each_entry(mapping, &bo_va->invalids, list) {
1040 		uint64_t update_flags = flags;
1041 
1042 		/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1043 		 * but in case of something, we filter the flags in first place
1044 		 */
1045 		if (!(mapping->flags & AMDGPU_PTE_READABLE))
1046 			update_flags &= ~AMDGPU_PTE_READABLE;
1047 		if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1048 			update_flags &= ~AMDGPU_PTE_WRITEABLE;
1049 
1050 		/* Apply ASIC specific mapping flags */
1051 		amdgpu_gmc_get_vm_pte(adev, mapping, &update_flags);
1052 
1053 		trace_amdgpu_vm_bo_update(mapping);
1054 
1055 		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb,
1056 					   resv, mapping->start, mapping->last,
1057 					   update_flags, mapping->offset,
1058 					   vram_base, mem, pages_addr,
1059 					   last_update);
1060 		if (r)
1061 			return r;
1062 	}
1063 
1064 	/* If the BO is not in its preferred location add it back to
1065 	 * the evicted list so that it gets validated again on the
1066 	 * next command submission.
1067 	 */
1068 	if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
1069 		uint32_t mem_type = bo->tbo.resource->mem_type;
1070 
1071 		if (!(bo->preferred_domains &
1072 		      amdgpu_mem_type_to_domain(mem_type)))
1073 			amdgpu_vm_bo_evicted(&bo_va->base);
1074 		else
1075 			amdgpu_vm_bo_idle(&bo_va->base);
1076 	} else {
1077 		amdgpu_vm_bo_done(&bo_va->base);
1078 	}
1079 
1080 	list_splice_init(&bo_va->invalids, &bo_va->valids);
1081 	bo_va->cleared = clear;
1082 	bo_va->base.moved = false;
1083 
1084 	if (trace_amdgpu_vm_bo_mapping_enabled()) {
1085 		list_for_each_entry(mapping, &bo_va->valids, list)
1086 			trace_amdgpu_vm_bo_mapping(mapping);
1087 	}
1088 
1089 	return 0;
1090 }
1091 
1092 /**
1093  * amdgpu_vm_update_prt_state - update the global PRT state
1094  *
1095  * @adev: amdgpu_device pointer
1096  */
1097 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1098 {
1099 	unsigned long flags;
1100 	bool enable;
1101 
1102 	spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1103 	enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1104 	adev->gmc.gmc_funcs->set_prt(adev, enable);
1105 	spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1106 }
1107 
1108 /**
1109  * amdgpu_vm_prt_get - add a PRT user
1110  *
1111  * @adev: amdgpu_device pointer
1112  */
1113 static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1114 {
1115 	if (!adev->gmc.gmc_funcs->set_prt)
1116 		return;
1117 
1118 	if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1119 		amdgpu_vm_update_prt_state(adev);
1120 }
1121 
1122 /**
1123  * amdgpu_vm_prt_put - drop a PRT user
1124  *
1125  * @adev: amdgpu_device pointer
1126  */
1127 static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1128 {
1129 	if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1130 		amdgpu_vm_update_prt_state(adev);
1131 }
1132 
1133 /**
1134  * amdgpu_vm_prt_cb - callback for updating the PRT status
1135  *
1136  * @fence: fence for the callback
1137  * @_cb: the callback function
1138  */
1139 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1140 {
1141 	struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1142 
1143 	amdgpu_vm_prt_put(cb->adev);
1144 	kfree(cb);
1145 }
1146 
1147 /**
1148  * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1149  *
1150  * @adev: amdgpu_device pointer
1151  * @fence: fence for the callback
1152  */
1153 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1154 				 struct dma_fence *fence)
1155 {
1156 	struct amdgpu_prt_cb *cb;
1157 
1158 	if (!adev->gmc.gmc_funcs->set_prt)
1159 		return;
1160 
1161 	cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1162 	if (!cb) {
1163 		/* Last resort when we are OOM */
1164 		if (fence)
1165 			dma_fence_wait(fence, false);
1166 
1167 		amdgpu_vm_prt_put(adev);
1168 	} else {
1169 		cb->adev = adev;
1170 		if (!fence || dma_fence_add_callback(fence, &cb->cb,
1171 						     amdgpu_vm_prt_cb))
1172 			amdgpu_vm_prt_cb(fence, &cb->cb);
1173 	}
1174 }
1175 
1176 /**
1177  * amdgpu_vm_free_mapping - free a mapping
1178  *
1179  * @adev: amdgpu_device pointer
1180  * @vm: requested vm
1181  * @mapping: mapping to be freed
1182  * @fence: fence of the unmap operation
1183  *
1184  * Free a mapping and make sure we decrease the PRT usage count if applicable.
1185  */
1186 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1187 				   struct amdgpu_vm *vm,
1188 				   struct amdgpu_bo_va_mapping *mapping,
1189 				   struct dma_fence *fence)
1190 {
1191 	if (mapping->flags & AMDGPU_PTE_PRT)
1192 		amdgpu_vm_add_prt_cb(adev, fence);
1193 	kfree(mapping);
1194 }
1195 
1196 /**
1197  * amdgpu_vm_prt_fini - finish all prt mappings
1198  *
1199  * @adev: amdgpu_device pointer
1200  * @vm: requested vm
1201  *
1202  * Register a cleanup callback to disable PRT support after VM dies.
1203  */
1204 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1205 {
1206 	struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1207 	struct dma_resv_iter cursor;
1208 	struct dma_fence *fence;
1209 
1210 	dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {
1211 		/* Add a callback for each fence in the reservation object */
1212 		amdgpu_vm_prt_get(adev);
1213 		amdgpu_vm_add_prt_cb(adev, fence);
1214 	}
1215 }
1216 
1217 /**
1218  * amdgpu_vm_clear_freed - clear freed BOs in the PT
1219  *
1220  * @adev: amdgpu_device pointer
1221  * @vm: requested vm
1222  * @fence: optional resulting fence (unchanged if no work needed to be done
1223  * or if an error occurred)
1224  *
1225  * Make sure all freed BOs are cleared in the PT.
1226  * PTs have to be reserved and mutex must be locked!
1227  *
1228  * Returns:
1229  * 0 for success.
1230  *
1231  */
1232 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1233 			  struct amdgpu_vm *vm,
1234 			  struct dma_fence **fence)
1235 {
1236 	struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1237 	struct amdgpu_bo_va_mapping *mapping;
1238 	uint64_t init_pte_value = 0;
1239 	struct dma_fence *f = NULL;
1240 	int r;
1241 
1242 	while (!list_empty(&vm->freed)) {
1243 		mapping = list_first_entry(&vm->freed,
1244 			struct amdgpu_bo_va_mapping, list);
1245 		list_del(&mapping->list);
1246 
1247 		if (vm->pte_support_ats &&
1248 		    mapping->start < AMDGPU_GMC_HOLE_START)
1249 			init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
1250 
1251 		r = amdgpu_vm_update_range(adev, vm, false, false, true, resv,
1252 					   mapping->start, mapping->last,
1253 					   init_pte_value, 0, 0, NULL, NULL,
1254 					   &f);
1255 		amdgpu_vm_free_mapping(adev, vm, mapping, f);
1256 		if (r) {
1257 			dma_fence_put(f);
1258 			return r;
1259 		}
1260 	}
1261 
1262 	if (fence && f) {
1263 		dma_fence_put(*fence);
1264 		*fence = f;
1265 	} else {
1266 		dma_fence_put(f);
1267 	}
1268 
1269 	return 0;
1270 
1271 }
1272 
1273 /**
1274  * amdgpu_vm_handle_moved - handle moved BOs in the PT
1275  *
1276  * @adev: amdgpu_device pointer
1277  * @vm: requested vm
1278  *
1279  * Make sure all BOs which are moved are updated in the PTs.
1280  *
1281  * Returns:
1282  * 0 for success.
1283  *
1284  * PTs have to be reserved!
1285  */
1286 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1287 			   struct amdgpu_vm *vm)
1288 {
1289 	struct amdgpu_bo_va *bo_va;
1290 	struct dma_resv *resv;
1291 	bool clear;
1292 	int r;
1293 
1294 	spin_lock(&vm->status_lock);
1295 	while (!list_empty(&vm->moved)) {
1296 		bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
1297 					 base.vm_status);
1298 		spin_unlock(&vm->status_lock);
1299 
1300 		/* Per VM BOs never need to bo cleared in the page tables */
1301 		r = amdgpu_vm_bo_update(adev, bo_va, false);
1302 		if (r)
1303 			return r;
1304 		spin_lock(&vm->status_lock);
1305 	}
1306 
1307 	while (!list_empty(&vm->invalidated)) {
1308 		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
1309 					 base.vm_status);
1310 		resv = bo_va->base.bo->tbo.base.resv;
1311 		spin_unlock(&vm->status_lock);
1312 
1313 		/* Try to reserve the BO to avoid clearing its ptes */
1314 		if (!amdgpu_vm_debug && dma_resv_trylock(resv))
1315 			clear = false;
1316 		/* Somebody else is using the BO right now */
1317 		else
1318 			clear = true;
1319 
1320 		r = amdgpu_vm_bo_update(adev, bo_va, clear);
1321 		if (r)
1322 			return r;
1323 
1324 		if (!clear)
1325 			dma_resv_unlock(resv);
1326 		spin_lock(&vm->status_lock);
1327 	}
1328 	spin_unlock(&vm->status_lock);
1329 
1330 	return 0;
1331 }
1332 
1333 /**
1334  * amdgpu_vm_bo_add - add a bo to a specific vm
1335  *
1336  * @adev: amdgpu_device pointer
1337  * @vm: requested vm
1338  * @bo: amdgpu buffer object
1339  *
1340  * Add @bo into the requested vm.
1341  * Add @bo to the list of bos associated with the vm
1342  *
1343  * Returns:
1344  * Newly added bo_va or NULL for failure
1345  *
1346  * Object has to be reserved!
1347  */
1348 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1349 				      struct amdgpu_vm *vm,
1350 				      struct amdgpu_bo *bo)
1351 {
1352 	struct amdgpu_bo_va *bo_va;
1353 
1354 	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1355 	if (bo_va == NULL) {
1356 		return NULL;
1357 	}
1358 	amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
1359 
1360 	bo_va->ref_count = 1;
1361 	INIT_LIST_HEAD(&bo_va->valids);
1362 	INIT_LIST_HEAD(&bo_va->invalids);
1363 
1364 	if (!bo)
1365 		return bo_va;
1366 
1367 	dma_resv_assert_held(bo->tbo.base.resv);
1368 	if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) {
1369 		bo_va->is_xgmi = true;
1370 		/* Power up XGMI if it can be potentially used */
1371 		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20);
1372 	}
1373 
1374 	return bo_va;
1375 }
1376 
1377 
1378 /**
1379  * amdgpu_vm_bo_insert_map - insert a new mapping
1380  *
1381  * @adev: amdgpu_device pointer
1382  * @bo_va: bo_va to store the address
1383  * @mapping: the mapping to insert
1384  *
1385  * Insert a new mapping into all structures.
1386  */
1387 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
1388 				    struct amdgpu_bo_va *bo_va,
1389 				    struct amdgpu_bo_va_mapping *mapping)
1390 {
1391 	struct amdgpu_vm *vm = bo_va->base.vm;
1392 	struct amdgpu_bo *bo = bo_va->base.bo;
1393 
1394 	mapping->bo_va = bo_va;
1395 	list_add(&mapping->list, &bo_va->invalids);
1396 	amdgpu_vm_it_insert(mapping, &vm->va);
1397 
1398 	if (mapping->flags & AMDGPU_PTE_PRT)
1399 		amdgpu_vm_prt_get(adev);
1400 
1401 	if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv &&
1402 	    !bo_va->base.moved) {
1403 		amdgpu_vm_bo_moved(&bo_va->base);
1404 	}
1405 	trace_amdgpu_vm_bo_map(bo_va, mapping);
1406 }
1407 
1408 /**
1409  * amdgpu_vm_bo_map - map bo inside a vm
1410  *
1411  * @adev: amdgpu_device pointer
1412  * @bo_va: bo_va to store the address
1413  * @saddr: where to map the BO
1414  * @offset: requested offset in the BO
1415  * @size: BO size in bytes
1416  * @flags: attributes of pages (read/write/valid/etc.)
1417  *
1418  * Add a mapping of the BO at the specefied addr into the VM.
1419  *
1420  * Returns:
1421  * 0 for success, error for failure.
1422  *
1423  * Object has to be reserved and unreserved outside!
1424  */
1425 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1426 		     struct amdgpu_bo_va *bo_va,
1427 		     uint64_t saddr, uint64_t offset,
1428 		     uint64_t size, uint64_t flags)
1429 {
1430 	struct amdgpu_bo_va_mapping *mapping, *tmp;
1431 	struct amdgpu_bo *bo = bo_va->base.bo;
1432 	struct amdgpu_vm *vm = bo_va->base.vm;
1433 	uint64_t eaddr;
1434 
1435 	/* validate the parameters */
1436 	if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK ||
1437 	    size == 0 || size & ~PAGE_MASK)
1438 		return -EINVAL;
1439 
1440 	/* make sure object fit at this offset */
1441 	eaddr = saddr + size - 1;
1442 	if (saddr >= eaddr ||
1443 	    (bo && offset + size > amdgpu_bo_size(bo)) ||
1444 	    (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
1445 		return -EINVAL;
1446 
1447 	saddr /= AMDGPU_GPU_PAGE_SIZE;
1448 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1449 
1450 	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1451 	if (tmp) {
1452 		/* bo and tmp overlap, invalid addr */
1453 		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1454 			"0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
1455 			tmp->start, tmp->last + 1);
1456 		return -EINVAL;
1457 	}
1458 
1459 	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1460 	if (!mapping)
1461 		return -ENOMEM;
1462 
1463 	mapping->start = saddr;
1464 	mapping->last = eaddr;
1465 	mapping->offset = offset;
1466 	mapping->flags = flags;
1467 
1468 	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1469 
1470 	return 0;
1471 }
1472 
1473 /**
1474  * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
1475  *
1476  * @adev: amdgpu_device pointer
1477  * @bo_va: bo_va to store the address
1478  * @saddr: where to map the BO
1479  * @offset: requested offset in the BO
1480  * @size: BO size in bytes
1481  * @flags: attributes of pages (read/write/valid/etc.)
1482  *
1483  * Add a mapping of the BO at the specefied addr into the VM. Replace existing
1484  * mappings as we do so.
1485  *
1486  * Returns:
1487  * 0 for success, error for failure.
1488  *
1489  * Object has to be reserved and unreserved outside!
1490  */
1491 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
1492 			     struct amdgpu_bo_va *bo_va,
1493 			     uint64_t saddr, uint64_t offset,
1494 			     uint64_t size, uint64_t flags)
1495 {
1496 	struct amdgpu_bo_va_mapping *mapping;
1497 	struct amdgpu_bo *bo = bo_va->base.bo;
1498 	uint64_t eaddr;
1499 	int r;
1500 
1501 	/* validate the parameters */
1502 	if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK ||
1503 	    size == 0 || size & ~PAGE_MASK)
1504 		return -EINVAL;
1505 
1506 	/* make sure object fit at this offset */
1507 	eaddr = saddr + size - 1;
1508 	if (saddr >= eaddr ||
1509 	    (bo && offset + size > amdgpu_bo_size(bo)) ||
1510 	    (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
1511 		return -EINVAL;
1512 
1513 	/* Allocate all the needed memory */
1514 	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1515 	if (!mapping)
1516 		return -ENOMEM;
1517 
1518 	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
1519 	if (r) {
1520 		kfree(mapping);
1521 		return r;
1522 	}
1523 
1524 	saddr /= AMDGPU_GPU_PAGE_SIZE;
1525 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1526 
1527 	mapping->start = saddr;
1528 	mapping->last = eaddr;
1529 	mapping->offset = offset;
1530 	mapping->flags = flags;
1531 
1532 	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1533 
1534 	return 0;
1535 }
1536 
1537 /**
1538  * amdgpu_vm_bo_unmap - remove bo mapping from vm
1539  *
1540  * @adev: amdgpu_device pointer
1541  * @bo_va: bo_va to remove the address from
1542  * @saddr: where to the BO is mapped
1543  *
1544  * Remove a mapping of the BO at the specefied addr from the VM.
1545  *
1546  * Returns:
1547  * 0 for success, error for failure.
1548  *
1549  * Object has to be reserved and unreserved outside!
1550  */
1551 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1552 		       struct amdgpu_bo_va *bo_va,
1553 		       uint64_t saddr)
1554 {
1555 	struct amdgpu_bo_va_mapping *mapping;
1556 	struct amdgpu_vm *vm = bo_va->base.vm;
1557 	bool valid = true;
1558 
1559 	saddr /= AMDGPU_GPU_PAGE_SIZE;
1560 
1561 	list_for_each_entry(mapping, &bo_va->valids, list) {
1562 		if (mapping->start == saddr)
1563 			break;
1564 	}
1565 
1566 	if (&mapping->list == &bo_va->valids) {
1567 		valid = false;
1568 
1569 		list_for_each_entry(mapping, &bo_va->invalids, list) {
1570 			if (mapping->start == saddr)
1571 				break;
1572 		}
1573 
1574 		if (&mapping->list == &bo_va->invalids)
1575 			return -ENOENT;
1576 	}
1577 
1578 	list_del(&mapping->list);
1579 	amdgpu_vm_it_remove(mapping, &vm->va);
1580 	mapping->bo_va = NULL;
1581 	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1582 
1583 	if (valid)
1584 		list_add(&mapping->list, &vm->freed);
1585 	else
1586 		amdgpu_vm_free_mapping(adev, vm, mapping,
1587 				       bo_va->last_pt_update);
1588 
1589 	return 0;
1590 }
1591 
1592 /**
1593  * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
1594  *
1595  * @adev: amdgpu_device pointer
1596  * @vm: VM structure to use
1597  * @saddr: start of the range
1598  * @size: size of the range
1599  *
1600  * Remove all mappings in a range, split them as appropriate.
1601  *
1602  * Returns:
1603  * 0 for success, error for failure.
1604  */
1605 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
1606 				struct amdgpu_vm *vm,
1607 				uint64_t saddr, uint64_t size)
1608 {
1609 	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
1610 	LIST_HEAD(removed);
1611 	uint64_t eaddr;
1612 
1613 	eaddr = saddr + size - 1;
1614 	saddr /= AMDGPU_GPU_PAGE_SIZE;
1615 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1616 
1617 	/* Allocate all the needed memory */
1618 	before = kzalloc(sizeof(*before), GFP_KERNEL);
1619 	if (!before)
1620 		return -ENOMEM;
1621 	INIT_LIST_HEAD(&before->list);
1622 
1623 	after = kzalloc(sizeof(*after), GFP_KERNEL);
1624 	if (!after) {
1625 		kfree(before);
1626 		return -ENOMEM;
1627 	}
1628 	INIT_LIST_HEAD(&after->list);
1629 
1630 	/* Now gather all removed mappings */
1631 	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1632 	while (tmp) {
1633 		/* Remember mapping split at the start */
1634 		if (tmp->start < saddr) {
1635 			before->start = tmp->start;
1636 			before->last = saddr - 1;
1637 			before->offset = tmp->offset;
1638 			before->flags = tmp->flags;
1639 			before->bo_va = tmp->bo_va;
1640 			list_add(&before->list, &tmp->bo_va->invalids);
1641 		}
1642 
1643 		/* Remember mapping split at the end */
1644 		if (tmp->last > eaddr) {
1645 			after->start = eaddr + 1;
1646 			after->last = tmp->last;
1647 			after->offset = tmp->offset;
1648 			after->offset += (after->start - tmp->start) << PAGE_SHIFT;
1649 			after->flags = tmp->flags;
1650 			after->bo_va = tmp->bo_va;
1651 			list_add(&after->list, &tmp->bo_va->invalids);
1652 		}
1653 
1654 		list_del(&tmp->list);
1655 		list_add(&tmp->list, &removed);
1656 
1657 		tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
1658 	}
1659 
1660 	/* And free them up */
1661 	list_for_each_entry_safe(tmp, next, &removed, list) {
1662 		amdgpu_vm_it_remove(tmp, &vm->va);
1663 		list_del(&tmp->list);
1664 
1665 		if (tmp->start < saddr)
1666 		    tmp->start = saddr;
1667 		if (tmp->last > eaddr)
1668 		    tmp->last = eaddr;
1669 
1670 		tmp->bo_va = NULL;
1671 		list_add(&tmp->list, &vm->freed);
1672 		trace_amdgpu_vm_bo_unmap(NULL, tmp);
1673 	}
1674 
1675 	/* Insert partial mapping before the range */
1676 	if (!list_empty(&before->list)) {
1677 		amdgpu_vm_it_insert(before, &vm->va);
1678 		if (before->flags & AMDGPU_PTE_PRT)
1679 			amdgpu_vm_prt_get(adev);
1680 	} else {
1681 		kfree(before);
1682 	}
1683 
1684 	/* Insert partial mapping after the range */
1685 	if (!list_empty(&after->list)) {
1686 		amdgpu_vm_it_insert(after, &vm->va);
1687 		if (after->flags & AMDGPU_PTE_PRT)
1688 			amdgpu_vm_prt_get(adev);
1689 	} else {
1690 		kfree(after);
1691 	}
1692 
1693 	return 0;
1694 }
1695 
1696 /**
1697  * amdgpu_vm_bo_lookup_mapping - find mapping by address
1698  *
1699  * @vm: the requested VM
1700  * @addr: the address
1701  *
1702  * Find a mapping by it's address.
1703  *
1704  * Returns:
1705  * The amdgpu_bo_va_mapping matching for addr or NULL
1706  *
1707  */
1708 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
1709 							 uint64_t addr)
1710 {
1711 	return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
1712 }
1713 
1714 /**
1715  * amdgpu_vm_bo_trace_cs - trace all reserved mappings
1716  *
1717  * @vm: the requested vm
1718  * @ticket: CS ticket
1719  *
1720  * Trace all mappings of BOs reserved during a command submission.
1721  */
1722 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
1723 {
1724 	struct amdgpu_bo_va_mapping *mapping;
1725 
1726 	if (!trace_amdgpu_vm_bo_cs_enabled())
1727 		return;
1728 
1729 	for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
1730 	     mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
1731 		if (mapping->bo_va && mapping->bo_va->base.bo) {
1732 			struct amdgpu_bo *bo;
1733 
1734 			bo = mapping->bo_va->base.bo;
1735 			if (dma_resv_locking_ctx(bo->tbo.base.resv) !=
1736 			    ticket)
1737 				continue;
1738 		}
1739 
1740 		trace_amdgpu_vm_bo_cs(mapping);
1741 	}
1742 }
1743 
1744 /**
1745  * amdgpu_vm_bo_del - remove a bo from a specific vm
1746  *
1747  * @adev: amdgpu_device pointer
1748  * @bo_va: requested bo_va
1749  *
1750  * Remove @bo_va->bo from the requested vm.
1751  *
1752  * Object have to be reserved!
1753  */
1754 void amdgpu_vm_bo_del(struct amdgpu_device *adev,
1755 		      struct amdgpu_bo_va *bo_va)
1756 {
1757 	struct amdgpu_bo_va_mapping *mapping, *next;
1758 	struct amdgpu_bo *bo = bo_va->base.bo;
1759 	struct amdgpu_vm *vm = bo_va->base.vm;
1760 	struct amdgpu_vm_bo_base **base;
1761 
1762 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
1763 
1764 	if (bo) {
1765 		dma_resv_assert_held(bo->tbo.base.resv);
1766 		if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
1767 			ttm_bo_set_bulk_move(&bo->tbo, NULL);
1768 
1769 		for (base = &bo_va->base.bo->vm_bo; *base;
1770 		     base = &(*base)->next) {
1771 			if (*base != &bo_va->base)
1772 				continue;
1773 
1774 			*base = bo_va->base.next;
1775 			break;
1776 		}
1777 	}
1778 
1779 	spin_lock(&vm->status_lock);
1780 	list_del(&bo_va->base.vm_status);
1781 	spin_unlock(&vm->status_lock);
1782 
1783 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
1784 		list_del(&mapping->list);
1785 		amdgpu_vm_it_remove(mapping, &vm->va);
1786 		mapping->bo_va = NULL;
1787 		trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1788 		list_add(&mapping->list, &vm->freed);
1789 	}
1790 	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
1791 		list_del(&mapping->list);
1792 		amdgpu_vm_it_remove(mapping, &vm->va);
1793 		amdgpu_vm_free_mapping(adev, vm, mapping,
1794 				       bo_va->last_pt_update);
1795 	}
1796 
1797 	dma_fence_put(bo_va->last_pt_update);
1798 
1799 	if (bo && bo_va->is_xgmi)
1800 		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN);
1801 
1802 	kfree(bo_va);
1803 }
1804 
1805 /**
1806  * amdgpu_vm_evictable - check if we can evict a VM
1807  *
1808  * @bo: A page table of the VM.
1809  *
1810  * Check if it is possible to evict a VM.
1811  */
1812 bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
1813 {
1814 	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
1815 
1816 	/* Page tables of a destroyed VM can go away immediately */
1817 	if (!bo_base || !bo_base->vm)
1818 		return true;
1819 
1820 	/* Don't evict VM page tables while they are busy */
1821 	if (!dma_resv_test_signaled(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP))
1822 		return false;
1823 
1824 	/* Try to block ongoing updates */
1825 	if (!amdgpu_vm_eviction_trylock(bo_base->vm))
1826 		return false;
1827 
1828 	/* Don't evict VM page tables while they are updated */
1829 	if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
1830 		amdgpu_vm_eviction_unlock(bo_base->vm);
1831 		return false;
1832 	}
1833 
1834 	bo_base->vm->evicting = true;
1835 	amdgpu_vm_eviction_unlock(bo_base->vm);
1836 	return true;
1837 }
1838 
1839 /**
1840  * amdgpu_vm_bo_invalidate - mark the bo as invalid
1841  *
1842  * @adev: amdgpu_device pointer
1843  * @bo: amdgpu buffer object
1844  * @evicted: is the BO evicted
1845  *
1846  * Mark @bo as invalid.
1847  */
1848 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
1849 			     struct amdgpu_bo *bo, bool evicted)
1850 {
1851 	struct amdgpu_vm_bo_base *bo_base;
1852 
1853 	/* shadow bo doesn't have bo base, its validation needs its parent */
1854 	if (bo->parent && (amdgpu_bo_shadowed(bo->parent) == bo))
1855 		bo = bo->parent;
1856 
1857 	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
1858 		struct amdgpu_vm *vm = bo_base->vm;
1859 
1860 		if (evicted && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
1861 			amdgpu_vm_bo_evicted(bo_base);
1862 			continue;
1863 		}
1864 
1865 		if (bo_base->moved)
1866 			continue;
1867 		bo_base->moved = true;
1868 
1869 		if (bo->tbo.type == ttm_bo_type_kernel)
1870 			amdgpu_vm_bo_relocated(bo_base);
1871 		else if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
1872 			amdgpu_vm_bo_moved(bo_base);
1873 		else
1874 			amdgpu_vm_bo_invalidated(bo_base);
1875 	}
1876 }
1877 
1878 /**
1879  * amdgpu_vm_get_block_size - calculate VM page table size as power of two
1880  *
1881  * @vm_size: VM size
1882  *
1883  * Returns:
1884  * VM page table as power of two
1885  */
1886 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
1887 {
1888 	/* Total bits covered by PD + PTs */
1889 	unsigned bits = ilog2(vm_size) + 18;
1890 
1891 	/* Make sure the PD is 4K in size up to 8GB address space.
1892 	   Above that split equal between PD and PTs */
1893 	if (vm_size <= 8)
1894 		return (bits - 9);
1895 	else
1896 		return ((bits + 3) / 2);
1897 }
1898 
1899 /**
1900  * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
1901  *
1902  * @adev: amdgpu_device pointer
1903  * @min_vm_size: the minimum vm size in GB if it's set auto
1904  * @fragment_size_default: Default PTE fragment size
1905  * @max_level: max VMPT level
1906  * @max_bits: max address space size in bits
1907  *
1908  */
1909 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
1910 			   uint32_t fragment_size_default, unsigned max_level,
1911 			   unsigned max_bits)
1912 {
1913 	unsigned int max_size = 1 << (max_bits - 30);
1914 	unsigned int vm_size;
1915 	uint64_t tmp;
1916 
1917 	/* adjust vm size first */
1918 	if (amdgpu_vm_size != -1) {
1919 		vm_size = amdgpu_vm_size;
1920 		if (vm_size > max_size) {
1921 			dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
1922 				 amdgpu_vm_size, max_size);
1923 			vm_size = max_size;
1924 		}
1925 	} else {
1926 		struct sysinfo si;
1927 		unsigned int phys_ram_gb;
1928 
1929 		/* Optimal VM size depends on the amount of physical
1930 		 * RAM available. Underlying requirements and
1931 		 * assumptions:
1932 		 *
1933 		 *  - Need to map system memory and VRAM from all GPUs
1934 		 *     - VRAM from other GPUs not known here
1935 		 *     - Assume VRAM <= system memory
1936 		 *  - On GFX8 and older, VM space can be segmented for
1937 		 *    different MTYPEs
1938 		 *  - Need to allow room for fragmentation, guard pages etc.
1939 		 *
1940 		 * This adds up to a rough guess of system memory x3.
1941 		 * Round up to power of two to maximize the available
1942 		 * VM size with the given page table size.
1943 		 */
1944 		si_meminfo(&si);
1945 		phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
1946 			       (1 << 30) - 1) >> 30;
1947 		vm_size = roundup_pow_of_two(
1948 			min(max(phys_ram_gb * 3, min_vm_size), max_size));
1949 	}
1950 
1951 	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
1952 
1953 	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
1954 	if (amdgpu_vm_block_size != -1)
1955 		tmp >>= amdgpu_vm_block_size - 9;
1956 	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
1957 	adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
1958 	switch (adev->vm_manager.num_level) {
1959 	case 3:
1960 		adev->vm_manager.root_level = AMDGPU_VM_PDB2;
1961 		break;
1962 	case 2:
1963 		adev->vm_manager.root_level = AMDGPU_VM_PDB1;
1964 		break;
1965 	case 1:
1966 		adev->vm_manager.root_level = AMDGPU_VM_PDB0;
1967 		break;
1968 	default:
1969 		dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
1970 	}
1971 	/* block size depends on vm size and hw setup*/
1972 	if (amdgpu_vm_block_size != -1)
1973 		adev->vm_manager.block_size =
1974 			min((unsigned)amdgpu_vm_block_size, max_bits
1975 			    - AMDGPU_GPU_PAGE_SHIFT
1976 			    - 9 * adev->vm_manager.num_level);
1977 	else if (adev->vm_manager.num_level > 1)
1978 		adev->vm_manager.block_size = 9;
1979 	else
1980 		adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
1981 
1982 	if (amdgpu_vm_fragment_size == -1)
1983 		adev->vm_manager.fragment_size = fragment_size_default;
1984 	else
1985 		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
1986 
1987 	DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
1988 		 vm_size, adev->vm_manager.num_level + 1,
1989 		 adev->vm_manager.block_size,
1990 		 adev->vm_manager.fragment_size);
1991 }
1992 
1993 /**
1994  * amdgpu_vm_wait_idle - wait for the VM to become idle
1995  *
1996  * @vm: VM object to wait for
1997  * @timeout: timeout to wait for VM to become idle
1998  */
1999 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
2000 {
2001 	timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv,
2002 					DMA_RESV_USAGE_BOOKKEEP,
2003 					true, timeout);
2004 	if (timeout <= 0)
2005 		return timeout;
2006 
2007 	return dma_fence_wait_timeout(vm->last_unlocked, true, timeout);
2008 }
2009 
2010 /**
2011  * amdgpu_vm_init - initialize a vm instance
2012  *
2013  * @adev: amdgpu_device pointer
2014  * @vm: requested vm
2015  *
2016  * Init @vm fields.
2017  *
2018  * Returns:
2019  * 0 for success, error for failure.
2020  */
2021 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2022 {
2023 	struct amdgpu_bo *root_bo;
2024 	struct amdgpu_bo_vm *root;
2025 	int r, i;
2026 
2027 	vm->va = RB_ROOT_CACHED;
2028 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2029 		vm->reserved_vmid[i] = NULL;
2030 	INIT_LIST_HEAD(&vm->evicted);
2031 	INIT_LIST_HEAD(&vm->relocated);
2032 	INIT_LIST_HEAD(&vm->moved);
2033 	INIT_LIST_HEAD(&vm->idle);
2034 	INIT_LIST_HEAD(&vm->invalidated);
2035 	spin_lock_init(&vm->status_lock);
2036 	INIT_LIST_HEAD(&vm->freed);
2037 	INIT_LIST_HEAD(&vm->done);
2038 	INIT_LIST_HEAD(&vm->pt_freed);
2039 	INIT_WORK(&vm->pt_free_work, amdgpu_vm_pt_free_work);
2040 
2041 	/* create scheduler entities for page table updates */
2042 	r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL,
2043 				  adev->vm_manager.vm_pte_scheds,
2044 				  adev->vm_manager.vm_pte_num_scheds, NULL);
2045 	if (r)
2046 		return r;
2047 
2048 	r = drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL,
2049 				  adev->vm_manager.vm_pte_scheds,
2050 				  adev->vm_manager.vm_pte_num_scheds, NULL);
2051 	if (r)
2052 		goto error_free_immediate;
2053 
2054 	vm->pte_support_ats = false;
2055 	vm->is_compute_context = false;
2056 
2057 	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2058 				    AMDGPU_VM_USE_CPU_FOR_GFX);
2059 
2060 	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2061 			 vm->use_cpu_for_update ? "CPU" : "SDMA");
2062 	WARN_ONCE((vm->use_cpu_for_update &&
2063 		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2064 		  "CPU update of VM recommended only for large BAR system\n");
2065 
2066 	if (vm->use_cpu_for_update)
2067 		vm->update_funcs = &amdgpu_vm_cpu_funcs;
2068 	else
2069 		vm->update_funcs = &amdgpu_vm_sdma_funcs;
2070 	vm->last_update = NULL;
2071 	vm->last_unlocked = dma_fence_get_stub();
2072 	vm->last_tlb_flush = dma_fence_get_stub();
2073 
2074 	mutex_init(&vm->eviction_lock);
2075 	vm->evicting = false;
2076 
2077 	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
2078 				false, &root);
2079 	if (r)
2080 		goto error_free_delayed;
2081 	root_bo = &root->bo;
2082 	r = amdgpu_bo_reserve(root_bo, true);
2083 	if (r)
2084 		goto error_free_root;
2085 
2086 	r = dma_resv_reserve_fences(root_bo->tbo.base.resv, 1);
2087 	if (r)
2088 		goto error_unreserve;
2089 
2090 	amdgpu_vm_bo_base_init(&vm->root, vm, root_bo);
2091 
2092 	r = amdgpu_vm_pt_clear(adev, vm, root, false);
2093 	if (r)
2094 		goto error_unreserve;
2095 
2096 	amdgpu_bo_unreserve(vm->root.bo);
2097 
2098 	INIT_KFIFO(vm->faults);
2099 
2100 	return 0;
2101 
2102 error_unreserve:
2103 	amdgpu_bo_unreserve(vm->root.bo);
2104 
2105 error_free_root:
2106 	amdgpu_bo_unref(&root->shadow);
2107 	amdgpu_bo_unref(&root_bo);
2108 	vm->root.bo = NULL;
2109 
2110 error_free_delayed:
2111 	dma_fence_put(vm->last_tlb_flush);
2112 	dma_fence_put(vm->last_unlocked);
2113 	drm_sched_entity_destroy(&vm->delayed);
2114 
2115 error_free_immediate:
2116 	drm_sched_entity_destroy(&vm->immediate);
2117 
2118 	return r;
2119 }
2120 
2121 /**
2122  * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2123  *
2124  * @adev: amdgpu_device pointer
2125  * @vm: requested vm
2126  *
2127  * This only works on GFX VMs that don't have any BOs added and no
2128  * page tables allocated yet.
2129  *
2130  * Changes the following VM parameters:
2131  * - use_cpu_for_update
2132  * - pte_supports_ats
2133  *
2134  * Reinitializes the page directory to reflect the changed ATS
2135  * setting.
2136  *
2137  * Returns:
2138  * 0 for success, -errno for errors.
2139  */
2140 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2141 {
2142 	bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
2143 	int r;
2144 
2145 	r = amdgpu_bo_reserve(vm->root.bo, true);
2146 	if (r)
2147 		return r;
2148 
2149 	/* Sanity checks */
2150 	if (!amdgpu_vm_pt_is_root_clean(adev, vm)) {
2151 		r = -EINVAL;
2152 		goto unreserve_bo;
2153 	}
2154 
2155 	/* Check if PD needs to be reinitialized and do it before
2156 	 * changing any other state, in case it fails.
2157 	 */
2158 	if (pte_support_ats != vm->pte_support_ats) {
2159 		vm->pte_support_ats = pte_support_ats;
2160 		r = amdgpu_vm_pt_clear(adev, vm, to_amdgpu_bo_vm(vm->root.bo),
2161 				       false);
2162 		if (r)
2163 			goto unreserve_bo;
2164 	}
2165 
2166 	/* Update VM state */
2167 	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2168 				    AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2169 	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2170 			 vm->use_cpu_for_update ? "CPU" : "SDMA");
2171 	WARN_ONCE((vm->use_cpu_for_update &&
2172 		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2173 		  "CPU update of VM recommended only for large BAR system\n");
2174 
2175 	if (vm->use_cpu_for_update) {
2176 		/* Sync with last SDMA update/clear before switching to CPU */
2177 		r = amdgpu_bo_sync_wait(vm->root.bo,
2178 					AMDGPU_FENCE_OWNER_UNDEFINED, true);
2179 		if (r)
2180 			goto unreserve_bo;
2181 
2182 		vm->update_funcs = &amdgpu_vm_cpu_funcs;
2183 	} else {
2184 		vm->update_funcs = &amdgpu_vm_sdma_funcs;
2185 	}
2186 	/*
2187 	 * Make sure root PD gets mapped. As vm_update_mode could be changed
2188 	 * when turning a GFX VM into a compute VM.
2189 	 */
2190 	r = vm->update_funcs->map_table(to_amdgpu_bo_vm(vm->root.bo));
2191 	if (r)
2192 		goto unreserve_bo;
2193 
2194 	dma_fence_put(vm->last_update);
2195 	vm->last_update = NULL;
2196 	vm->is_compute_context = true;
2197 
2198 	/* Free the shadow bo for compute VM */
2199 	amdgpu_bo_unref(&to_amdgpu_bo_vm(vm->root.bo)->shadow);
2200 
2201 	goto unreserve_bo;
2202 
2203 unreserve_bo:
2204 	amdgpu_bo_unreserve(vm->root.bo);
2205 	return r;
2206 }
2207 
2208 /**
2209  * amdgpu_vm_release_compute - release a compute vm
2210  * @adev: amdgpu_device pointer
2211  * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
2212  *
2213  * This is a correspondant of amdgpu_vm_make_compute. It decouples compute
2214  * pasid from vm. Compute should stop use of vm after this call.
2215  */
2216 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2217 {
2218 	amdgpu_vm_set_pasid(adev, vm, 0);
2219 	vm->is_compute_context = false;
2220 }
2221 
2222 /**
2223  * amdgpu_vm_fini - tear down a vm instance
2224  *
2225  * @adev: amdgpu_device pointer
2226  * @vm: requested vm
2227  *
2228  * Tear down @vm.
2229  * Unbind the VM and remove all bos from the vm bo list
2230  */
2231 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2232 {
2233 	struct amdgpu_bo_va_mapping *mapping, *tmp;
2234 	bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2235 	struct amdgpu_bo *root;
2236 	unsigned long flags;
2237 	int i;
2238 
2239 	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2240 
2241 	flush_work(&vm->pt_free_work);
2242 
2243 	root = amdgpu_bo_ref(vm->root.bo);
2244 	amdgpu_bo_reserve(root, true);
2245 	amdgpu_vm_set_pasid(adev, vm, 0);
2246 	dma_fence_wait(vm->last_unlocked, false);
2247 	dma_fence_put(vm->last_unlocked);
2248 	dma_fence_wait(vm->last_tlb_flush, false);
2249 	/* Make sure that all fence callbacks have completed */
2250 	spin_lock_irqsave(vm->last_tlb_flush->lock, flags);
2251 	spin_unlock_irqrestore(vm->last_tlb_flush->lock, flags);
2252 	dma_fence_put(vm->last_tlb_flush);
2253 
2254 	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2255 		if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2256 			amdgpu_vm_prt_fini(adev, vm);
2257 			prt_fini_needed = false;
2258 		}
2259 
2260 		list_del(&mapping->list);
2261 		amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2262 	}
2263 
2264 	amdgpu_vm_pt_free_root(adev, vm);
2265 	amdgpu_bo_unreserve(root);
2266 	amdgpu_bo_unref(&root);
2267 	WARN_ON(vm->root.bo);
2268 
2269 	drm_sched_entity_destroy(&vm->immediate);
2270 	drm_sched_entity_destroy(&vm->delayed);
2271 
2272 	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2273 		dev_err(adev->dev, "still active bo inside vm\n");
2274 	}
2275 	rbtree_postorder_for_each_entry_safe(mapping, tmp,
2276 					     &vm->va.rb_root, rb) {
2277 		/* Don't remove the mapping here, we don't want to trigger a
2278 		 * rebalance and the tree is about to be destroyed anyway.
2279 		 */
2280 		list_del(&mapping->list);
2281 		kfree(mapping);
2282 	}
2283 
2284 	dma_fence_put(vm->last_update);
2285 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2286 		amdgpu_vmid_free_reserved(adev, vm, i);
2287 }
2288 
2289 /**
2290  * amdgpu_vm_manager_init - init the VM manager
2291  *
2292  * @adev: amdgpu_device pointer
2293  *
2294  * Initialize the VM manager structures
2295  */
2296 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2297 {
2298 	unsigned i;
2299 
2300 	/* Concurrent flushes are only possible starting with Vega10 and
2301 	 * are broken on Navi10 and Navi14.
2302 	 */
2303 	adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 ||
2304 					      adev->asic_type == CHIP_NAVI10 ||
2305 					      adev->asic_type == CHIP_NAVI14);
2306 	amdgpu_vmid_mgr_init(adev);
2307 
2308 	adev->vm_manager.fence_context =
2309 		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2310 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
2311 		adev->vm_manager.seqno[i] = 0;
2312 
2313 	spin_lock_init(&adev->vm_manager.prt_lock);
2314 	atomic_set(&adev->vm_manager.num_prt_users, 0);
2315 
2316 	/* If not overridden by the user, by default, only in large BAR systems
2317 	 * Compute VM tables will be updated by CPU
2318 	 */
2319 #ifdef CONFIG_X86_64
2320 	if (amdgpu_vm_update_mode == -1) {
2321 		/* For asic with VF MMIO access protection
2322 		 * avoid using CPU for VM table updates
2323 		 */
2324 		if (amdgpu_gmc_vram_full_visible(&adev->gmc) &&
2325 		    !amdgpu_sriov_vf_mmio_access_protection(adev))
2326 			adev->vm_manager.vm_update_mode =
2327 				AMDGPU_VM_USE_CPU_FOR_COMPUTE;
2328 		else
2329 			adev->vm_manager.vm_update_mode = 0;
2330 	} else
2331 		adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
2332 #else
2333 	adev->vm_manager.vm_update_mode = 0;
2334 #endif
2335 
2336 	xa_init_flags(&adev->vm_manager.pasids, XA_FLAGS_LOCK_IRQ);
2337 }
2338 
2339 /**
2340  * amdgpu_vm_manager_fini - cleanup VM manager
2341  *
2342  * @adev: amdgpu_device pointer
2343  *
2344  * Cleanup the VM manager and free resources.
2345  */
2346 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2347 {
2348 	WARN_ON(!xa_empty(&adev->vm_manager.pasids));
2349 	xa_destroy(&adev->vm_manager.pasids);
2350 
2351 	amdgpu_vmid_mgr_fini(adev);
2352 }
2353 
2354 /**
2355  * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
2356  *
2357  * @dev: drm device pointer
2358  * @data: drm_amdgpu_vm
2359  * @filp: drm file pointer
2360  *
2361  * Returns:
2362  * 0 for success, -errno for errors.
2363  */
2364 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2365 {
2366 	union drm_amdgpu_vm *args = data;
2367 	struct amdgpu_device *adev = drm_to_adev(dev);
2368 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
2369 	int r;
2370 
2371 	switch (args->in.op) {
2372 	case AMDGPU_VM_OP_RESERVE_VMID:
2373 		/* We only have requirement to reserve vmid from gfxhub */
2374 		r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm,
2375 					       AMDGPU_GFXHUB_0);
2376 		if (r)
2377 			return r;
2378 		break;
2379 	case AMDGPU_VM_OP_UNRESERVE_VMID:
2380 		amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0);
2381 		break;
2382 	default:
2383 		return -EINVAL;
2384 	}
2385 
2386 	return 0;
2387 }
2388 
2389 /**
2390  * amdgpu_vm_get_task_info - Extracts task info for a PASID.
2391  *
2392  * @adev: drm device pointer
2393  * @pasid: PASID identifier for VM
2394  * @task_info: task_info to fill.
2395  */
2396 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid,
2397 			 struct amdgpu_task_info *task_info)
2398 {
2399 	struct amdgpu_vm *vm;
2400 	unsigned long flags;
2401 
2402 	xa_lock_irqsave(&adev->vm_manager.pasids, flags);
2403 
2404 	vm = xa_load(&adev->vm_manager.pasids, pasid);
2405 	if (vm)
2406 		*task_info = vm->task_info;
2407 
2408 	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
2409 }
2410 
2411 /**
2412  * amdgpu_vm_set_task_info - Sets VMs task info.
2413  *
2414  * @vm: vm for which to set the info
2415  */
2416 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
2417 {
2418 	if (vm->task_info.pid)
2419 		return;
2420 
2421 	vm->task_info.pid = current->pid;
2422 	get_task_comm(vm->task_info.task_name, current);
2423 
2424 	if (current->group_leader->mm != current->mm)
2425 		return;
2426 
2427 	vm->task_info.tgid = current->group_leader->pid;
2428 	get_task_comm(vm->task_info.process_name, current->group_leader);
2429 }
2430 
2431 /**
2432  * amdgpu_vm_handle_fault - graceful handling of VM faults.
2433  * @adev: amdgpu device pointer
2434  * @pasid: PASID of the VM
2435  * @addr: Address of the fault
2436  * @write_fault: true is write fault, false is read fault
2437  *
2438  * Try to gracefully handle a VM fault. Return true if the fault was handled and
2439  * shouldn't be reported any more.
2440  */
2441 bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
2442 			    uint64_t addr, bool write_fault)
2443 {
2444 	bool is_compute_context = false;
2445 	struct amdgpu_bo *root;
2446 	unsigned long irqflags;
2447 	uint64_t value, flags;
2448 	struct amdgpu_vm *vm;
2449 	int r;
2450 
2451 	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2452 	vm = xa_load(&adev->vm_manager.pasids, pasid);
2453 	if (vm) {
2454 		root = amdgpu_bo_ref(vm->root.bo);
2455 		is_compute_context = vm->is_compute_context;
2456 	} else {
2457 		root = NULL;
2458 	}
2459 	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2460 
2461 	if (!root)
2462 		return false;
2463 
2464 	addr /= AMDGPU_GPU_PAGE_SIZE;
2465 
2466 	if (is_compute_context &&
2467 	    !svm_range_restore_pages(adev, pasid, addr, write_fault)) {
2468 		amdgpu_bo_unref(&root);
2469 		return true;
2470 	}
2471 
2472 	r = amdgpu_bo_reserve(root, true);
2473 	if (r)
2474 		goto error_unref;
2475 
2476 	/* Double check that the VM still exists */
2477 	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2478 	vm = xa_load(&adev->vm_manager.pasids, pasid);
2479 	if (vm && vm->root.bo != root)
2480 		vm = NULL;
2481 	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2482 	if (!vm)
2483 		goto error_unlock;
2484 
2485 	flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
2486 		AMDGPU_PTE_SYSTEM;
2487 
2488 	if (is_compute_context) {
2489 		/* Intentionally setting invalid PTE flag
2490 		 * combination to force a no-retry-fault
2491 		 */
2492 		flags = AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT;
2493 		value = 0;
2494 	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
2495 		/* Redirect the access to the dummy page */
2496 		value = adev->dummy_page_addr;
2497 		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
2498 			AMDGPU_PTE_WRITEABLE;
2499 
2500 	} else {
2501 		/* Let the hw retry silently on the PTE */
2502 		value = 0;
2503 	}
2504 
2505 	r = dma_resv_reserve_fences(root->tbo.base.resv, 1);
2506 	if (r) {
2507 		pr_debug("failed %d to reserve fence slot\n", r);
2508 		goto error_unlock;
2509 	}
2510 
2511 	r = amdgpu_vm_update_range(adev, vm, true, false, false, NULL, addr,
2512 				   addr, flags, value, 0, NULL, NULL, NULL);
2513 	if (r)
2514 		goto error_unlock;
2515 
2516 	r = amdgpu_vm_update_pdes(adev, vm, true);
2517 
2518 error_unlock:
2519 	amdgpu_bo_unreserve(root);
2520 	if (r < 0)
2521 		DRM_ERROR("Can't handle page fault (%d)\n", r);
2522 
2523 error_unref:
2524 	amdgpu_bo_unref(&root);
2525 
2526 	return false;
2527 }
2528 
2529 #if defined(CONFIG_DEBUG_FS)
2530 /**
2531  * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
2532  *
2533  * @vm: Requested VM for printing BO info
2534  * @m: debugfs file
2535  *
2536  * Print BO information in debugfs file for the VM
2537  */
2538 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
2539 {
2540 	struct amdgpu_bo_va *bo_va, *tmp;
2541 	u64 total_idle = 0;
2542 	u64 total_evicted = 0;
2543 	u64 total_relocated = 0;
2544 	u64 total_moved = 0;
2545 	u64 total_invalidated = 0;
2546 	u64 total_done = 0;
2547 	unsigned int total_idle_objs = 0;
2548 	unsigned int total_evicted_objs = 0;
2549 	unsigned int total_relocated_objs = 0;
2550 	unsigned int total_moved_objs = 0;
2551 	unsigned int total_invalidated_objs = 0;
2552 	unsigned int total_done_objs = 0;
2553 	unsigned int id = 0;
2554 
2555 	spin_lock(&vm->status_lock);
2556 	seq_puts(m, "\tIdle BOs:\n");
2557 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
2558 		if (!bo_va->base.bo)
2559 			continue;
2560 		total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2561 	}
2562 	total_idle_objs = id;
2563 	id = 0;
2564 
2565 	seq_puts(m, "\tEvicted BOs:\n");
2566 	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
2567 		if (!bo_va->base.bo)
2568 			continue;
2569 		total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2570 	}
2571 	total_evicted_objs = id;
2572 	id = 0;
2573 
2574 	seq_puts(m, "\tRelocated BOs:\n");
2575 	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
2576 		if (!bo_va->base.bo)
2577 			continue;
2578 		total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2579 	}
2580 	total_relocated_objs = id;
2581 	id = 0;
2582 
2583 	seq_puts(m, "\tMoved BOs:\n");
2584 	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
2585 		if (!bo_va->base.bo)
2586 			continue;
2587 		total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2588 	}
2589 	total_moved_objs = id;
2590 	id = 0;
2591 
2592 	seq_puts(m, "\tInvalidated BOs:\n");
2593 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
2594 		if (!bo_va->base.bo)
2595 			continue;
2596 		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
2597 	}
2598 	total_invalidated_objs = id;
2599 	id = 0;
2600 
2601 	seq_puts(m, "\tDone BOs:\n");
2602 	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
2603 		if (!bo_va->base.bo)
2604 			continue;
2605 		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2606 	}
2607 	spin_unlock(&vm->status_lock);
2608 	total_done_objs = id;
2609 
2610 	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
2611 		   total_idle_objs);
2612 	seq_printf(m, "\tTotal evicted size:     %12lld\tobjs:\t%d\n", total_evicted,
2613 		   total_evicted_objs);
2614 	seq_printf(m, "\tTotal relocated size:   %12lld\tobjs:\t%d\n", total_relocated,
2615 		   total_relocated_objs);
2616 	seq_printf(m, "\tTotal moved size:       %12lld\tobjs:\t%d\n", total_moved,
2617 		   total_moved_objs);
2618 	seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
2619 		   total_invalidated_objs);
2620 	seq_printf(m, "\tTotal done size:        %12lld\tobjs:\t%d\n", total_done,
2621 		   total_done_objs);
2622 }
2623 #endif
2624