1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 
29 #include <linux/dma-fence-array.h>
30 #include <linux/interval_tree_generic.h>
31 #include <linux/idr.h>
32 #include <linux/dma-buf.h>
33 
34 #include <drm/amdgpu_drm.h>
35 #include <drm/drm_drv.h>
36 #include "amdgpu.h"
37 #include "amdgpu_trace.h"
38 #include "amdgpu_amdkfd.h"
39 #include "amdgpu_gmc.h"
40 #include "amdgpu_xgmi.h"
41 #include "amdgpu_dma_buf.h"
42 #include "amdgpu_res_cursor.h"
43 #include "kfd_svm.h"
44 
45 /**
46  * DOC: GPUVM
47  *
48  * GPUVM is similar to the legacy gart on older asics, however
49  * rather than there being a single global gart table
50  * for the entire GPU, there are multiple VM page tables active
51  * at any given time.  The VM page tables can contain a mix
52  * vram pages and system memory pages and system memory pages
53  * can be mapped as snooped (cached system pages) or unsnooped
54  * (uncached system pages).
55  * Each VM has an ID associated with it and there is a page table
56  * associated with each VMID.  When executing a command buffer,
57  * the kernel tells the the ring what VMID to use for that command
58  * buffer.  VMIDs are allocated dynamically as commands are submitted.
59  * The userspace drivers maintain their own address space and the kernel
60  * sets up their pages tables accordingly when they submit their
61  * command buffers and a VMID is assigned.
62  * Cayman/Trinity support up to 8 active VMs at any given time;
63  * SI supports 16.
64  */
65 
66 #define START(node) ((node)->start)
67 #define LAST(node) ((node)->last)
68 
69 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
70 		     START, LAST, static, amdgpu_vm_it)
71 
72 #undef START
73 #undef LAST
74 
75 /**
76  * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
77  */
78 struct amdgpu_prt_cb {
79 
80 	/**
81 	 * @adev: amdgpu device
82 	 */
83 	struct amdgpu_device *adev;
84 
85 	/**
86 	 * @cb: callback
87 	 */
88 	struct dma_fence_cb cb;
89 };
90 
91 /**
92  * amdgpu_vm_set_pasid - manage pasid and vm ptr mapping
93  *
94  * @adev: amdgpu_device pointer
95  * @vm: amdgpu_vm pointer
96  * @pasid: the pasid the VM is using on this GPU
97  *
98  * Set the pasid this VM is using on this GPU, can also be used to remove the
99  * pasid by passing in zero.
100  *
101  */
102 int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm,
103 			u32 pasid)
104 {
105 	int r;
106 
107 	if (vm->pasid == pasid)
108 		return 0;
109 
110 	if (vm->pasid) {
111 		r = xa_err(xa_erase_irq(&adev->vm_manager.pasids, vm->pasid));
112 		if (r < 0)
113 			return r;
114 
115 		vm->pasid = 0;
116 	}
117 
118 	if (pasid) {
119 		r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm,
120 					GFP_KERNEL));
121 		if (r < 0)
122 			return r;
123 
124 		vm->pasid = pasid;
125 	}
126 
127 
128 	return 0;
129 }
130 
131 /*
132  * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
133  * happens while holding this lock anywhere to prevent deadlocks when
134  * an MMU notifier runs in reclaim-FS context.
135  */
136 static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
137 {
138 	mutex_lock(&vm->eviction_lock);
139 	vm->saved_flags = memalloc_noreclaim_save();
140 }
141 
142 static inline int amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
143 {
144 	if (mutex_trylock(&vm->eviction_lock)) {
145 		vm->saved_flags = memalloc_noreclaim_save();
146 		return 1;
147 	}
148 	return 0;
149 }
150 
151 static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
152 {
153 	memalloc_noreclaim_restore(vm->saved_flags);
154 	mutex_unlock(&vm->eviction_lock);
155 }
156 
157 /**
158  * amdgpu_vm_level_shift - return the addr shift for each level
159  *
160  * @adev: amdgpu_device pointer
161  * @level: VMPT level
162  *
163  * Returns:
164  * The number of bits the pfn needs to be right shifted for a level.
165  */
166 static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
167 				      unsigned level)
168 {
169 	switch (level) {
170 	case AMDGPU_VM_PDB2:
171 	case AMDGPU_VM_PDB1:
172 	case AMDGPU_VM_PDB0:
173 		return 9 * (AMDGPU_VM_PDB0 - level) +
174 			adev->vm_manager.block_size;
175 	case AMDGPU_VM_PTB:
176 		return 0;
177 	default:
178 		return ~0;
179 	}
180 }
181 
182 /**
183  * amdgpu_vm_num_entries - return the number of entries in a PD/PT
184  *
185  * @adev: amdgpu_device pointer
186  * @level: VMPT level
187  *
188  * Returns:
189  * The number of entries in a page directory or page table.
190  */
191 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
192 				      unsigned level)
193 {
194 	unsigned shift = amdgpu_vm_level_shift(adev,
195 					       adev->vm_manager.root_level);
196 
197 	if (level == adev->vm_manager.root_level)
198 		/* For the root directory */
199 		return round_up(adev->vm_manager.max_pfn, 1ULL << shift)
200 			>> shift;
201 	else if (level != AMDGPU_VM_PTB)
202 		/* Everything in between */
203 		return 512;
204 	else
205 		/* For the page tables on the leaves */
206 		return AMDGPU_VM_PTE_COUNT(adev);
207 }
208 
209 /**
210  * amdgpu_vm_num_ats_entries - return the number of ATS entries in the root PD
211  *
212  * @adev: amdgpu_device pointer
213  *
214  * Returns:
215  * The number of entries in the root page directory which needs the ATS setting.
216  */
217 static unsigned amdgpu_vm_num_ats_entries(struct amdgpu_device *adev)
218 {
219 	unsigned shift;
220 
221 	shift = amdgpu_vm_level_shift(adev, adev->vm_manager.root_level);
222 	return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT);
223 }
224 
225 /**
226  * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT
227  *
228  * @adev: amdgpu_device pointer
229  * @level: VMPT level
230  *
231  * Returns:
232  * The mask to extract the entry number of a PD/PT from an address.
233  */
234 static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device *adev,
235 				       unsigned int level)
236 {
237 	if (level <= adev->vm_manager.root_level)
238 		return 0xffffffff;
239 	else if (level != AMDGPU_VM_PTB)
240 		return 0x1ff;
241 	else
242 		return AMDGPU_VM_PTE_COUNT(adev) - 1;
243 }
244 
245 /**
246  * amdgpu_vm_bo_size - returns the size of the BOs in bytes
247  *
248  * @adev: amdgpu_device pointer
249  * @level: VMPT level
250  *
251  * Returns:
252  * The size of the BO for a page directory or page table in bytes.
253  */
254 static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
255 {
256 	return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
257 }
258 
259 /**
260  * amdgpu_vm_bo_evicted - vm_bo is evicted
261  *
262  * @vm_bo: vm_bo which is evicted
263  *
264  * State for PDs/PTs and per VM BOs which are not at the location they should
265  * be.
266  */
267 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
268 {
269 	struct amdgpu_vm *vm = vm_bo->vm;
270 	struct amdgpu_bo *bo = vm_bo->bo;
271 
272 	vm_bo->moved = true;
273 	if (bo->tbo.type == ttm_bo_type_kernel)
274 		list_move(&vm_bo->vm_status, &vm->evicted);
275 	else
276 		list_move_tail(&vm_bo->vm_status, &vm->evicted);
277 }
278 /**
279  * amdgpu_vm_bo_moved - vm_bo is moved
280  *
281  * @vm_bo: vm_bo which is moved
282  *
283  * State for per VM BOs which are moved, but that change is not yet reflected
284  * in the page tables.
285  */
286 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
287 {
288 	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
289 }
290 
291 /**
292  * amdgpu_vm_bo_idle - vm_bo is idle
293  *
294  * @vm_bo: vm_bo which is now idle
295  *
296  * State for PDs/PTs and per VM BOs which have gone through the state machine
297  * and are now idle.
298  */
299 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
300 {
301 	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
302 	vm_bo->moved = false;
303 }
304 
305 /**
306  * amdgpu_vm_bo_invalidated - vm_bo is invalidated
307  *
308  * @vm_bo: vm_bo which is now invalidated
309  *
310  * State for normal BOs which are invalidated and that change not yet reflected
311  * in the PTs.
312  */
313 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
314 {
315 	spin_lock(&vm_bo->vm->invalidated_lock);
316 	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
317 	spin_unlock(&vm_bo->vm->invalidated_lock);
318 }
319 
320 /**
321  * amdgpu_vm_bo_relocated - vm_bo is reloacted
322  *
323  * @vm_bo: vm_bo which is relocated
324  *
325  * State for PDs/PTs which needs to update their parent PD.
326  * For the root PD, just move to idle state.
327  */
328 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
329 {
330 	if (vm_bo->bo->parent)
331 		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
332 	else
333 		amdgpu_vm_bo_idle(vm_bo);
334 }
335 
336 /**
337  * amdgpu_vm_bo_done - vm_bo is done
338  *
339  * @vm_bo: vm_bo which is now done
340  *
341  * State for normal BOs which are invalidated and that change has been updated
342  * in the PTs.
343  */
344 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
345 {
346 	spin_lock(&vm_bo->vm->invalidated_lock);
347 	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
348 	spin_unlock(&vm_bo->vm->invalidated_lock);
349 }
350 
351 /**
352  * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
353  *
354  * @base: base structure for tracking BO usage in a VM
355  * @vm: vm to which bo is to be added
356  * @bo: amdgpu buffer object
357  *
358  * Initialize a bo_va_base structure and add it to the appropriate lists
359  *
360  */
361 static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
362 				   struct amdgpu_vm *vm,
363 				   struct amdgpu_bo *bo)
364 {
365 	base->vm = vm;
366 	base->bo = bo;
367 	base->next = NULL;
368 	INIT_LIST_HEAD(&base->vm_status);
369 
370 	if (!bo)
371 		return;
372 	base->next = bo->vm_bo;
373 	bo->vm_bo = base;
374 
375 	if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv)
376 		return;
377 
378 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
379 
380 	vm->bulk_moveable = false;
381 	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
382 		amdgpu_vm_bo_relocated(base);
383 	else
384 		amdgpu_vm_bo_idle(base);
385 
386 	if (bo->preferred_domains &
387 	    amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
388 		return;
389 
390 	/*
391 	 * we checked all the prerequisites, but it looks like this per vm bo
392 	 * is currently evicted. add the bo to the evicted list to make sure it
393 	 * is validated on next vm use to avoid fault.
394 	 * */
395 	amdgpu_vm_bo_evicted(base);
396 }
397 
398 /**
399  * amdgpu_vm_pt_parent - get the parent page directory
400  *
401  * @pt: child page table
402  *
403  * Helper to get the parent entry for the child page table. NULL if we are at
404  * the root page directory.
405  */
406 static struct amdgpu_vm_bo_base *amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt)
407 {
408 	struct amdgpu_bo *parent = pt->bo->parent;
409 
410 	if (!parent)
411 		return NULL;
412 
413 	return parent->vm_bo;
414 }
415 
416 /*
417  * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt
418  */
419 struct amdgpu_vm_pt_cursor {
420 	uint64_t pfn;
421 	struct amdgpu_vm_bo_base *parent;
422 	struct amdgpu_vm_bo_base *entry;
423 	unsigned level;
424 };
425 
426 /**
427  * amdgpu_vm_pt_start - start PD/PT walk
428  *
429  * @adev: amdgpu_device pointer
430  * @vm: amdgpu_vm structure
431  * @start: start address of the walk
432  * @cursor: state to initialize
433  *
434  * Initialize a amdgpu_vm_pt_cursor to start a walk.
435  */
436 static void amdgpu_vm_pt_start(struct amdgpu_device *adev,
437 			       struct amdgpu_vm *vm, uint64_t start,
438 			       struct amdgpu_vm_pt_cursor *cursor)
439 {
440 	cursor->pfn = start;
441 	cursor->parent = NULL;
442 	cursor->entry = &vm->root;
443 	cursor->level = adev->vm_manager.root_level;
444 }
445 
446 /**
447  * amdgpu_vm_pt_descendant - go to child node
448  *
449  * @adev: amdgpu_device pointer
450  * @cursor: current state
451  *
452  * Walk to the child node of the current node.
453  * Returns:
454  * True if the walk was possible, false otherwise.
455  */
456 static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev,
457 				    struct amdgpu_vm_pt_cursor *cursor)
458 {
459 	unsigned mask, shift, idx;
460 
461 	if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry ||
462 	    !cursor->entry->bo)
463 		return false;
464 
465 	mask = amdgpu_vm_entries_mask(adev, cursor->level);
466 	shift = amdgpu_vm_level_shift(adev, cursor->level);
467 
468 	++cursor->level;
469 	idx = (cursor->pfn >> shift) & mask;
470 	cursor->parent = cursor->entry;
471 	cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx];
472 	return true;
473 }
474 
475 /**
476  * amdgpu_vm_pt_sibling - go to sibling node
477  *
478  * @adev: amdgpu_device pointer
479  * @cursor: current state
480  *
481  * Walk to the sibling node of the current node.
482  * Returns:
483  * True if the walk was possible, false otherwise.
484  */
485 static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev,
486 				 struct amdgpu_vm_pt_cursor *cursor)
487 {
488 	unsigned shift, num_entries;
489 
490 	/* Root doesn't have a sibling */
491 	if (!cursor->parent)
492 		return false;
493 
494 	/* Go to our parents and see if we got a sibling */
495 	shift = amdgpu_vm_level_shift(adev, cursor->level - 1);
496 	num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1);
497 
498 	if (cursor->entry == &to_amdgpu_bo_vm(cursor->parent->bo)->entries[num_entries - 1])
499 		return false;
500 
501 	cursor->pfn += 1ULL << shift;
502 	cursor->pfn &= ~((1ULL << shift) - 1);
503 	++cursor->entry;
504 	return true;
505 }
506 
507 /**
508  * amdgpu_vm_pt_ancestor - go to parent node
509  *
510  * @cursor: current state
511  *
512  * Walk to the parent node of the current node.
513  * Returns:
514  * True if the walk was possible, false otherwise.
515  */
516 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor)
517 {
518 	if (!cursor->parent)
519 		return false;
520 
521 	--cursor->level;
522 	cursor->entry = cursor->parent;
523 	cursor->parent = amdgpu_vm_pt_parent(cursor->parent);
524 	return true;
525 }
526 
527 /**
528  * amdgpu_vm_pt_next - get next PD/PT in hieratchy
529  *
530  * @adev: amdgpu_device pointer
531  * @cursor: current state
532  *
533  * Walk the PD/PT tree to the next node.
534  */
535 static void amdgpu_vm_pt_next(struct amdgpu_device *adev,
536 			      struct amdgpu_vm_pt_cursor *cursor)
537 {
538 	/* First try a newborn child */
539 	if (amdgpu_vm_pt_descendant(adev, cursor))
540 		return;
541 
542 	/* If that didn't worked try to find a sibling */
543 	while (!amdgpu_vm_pt_sibling(adev, cursor)) {
544 		/* No sibling, go to our parents and grandparents */
545 		if (!amdgpu_vm_pt_ancestor(cursor)) {
546 			cursor->pfn = ~0ll;
547 			return;
548 		}
549 	}
550 }
551 
552 /**
553  * amdgpu_vm_pt_first_dfs - start a deep first search
554  *
555  * @adev: amdgpu_device structure
556  * @vm: amdgpu_vm structure
557  * @start: optional cursor to start with
558  * @cursor: state to initialize
559  *
560  * Starts a deep first traversal of the PD/PT tree.
561  */
562 static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev,
563 				   struct amdgpu_vm *vm,
564 				   struct amdgpu_vm_pt_cursor *start,
565 				   struct amdgpu_vm_pt_cursor *cursor)
566 {
567 	if (start)
568 		*cursor = *start;
569 	else
570 		amdgpu_vm_pt_start(adev, vm, 0, cursor);
571 	while (amdgpu_vm_pt_descendant(adev, cursor));
572 }
573 
574 /**
575  * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue
576  *
577  * @start: starting point for the search
578  * @entry: current entry
579  *
580  * Returns:
581  * True when the search should continue, false otherwise.
582  */
583 static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start,
584 				      struct amdgpu_vm_bo_base *entry)
585 {
586 	return entry && (!start || entry != start->entry);
587 }
588 
589 /**
590  * amdgpu_vm_pt_next_dfs - get the next node for a deep first search
591  *
592  * @adev: amdgpu_device structure
593  * @cursor: current state
594  *
595  * Move the cursor to the next node in a deep first search.
596  */
597 static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
598 				  struct amdgpu_vm_pt_cursor *cursor)
599 {
600 	if (!cursor->entry)
601 		return;
602 
603 	if (!cursor->parent)
604 		cursor->entry = NULL;
605 	else if (amdgpu_vm_pt_sibling(adev, cursor))
606 		while (amdgpu_vm_pt_descendant(adev, cursor));
607 	else
608 		amdgpu_vm_pt_ancestor(cursor);
609 }
610 
611 /*
612  * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs
613  */
614 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)		\
615 	for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)),		\
616 	     (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
617 	     amdgpu_vm_pt_continue_dfs((start), (entry));			\
618 	     (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor)))
619 
620 /**
621  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
622  *
623  * @vm: vm providing the BOs
624  * @validated: head of validation list
625  * @entry: entry to add
626  *
627  * Add the page directory to the list of BOs to
628  * validate for command submission.
629  */
630 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
631 			 struct list_head *validated,
632 			 struct amdgpu_bo_list_entry *entry)
633 {
634 	entry->priority = 0;
635 	entry->tv.bo = &vm->root.bo->tbo;
636 	/* Two for VM updates, one for TTM and one for the CS job */
637 	entry->tv.num_shared = 4;
638 	entry->user_pages = NULL;
639 	list_add(&entry->tv.head, validated);
640 }
641 
642 /**
643  * amdgpu_vm_del_from_lru_notify - update bulk_moveable flag
644  *
645  * @bo: BO which was removed from the LRU
646  *
647  * Make sure the bulk_moveable flag is updated when a BO is removed from the
648  * LRU.
649  */
650 void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo)
651 {
652 	struct amdgpu_bo *abo;
653 	struct amdgpu_vm_bo_base *bo_base;
654 
655 	if (!amdgpu_bo_is_amdgpu_bo(bo))
656 		return;
657 
658 	if (bo->pin_count)
659 		return;
660 
661 	abo = ttm_to_amdgpu_bo(bo);
662 	if (!abo->parent)
663 		return;
664 	for (bo_base = abo->vm_bo; bo_base; bo_base = bo_base->next) {
665 		struct amdgpu_vm *vm = bo_base->vm;
666 
667 		if (abo->tbo.base.resv == vm->root.bo->tbo.base.resv)
668 			vm->bulk_moveable = false;
669 	}
670 
671 }
672 /**
673  * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
674  *
675  * @adev: amdgpu device pointer
676  * @vm: vm providing the BOs
677  *
678  * Move all BOs to the end of LRU and remember their positions to put them
679  * together.
680  */
681 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
682 				struct amdgpu_vm *vm)
683 {
684 	struct amdgpu_vm_bo_base *bo_base;
685 
686 	if (vm->bulk_moveable) {
687 		spin_lock(&adev->mman.bdev.lru_lock);
688 		ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move);
689 		spin_unlock(&adev->mman.bdev.lru_lock);
690 		return;
691 	}
692 
693 	memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move));
694 
695 	spin_lock(&adev->mman.bdev.lru_lock);
696 	list_for_each_entry(bo_base, &vm->idle, vm_status) {
697 		struct amdgpu_bo *bo = bo_base->bo;
698 		struct amdgpu_bo *shadow = amdgpu_bo_shadowed(bo);
699 
700 		if (!bo->parent)
701 			continue;
702 
703 		ttm_bo_move_to_lru_tail(&bo->tbo, bo->tbo.resource,
704 					&vm->lru_bulk_move);
705 		if (shadow)
706 			ttm_bo_move_to_lru_tail(&shadow->tbo,
707 						shadow->tbo.resource,
708 						&vm->lru_bulk_move);
709 	}
710 	spin_unlock(&adev->mman.bdev.lru_lock);
711 
712 	vm->bulk_moveable = true;
713 }
714 
715 /**
716  * amdgpu_vm_validate_pt_bos - validate the page table BOs
717  *
718  * @adev: amdgpu device pointer
719  * @vm: vm providing the BOs
720  * @validate: callback to do the validation
721  * @param: parameter for the validation callback
722  *
723  * Validate the page table BOs on command submission if neccessary.
724  *
725  * Returns:
726  * Validation result.
727  */
728 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
729 			      int (*validate)(void *p, struct amdgpu_bo *bo),
730 			      void *param)
731 {
732 	struct amdgpu_vm_bo_base *bo_base, *tmp;
733 	int r;
734 
735 	vm->bulk_moveable &= list_empty(&vm->evicted);
736 
737 	list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
738 		struct amdgpu_bo *bo = bo_base->bo;
739 		struct amdgpu_bo *shadow = amdgpu_bo_shadowed(bo);
740 
741 		r = validate(param, bo);
742 		if (r)
743 			return r;
744 		if (shadow) {
745 			r = validate(param, shadow);
746 			if (r)
747 				return r;
748 		}
749 
750 		if (bo->tbo.type != ttm_bo_type_kernel) {
751 			amdgpu_vm_bo_moved(bo_base);
752 		} else {
753 			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
754 			amdgpu_vm_bo_relocated(bo_base);
755 		}
756 	}
757 
758 	amdgpu_vm_eviction_lock(vm);
759 	vm->evicting = false;
760 	amdgpu_vm_eviction_unlock(vm);
761 
762 	return 0;
763 }
764 
765 /**
766  * amdgpu_vm_ready - check VM is ready for updates
767  *
768  * @vm: VM to check
769  *
770  * Check if all VM PDs/PTs are ready for updates
771  *
772  * Returns:
773  * True if VM is not evicting.
774  */
775 bool amdgpu_vm_ready(struct amdgpu_vm *vm)
776 {
777 	bool ret;
778 
779 	amdgpu_vm_eviction_lock(vm);
780 	ret = !vm->evicting;
781 	amdgpu_vm_eviction_unlock(vm);
782 
783 	return ret && list_empty(&vm->evicted);
784 }
785 
786 /**
787  * amdgpu_vm_clear_bo - initially clear the PDs/PTs
788  *
789  * @adev: amdgpu_device pointer
790  * @vm: VM to clear BO from
791  * @vmbo: BO to clear
792  * @immediate: use an immediate update
793  *
794  * Root PD needs to be reserved when calling this.
795  *
796  * Returns:
797  * 0 on success, errno otherwise.
798  */
799 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
800 			      struct amdgpu_vm *vm,
801 			      struct amdgpu_bo_vm *vmbo,
802 			      bool immediate)
803 {
804 	struct ttm_operation_ctx ctx = { true, false };
805 	unsigned level = adev->vm_manager.root_level;
806 	struct amdgpu_vm_update_params params;
807 	struct amdgpu_bo *ancestor = &vmbo->bo;
808 	struct amdgpu_bo *bo = &vmbo->bo;
809 	unsigned entries, ats_entries;
810 	uint64_t addr;
811 	int r, idx;
812 
813 	/* Figure out our place in the hierarchy */
814 	if (ancestor->parent) {
815 		++level;
816 		while (ancestor->parent->parent) {
817 			++level;
818 			ancestor = ancestor->parent;
819 		}
820 	}
821 
822 	entries = amdgpu_bo_size(bo) / 8;
823 	if (!vm->pte_support_ats) {
824 		ats_entries = 0;
825 
826 	} else if (!bo->parent) {
827 		ats_entries = amdgpu_vm_num_ats_entries(adev);
828 		ats_entries = min(ats_entries, entries);
829 		entries -= ats_entries;
830 
831 	} else {
832 		struct amdgpu_vm_bo_base *pt;
833 
834 		pt = ancestor->vm_bo;
835 		ats_entries = amdgpu_vm_num_ats_entries(adev);
836 		if ((pt - to_amdgpu_bo_vm(vm->root.bo)->entries) >= ats_entries) {
837 			ats_entries = 0;
838 		} else {
839 			ats_entries = entries;
840 			entries = 0;
841 		}
842 	}
843 
844 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
845 	if (r)
846 		return r;
847 
848 	if (vmbo->shadow) {
849 		struct amdgpu_bo *shadow = vmbo->shadow;
850 
851 		r = ttm_bo_validate(&shadow->tbo, &shadow->placement, &ctx);
852 		if (r)
853 			return r;
854 	}
855 
856 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
857 		return -ENODEV;
858 
859 	r = vm->update_funcs->map_table(vmbo);
860 	if (r)
861 		goto exit;
862 
863 	memset(&params, 0, sizeof(params));
864 	params.adev = adev;
865 	params.vm = vm;
866 	params.immediate = immediate;
867 
868 	r = vm->update_funcs->prepare(&params, NULL, AMDGPU_SYNC_EXPLICIT);
869 	if (r)
870 		goto exit;
871 
872 	addr = 0;
873 	if (ats_entries) {
874 		uint64_t value = 0, flags;
875 
876 		flags = AMDGPU_PTE_DEFAULT_ATC;
877 		if (level != AMDGPU_VM_PTB) {
878 			/* Handle leaf PDEs as PTEs */
879 			flags |= AMDGPU_PDE_PTE;
880 			amdgpu_gmc_get_vm_pde(adev, level, &value, &flags);
881 		}
882 
883 		r = vm->update_funcs->update(&params, vmbo, addr, 0, ats_entries,
884 					     value, flags);
885 		if (r)
886 			goto exit;
887 
888 		addr += ats_entries * 8;
889 	}
890 
891 	if (entries) {
892 		uint64_t value = 0, flags = 0;
893 
894 		if (adev->asic_type >= CHIP_VEGA10) {
895 			if (level != AMDGPU_VM_PTB) {
896 				/* Handle leaf PDEs as PTEs */
897 				flags |= AMDGPU_PDE_PTE;
898 				amdgpu_gmc_get_vm_pde(adev, level,
899 						      &value, &flags);
900 			} else {
901 				/* Workaround for fault priority problem on GMC9 */
902 				flags = AMDGPU_PTE_EXECUTABLE;
903 			}
904 		}
905 
906 		r = vm->update_funcs->update(&params, vmbo, addr, 0, entries,
907 					     value, flags);
908 		if (r)
909 			goto exit;
910 	}
911 
912 	r = vm->update_funcs->commit(&params, NULL);
913 exit:
914 	drm_dev_exit(idx);
915 	return r;
916 }
917 
918 /**
919  * amdgpu_vm_pt_create - create bo for PD/PT
920  *
921  * @adev: amdgpu_device pointer
922  * @vm: requesting vm
923  * @level: the page table level
924  * @immediate: use a immediate update
925  * @vmbo: pointer to the buffer object pointer
926  */
927 static int amdgpu_vm_pt_create(struct amdgpu_device *adev,
928 			       struct amdgpu_vm *vm,
929 			       int level, bool immediate,
930 			       struct amdgpu_bo_vm **vmbo)
931 {
932 	struct amdgpu_bo_param bp;
933 	struct amdgpu_bo *bo;
934 	struct dma_resv *resv;
935 	unsigned int num_entries;
936 	int r;
937 
938 	memset(&bp, 0, sizeof(bp));
939 
940 	bp.size = amdgpu_vm_bo_size(adev, level);
941 	bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
942 	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
943 	bp.domain = amdgpu_bo_get_preferred_domain(adev, bp.domain);
944 	bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
945 		AMDGPU_GEM_CREATE_CPU_GTT_USWC;
946 
947 	if (level < AMDGPU_VM_PTB)
948 		num_entries = amdgpu_vm_num_entries(adev, level);
949 	else
950 		num_entries = 0;
951 
952 	bp.bo_ptr_size = struct_size((*vmbo), entries, num_entries);
953 
954 	if (vm->use_cpu_for_update)
955 		bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
956 
957 	bp.type = ttm_bo_type_kernel;
958 	bp.no_wait_gpu = immediate;
959 	if (vm->root.bo)
960 		bp.resv = vm->root.bo->tbo.base.resv;
961 
962 	r = amdgpu_bo_create_vm(adev, &bp, vmbo);
963 	if (r)
964 		return r;
965 
966 	bo = &(*vmbo)->bo;
967 	if (vm->is_compute_context || (adev->flags & AMD_IS_APU)) {
968 		(*vmbo)->shadow = NULL;
969 		return 0;
970 	}
971 
972 	if (!bp.resv)
973 		WARN_ON(dma_resv_lock(bo->tbo.base.resv,
974 				      NULL));
975 	resv = bp.resv;
976 	memset(&bp, 0, sizeof(bp));
977 	bp.size = amdgpu_vm_bo_size(adev, level);
978 	bp.domain = AMDGPU_GEM_DOMAIN_GTT;
979 	bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
980 	bp.type = ttm_bo_type_kernel;
981 	bp.resv = bo->tbo.base.resv;
982 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
983 
984 	r = amdgpu_bo_create(adev, &bp, &(*vmbo)->shadow);
985 
986 	if (!resv)
987 		dma_resv_unlock(bo->tbo.base.resv);
988 
989 	if (r) {
990 		amdgpu_bo_unref(&bo);
991 		return r;
992 	}
993 
994 	(*vmbo)->shadow->parent = amdgpu_bo_ref(bo);
995 	amdgpu_bo_add_to_shadow_list(*vmbo);
996 
997 	return 0;
998 }
999 
1000 /**
1001  * amdgpu_vm_alloc_pts - Allocate a specific page table
1002  *
1003  * @adev: amdgpu_device pointer
1004  * @vm: VM to allocate page tables for
1005  * @cursor: Which page table to allocate
1006  * @immediate: use an immediate update
1007  *
1008  * Make sure a specific page table or directory is allocated.
1009  *
1010  * Returns:
1011  * 1 if page table needed to be allocated, 0 if page table was already
1012  * allocated, negative errno if an error occurred.
1013  */
1014 static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
1015 			       struct amdgpu_vm *vm,
1016 			       struct amdgpu_vm_pt_cursor *cursor,
1017 			       bool immediate)
1018 {
1019 	struct amdgpu_vm_bo_base *entry = cursor->entry;
1020 	struct amdgpu_bo *pt_bo;
1021 	struct amdgpu_bo_vm *pt;
1022 	int r;
1023 
1024 	if (entry->bo)
1025 		return 0;
1026 
1027 	r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt);
1028 	if (r)
1029 		return r;
1030 
1031 	/* Keep a reference to the root directory to avoid
1032 	 * freeing them up in the wrong order.
1033 	 */
1034 	pt_bo = &pt->bo;
1035 	pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
1036 	amdgpu_vm_bo_base_init(entry, vm, pt_bo);
1037 	r = amdgpu_vm_clear_bo(adev, vm, pt, immediate);
1038 	if (r)
1039 		goto error_free_pt;
1040 
1041 	return 0;
1042 
1043 error_free_pt:
1044 	amdgpu_bo_unref(&pt->shadow);
1045 	amdgpu_bo_unref(&pt_bo);
1046 	return r;
1047 }
1048 
1049 /**
1050  * amdgpu_vm_free_table - fre one PD/PT
1051  *
1052  * @entry: PDE to free
1053  */
1054 static void amdgpu_vm_free_table(struct amdgpu_vm_bo_base *entry)
1055 {
1056 	struct amdgpu_bo *shadow;
1057 
1058 	if (!entry->bo)
1059 		return;
1060 	shadow = amdgpu_bo_shadowed(entry->bo);
1061 	entry->bo->vm_bo = NULL;
1062 	list_del(&entry->vm_status);
1063 	amdgpu_bo_unref(&shadow);
1064 	amdgpu_bo_unref(&entry->bo);
1065 }
1066 
1067 /**
1068  * amdgpu_vm_free_pts - free PD/PT levels
1069  *
1070  * @adev: amdgpu device structure
1071  * @vm: amdgpu vm structure
1072  * @start: optional cursor where to start freeing PDs/PTs
1073  *
1074  * Free the page directory or page table level and all sub levels.
1075  */
1076 static void amdgpu_vm_free_pts(struct amdgpu_device *adev,
1077 			       struct amdgpu_vm *vm,
1078 			       struct amdgpu_vm_pt_cursor *start)
1079 {
1080 	struct amdgpu_vm_pt_cursor cursor;
1081 	struct amdgpu_vm_bo_base *entry;
1082 
1083 	vm->bulk_moveable = false;
1084 
1085 	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
1086 		amdgpu_vm_free_table(entry);
1087 
1088 	if (start)
1089 		amdgpu_vm_free_table(start->entry);
1090 }
1091 
1092 /**
1093  * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
1094  *
1095  * @adev: amdgpu_device pointer
1096  */
1097 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
1098 {
1099 	const struct amdgpu_ip_block *ip_block;
1100 	bool has_compute_vm_bug;
1101 	struct amdgpu_ring *ring;
1102 	int i;
1103 
1104 	has_compute_vm_bug = false;
1105 
1106 	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
1107 	if (ip_block) {
1108 		/* Compute has a VM bug for GFX version < 7.
1109 		   Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
1110 		if (ip_block->version->major <= 7)
1111 			has_compute_vm_bug = true;
1112 		else if (ip_block->version->major == 8)
1113 			if (adev->gfx.mec_fw_version < 673)
1114 				has_compute_vm_bug = true;
1115 	}
1116 
1117 	for (i = 0; i < adev->num_rings; i++) {
1118 		ring = adev->rings[i];
1119 		if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
1120 			/* only compute rings */
1121 			ring->has_compute_vm_bug = has_compute_vm_bug;
1122 		else
1123 			ring->has_compute_vm_bug = false;
1124 	}
1125 }
1126 
1127 /**
1128  * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
1129  *
1130  * @ring: ring on which the job will be submitted
1131  * @job: job to submit
1132  *
1133  * Returns:
1134  * True if sync is needed.
1135  */
1136 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
1137 				  struct amdgpu_job *job)
1138 {
1139 	struct amdgpu_device *adev = ring->adev;
1140 	unsigned vmhub = ring->funcs->vmhub;
1141 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
1142 	struct amdgpu_vmid *id;
1143 	bool gds_switch_needed;
1144 	bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
1145 
1146 	if (job->vmid == 0)
1147 		return false;
1148 	id = &id_mgr->ids[job->vmid];
1149 	gds_switch_needed = ring->funcs->emit_gds_switch && (
1150 		id->gds_base != job->gds_base ||
1151 		id->gds_size != job->gds_size ||
1152 		id->gws_base != job->gws_base ||
1153 		id->gws_size != job->gws_size ||
1154 		id->oa_base != job->oa_base ||
1155 		id->oa_size != job->oa_size);
1156 
1157 	if (amdgpu_vmid_had_gpu_reset(adev, id))
1158 		return true;
1159 
1160 	return vm_flush_needed || gds_switch_needed;
1161 }
1162 
1163 /**
1164  * amdgpu_vm_flush - hardware flush the vm
1165  *
1166  * @ring: ring to use for flush
1167  * @job:  related job
1168  * @need_pipe_sync: is pipe sync needed
1169  *
1170  * Emit a VM flush when it is necessary.
1171  *
1172  * Returns:
1173  * 0 on success, errno otherwise.
1174  */
1175 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
1176 		    bool need_pipe_sync)
1177 {
1178 	struct amdgpu_device *adev = ring->adev;
1179 	unsigned vmhub = ring->funcs->vmhub;
1180 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
1181 	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
1182 	bool gds_switch_needed = ring->funcs->emit_gds_switch && (
1183 		id->gds_base != job->gds_base ||
1184 		id->gds_size != job->gds_size ||
1185 		id->gws_base != job->gws_base ||
1186 		id->gws_size != job->gws_size ||
1187 		id->oa_base != job->oa_base ||
1188 		id->oa_size != job->oa_size);
1189 	bool vm_flush_needed = job->vm_needs_flush;
1190 	struct dma_fence *fence = NULL;
1191 	bool pasid_mapping_needed = false;
1192 	unsigned patch_offset = 0;
1193 	bool update_spm_vmid_needed = (job->vm && (job->vm->reserved_vmid[vmhub] != NULL));
1194 	int r;
1195 
1196 	if (update_spm_vmid_needed && adev->gfx.rlc.funcs->update_spm_vmid)
1197 		adev->gfx.rlc.funcs->update_spm_vmid(adev, job->vmid);
1198 
1199 	if (amdgpu_vmid_had_gpu_reset(adev, id)) {
1200 		gds_switch_needed = true;
1201 		vm_flush_needed = true;
1202 		pasid_mapping_needed = true;
1203 	}
1204 
1205 	mutex_lock(&id_mgr->lock);
1206 	if (id->pasid != job->pasid || !id->pasid_mapping ||
1207 	    !dma_fence_is_signaled(id->pasid_mapping))
1208 		pasid_mapping_needed = true;
1209 	mutex_unlock(&id_mgr->lock);
1210 
1211 	gds_switch_needed &= !!ring->funcs->emit_gds_switch;
1212 	vm_flush_needed &= !!ring->funcs->emit_vm_flush  &&
1213 			job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
1214 	pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
1215 		ring->funcs->emit_wreg;
1216 
1217 	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
1218 		return 0;
1219 
1220 	if (ring->funcs->init_cond_exec)
1221 		patch_offset = amdgpu_ring_init_cond_exec(ring);
1222 
1223 	if (need_pipe_sync)
1224 		amdgpu_ring_emit_pipeline_sync(ring);
1225 
1226 	if (vm_flush_needed) {
1227 		trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
1228 		amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
1229 	}
1230 
1231 	if (pasid_mapping_needed)
1232 		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
1233 
1234 	if (vm_flush_needed || pasid_mapping_needed) {
1235 		r = amdgpu_fence_emit(ring, &fence, NULL, 0);
1236 		if (r)
1237 			return r;
1238 	}
1239 
1240 	if (vm_flush_needed) {
1241 		mutex_lock(&id_mgr->lock);
1242 		dma_fence_put(id->last_flush);
1243 		id->last_flush = dma_fence_get(fence);
1244 		id->current_gpu_reset_count =
1245 			atomic_read(&adev->gpu_reset_counter);
1246 		mutex_unlock(&id_mgr->lock);
1247 	}
1248 
1249 	if (pasid_mapping_needed) {
1250 		mutex_lock(&id_mgr->lock);
1251 		id->pasid = job->pasid;
1252 		dma_fence_put(id->pasid_mapping);
1253 		id->pasid_mapping = dma_fence_get(fence);
1254 		mutex_unlock(&id_mgr->lock);
1255 	}
1256 	dma_fence_put(fence);
1257 
1258 	if (ring->funcs->emit_gds_switch && gds_switch_needed) {
1259 		id->gds_base = job->gds_base;
1260 		id->gds_size = job->gds_size;
1261 		id->gws_base = job->gws_base;
1262 		id->gws_size = job->gws_size;
1263 		id->oa_base = job->oa_base;
1264 		id->oa_size = job->oa_size;
1265 		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
1266 					    job->gds_size, job->gws_base,
1267 					    job->gws_size, job->oa_base,
1268 					    job->oa_size);
1269 	}
1270 
1271 	if (ring->funcs->patch_cond_exec)
1272 		amdgpu_ring_patch_cond_exec(ring, patch_offset);
1273 
1274 	/* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
1275 	if (ring->funcs->emit_switch_buffer) {
1276 		amdgpu_ring_emit_switch_buffer(ring);
1277 		amdgpu_ring_emit_switch_buffer(ring);
1278 	}
1279 	return 0;
1280 }
1281 
1282 /**
1283  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
1284  *
1285  * @vm: requested vm
1286  * @bo: requested buffer object
1287  *
1288  * Find @bo inside the requested vm.
1289  * Search inside the @bos vm list for the requested vm
1290  * Returns the found bo_va or NULL if none is found
1291  *
1292  * Object has to be reserved!
1293  *
1294  * Returns:
1295  * Found bo_va or NULL.
1296  */
1297 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
1298 				       struct amdgpu_bo *bo)
1299 {
1300 	struct amdgpu_vm_bo_base *base;
1301 
1302 	for (base = bo->vm_bo; base; base = base->next) {
1303 		if (base->vm != vm)
1304 			continue;
1305 
1306 		return container_of(base, struct amdgpu_bo_va, base);
1307 	}
1308 	return NULL;
1309 }
1310 
1311 /**
1312  * amdgpu_vm_map_gart - Resolve gart mapping of addr
1313  *
1314  * @pages_addr: optional DMA address to use for lookup
1315  * @addr: the unmapped addr
1316  *
1317  * Look up the physical address of the page that the pte resolves
1318  * to.
1319  *
1320  * Returns:
1321  * The pointer for the page table entry.
1322  */
1323 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
1324 {
1325 	uint64_t result;
1326 
1327 	/* page table offset */
1328 	result = pages_addr[addr >> PAGE_SHIFT];
1329 
1330 	/* in case cpu page size != gpu page size*/
1331 	result |= addr & (~PAGE_MASK);
1332 
1333 	result &= 0xFFFFFFFFFFFFF000ULL;
1334 
1335 	return result;
1336 }
1337 
1338 /**
1339  * amdgpu_vm_update_pde - update a single level in the hierarchy
1340  *
1341  * @params: parameters for the update
1342  * @vm: requested vm
1343  * @entry: entry to update
1344  *
1345  * Makes sure the requested entry in parent is up to date.
1346  */
1347 static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params,
1348 				struct amdgpu_vm *vm,
1349 				struct amdgpu_vm_bo_base *entry)
1350 {
1351 	struct amdgpu_vm_bo_base *parent = amdgpu_vm_pt_parent(entry);
1352 	struct amdgpu_bo *bo = parent->bo, *pbo;
1353 	uint64_t pde, pt, flags;
1354 	unsigned level;
1355 
1356 	for (level = 0, pbo = bo->parent; pbo; ++level)
1357 		pbo = pbo->parent;
1358 
1359 	level += params->adev->vm_manager.root_level;
1360 	amdgpu_gmc_get_pde_for_bo(entry->bo, level, &pt, &flags);
1361 	pde = (entry - to_amdgpu_bo_vm(parent->bo)->entries) * 8;
1362 	return vm->update_funcs->update(params, to_amdgpu_bo_vm(bo), pde, pt,
1363 					1, 0, flags);
1364 }
1365 
1366 /**
1367  * amdgpu_vm_invalidate_pds - mark all PDs as invalid
1368  *
1369  * @adev: amdgpu_device pointer
1370  * @vm: related vm
1371  *
1372  * Mark all PD level as invalid after an error.
1373  */
1374 static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev,
1375 				     struct amdgpu_vm *vm)
1376 {
1377 	struct amdgpu_vm_pt_cursor cursor;
1378 	struct amdgpu_vm_bo_base *entry;
1379 
1380 	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry)
1381 		if (entry->bo && !entry->moved)
1382 			amdgpu_vm_bo_relocated(entry);
1383 }
1384 
1385 /**
1386  * amdgpu_vm_update_pdes - make sure that all directories are valid
1387  *
1388  * @adev: amdgpu_device pointer
1389  * @vm: requested vm
1390  * @immediate: submit immediately to the paging queue
1391  *
1392  * Makes sure all directories are up to date.
1393  *
1394  * Returns:
1395  * 0 for success, error for failure.
1396  */
1397 int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
1398 			  struct amdgpu_vm *vm, bool immediate)
1399 {
1400 	struct amdgpu_vm_update_params params;
1401 	int r, idx;
1402 
1403 	if (list_empty(&vm->relocated))
1404 		return 0;
1405 
1406 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
1407 		return -ENODEV;
1408 
1409 	memset(&params, 0, sizeof(params));
1410 	params.adev = adev;
1411 	params.vm = vm;
1412 	params.immediate = immediate;
1413 
1414 	r = vm->update_funcs->prepare(&params, NULL, AMDGPU_SYNC_EXPLICIT);
1415 	if (r)
1416 		goto exit;
1417 
1418 	while (!list_empty(&vm->relocated)) {
1419 		struct amdgpu_vm_bo_base *entry;
1420 
1421 		entry = list_first_entry(&vm->relocated,
1422 					 struct amdgpu_vm_bo_base,
1423 					 vm_status);
1424 		amdgpu_vm_bo_idle(entry);
1425 
1426 		r = amdgpu_vm_update_pde(&params, vm, entry);
1427 		if (r)
1428 			goto error;
1429 	}
1430 
1431 	r = vm->update_funcs->commit(&params, &vm->last_update);
1432 	if (r)
1433 		goto error;
1434 	drm_dev_exit(idx);
1435 	return 0;
1436 
1437 error:
1438 	amdgpu_vm_invalidate_pds(adev, vm);
1439 exit:
1440 	drm_dev_exit(idx);
1441 	return r;
1442 }
1443 
1444 /*
1445  * amdgpu_vm_update_flags - figure out flags for PTE updates
1446  *
1447  * Make sure to set the right flags for the PTEs at the desired level.
1448  */
1449 static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params,
1450 				   struct amdgpu_bo_vm *pt, unsigned int level,
1451 				   uint64_t pe, uint64_t addr,
1452 				   unsigned int count, uint32_t incr,
1453 				   uint64_t flags)
1454 
1455 {
1456 	if (level != AMDGPU_VM_PTB) {
1457 		flags |= AMDGPU_PDE_PTE;
1458 		amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags);
1459 
1460 	} else if (params->adev->asic_type >= CHIP_VEGA10 &&
1461 		   !(flags & AMDGPU_PTE_VALID) &&
1462 		   !(flags & AMDGPU_PTE_PRT)) {
1463 
1464 		/* Workaround for fault priority problem on GMC9 */
1465 		flags |= AMDGPU_PTE_EXECUTABLE;
1466 	}
1467 
1468 	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
1469 					 flags);
1470 }
1471 
1472 /**
1473  * amdgpu_vm_fragment - get fragment for PTEs
1474  *
1475  * @params: see amdgpu_vm_update_params definition
1476  * @start: first PTE to handle
1477  * @end: last PTE to handle
1478  * @flags: hw mapping flags
1479  * @frag: resulting fragment size
1480  * @frag_end: end of this fragment
1481  *
1482  * Returns the first possible fragment for the start and end address.
1483  */
1484 static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params,
1485 			       uint64_t start, uint64_t end, uint64_t flags,
1486 			       unsigned int *frag, uint64_t *frag_end)
1487 {
1488 	/**
1489 	 * The MC L1 TLB supports variable sized pages, based on a fragment
1490 	 * field in the PTE. When this field is set to a non-zero value, page
1491 	 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
1492 	 * flags are considered valid for all PTEs within the fragment range
1493 	 * and corresponding mappings are assumed to be physically contiguous.
1494 	 *
1495 	 * The L1 TLB can store a single PTE for the whole fragment,
1496 	 * significantly increasing the space available for translation
1497 	 * caching. This leads to large improvements in throughput when the
1498 	 * TLB is under pressure.
1499 	 *
1500 	 * The L2 TLB distributes small and large fragments into two
1501 	 * asymmetric partitions. The large fragment cache is significantly
1502 	 * larger. Thus, we try to use large fragments wherever possible.
1503 	 * Userspace can support this by aligning virtual base address and
1504 	 * allocation size to the fragment size.
1505 	 *
1506 	 * Starting with Vega10 the fragment size only controls the L1. The L2
1507 	 * is now directly feed with small/huge/giant pages from the walker.
1508 	 */
1509 	unsigned max_frag;
1510 
1511 	if (params->adev->asic_type < CHIP_VEGA10)
1512 		max_frag = params->adev->vm_manager.fragment_size;
1513 	else
1514 		max_frag = 31;
1515 
1516 	/* system pages are non continuously */
1517 	if (params->pages_addr) {
1518 		*frag = 0;
1519 		*frag_end = end;
1520 		return;
1521 	}
1522 
1523 	/* This intentionally wraps around if no bit is set */
1524 	*frag = min((unsigned)ffs(start) - 1, (unsigned)fls64(end - start) - 1);
1525 	if (*frag >= max_frag) {
1526 		*frag = max_frag;
1527 		*frag_end = end & ~((1ULL << max_frag) - 1);
1528 	} else {
1529 		*frag_end = start + (1 << *frag);
1530 	}
1531 }
1532 
1533 /**
1534  * amdgpu_vm_update_ptes - make sure that page tables are valid
1535  *
1536  * @params: see amdgpu_vm_update_params definition
1537  * @start: start of GPU address range
1538  * @end: end of GPU address range
1539  * @dst: destination address to map to, the next dst inside the function
1540  * @flags: mapping flags
1541  *
1542  * Update the page tables in the range @start - @end.
1543  *
1544  * Returns:
1545  * 0 for success, -EINVAL for failure.
1546  */
1547 static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params,
1548 				 uint64_t start, uint64_t end,
1549 				 uint64_t dst, uint64_t flags)
1550 {
1551 	struct amdgpu_device *adev = params->adev;
1552 	struct amdgpu_vm_pt_cursor cursor;
1553 	uint64_t frag_start = start, frag_end;
1554 	unsigned int frag;
1555 	int r;
1556 
1557 	/* figure out the initial fragment */
1558 	amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end);
1559 
1560 	/* walk over the address space and update the PTs */
1561 	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
1562 	while (cursor.pfn < end) {
1563 		unsigned shift, parent_shift, mask;
1564 		uint64_t incr, entry_end, pe_start;
1565 		struct amdgpu_bo *pt;
1566 
1567 		if (!params->unlocked) {
1568 			/* make sure that the page tables covering the
1569 			 * address range are actually allocated
1570 			 */
1571 			r = amdgpu_vm_alloc_pts(params->adev, params->vm,
1572 						&cursor, params->immediate);
1573 			if (r)
1574 				return r;
1575 		}
1576 
1577 		shift = amdgpu_vm_level_shift(adev, cursor.level);
1578 		parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1);
1579 		if (params->unlocked) {
1580 			/* Unlocked updates are only allowed on the leaves */
1581 			if (amdgpu_vm_pt_descendant(adev, &cursor))
1582 				continue;
1583 		} else if (adev->asic_type < CHIP_VEGA10 &&
1584 			   (flags & AMDGPU_PTE_VALID)) {
1585 			/* No huge page support before GMC v9 */
1586 			if (cursor.level != AMDGPU_VM_PTB) {
1587 				if (!amdgpu_vm_pt_descendant(adev, &cursor))
1588 					return -ENOENT;
1589 				continue;
1590 			}
1591 		} else if (frag < shift) {
1592 			/* We can't use this level when the fragment size is
1593 			 * smaller than the address shift. Go to the next
1594 			 * child entry and try again.
1595 			 */
1596 			if (amdgpu_vm_pt_descendant(adev, &cursor))
1597 				continue;
1598 		} else if (frag >= parent_shift) {
1599 			/* If the fragment size is even larger than the parent
1600 			 * shift we should go up one level and check it again.
1601 			 */
1602 			if (!amdgpu_vm_pt_ancestor(&cursor))
1603 				return -EINVAL;
1604 			continue;
1605 		}
1606 
1607 		pt = cursor.entry->bo;
1608 		if (!pt) {
1609 			/* We need all PDs and PTs for mapping something, */
1610 			if (flags & AMDGPU_PTE_VALID)
1611 				return -ENOENT;
1612 
1613 			/* but unmapping something can happen at a higher
1614 			 * level.
1615 			 */
1616 			if (!amdgpu_vm_pt_ancestor(&cursor))
1617 				return -EINVAL;
1618 
1619 			pt = cursor.entry->bo;
1620 			shift = parent_shift;
1621 			frag_end = max(frag_end, ALIGN(frag_start + 1,
1622 				   1ULL << shift));
1623 		}
1624 
1625 		/* Looks good so far, calculate parameters for the update */
1626 		incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift;
1627 		mask = amdgpu_vm_entries_mask(adev, cursor.level);
1628 		pe_start = ((cursor.pfn >> shift) & mask) * 8;
1629 		entry_end = ((uint64_t)mask + 1) << shift;
1630 		entry_end += cursor.pfn & ~(entry_end - 1);
1631 		entry_end = min(entry_end, end);
1632 
1633 		do {
1634 			struct amdgpu_vm *vm = params->vm;
1635 			uint64_t upd_end = min(entry_end, frag_end);
1636 			unsigned nptes = (upd_end - frag_start) >> shift;
1637 			uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag);
1638 
1639 			/* This can happen when we set higher level PDs to
1640 			 * silent to stop fault floods.
1641 			 */
1642 			nptes = max(nptes, 1u);
1643 
1644 			trace_amdgpu_vm_update_ptes(params, frag_start, upd_end,
1645 						    min(nptes, 32u), dst, incr, upd_flags,
1646 						    vm->task_info.pid,
1647 						    vm->immediate.fence_context);
1648 			amdgpu_vm_update_flags(params, to_amdgpu_bo_vm(pt),
1649 					       cursor.level, pe_start, dst,
1650 					       nptes, incr, upd_flags);
1651 
1652 			pe_start += nptes * 8;
1653 			dst += nptes * incr;
1654 
1655 			frag_start = upd_end;
1656 			if (frag_start >= frag_end) {
1657 				/* figure out the next fragment */
1658 				amdgpu_vm_fragment(params, frag_start, end,
1659 						   flags, &frag, &frag_end);
1660 				if (frag < shift)
1661 					break;
1662 			}
1663 		} while (frag_start < entry_end);
1664 
1665 		if (amdgpu_vm_pt_descendant(adev, &cursor)) {
1666 			/* Free all child entries.
1667 			 * Update the tables with the flags and addresses and free up subsequent
1668 			 * tables in the case of huge pages or freed up areas.
1669 			 * This is the maximum you can free, because all other page tables are not
1670 			 * completely covered by the range and so potentially still in use.
1671 			 */
1672 			while (cursor.pfn < frag_start) {
1673 				/* Make sure previous mapping is freed */
1674 				if (cursor.entry->bo) {
1675 					params->table_freed = true;
1676 					amdgpu_vm_free_pts(adev, params->vm, &cursor);
1677 				}
1678 				amdgpu_vm_pt_next(adev, &cursor);
1679 			}
1680 
1681 		} else if (frag >= shift) {
1682 			/* or just move on to the next on the same level. */
1683 			amdgpu_vm_pt_next(adev, &cursor);
1684 		}
1685 	}
1686 
1687 	return 0;
1688 }
1689 
1690 /**
1691  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1692  *
1693  * @adev: amdgpu_device pointer of the VM
1694  * @bo_adev: amdgpu_device pointer of the mapped BO
1695  * @vm: requested vm
1696  * @immediate: immediate submission in a page fault
1697  * @unlocked: unlocked invalidation during MM callback
1698  * @resv: fences we need to sync to
1699  * @start: start of mapped range
1700  * @last: last mapped entry
1701  * @flags: flags for the entries
1702  * @offset: offset into nodes and pages_addr
1703  * @res: ttm_resource to map
1704  * @pages_addr: DMA addresses to use for mapping
1705  * @fence: optional resulting fence
1706  * @table_freed: return true if page table is freed
1707  *
1708  * Fill in the page table entries between @start and @last.
1709  *
1710  * Returns:
1711  * 0 for success, -EINVAL for failure.
1712  */
1713 int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1714 				struct amdgpu_device *bo_adev,
1715 				struct amdgpu_vm *vm, bool immediate,
1716 				bool unlocked, struct dma_resv *resv,
1717 				uint64_t start, uint64_t last,
1718 				uint64_t flags, uint64_t offset,
1719 				struct ttm_resource *res,
1720 				dma_addr_t *pages_addr,
1721 				struct dma_fence **fence,
1722 				bool *table_freed)
1723 {
1724 	struct amdgpu_vm_update_params params;
1725 	struct amdgpu_res_cursor cursor;
1726 	enum amdgpu_sync_mode sync_mode;
1727 	int r, idx;
1728 
1729 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
1730 		return -ENODEV;
1731 
1732 	memset(&params, 0, sizeof(params));
1733 	params.adev = adev;
1734 	params.vm = vm;
1735 	params.immediate = immediate;
1736 	params.pages_addr = pages_addr;
1737 	params.unlocked = unlocked;
1738 
1739 	/* Implicitly sync to command submissions in the same VM before
1740 	 * unmapping. Sync to moving fences before mapping.
1741 	 */
1742 	if (!(flags & AMDGPU_PTE_VALID))
1743 		sync_mode = AMDGPU_SYNC_EQ_OWNER;
1744 	else
1745 		sync_mode = AMDGPU_SYNC_EXPLICIT;
1746 
1747 	amdgpu_vm_eviction_lock(vm);
1748 	if (vm->evicting) {
1749 		r = -EBUSY;
1750 		goto error_unlock;
1751 	}
1752 
1753 	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
1754 		struct dma_fence *tmp = dma_fence_get_stub();
1755 
1756 		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
1757 		swap(vm->last_unlocked, tmp);
1758 		dma_fence_put(tmp);
1759 	}
1760 
1761 	r = vm->update_funcs->prepare(&params, resv, sync_mode);
1762 	if (r)
1763 		goto error_unlock;
1764 
1765 	amdgpu_res_first(pages_addr ? NULL : res, offset,
1766 			 (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor);
1767 	while (cursor.remaining) {
1768 		uint64_t tmp, num_entries, addr;
1769 
1770 		num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;
1771 		if (pages_addr) {
1772 			bool contiguous = true;
1773 
1774 			if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) {
1775 				uint64_t pfn = cursor.start >> PAGE_SHIFT;
1776 				uint64_t count;
1777 
1778 				contiguous = pages_addr[pfn + 1] ==
1779 					pages_addr[pfn] + PAGE_SIZE;
1780 
1781 				tmp = num_entries /
1782 					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1783 				for (count = 2; count < tmp; ++count) {
1784 					uint64_t idx = pfn + count;
1785 
1786 					if (contiguous != (pages_addr[idx] ==
1787 					    pages_addr[idx - 1] + PAGE_SIZE))
1788 						break;
1789 				}
1790 				num_entries = count *
1791 					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1792 			}
1793 
1794 			if (!contiguous) {
1795 				addr = cursor.start;
1796 				params.pages_addr = pages_addr;
1797 			} else {
1798 				addr = pages_addr[cursor.start >> PAGE_SHIFT];
1799 				params.pages_addr = NULL;
1800 			}
1801 
1802 		} else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) {
1803 			addr = bo_adev->vm_manager.vram_base_offset +
1804 				cursor.start;
1805 		} else {
1806 			addr = 0;
1807 		}
1808 
1809 		tmp = start + num_entries;
1810 		r = amdgpu_vm_update_ptes(&params, start, tmp, addr, flags);
1811 		if (r)
1812 			goto error_unlock;
1813 
1814 		amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE);
1815 		start = tmp;
1816 	}
1817 
1818 	r = vm->update_funcs->commit(&params, fence);
1819 
1820 	if (table_freed)
1821 		*table_freed = *table_freed || params.table_freed;
1822 
1823 error_unlock:
1824 	amdgpu_vm_eviction_unlock(vm);
1825 	drm_dev_exit(idx);
1826 	return r;
1827 }
1828 
1829 void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
1830 				uint64_t *gtt_mem, uint64_t *cpu_mem)
1831 {
1832 	struct amdgpu_bo_va *bo_va, *tmp;
1833 
1834 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
1835 		if (!bo_va->base.bo)
1836 			continue;
1837 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
1838 				gtt_mem, cpu_mem);
1839 	}
1840 	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
1841 		if (!bo_va->base.bo)
1842 			continue;
1843 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
1844 				gtt_mem, cpu_mem);
1845 	}
1846 	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
1847 		if (!bo_va->base.bo)
1848 			continue;
1849 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
1850 				gtt_mem, cpu_mem);
1851 	}
1852 	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
1853 		if (!bo_va->base.bo)
1854 			continue;
1855 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
1856 				gtt_mem, cpu_mem);
1857 	}
1858 	spin_lock(&vm->invalidated_lock);
1859 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
1860 		if (!bo_va->base.bo)
1861 			continue;
1862 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
1863 				gtt_mem, cpu_mem);
1864 	}
1865 	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
1866 		if (!bo_va->base.bo)
1867 			continue;
1868 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
1869 				gtt_mem, cpu_mem);
1870 	}
1871 	spin_unlock(&vm->invalidated_lock);
1872 }
1873 /**
1874  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1875  *
1876  * @adev: amdgpu_device pointer
1877  * @bo_va: requested BO and VM object
1878  * @clear: if true clear the entries
1879  * @table_freed: return true if page table is freed
1880  *
1881  * Fill in the page table entries for @bo_va.
1882  *
1883  * Returns:
1884  * 0 for success, -EINVAL for failure.
1885  */
1886 int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
1887 			bool clear, bool *table_freed)
1888 {
1889 	struct amdgpu_bo *bo = bo_va->base.bo;
1890 	struct amdgpu_vm *vm = bo_va->base.vm;
1891 	struct amdgpu_bo_va_mapping *mapping;
1892 	dma_addr_t *pages_addr = NULL;
1893 	struct ttm_resource *mem;
1894 	struct dma_fence **last_update;
1895 	struct dma_resv *resv;
1896 	uint64_t flags;
1897 	struct amdgpu_device *bo_adev = adev;
1898 	int r;
1899 
1900 	if (clear || !bo) {
1901 		mem = NULL;
1902 		resv = vm->root.bo->tbo.base.resv;
1903 	} else {
1904 		struct drm_gem_object *obj = &bo->tbo.base;
1905 
1906 		resv = bo->tbo.base.resv;
1907 		if (obj->import_attach && bo_va->is_xgmi) {
1908 			struct dma_buf *dma_buf = obj->import_attach->dmabuf;
1909 			struct drm_gem_object *gobj = dma_buf->priv;
1910 			struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
1911 
1912 			if (abo->tbo.resource->mem_type == TTM_PL_VRAM)
1913 				bo = gem_to_amdgpu_bo(gobj);
1914 		}
1915 		mem = bo->tbo.resource;
1916 		if (mem->mem_type == TTM_PL_TT ||
1917 		    mem->mem_type == AMDGPU_PL_PREEMPT)
1918 			pages_addr = bo->tbo.ttm->dma_address;
1919 	}
1920 
1921 	if (bo) {
1922 		flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1923 
1924 		if (amdgpu_bo_encrypted(bo))
1925 			flags |= AMDGPU_PTE_TMZ;
1926 
1927 		bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1928 	} else {
1929 		flags = 0x0;
1930 	}
1931 
1932 	if (clear || (bo && bo->tbo.base.resv ==
1933 		      vm->root.bo->tbo.base.resv))
1934 		last_update = &vm->last_update;
1935 	else
1936 		last_update = &bo_va->last_pt_update;
1937 
1938 	if (!clear && bo_va->base.moved) {
1939 		bo_va->base.moved = false;
1940 		list_splice_init(&bo_va->valids, &bo_va->invalids);
1941 
1942 	} else if (bo_va->cleared != clear) {
1943 		list_splice_init(&bo_va->valids, &bo_va->invalids);
1944 	}
1945 
1946 	list_for_each_entry(mapping, &bo_va->invalids, list) {
1947 		uint64_t update_flags = flags;
1948 
1949 		/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1950 		 * but in case of something, we filter the flags in first place
1951 		 */
1952 		if (!(mapping->flags & AMDGPU_PTE_READABLE))
1953 			update_flags &= ~AMDGPU_PTE_READABLE;
1954 		if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1955 			update_flags &= ~AMDGPU_PTE_WRITEABLE;
1956 
1957 		/* Apply ASIC specific mapping flags */
1958 		amdgpu_gmc_get_vm_pte(adev, mapping, &update_flags);
1959 
1960 		trace_amdgpu_vm_bo_update(mapping);
1961 
1962 		r = amdgpu_vm_bo_update_mapping(adev, bo_adev, vm, false, false,
1963 						resv, mapping->start,
1964 						mapping->last, update_flags,
1965 						mapping->offset, mem,
1966 						pages_addr, last_update, table_freed);
1967 		if (r)
1968 			return r;
1969 	}
1970 
1971 	/* If the BO is not in its preferred location add it back to
1972 	 * the evicted list so that it gets validated again on the
1973 	 * next command submission.
1974 	 */
1975 	if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
1976 		uint32_t mem_type = bo->tbo.resource->mem_type;
1977 
1978 		if (!(bo->preferred_domains &
1979 		      amdgpu_mem_type_to_domain(mem_type)))
1980 			amdgpu_vm_bo_evicted(&bo_va->base);
1981 		else
1982 			amdgpu_vm_bo_idle(&bo_va->base);
1983 	} else {
1984 		amdgpu_vm_bo_done(&bo_va->base);
1985 	}
1986 
1987 	list_splice_init(&bo_va->invalids, &bo_va->valids);
1988 	bo_va->cleared = clear;
1989 
1990 	if (trace_amdgpu_vm_bo_mapping_enabled()) {
1991 		list_for_each_entry(mapping, &bo_va->valids, list)
1992 			trace_amdgpu_vm_bo_mapping(mapping);
1993 	}
1994 
1995 	return 0;
1996 }
1997 
1998 /**
1999  * amdgpu_vm_update_prt_state - update the global PRT state
2000  *
2001  * @adev: amdgpu_device pointer
2002  */
2003 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
2004 {
2005 	unsigned long flags;
2006 	bool enable;
2007 
2008 	spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
2009 	enable = !!atomic_read(&adev->vm_manager.num_prt_users);
2010 	adev->gmc.gmc_funcs->set_prt(adev, enable);
2011 	spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
2012 }
2013 
2014 /**
2015  * amdgpu_vm_prt_get - add a PRT user
2016  *
2017  * @adev: amdgpu_device pointer
2018  */
2019 static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
2020 {
2021 	if (!adev->gmc.gmc_funcs->set_prt)
2022 		return;
2023 
2024 	if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
2025 		amdgpu_vm_update_prt_state(adev);
2026 }
2027 
2028 /**
2029  * amdgpu_vm_prt_put - drop a PRT user
2030  *
2031  * @adev: amdgpu_device pointer
2032  */
2033 static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
2034 {
2035 	if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
2036 		amdgpu_vm_update_prt_state(adev);
2037 }
2038 
2039 /**
2040  * amdgpu_vm_prt_cb - callback for updating the PRT status
2041  *
2042  * @fence: fence for the callback
2043  * @_cb: the callback function
2044  */
2045 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
2046 {
2047 	struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
2048 
2049 	amdgpu_vm_prt_put(cb->adev);
2050 	kfree(cb);
2051 }
2052 
2053 /**
2054  * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
2055  *
2056  * @adev: amdgpu_device pointer
2057  * @fence: fence for the callback
2058  */
2059 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
2060 				 struct dma_fence *fence)
2061 {
2062 	struct amdgpu_prt_cb *cb;
2063 
2064 	if (!adev->gmc.gmc_funcs->set_prt)
2065 		return;
2066 
2067 	cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
2068 	if (!cb) {
2069 		/* Last resort when we are OOM */
2070 		if (fence)
2071 			dma_fence_wait(fence, false);
2072 
2073 		amdgpu_vm_prt_put(adev);
2074 	} else {
2075 		cb->adev = adev;
2076 		if (!fence || dma_fence_add_callback(fence, &cb->cb,
2077 						     amdgpu_vm_prt_cb))
2078 			amdgpu_vm_prt_cb(fence, &cb->cb);
2079 	}
2080 }
2081 
2082 /**
2083  * amdgpu_vm_free_mapping - free a mapping
2084  *
2085  * @adev: amdgpu_device pointer
2086  * @vm: requested vm
2087  * @mapping: mapping to be freed
2088  * @fence: fence of the unmap operation
2089  *
2090  * Free a mapping and make sure we decrease the PRT usage count if applicable.
2091  */
2092 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
2093 				   struct amdgpu_vm *vm,
2094 				   struct amdgpu_bo_va_mapping *mapping,
2095 				   struct dma_fence *fence)
2096 {
2097 	if (mapping->flags & AMDGPU_PTE_PRT)
2098 		amdgpu_vm_add_prt_cb(adev, fence);
2099 	kfree(mapping);
2100 }
2101 
2102 /**
2103  * amdgpu_vm_prt_fini - finish all prt mappings
2104  *
2105  * @adev: amdgpu_device pointer
2106  * @vm: requested vm
2107  *
2108  * Register a cleanup callback to disable PRT support after VM dies.
2109  */
2110 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2111 {
2112 	struct dma_resv *resv = vm->root.bo->tbo.base.resv;
2113 	struct dma_resv_iter cursor;
2114 	struct dma_fence *fence;
2115 
2116 	dma_resv_for_each_fence(&cursor, resv, true, fence) {
2117 		/* Add a callback for each fence in the reservation object */
2118 		amdgpu_vm_prt_get(adev);
2119 		amdgpu_vm_add_prt_cb(adev, fence);
2120 	}
2121 }
2122 
2123 /**
2124  * amdgpu_vm_clear_freed - clear freed BOs in the PT
2125  *
2126  * @adev: amdgpu_device pointer
2127  * @vm: requested vm
2128  * @fence: optional resulting fence (unchanged if no work needed to be done
2129  * or if an error occurred)
2130  *
2131  * Make sure all freed BOs are cleared in the PT.
2132  * PTs have to be reserved and mutex must be locked!
2133  *
2134  * Returns:
2135  * 0 for success.
2136  *
2137  */
2138 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
2139 			  struct amdgpu_vm *vm,
2140 			  struct dma_fence **fence)
2141 {
2142 	struct dma_resv *resv = vm->root.bo->tbo.base.resv;
2143 	struct amdgpu_bo_va_mapping *mapping;
2144 	uint64_t init_pte_value = 0;
2145 	struct dma_fence *f = NULL;
2146 	int r;
2147 
2148 	while (!list_empty(&vm->freed)) {
2149 		mapping = list_first_entry(&vm->freed,
2150 			struct amdgpu_bo_va_mapping, list);
2151 		list_del(&mapping->list);
2152 
2153 		if (vm->pte_support_ats &&
2154 		    mapping->start < AMDGPU_GMC_HOLE_START)
2155 			init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
2156 
2157 		r = amdgpu_vm_bo_update_mapping(adev, adev, vm, false, false,
2158 						resv, mapping->start,
2159 						mapping->last, init_pte_value,
2160 						0, NULL, NULL, &f, NULL);
2161 		amdgpu_vm_free_mapping(adev, vm, mapping, f);
2162 		if (r) {
2163 			dma_fence_put(f);
2164 			return r;
2165 		}
2166 	}
2167 
2168 	if (fence && f) {
2169 		dma_fence_put(*fence);
2170 		*fence = f;
2171 	} else {
2172 		dma_fence_put(f);
2173 	}
2174 
2175 	return 0;
2176 
2177 }
2178 
2179 /**
2180  * amdgpu_vm_handle_moved - handle moved BOs in the PT
2181  *
2182  * @adev: amdgpu_device pointer
2183  * @vm: requested vm
2184  *
2185  * Make sure all BOs which are moved are updated in the PTs.
2186  *
2187  * Returns:
2188  * 0 for success.
2189  *
2190  * PTs have to be reserved!
2191  */
2192 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
2193 			   struct amdgpu_vm *vm)
2194 {
2195 	struct amdgpu_bo_va *bo_va, *tmp;
2196 	struct dma_resv *resv;
2197 	bool clear;
2198 	int r;
2199 
2200 	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
2201 		/* Per VM BOs never need to bo cleared in the page tables */
2202 		r = amdgpu_vm_bo_update(adev, bo_va, false, NULL);
2203 		if (r)
2204 			return r;
2205 	}
2206 
2207 	spin_lock(&vm->invalidated_lock);
2208 	while (!list_empty(&vm->invalidated)) {
2209 		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
2210 					 base.vm_status);
2211 		resv = bo_va->base.bo->tbo.base.resv;
2212 		spin_unlock(&vm->invalidated_lock);
2213 
2214 		/* Try to reserve the BO to avoid clearing its ptes */
2215 		if (!amdgpu_vm_debug && dma_resv_trylock(resv))
2216 			clear = false;
2217 		/* Somebody else is using the BO right now */
2218 		else
2219 			clear = true;
2220 
2221 		r = amdgpu_vm_bo_update(adev, bo_va, clear, NULL);
2222 		if (r)
2223 			return r;
2224 
2225 		if (!clear)
2226 			dma_resv_unlock(resv);
2227 		spin_lock(&vm->invalidated_lock);
2228 	}
2229 	spin_unlock(&vm->invalidated_lock);
2230 
2231 	return 0;
2232 }
2233 
2234 /**
2235  * amdgpu_vm_bo_add - add a bo to a specific vm
2236  *
2237  * @adev: amdgpu_device pointer
2238  * @vm: requested vm
2239  * @bo: amdgpu buffer object
2240  *
2241  * Add @bo into the requested vm.
2242  * Add @bo to the list of bos associated with the vm
2243  *
2244  * Returns:
2245  * Newly added bo_va or NULL for failure
2246  *
2247  * Object has to be reserved!
2248  */
2249 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
2250 				      struct amdgpu_vm *vm,
2251 				      struct amdgpu_bo *bo)
2252 {
2253 	struct amdgpu_bo_va *bo_va;
2254 
2255 	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
2256 	if (bo_va == NULL) {
2257 		return NULL;
2258 	}
2259 	amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
2260 
2261 	bo_va->ref_count = 1;
2262 	INIT_LIST_HEAD(&bo_va->valids);
2263 	INIT_LIST_HEAD(&bo_va->invalids);
2264 
2265 	if (!bo)
2266 		return bo_va;
2267 
2268 	dma_resv_assert_held(bo->tbo.base.resv);
2269 	if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) {
2270 		bo_va->is_xgmi = true;
2271 		/* Power up XGMI if it can be potentially used */
2272 		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20);
2273 	}
2274 
2275 	return bo_va;
2276 }
2277 
2278 
2279 /**
2280  * amdgpu_vm_bo_insert_map - insert a new mapping
2281  *
2282  * @adev: amdgpu_device pointer
2283  * @bo_va: bo_va to store the address
2284  * @mapping: the mapping to insert
2285  *
2286  * Insert a new mapping into all structures.
2287  */
2288 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
2289 				    struct amdgpu_bo_va *bo_va,
2290 				    struct amdgpu_bo_va_mapping *mapping)
2291 {
2292 	struct amdgpu_vm *vm = bo_va->base.vm;
2293 	struct amdgpu_bo *bo = bo_va->base.bo;
2294 
2295 	mapping->bo_va = bo_va;
2296 	list_add(&mapping->list, &bo_va->invalids);
2297 	amdgpu_vm_it_insert(mapping, &vm->va);
2298 
2299 	if (mapping->flags & AMDGPU_PTE_PRT)
2300 		amdgpu_vm_prt_get(adev);
2301 
2302 	if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv &&
2303 	    !bo_va->base.moved) {
2304 		list_move(&bo_va->base.vm_status, &vm->moved);
2305 	}
2306 	trace_amdgpu_vm_bo_map(bo_va, mapping);
2307 }
2308 
2309 /**
2310  * amdgpu_vm_bo_map - map bo inside a vm
2311  *
2312  * @adev: amdgpu_device pointer
2313  * @bo_va: bo_va to store the address
2314  * @saddr: where to map the BO
2315  * @offset: requested offset in the BO
2316  * @size: BO size in bytes
2317  * @flags: attributes of pages (read/write/valid/etc.)
2318  *
2319  * Add a mapping of the BO at the specefied addr into the VM.
2320  *
2321  * Returns:
2322  * 0 for success, error for failure.
2323  *
2324  * Object has to be reserved and unreserved outside!
2325  */
2326 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
2327 		     struct amdgpu_bo_va *bo_va,
2328 		     uint64_t saddr, uint64_t offset,
2329 		     uint64_t size, uint64_t flags)
2330 {
2331 	struct amdgpu_bo_va_mapping *mapping, *tmp;
2332 	struct amdgpu_bo *bo = bo_va->base.bo;
2333 	struct amdgpu_vm *vm = bo_va->base.vm;
2334 	uint64_t eaddr;
2335 
2336 	/* validate the parameters */
2337 	if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK ||
2338 	    size == 0 || size & ~PAGE_MASK)
2339 		return -EINVAL;
2340 
2341 	/* make sure object fit at this offset */
2342 	eaddr = saddr + size - 1;
2343 	if (saddr >= eaddr ||
2344 	    (bo && offset + size > amdgpu_bo_size(bo)) ||
2345 	    (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
2346 		return -EINVAL;
2347 
2348 	saddr /= AMDGPU_GPU_PAGE_SIZE;
2349 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
2350 
2351 	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2352 	if (tmp) {
2353 		/* bo and tmp overlap, invalid addr */
2354 		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
2355 			"0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
2356 			tmp->start, tmp->last + 1);
2357 		return -EINVAL;
2358 	}
2359 
2360 	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2361 	if (!mapping)
2362 		return -ENOMEM;
2363 
2364 	mapping->start = saddr;
2365 	mapping->last = eaddr;
2366 	mapping->offset = offset;
2367 	mapping->flags = flags;
2368 
2369 	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2370 
2371 	return 0;
2372 }
2373 
2374 /**
2375  * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
2376  *
2377  * @adev: amdgpu_device pointer
2378  * @bo_va: bo_va to store the address
2379  * @saddr: where to map the BO
2380  * @offset: requested offset in the BO
2381  * @size: BO size in bytes
2382  * @flags: attributes of pages (read/write/valid/etc.)
2383  *
2384  * Add a mapping of the BO at the specefied addr into the VM. Replace existing
2385  * mappings as we do so.
2386  *
2387  * Returns:
2388  * 0 for success, error for failure.
2389  *
2390  * Object has to be reserved and unreserved outside!
2391  */
2392 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
2393 			     struct amdgpu_bo_va *bo_va,
2394 			     uint64_t saddr, uint64_t offset,
2395 			     uint64_t size, uint64_t flags)
2396 {
2397 	struct amdgpu_bo_va_mapping *mapping;
2398 	struct amdgpu_bo *bo = bo_va->base.bo;
2399 	uint64_t eaddr;
2400 	int r;
2401 
2402 	/* validate the parameters */
2403 	if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK ||
2404 	    size == 0 || size & ~PAGE_MASK)
2405 		return -EINVAL;
2406 
2407 	/* make sure object fit at this offset */
2408 	eaddr = saddr + size - 1;
2409 	if (saddr >= eaddr ||
2410 	    (bo && offset + size > amdgpu_bo_size(bo)) ||
2411 	    (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
2412 		return -EINVAL;
2413 
2414 	/* Allocate all the needed memory */
2415 	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2416 	if (!mapping)
2417 		return -ENOMEM;
2418 
2419 	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
2420 	if (r) {
2421 		kfree(mapping);
2422 		return r;
2423 	}
2424 
2425 	saddr /= AMDGPU_GPU_PAGE_SIZE;
2426 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
2427 
2428 	mapping->start = saddr;
2429 	mapping->last = eaddr;
2430 	mapping->offset = offset;
2431 	mapping->flags = flags;
2432 
2433 	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2434 
2435 	return 0;
2436 }
2437 
2438 /**
2439  * amdgpu_vm_bo_unmap - remove bo mapping from vm
2440  *
2441  * @adev: amdgpu_device pointer
2442  * @bo_va: bo_va to remove the address from
2443  * @saddr: where to the BO is mapped
2444  *
2445  * Remove a mapping of the BO at the specefied addr from the VM.
2446  *
2447  * Returns:
2448  * 0 for success, error for failure.
2449  *
2450  * Object has to be reserved and unreserved outside!
2451  */
2452 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
2453 		       struct amdgpu_bo_va *bo_va,
2454 		       uint64_t saddr)
2455 {
2456 	struct amdgpu_bo_va_mapping *mapping;
2457 	struct amdgpu_vm *vm = bo_va->base.vm;
2458 	bool valid = true;
2459 
2460 	saddr /= AMDGPU_GPU_PAGE_SIZE;
2461 
2462 	list_for_each_entry(mapping, &bo_va->valids, list) {
2463 		if (mapping->start == saddr)
2464 			break;
2465 	}
2466 
2467 	if (&mapping->list == &bo_va->valids) {
2468 		valid = false;
2469 
2470 		list_for_each_entry(mapping, &bo_va->invalids, list) {
2471 			if (mapping->start == saddr)
2472 				break;
2473 		}
2474 
2475 		if (&mapping->list == &bo_va->invalids)
2476 			return -ENOENT;
2477 	}
2478 
2479 	list_del(&mapping->list);
2480 	amdgpu_vm_it_remove(mapping, &vm->va);
2481 	mapping->bo_va = NULL;
2482 	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2483 
2484 	if (valid)
2485 		list_add(&mapping->list, &vm->freed);
2486 	else
2487 		amdgpu_vm_free_mapping(adev, vm, mapping,
2488 				       bo_va->last_pt_update);
2489 
2490 	return 0;
2491 }
2492 
2493 /**
2494  * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2495  *
2496  * @adev: amdgpu_device pointer
2497  * @vm: VM structure to use
2498  * @saddr: start of the range
2499  * @size: size of the range
2500  *
2501  * Remove all mappings in a range, split them as appropriate.
2502  *
2503  * Returns:
2504  * 0 for success, error for failure.
2505  */
2506 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2507 				struct amdgpu_vm *vm,
2508 				uint64_t saddr, uint64_t size)
2509 {
2510 	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
2511 	LIST_HEAD(removed);
2512 	uint64_t eaddr;
2513 
2514 	eaddr = saddr + size - 1;
2515 	saddr /= AMDGPU_GPU_PAGE_SIZE;
2516 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
2517 
2518 	/* Allocate all the needed memory */
2519 	before = kzalloc(sizeof(*before), GFP_KERNEL);
2520 	if (!before)
2521 		return -ENOMEM;
2522 	INIT_LIST_HEAD(&before->list);
2523 
2524 	after = kzalloc(sizeof(*after), GFP_KERNEL);
2525 	if (!after) {
2526 		kfree(before);
2527 		return -ENOMEM;
2528 	}
2529 	INIT_LIST_HEAD(&after->list);
2530 
2531 	/* Now gather all removed mappings */
2532 	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2533 	while (tmp) {
2534 		/* Remember mapping split at the start */
2535 		if (tmp->start < saddr) {
2536 			before->start = tmp->start;
2537 			before->last = saddr - 1;
2538 			before->offset = tmp->offset;
2539 			before->flags = tmp->flags;
2540 			before->bo_va = tmp->bo_va;
2541 			list_add(&before->list, &tmp->bo_va->invalids);
2542 		}
2543 
2544 		/* Remember mapping split at the end */
2545 		if (tmp->last > eaddr) {
2546 			after->start = eaddr + 1;
2547 			after->last = tmp->last;
2548 			after->offset = tmp->offset;
2549 			after->offset += (after->start - tmp->start) << PAGE_SHIFT;
2550 			after->flags = tmp->flags;
2551 			after->bo_va = tmp->bo_va;
2552 			list_add(&after->list, &tmp->bo_va->invalids);
2553 		}
2554 
2555 		list_del(&tmp->list);
2556 		list_add(&tmp->list, &removed);
2557 
2558 		tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2559 	}
2560 
2561 	/* And free them up */
2562 	list_for_each_entry_safe(tmp, next, &removed, list) {
2563 		amdgpu_vm_it_remove(tmp, &vm->va);
2564 		list_del(&tmp->list);
2565 
2566 		if (tmp->start < saddr)
2567 		    tmp->start = saddr;
2568 		if (tmp->last > eaddr)
2569 		    tmp->last = eaddr;
2570 
2571 		tmp->bo_va = NULL;
2572 		list_add(&tmp->list, &vm->freed);
2573 		trace_amdgpu_vm_bo_unmap(NULL, tmp);
2574 	}
2575 
2576 	/* Insert partial mapping before the range */
2577 	if (!list_empty(&before->list)) {
2578 		amdgpu_vm_it_insert(before, &vm->va);
2579 		if (before->flags & AMDGPU_PTE_PRT)
2580 			amdgpu_vm_prt_get(adev);
2581 	} else {
2582 		kfree(before);
2583 	}
2584 
2585 	/* Insert partial mapping after the range */
2586 	if (!list_empty(&after->list)) {
2587 		amdgpu_vm_it_insert(after, &vm->va);
2588 		if (after->flags & AMDGPU_PTE_PRT)
2589 			amdgpu_vm_prt_get(adev);
2590 	} else {
2591 		kfree(after);
2592 	}
2593 
2594 	return 0;
2595 }
2596 
2597 /**
2598  * amdgpu_vm_bo_lookup_mapping - find mapping by address
2599  *
2600  * @vm: the requested VM
2601  * @addr: the address
2602  *
2603  * Find a mapping by it's address.
2604  *
2605  * Returns:
2606  * The amdgpu_bo_va_mapping matching for addr or NULL
2607  *
2608  */
2609 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2610 							 uint64_t addr)
2611 {
2612 	return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
2613 }
2614 
2615 /**
2616  * amdgpu_vm_bo_trace_cs - trace all reserved mappings
2617  *
2618  * @vm: the requested vm
2619  * @ticket: CS ticket
2620  *
2621  * Trace all mappings of BOs reserved during a command submission.
2622  */
2623 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
2624 {
2625 	struct amdgpu_bo_va_mapping *mapping;
2626 
2627 	if (!trace_amdgpu_vm_bo_cs_enabled())
2628 		return;
2629 
2630 	for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
2631 	     mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
2632 		if (mapping->bo_va && mapping->bo_va->base.bo) {
2633 			struct amdgpu_bo *bo;
2634 
2635 			bo = mapping->bo_va->base.bo;
2636 			if (dma_resv_locking_ctx(bo->tbo.base.resv) !=
2637 			    ticket)
2638 				continue;
2639 		}
2640 
2641 		trace_amdgpu_vm_bo_cs(mapping);
2642 	}
2643 }
2644 
2645 /**
2646  * amdgpu_vm_bo_del - remove a bo from a specific vm
2647  *
2648  * @adev: amdgpu_device pointer
2649  * @bo_va: requested bo_va
2650  *
2651  * Remove @bo_va->bo from the requested vm.
2652  *
2653  * Object have to be reserved!
2654  */
2655 void amdgpu_vm_bo_del(struct amdgpu_device *adev,
2656 		      struct amdgpu_bo_va *bo_va)
2657 {
2658 	struct amdgpu_bo_va_mapping *mapping, *next;
2659 	struct amdgpu_bo *bo = bo_va->base.bo;
2660 	struct amdgpu_vm *vm = bo_va->base.vm;
2661 	struct amdgpu_vm_bo_base **base;
2662 
2663 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
2664 
2665 	if (bo) {
2666 		dma_resv_assert_held(bo->tbo.base.resv);
2667 		if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
2668 			vm->bulk_moveable = false;
2669 
2670 		for (base = &bo_va->base.bo->vm_bo; *base;
2671 		     base = &(*base)->next) {
2672 			if (*base != &bo_va->base)
2673 				continue;
2674 
2675 			*base = bo_va->base.next;
2676 			break;
2677 		}
2678 	}
2679 
2680 	spin_lock(&vm->invalidated_lock);
2681 	list_del(&bo_va->base.vm_status);
2682 	spin_unlock(&vm->invalidated_lock);
2683 
2684 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2685 		list_del(&mapping->list);
2686 		amdgpu_vm_it_remove(mapping, &vm->va);
2687 		mapping->bo_va = NULL;
2688 		trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2689 		list_add(&mapping->list, &vm->freed);
2690 	}
2691 	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2692 		list_del(&mapping->list);
2693 		amdgpu_vm_it_remove(mapping, &vm->va);
2694 		amdgpu_vm_free_mapping(adev, vm, mapping,
2695 				       bo_va->last_pt_update);
2696 	}
2697 
2698 	dma_fence_put(bo_va->last_pt_update);
2699 
2700 	if (bo && bo_va->is_xgmi)
2701 		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN);
2702 
2703 	kfree(bo_va);
2704 }
2705 
2706 /**
2707  * amdgpu_vm_evictable - check if we can evict a VM
2708  *
2709  * @bo: A page table of the VM.
2710  *
2711  * Check if it is possible to evict a VM.
2712  */
2713 bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
2714 {
2715 	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
2716 
2717 	/* Page tables of a destroyed VM can go away immediately */
2718 	if (!bo_base || !bo_base->vm)
2719 		return true;
2720 
2721 	/* Don't evict VM page tables while they are busy */
2722 	if (!dma_resv_test_signaled(bo->tbo.base.resv, true))
2723 		return false;
2724 
2725 	/* Try to block ongoing updates */
2726 	if (!amdgpu_vm_eviction_trylock(bo_base->vm))
2727 		return false;
2728 
2729 	/* Don't evict VM page tables while they are updated */
2730 	if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
2731 		amdgpu_vm_eviction_unlock(bo_base->vm);
2732 		return false;
2733 	}
2734 
2735 	bo_base->vm->evicting = true;
2736 	amdgpu_vm_eviction_unlock(bo_base->vm);
2737 	return true;
2738 }
2739 
2740 /**
2741  * amdgpu_vm_bo_invalidate - mark the bo as invalid
2742  *
2743  * @adev: amdgpu_device pointer
2744  * @bo: amdgpu buffer object
2745  * @evicted: is the BO evicted
2746  *
2747  * Mark @bo as invalid.
2748  */
2749 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
2750 			     struct amdgpu_bo *bo, bool evicted)
2751 {
2752 	struct amdgpu_vm_bo_base *bo_base;
2753 
2754 	/* shadow bo doesn't have bo base, its validation needs its parent */
2755 	if (bo->parent && (amdgpu_bo_shadowed(bo->parent) == bo))
2756 		bo = bo->parent;
2757 
2758 	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
2759 		struct amdgpu_vm *vm = bo_base->vm;
2760 
2761 		if (evicted && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
2762 			amdgpu_vm_bo_evicted(bo_base);
2763 			continue;
2764 		}
2765 
2766 		if (bo_base->moved)
2767 			continue;
2768 		bo_base->moved = true;
2769 
2770 		if (bo->tbo.type == ttm_bo_type_kernel)
2771 			amdgpu_vm_bo_relocated(bo_base);
2772 		else if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
2773 			amdgpu_vm_bo_moved(bo_base);
2774 		else
2775 			amdgpu_vm_bo_invalidated(bo_base);
2776 	}
2777 }
2778 
2779 /**
2780  * amdgpu_vm_get_block_size - calculate VM page table size as power of two
2781  *
2782  * @vm_size: VM size
2783  *
2784  * Returns:
2785  * VM page table as power of two
2786  */
2787 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2788 {
2789 	/* Total bits covered by PD + PTs */
2790 	unsigned bits = ilog2(vm_size) + 18;
2791 
2792 	/* Make sure the PD is 4K in size up to 8GB address space.
2793 	   Above that split equal between PD and PTs */
2794 	if (vm_size <= 8)
2795 		return (bits - 9);
2796 	else
2797 		return ((bits + 3) / 2);
2798 }
2799 
2800 /**
2801  * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2802  *
2803  * @adev: amdgpu_device pointer
2804  * @min_vm_size: the minimum vm size in GB if it's set auto
2805  * @fragment_size_default: Default PTE fragment size
2806  * @max_level: max VMPT level
2807  * @max_bits: max address space size in bits
2808  *
2809  */
2810 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
2811 			   uint32_t fragment_size_default, unsigned max_level,
2812 			   unsigned max_bits)
2813 {
2814 	unsigned int max_size = 1 << (max_bits - 30);
2815 	unsigned int vm_size;
2816 	uint64_t tmp;
2817 
2818 	/* adjust vm size first */
2819 	if (amdgpu_vm_size != -1) {
2820 		vm_size = amdgpu_vm_size;
2821 		if (vm_size > max_size) {
2822 			dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2823 				 amdgpu_vm_size, max_size);
2824 			vm_size = max_size;
2825 		}
2826 	} else {
2827 		struct sysinfo si;
2828 		unsigned int phys_ram_gb;
2829 
2830 		/* Optimal VM size depends on the amount of physical
2831 		 * RAM available. Underlying requirements and
2832 		 * assumptions:
2833 		 *
2834 		 *  - Need to map system memory and VRAM from all GPUs
2835 		 *     - VRAM from other GPUs not known here
2836 		 *     - Assume VRAM <= system memory
2837 		 *  - On GFX8 and older, VM space can be segmented for
2838 		 *    different MTYPEs
2839 		 *  - Need to allow room for fragmentation, guard pages etc.
2840 		 *
2841 		 * This adds up to a rough guess of system memory x3.
2842 		 * Round up to power of two to maximize the available
2843 		 * VM size with the given page table size.
2844 		 */
2845 		si_meminfo(&si);
2846 		phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
2847 			       (1 << 30) - 1) >> 30;
2848 		vm_size = roundup_pow_of_two(
2849 			min(max(phys_ram_gb * 3, min_vm_size), max_size));
2850 	}
2851 
2852 	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2853 
2854 	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2855 	if (amdgpu_vm_block_size != -1)
2856 		tmp >>= amdgpu_vm_block_size - 9;
2857 	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2858 	adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
2859 	switch (adev->vm_manager.num_level) {
2860 	case 3:
2861 		adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2862 		break;
2863 	case 2:
2864 		adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2865 		break;
2866 	case 1:
2867 		adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2868 		break;
2869 	default:
2870 		dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2871 	}
2872 	/* block size depends on vm size and hw setup*/
2873 	if (amdgpu_vm_block_size != -1)
2874 		adev->vm_manager.block_size =
2875 			min((unsigned)amdgpu_vm_block_size, max_bits
2876 			    - AMDGPU_GPU_PAGE_SHIFT
2877 			    - 9 * adev->vm_manager.num_level);
2878 	else if (adev->vm_manager.num_level > 1)
2879 		adev->vm_manager.block_size = 9;
2880 	else
2881 		adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2882 
2883 	if (amdgpu_vm_fragment_size == -1)
2884 		adev->vm_manager.fragment_size = fragment_size_default;
2885 	else
2886 		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2887 
2888 	DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2889 		 vm_size, adev->vm_manager.num_level + 1,
2890 		 adev->vm_manager.block_size,
2891 		 adev->vm_manager.fragment_size);
2892 }
2893 
2894 /**
2895  * amdgpu_vm_wait_idle - wait for the VM to become idle
2896  *
2897  * @vm: VM object to wait for
2898  * @timeout: timeout to wait for VM to become idle
2899  */
2900 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
2901 {
2902 	timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv, true,
2903 					true, timeout);
2904 	if (timeout <= 0)
2905 		return timeout;
2906 
2907 	return dma_fence_wait_timeout(vm->last_unlocked, true, timeout);
2908 }
2909 
2910 /**
2911  * amdgpu_vm_init - initialize a vm instance
2912  *
2913  * @adev: amdgpu_device pointer
2914  * @vm: requested vm
2915  *
2916  * Init @vm fields.
2917  *
2918  * Returns:
2919  * 0 for success, error for failure.
2920  */
2921 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2922 {
2923 	struct amdgpu_bo *root_bo;
2924 	struct amdgpu_bo_vm *root;
2925 	int r, i;
2926 
2927 	vm->va = RB_ROOT_CACHED;
2928 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2929 		vm->reserved_vmid[i] = NULL;
2930 	INIT_LIST_HEAD(&vm->evicted);
2931 	INIT_LIST_HEAD(&vm->relocated);
2932 	INIT_LIST_HEAD(&vm->moved);
2933 	INIT_LIST_HEAD(&vm->idle);
2934 	INIT_LIST_HEAD(&vm->invalidated);
2935 	spin_lock_init(&vm->invalidated_lock);
2936 	INIT_LIST_HEAD(&vm->freed);
2937 	INIT_LIST_HEAD(&vm->done);
2938 
2939 	/* create scheduler entities for page table updates */
2940 	r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL,
2941 				  adev->vm_manager.vm_pte_scheds,
2942 				  adev->vm_manager.vm_pte_num_scheds, NULL);
2943 	if (r)
2944 		return r;
2945 
2946 	r = drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL,
2947 				  adev->vm_manager.vm_pte_scheds,
2948 				  adev->vm_manager.vm_pte_num_scheds, NULL);
2949 	if (r)
2950 		goto error_free_immediate;
2951 
2952 	vm->pte_support_ats = false;
2953 	vm->is_compute_context = false;
2954 
2955 	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2956 				    AMDGPU_VM_USE_CPU_FOR_GFX);
2957 
2958 	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2959 			 vm->use_cpu_for_update ? "CPU" : "SDMA");
2960 	WARN_ONCE((vm->use_cpu_for_update &&
2961 		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2962 		  "CPU update of VM recommended only for large BAR system\n");
2963 
2964 	if (vm->use_cpu_for_update)
2965 		vm->update_funcs = &amdgpu_vm_cpu_funcs;
2966 	else
2967 		vm->update_funcs = &amdgpu_vm_sdma_funcs;
2968 	vm->last_update = NULL;
2969 	vm->last_unlocked = dma_fence_get_stub();
2970 
2971 	mutex_init(&vm->eviction_lock);
2972 	vm->evicting = false;
2973 
2974 	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
2975 				false, &root);
2976 	if (r)
2977 		goto error_free_delayed;
2978 	root_bo = &root->bo;
2979 	r = amdgpu_bo_reserve(root_bo, true);
2980 	if (r)
2981 		goto error_free_root;
2982 
2983 	r = dma_resv_reserve_shared(root_bo->tbo.base.resv, 1);
2984 	if (r)
2985 		goto error_unreserve;
2986 
2987 	amdgpu_vm_bo_base_init(&vm->root, vm, root_bo);
2988 
2989 	r = amdgpu_vm_clear_bo(adev, vm, root, false);
2990 	if (r)
2991 		goto error_unreserve;
2992 
2993 	amdgpu_bo_unreserve(vm->root.bo);
2994 
2995 	INIT_KFIFO(vm->faults);
2996 
2997 	return 0;
2998 
2999 error_unreserve:
3000 	amdgpu_bo_unreserve(vm->root.bo);
3001 
3002 error_free_root:
3003 	amdgpu_bo_unref(&root->shadow);
3004 	amdgpu_bo_unref(&root_bo);
3005 	vm->root.bo = NULL;
3006 
3007 error_free_delayed:
3008 	dma_fence_put(vm->last_unlocked);
3009 	drm_sched_entity_destroy(&vm->delayed);
3010 
3011 error_free_immediate:
3012 	drm_sched_entity_destroy(&vm->immediate);
3013 
3014 	return r;
3015 }
3016 
3017 /**
3018  * amdgpu_vm_check_clean_reserved - check if a VM is clean
3019  *
3020  * @adev: amdgpu_device pointer
3021  * @vm: the VM to check
3022  *
3023  * check all entries of the root PD, if any subsequent PDs are allocated,
3024  * it means there are page table creating and filling, and is no a clean
3025  * VM
3026  *
3027  * Returns:
3028  *	0 if this VM is clean
3029  */
3030 static int amdgpu_vm_check_clean_reserved(struct amdgpu_device *adev,
3031 					  struct amdgpu_vm *vm)
3032 {
3033 	enum amdgpu_vm_level root = adev->vm_manager.root_level;
3034 	unsigned int entries = amdgpu_vm_num_entries(adev, root);
3035 	unsigned int i = 0;
3036 
3037 	for (i = 0; i < entries; i++) {
3038 		if (to_amdgpu_bo_vm(vm->root.bo)->entries[i].bo)
3039 			return -EINVAL;
3040 	}
3041 
3042 	return 0;
3043 }
3044 
3045 /**
3046  * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
3047  *
3048  * @adev: amdgpu_device pointer
3049  * @vm: requested vm
3050  *
3051  * This only works on GFX VMs that don't have any BOs added and no
3052  * page tables allocated yet.
3053  *
3054  * Changes the following VM parameters:
3055  * - use_cpu_for_update
3056  * - pte_supports_ats
3057  *
3058  * Reinitializes the page directory to reflect the changed ATS
3059  * setting.
3060  *
3061  * Returns:
3062  * 0 for success, -errno for errors.
3063  */
3064 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
3065 {
3066 	bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
3067 	int r;
3068 
3069 	r = amdgpu_bo_reserve(vm->root.bo, true);
3070 	if (r)
3071 		return r;
3072 
3073 	/* Sanity checks */
3074 	r = amdgpu_vm_check_clean_reserved(adev, vm);
3075 	if (r)
3076 		goto unreserve_bo;
3077 
3078 	/* Check if PD needs to be reinitialized and do it before
3079 	 * changing any other state, in case it fails.
3080 	 */
3081 	if (pte_support_ats != vm->pte_support_ats) {
3082 		vm->pte_support_ats = pte_support_ats;
3083 		r = amdgpu_vm_clear_bo(adev, vm,
3084 				       to_amdgpu_bo_vm(vm->root.bo),
3085 				       false);
3086 		if (r)
3087 			goto unreserve_bo;
3088 	}
3089 
3090 	/* Update VM state */
3091 	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
3092 				    AMDGPU_VM_USE_CPU_FOR_COMPUTE);
3093 	DRM_DEBUG_DRIVER("VM update mode is %s\n",
3094 			 vm->use_cpu_for_update ? "CPU" : "SDMA");
3095 	WARN_ONCE((vm->use_cpu_for_update &&
3096 		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),
3097 		  "CPU update of VM recommended only for large BAR system\n");
3098 
3099 	if (vm->use_cpu_for_update) {
3100 		/* Sync with last SDMA update/clear before switching to CPU */
3101 		r = amdgpu_bo_sync_wait(vm->root.bo,
3102 					AMDGPU_FENCE_OWNER_UNDEFINED, true);
3103 		if (r)
3104 			goto unreserve_bo;
3105 
3106 		vm->update_funcs = &amdgpu_vm_cpu_funcs;
3107 	} else {
3108 		vm->update_funcs = &amdgpu_vm_sdma_funcs;
3109 	}
3110 	dma_fence_put(vm->last_update);
3111 	vm->last_update = NULL;
3112 	vm->is_compute_context = true;
3113 
3114 	/* Free the shadow bo for compute VM */
3115 	amdgpu_bo_unref(&to_amdgpu_bo_vm(vm->root.bo)->shadow);
3116 
3117 	goto unreserve_bo;
3118 
3119 unreserve_bo:
3120 	amdgpu_bo_unreserve(vm->root.bo);
3121 	return r;
3122 }
3123 
3124 /**
3125  * amdgpu_vm_release_compute - release a compute vm
3126  * @adev: amdgpu_device pointer
3127  * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
3128  *
3129  * This is a correspondant of amdgpu_vm_make_compute. It decouples compute
3130  * pasid from vm. Compute should stop use of vm after this call.
3131  */
3132 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
3133 {
3134 	amdgpu_vm_set_pasid(adev, vm, 0);
3135 	vm->is_compute_context = false;
3136 }
3137 
3138 /**
3139  * amdgpu_vm_fini - tear down a vm instance
3140  *
3141  * @adev: amdgpu_device pointer
3142  * @vm: requested vm
3143  *
3144  * Tear down @vm.
3145  * Unbind the VM and remove all bos from the vm bo list
3146  */
3147 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
3148 {
3149 	struct amdgpu_bo_va_mapping *mapping, *tmp;
3150 	bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
3151 	struct amdgpu_bo *root;
3152 	int i;
3153 
3154 	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
3155 
3156 	root = amdgpu_bo_ref(vm->root.bo);
3157 	amdgpu_bo_reserve(root, true);
3158 	amdgpu_vm_set_pasid(adev, vm, 0);
3159 	dma_fence_wait(vm->last_unlocked, false);
3160 	dma_fence_put(vm->last_unlocked);
3161 
3162 	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
3163 		if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
3164 			amdgpu_vm_prt_fini(adev, vm);
3165 			prt_fini_needed = false;
3166 		}
3167 
3168 		list_del(&mapping->list);
3169 		amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
3170 	}
3171 
3172 	amdgpu_vm_free_pts(adev, vm, NULL);
3173 	amdgpu_bo_unreserve(root);
3174 	amdgpu_bo_unref(&root);
3175 	WARN_ON(vm->root.bo);
3176 
3177 	drm_sched_entity_destroy(&vm->immediate);
3178 	drm_sched_entity_destroy(&vm->delayed);
3179 
3180 	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
3181 		dev_err(adev->dev, "still active bo inside vm\n");
3182 	}
3183 	rbtree_postorder_for_each_entry_safe(mapping, tmp,
3184 					     &vm->va.rb_root, rb) {
3185 		/* Don't remove the mapping here, we don't want to trigger a
3186 		 * rebalance and the tree is about to be destroyed anyway.
3187 		 */
3188 		list_del(&mapping->list);
3189 		kfree(mapping);
3190 	}
3191 
3192 	dma_fence_put(vm->last_update);
3193 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
3194 		amdgpu_vmid_free_reserved(adev, vm, i);
3195 }
3196 
3197 /**
3198  * amdgpu_vm_manager_init - init the VM manager
3199  *
3200  * @adev: amdgpu_device pointer
3201  *
3202  * Initialize the VM manager structures
3203  */
3204 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
3205 {
3206 	unsigned i;
3207 
3208 	/* Concurrent flushes are only possible starting with Vega10 and
3209 	 * are broken on Navi10 and Navi14.
3210 	 */
3211 	adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 ||
3212 					      adev->asic_type == CHIP_NAVI10 ||
3213 					      adev->asic_type == CHIP_NAVI14);
3214 	amdgpu_vmid_mgr_init(adev);
3215 
3216 	adev->vm_manager.fence_context =
3217 		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3218 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
3219 		adev->vm_manager.seqno[i] = 0;
3220 
3221 	spin_lock_init(&adev->vm_manager.prt_lock);
3222 	atomic_set(&adev->vm_manager.num_prt_users, 0);
3223 
3224 	/* If not overridden by the user, by default, only in large BAR systems
3225 	 * Compute VM tables will be updated by CPU
3226 	 */
3227 #ifdef CONFIG_X86_64
3228 	if (amdgpu_vm_update_mode == -1) {
3229 		if (amdgpu_gmc_vram_full_visible(&adev->gmc))
3230 			adev->vm_manager.vm_update_mode =
3231 				AMDGPU_VM_USE_CPU_FOR_COMPUTE;
3232 		else
3233 			adev->vm_manager.vm_update_mode = 0;
3234 	} else
3235 		adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
3236 #else
3237 	adev->vm_manager.vm_update_mode = 0;
3238 #endif
3239 
3240 	xa_init_flags(&adev->vm_manager.pasids, XA_FLAGS_LOCK_IRQ);
3241 }
3242 
3243 /**
3244  * amdgpu_vm_manager_fini - cleanup VM manager
3245  *
3246  * @adev: amdgpu_device pointer
3247  *
3248  * Cleanup the VM manager and free resources.
3249  */
3250 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
3251 {
3252 	WARN_ON(!xa_empty(&adev->vm_manager.pasids));
3253 	xa_destroy(&adev->vm_manager.pasids);
3254 
3255 	amdgpu_vmid_mgr_fini(adev);
3256 }
3257 
3258 /**
3259  * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
3260  *
3261  * @dev: drm device pointer
3262  * @data: drm_amdgpu_vm
3263  * @filp: drm file pointer
3264  *
3265  * Returns:
3266  * 0 for success, -errno for errors.
3267  */
3268 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
3269 {
3270 	union drm_amdgpu_vm *args = data;
3271 	struct amdgpu_device *adev = drm_to_adev(dev);
3272 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
3273 	long timeout = msecs_to_jiffies(2000);
3274 	int r;
3275 
3276 	switch (args->in.op) {
3277 	case AMDGPU_VM_OP_RESERVE_VMID:
3278 		/* We only have requirement to reserve vmid from gfxhub */
3279 		r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm,
3280 					       AMDGPU_GFXHUB_0);
3281 		if (r)
3282 			return r;
3283 		break;
3284 	case AMDGPU_VM_OP_UNRESERVE_VMID:
3285 		if (amdgpu_sriov_runtime(adev))
3286 			timeout = 8 * timeout;
3287 
3288 		/* Wait vm idle to make sure the vmid set in SPM_VMID is
3289 		 * not referenced anymore.
3290 		 */
3291 		r = amdgpu_bo_reserve(fpriv->vm.root.bo, true);
3292 		if (r)
3293 			return r;
3294 
3295 		r = amdgpu_vm_wait_idle(&fpriv->vm, timeout);
3296 		if (r < 0)
3297 			return r;
3298 
3299 		amdgpu_bo_unreserve(fpriv->vm.root.bo);
3300 		amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0);
3301 		break;
3302 	default:
3303 		return -EINVAL;
3304 	}
3305 
3306 	return 0;
3307 }
3308 
3309 /**
3310  * amdgpu_vm_get_task_info - Extracts task info for a PASID.
3311  *
3312  * @adev: drm device pointer
3313  * @pasid: PASID identifier for VM
3314  * @task_info: task_info to fill.
3315  */
3316 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid,
3317 			 struct amdgpu_task_info *task_info)
3318 {
3319 	struct amdgpu_vm *vm;
3320 	unsigned long flags;
3321 
3322 	xa_lock_irqsave(&adev->vm_manager.pasids, flags);
3323 
3324 	vm = xa_load(&adev->vm_manager.pasids, pasid);
3325 	if (vm)
3326 		*task_info = vm->task_info;
3327 
3328 	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
3329 }
3330 
3331 /**
3332  * amdgpu_vm_set_task_info - Sets VMs task info.
3333  *
3334  * @vm: vm for which to set the info
3335  */
3336 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
3337 {
3338 	if (vm->task_info.pid)
3339 		return;
3340 
3341 	vm->task_info.pid = current->pid;
3342 	get_task_comm(vm->task_info.task_name, current);
3343 
3344 	if (current->group_leader->mm != current->mm)
3345 		return;
3346 
3347 	vm->task_info.tgid = current->group_leader->pid;
3348 	get_task_comm(vm->task_info.process_name, current->group_leader);
3349 }
3350 
3351 /**
3352  * amdgpu_vm_handle_fault - graceful handling of VM faults.
3353  * @adev: amdgpu device pointer
3354  * @pasid: PASID of the VM
3355  * @addr: Address of the fault
3356  * @write_fault: true is write fault, false is read fault
3357  *
3358  * Try to gracefully handle a VM fault. Return true if the fault was handled and
3359  * shouldn't be reported any more.
3360  */
3361 bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
3362 			    uint64_t addr, bool write_fault)
3363 {
3364 	bool is_compute_context = false;
3365 	struct amdgpu_bo *root;
3366 	unsigned long irqflags;
3367 	uint64_t value, flags;
3368 	struct amdgpu_vm *vm;
3369 	int r;
3370 
3371 	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
3372 	vm = xa_load(&adev->vm_manager.pasids, pasid);
3373 	if (vm) {
3374 		root = amdgpu_bo_ref(vm->root.bo);
3375 		is_compute_context = vm->is_compute_context;
3376 	} else {
3377 		root = NULL;
3378 	}
3379 	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
3380 
3381 	if (!root)
3382 		return false;
3383 
3384 	addr /= AMDGPU_GPU_PAGE_SIZE;
3385 
3386 	if (is_compute_context &&
3387 	    !svm_range_restore_pages(adev, pasid, addr, write_fault)) {
3388 		amdgpu_bo_unref(&root);
3389 		return true;
3390 	}
3391 
3392 	r = amdgpu_bo_reserve(root, true);
3393 	if (r)
3394 		goto error_unref;
3395 
3396 	/* Double check that the VM still exists */
3397 	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
3398 	vm = xa_load(&adev->vm_manager.pasids, pasid);
3399 	if (vm && vm->root.bo != root)
3400 		vm = NULL;
3401 	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
3402 	if (!vm)
3403 		goto error_unlock;
3404 
3405 	flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
3406 		AMDGPU_PTE_SYSTEM;
3407 
3408 	if (is_compute_context) {
3409 		/* Intentionally setting invalid PTE flag
3410 		 * combination to force a no-retry-fault
3411 		 */
3412 		flags = AMDGPU_PTE_EXECUTABLE | AMDGPU_PDE_PTE |
3413 			AMDGPU_PTE_TF;
3414 		value = 0;
3415 	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
3416 		/* Redirect the access to the dummy page */
3417 		value = adev->dummy_page_addr;
3418 		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
3419 			AMDGPU_PTE_WRITEABLE;
3420 
3421 	} else {
3422 		/* Let the hw retry silently on the PTE */
3423 		value = 0;
3424 	}
3425 
3426 	r = dma_resv_reserve_shared(root->tbo.base.resv, 1);
3427 	if (r) {
3428 		pr_debug("failed %d to reserve fence slot\n", r);
3429 		goto error_unlock;
3430 	}
3431 
3432 	r = amdgpu_vm_bo_update_mapping(adev, adev, vm, true, false, NULL, addr,
3433 					addr, flags, value, NULL, NULL, NULL,
3434 					NULL);
3435 	if (r)
3436 		goto error_unlock;
3437 
3438 	r = amdgpu_vm_update_pdes(adev, vm, true);
3439 
3440 error_unlock:
3441 	amdgpu_bo_unreserve(root);
3442 	if (r < 0)
3443 		DRM_ERROR("Can't handle page fault (%d)\n", r);
3444 
3445 error_unref:
3446 	amdgpu_bo_unref(&root);
3447 
3448 	return false;
3449 }
3450 
3451 #if defined(CONFIG_DEBUG_FS)
3452 /**
3453  * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
3454  *
3455  * @vm: Requested VM for printing BO info
3456  * @m: debugfs file
3457  *
3458  * Print BO information in debugfs file for the VM
3459  */
3460 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
3461 {
3462 	struct amdgpu_bo_va *bo_va, *tmp;
3463 	u64 total_idle = 0;
3464 	u64 total_evicted = 0;
3465 	u64 total_relocated = 0;
3466 	u64 total_moved = 0;
3467 	u64 total_invalidated = 0;
3468 	u64 total_done = 0;
3469 	unsigned int total_idle_objs = 0;
3470 	unsigned int total_evicted_objs = 0;
3471 	unsigned int total_relocated_objs = 0;
3472 	unsigned int total_moved_objs = 0;
3473 	unsigned int total_invalidated_objs = 0;
3474 	unsigned int total_done_objs = 0;
3475 	unsigned int id = 0;
3476 
3477 	seq_puts(m, "\tIdle BOs:\n");
3478 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
3479 		if (!bo_va->base.bo)
3480 			continue;
3481 		total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
3482 	}
3483 	total_idle_objs = id;
3484 	id = 0;
3485 
3486 	seq_puts(m, "\tEvicted BOs:\n");
3487 	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
3488 		if (!bo_va->base.bo)
3489 			continue;
3490 		total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
3491 	}
3492 	total_evicted_objs = id;
3493 	id = 0;
3494 
3495 	seq_puts(m, "\tRelocated BOs:\n");
3496 	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
3497 		if (!bo_va->base.bo)
3498 			continue;
3499 		total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
3500 	}
3501 	total_relocated_objs = id;
3502 	id = 0;
3503 
3504 	seq_puts(m, "\tMoved BOs:\n");
3505 	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
3506 		if (!bo_va->base.bo)
3507 			continue;
3508 		total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
3509 	}
3510 	total_moved_objs = id;
3511 	id = 0;
3512 
3513 	seq_puts(m, "\tInvalidated BOs:\n");
3514 	spin_lock(&vm->invalidated_lock);
3515 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
3516 		if (!bo_va->base.bo)
3517 			continue;
3518 		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
3519 	}
3520 	total_invalidated_objs = id;
3521 	id = 0;
3522 
3523 	seq_puts(m, "\tDone BOs:\n");
3524 	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
3525 		if (!bo_va->base.bo)
3526 			continue;
3527 		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
3528 	}
3529 	spin_unlock(&vm->invalidated_lock);
3530 	total_done_objs = id;
3531 
3532 	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
3533 		   total_idle_objs);
3534 	seq_printf(m, "\tTotal evicted size:     %12lld\tobjs:\t%d\n", total_evicted,
3535 		   total_evicted_objs);
3536 	seq_printf(m, "\tTotal relocated size:   %12lld\tobjs:\t%d\n", total_relocated,
3537 		   total_relocated_objs);
3538 	seq_printf(m, "\tTotal moved size:       %12lld\tobjs:\t%d\n", total_moved,
3539 		   total_moved_objs);
3540 	seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
3541 		   total_invalidated_objs);
3542 	seq_printf(m, "\tTotal done size:        %12lld\tobjs:\t%d\n", total_done,
3543 		   total_done_objs);
3544 }
3545 #endif
3546