1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3 * Copyright 2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include <drm/drm_drv.h>
25
26 #include "amdgpu.h"
27 #include "amdgpu_trace.h"
28 #include "amdgpu_vm.h"
29
30 /*
31 * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt
32 */
33 struct amdgpu_vm_pt_cursor {
34 uint64_t pfn;
35 struct amdgpu_vm_bo_base *parent;
36 struct amdgpu_vm_bo_base *entry;
37 unsigned int level;
38 };
39
40 /**
41 * amdgpu_vm_pt_level_shift - return the addr shift for each level
42 *
43 * @adev: amdgpu_device pointer
44 * @level: VMPT level
45 *
46 * Returns:
47 * The number of bits the pfn needs to be right shifted for a level.
48 */
amdgpu_vm_pt_level_shift(struct amdgpu_device * adev,unsigned int level)49 static unsigned int amdgpu_vm_pt_level_shift(struct amdgpu_device *adev,
50 unsigned int level)
51 {
52 switch (level) {
53 case AMDGPU_VM_PDB2:
54 case AMDGPU_VM_PDB1:
55 case AMDGPU_VM_PDB0:
56 return 9 * (AMDGPU_VM_PDB0 - level) +
57 adev->vm_manager.block_size;
58 case AMDGPU_VM_PTB:
59 return 0;
60 default:
61 return ~0;
62 }
63 }
64
65 /**
66 * amdgpu_vm_pt_num_entries - return the number of entries in a PD/PT
67 *
68 * @adev: amdgpu_device pointer
69 * @level: VMPT level
70 *
71 * Returns:
72 * The number of entries in a page directory or page table.
73 */
amdgpu_vm_pt_num_entries(struct amdgpu_device * adev,unsigned int level)74 static unsigned int amdgpu_vm_pt_num_entries(struct amdgpu_device *adev,
75 unsigned int level)
76 {
77 unsigned int shift;
78
79 shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level);
80 if (level == adev->vm_manager.root_level)
81 /* For the root directory */
82 return round_up(adev->vm_manager.max_pfn, 1ULL << shift)
83 >> shift;
84 else if (level != AMDGPU_VM_PTB)
85 /* Everything in between */
86 return 512;
87
88 /* For the page tables on the leaves */
89 return AMDGPU_VM_PTE_COUNT(adev);
90 }
91
92 /**
93 * amdgpu_vm_pt_num_ats_entries - return the number of ATS entries in the root PD
94 *
95 * @adev: amdgpu_device pointer
96 *
97 * Returns:
98 * The number of entries in the root page directory which needs the ATS setting.
99 */
amdgpu_vm_pt_num_ats_entries(struct amdgpu_device * adev)100 static unsigned int amdgpu_vm_pt_num_ats_entries(struct amdgpu_device *adev)
101 {
102 unsigned int shift;
103
104 shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level);
105 return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT);
106 }
107
108 /**
109 * amdgpu_vm_pt_entries_mask - the mask to get the entry number of a PD/PT
110 *
111 * @adev: amdgpu_device pointer
112 * @level: VMPT level
113 *
114 * Returns:
115 * The mask to extract the entry number of a PD/PT from an address.
116 */
amdgpu_vm_pt_entries_mask(struct amdgpu_device * adev,unsigned int level)117 static uint32_t amdgpu_vm_pt_entries_mask(struct amdgpu_device *adev,
118 unsigned int level)
119 {
120 if (level <= adev->vm_manager.root_level)
121 return 0xffffffff;
122 else if (level != AMDGPU_VM_PTB)
123 return 0x1ff;
124 else
125 return AMDGPU_VM_PTE_COUNT(adev) - 1;
126 }
127
128 /**
129 * amdgpu_vm_pt_size - returns the size of the page table in bytes
130 *
131 * @adev: amdgpu_device pointer
132 * @level: VMPT level
133 *
134 * Returns:
135 * The size of the BO for a page directory or page table in bytes.
136 */
amdgpu_vm_pt_size(struct amdgpu_device * adev,unsigned int level)137 static unsigned int amdgpu_vm_pt_size(struct amdgpu_device *adev,
138 unsigned int level)
139 {
140 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_pt_num_entries(adev, level) * 8);
141 }
142
143 /**
144 * amdgpu_vm_pt_parent - get the parent page directory
145 *
146 * @pt: child page table
147 *
148 * Helper to get the parent entry for the child page table. NULL if we are at
149 * the root page directory.
150 */
151 static struct amdgpu_vm_bo_base *
amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base * pt)152 amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt)
153 {
154 struct amdgpu_bo *parent = pt->bo->parent;
155
156 if (!parent)
157 return NULL;
158
159 return parent->vm_bo;
160 }
161
162 /**
163 * amdgpu_vm_pt_start - start PD/PT walk
164 *
165 * @adev: amdgpu_device pointer
166 * @vm: amdgpu_vm structure
167 * @start: start address of the walk
168 * @cursor: state to initialize
169 *
170 * Initialize a amdgpu_vm_pt_cursor to start a walk.
171 */
amdgpu_vm_pt_start(struct amdgpu_device * adev,struct amdgpu_vm * vm,uint64_t start,struct amdgpu_vm_pt_cursor * cursor)172 static void amdgpu_vm_pt_start(struct amdgpu_device *adev,
173 struct amdgpu_vm *vm, uint64_t start,
174 struct amdgpu_vm_pt_cursor *cursor)
175 {
176 cursor->pfn = start;
177 cursor->parent = NULL;
178 cursor->entry = &vm->root;
179 cursor->level = adev->vm_manager.root_level;
180 }
181
182 /**
183 * amdgpu_vm_pt_descendant - go to child node
184 *
185 * @adev: amdgpu_device pointer
186 * @cursor: current state
187 *
188 * Walk to the child node of the current node.
189 * Returns:
190 * True if the walk was possible, false otherwise.
191 */
amdgpu_vm_pt_descendant(struct amdgpu_device * adev,struct amdgpu_vm_pt_cursor * cursor)192 static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev,
193 struct amdgpu_vm_pt_cursor *cursor)
194 {
195 unsigned int mask, shift, idx;
196
197 if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry ||
198 !cursor->entry->bo)
199 return false;
200
201 mask = amdgpu_vm_pt_entries_mask(adev, cursor->level);
202 shift = amdgpu_vm_pt_level_shift(adev, cursor->level);
203
204 ++cursor->level;
205 idx = (cursor->pfn >> shift) & mask;
206 cursor->parent = cursor->entry;
207 cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx];
208 return true;
209 }
210
211 /**
212 * amdgpu_vm_pt_sibling - go to sibling node
213 *
214 * @adev: amdgpu_device pointer
215 * @cursor: current state
216 *
217 * Walk to the sibling node of the current node.
218 * Returns:
219 * True if the walk was possible, false otherwise.
220 */
amdgpu_vm_pt_sibling(struct amdgpu_device * adev,struct amdgpu_vm_pt_cursor * cursor)221 static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev,
222 struct amdgpu_vm_pt_cursor *cursor)
223 {
224
225 unsigned int shift, num_entries;
226 struct amdgpu_bo_vm *parent;
227
228 /* Root doesn't have a sibling */
229 if (!cursor->parent)
230 return false;
231
232 /* Go to our parents and see if we got a sibling */
233 shift = amdgpu_vm_pt_level_shift(adev, cursor->level - 1);
234 num_entries = amdgpu_vm_pt_num_entries(adev, cursor->level - 1);
235 parent = to_amdgpu_bo_vm(cursor->parent->bo);
236
237 if (cursor->entry == &parent->entries[num_entries - 1])
238 return false;
239
240 cursor->pfn += 1ULL << shift;
241 cursor->pfn &= ~((1ULL << shift) - 1);
242 ++cursor->entry;
243 return true;
244 }
245
246 /**
247 * amdgpu_vm_pt_ancestor - go to parent node
248 *
249 * @cursor: current state
250 *
251 * Walk to the parent node of the current node.
252 * Returns:
253 * True if the walk was possible, false otherwise.
254 */
amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor * cursor)255 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor)
256 {
257 if (!cursor->parent)
258 return false;
259
260 --cursor->level;
261 cursor->entry = cursor->parent;
262 cursor->parent = amdgpu_vm_pt_parent(cursor->parent);
263 return true;
264 }
265
266 /**
267 * amdgpu_vm_pt_next - get next PD/PT in hieratchy
268 *
269 * @adev: amdgpu_device pointer
270 * @cursor: current state
271 *
272 * Walk the PD/PT tree to the next node.
273 */
amdgpu_vm_pt_next(struct amdgpu_device * adev,struct amdgpu_vm_pt_cursor * cursor)274 static void amdgpu_vm_pt_next(struct amdgpu_device *adev,
275 struct amdgpu_vm_pt_cursor *cursor)
276 {
277 /* First try a newborn child */
278 if (amdgpu_vm_pt_descendant(adev, cursor))
279 return;
280
281 /* If that didn't worked try to find a sibling */
282 while (!amdgpu_vm_pt_sibling(adev, cursor)) {
283 /* No sibling, go to our parents and grandparents */
284 if (!amdgpu_vm_pt_ancestor(cursor)) {
285 cursor->pfn = ~0ll;
286 return;
287 }
288 }
289 }
290
291 /**
292 * amdgpu_vm_pt_first_dfs - start a deep first search
293 *
294 * @adev: amdgpu_device structure
295 * @vm: amdgpu_vm structure
296 * @start: optional cursor to start with
297 * @cursor: state to initialize
298 *
299 * Starts a deep first traversal of the PD/PT tree.
300 */
amdgpu_vm_pt_first_dfs(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_vm_pt_cursor * start,struct amdgpu_vm_pt_cursor * cursor)301 static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev,
302 struct amdgpu_vm *vm,
303 struct amdgpu_vm_pt_cursor *start,
304 struct amdgpu_vm_pt_cursor *cursor)
305 {
306 if (start)
307 *cursor = *start;
308 else
309 amdgpu_vm_pt_start(adev, vm, 0, cursor);
310
311 while (amdgpu_vm_pt_descendant(adev, cursor))
312 ;
313 }
314
315 /**
316 * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue
317 *
318 * @start: starting point for the search
319 * @entry: current entry
320 *
321 * Returns:
322 * True when the search should continue, false otherwise.
323 */
amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor * start,struct amdgpu_vm_bo_base * entry)324 static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start,
325 struct amdgpu_vm_bo_base *entry)
326 {
327 return entry && (!start || entry != start->entry);
328 }
329
330 /**
331 * amdgpu_vm_pt_next_dfs - get the next node for a deep first search
332 *
333 * @adev: amdgpu_device structure
334 * @cursor: current state
335 *
336 * Move the cursor to the next node in a deep first search.
337 */
amdgpu_vm_pt_next_dfs(struct amdgpu_device * adev,struct amdgpu_vm_pt_cursor * cursor)338 static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
339 struct amdgpu_vm_pt_cursor *cursor)
340 {
341 if (!cursor->entry)
342 return;
343
344 if (!cursor->parent)
345 cursor->entry = NULL;
346 else if (amdgpu_vm_pt_sibling(adev, cursor))
347 while (amdgpu_vm_pt_descendant(adev, cursor))
348 ;
349 else
350 amdgpu_vm_pt_ancestor(cursor);
351 }
352
353 /*
354 * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs
355 */
356 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \
357 for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \
358 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
359 amdgpu_vm_pt_continue_dfs((start), (entry)); \
360 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor)))
361
362 /**
363 * amdgpu_vm_pt_clear - initially clear the PDs/PTs
364 *
365 * @adev: amdgpu_device pointer
366 * @vm: VM to clear BO from
367 * @vmbo: BO to clear
368 * @immediate: use an immediate update
369 *
370 * Root PD needs to be reserved when calling this.
371 *
372 * Returns:
373 * 0 on success, errno otherwise.
374 */
amdgpu_vm_pt_clear(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo_vm * vmbo,bool immediate)375 int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
376 struct amdgpu_bo_vm *vmbo, bool immediate)
377 {
378 unsigned int level = adev->vm_manager.root_level;
379 struct ttm_operation_ctx ctx = { true, false };
380 struct amdgpu_vm_update_params params;
381 struct amdgpu_bo *ancestor = &vmbo->bo;
382 unsigned int entries, ats_entries;
383 struct amdgpu_bo *bo = &vmbo->bo;
384 uint64_t addr;
385 int r, idx;
386
387 /* Figure out our place in the hierarchy */
388 if (ancestor->parent) {
389 ++level;
390 while (ancestor->parent->parent) {
391 ++level;
392 ancestor = ancestor->parent;
393 }
394 }
395
396 entries = amdgpu_bo_size(bo) / 8;
397 if (!vm->pte_support_ats) {
398 ats_entries = 0;
399
400 } else if (!bo->parent) {
401 ats_entries = amdgpu_vm_pt_num_ats_entries(adev);
402 ats_entries = min(ats_entries, entries);
403 entries -= ats_entries;
404
405 } else {
406 struct amdgpu_vm_bo_base *pt;
407
408 pt = ancestor->vm_bo;
409 ats_entries = amdgpu_vm_pt_num_ats_entries(adev);
410 if ((pt - to_amdgpu_bo_vm(vm->root.bo)->entries) >=
411 ats_entries) {
412 ats_entries = 0;
413 } else {
414 ats_entries = entries;
415 entries = 0;
416 }
417 }
418
419 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
420 if (r)
421 return r;
422
423 if (vmbo->shadow) {
424 struct amdgpu_bo *shadow = vmbo->shadow;
425
426 r = ttm_bo_validate(&shadow->tbo, &shadow->placement, &ctx);
427 if (r)
428 return r;
429 }
430
431 if (!drm_dev_enter(adev_to_drm(adev), &idx))
432 return -ENODEV;
433
434 r = vm->update_funcs->map_table(vmbo);
435 if (r)
436 goto exit;
437
438 memset(¶ms, 0, sizeof(params));
439 params.adev = adev;
440 params.vm = vm;
441 params.immediate = immediate;
442
443 r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT);
444 if (r)
445 goto exit;
446
447 addr = 0;
448 if (ats_entries) {
449 uint64_t value = 0, flags;
450
451 flags = AMDGPU_PTE_DEFAULT_ATC;
452 if (level != AMDGPU_VM_PTB) {
453 /* Handle leaf PDEs as PTEs */
454 flags |= AMDGPU_PDE_PTE;
455 amdgpu_gmc_get_vm_pde(adev, level, &value, &flags);
456 }
457
458 r = vm->update_funcs->update(¶ms, vmbo, addr, 0,
459 ats_entries, value, flags);
460 if (r)
461 goto exit;
462
463 addr += ats_entries * 8;
464 }
465
466 if (entries) {
467 uint64_t value = 0, flags = 0;
468
469 if (adev->asic_type >= CHIP_VEGA10) {
470 if (level != AMDGPU_VM_PTB) {
471 /* Handle leaf PDEs as PTEs */
472 flags |= AMDGPU_PDE_PTE;
473 amdgpu_gmc_get_vm_pde(adev, level,
474 &value, &flags);
475 } else {
476 /* Workaround for fault priority problem on GMC9 */
477 flags = AMDGPU_PTE_EXECUTABLE;
478 }
479 }
480
481 r = vm->update_funcs->update(¶ms, vmbo, addr, 0, entries,
482 value, flags);
483 if (r)
484 goto exit;
485 }
486
487 r = vm->update_funcs->commit(¶ms, NULL);
488 exit:
489 drm_dev_exit(idx);
490 return r;
491 }
492
493 /**
494 * amdgpu_vm_pt_create - create bo for PD/PT
495 *
496 * @adev: amdgpu_device pointer
497 * @vm: requesting vm
498 * @level: the page table level
499 * @immediate: use a immediate update
500 * @vmbo: pointer to the buffer object pointer
501 * @xcp_id: GPU partition id
502 */
amdgpu_vm_pt_create(struct amdgpu_device * adev,struct amdgpu_vm * vm,int level,bool immediate,struct amdgpu_bo_vm ** vmbo,int32_t xcp_id)503 int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
504 int level, bool immediate, struct amdgpu_bo_vm **vmbo,
505 int32_t xcp_id)
506 {
507 struct amdgpu_bo_param bp;
508 struct amdgpu_bo *bo;
509 struct dma_resv *resv;
510 unsigned int num_entries;
511 int r;
512
513 memset(&bp, 0, sizeof(bp));
514
515 bp.size = amdgpu_vm_pt_size(adev, level);
516 bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
517
518 if (!adev->gmc.is_app_apu)
519 bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
520 else
521 bp.domain = AMDGPU_GEM_DOMAIN_GTT;
522
523 bp.domain = amdgpu_bo_get_preferred_domain(adev, bp.domain);
524 bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
525 AMDGPU_GEM_CREATE_CPU_GTT_USWC;
526
527 if (level < AMDGPU_VM_PTB)
528 num_entries = amdgpu_vm_pt_num_entries(adev, level);
529 else
530 num_entries = 0;
531
532 bp.bo_ptr_size = struct_size((*vmbo), entries, num_entries);
533
534 if (vm->use_cpu_for_update)
535 bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
536
537 bp.type = ttm_bo_type_kernel;
538 bp.no_wait_gpu = immediate;
539 bp.xcp_id_plus1 = xcp_id + 1;
540
541 if (vm->root.bo)
542 bp.resv = vm->root.bo->tbo.base.resv;
543
544 r = amdgpu_bo_create_vm(adev, &bp, vmbo);
545 if (r)
546 return r;
547
548 bo = &(*vmbo)->bo;
549 if (vm->is_compute_context || (adev->flags & AMD_IS_APU)) {
550 (*vmbo)->shadow = NULL;
551 return 0;
552 }
553
554 if (!bp.resv)
555 WARN_ON(dma_resv_lock(bo->tbo.base.resv,
556 NULL));
557 resv = bp.resv;
558 memset(&bp, 0, sizeof(bp));
559 bp.size = amdgpu_vm_pt_size(adev, level);
560 bp.domain = AMDGPU_GEM_DOMAIN_GTT;
561 bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
562 bp.type = ttm_bo_type_kernel;
563 bp.resv = bo->tbo.base.resv;
564 bp.bo_ptr_size = sizeof(struct amdgpu_bo);
565 bp.xcp_id_plus1 = xcp_id + 1;
566
567 r = amdgpu_bo_create(adev, &bp, &(*vmbo)->shadow);
568
569 if (!resv)
570 dma_resv_unlock(bo->tbo.base.resv);
571
572 if (r) {
573 amdgpu_bo_unref(&bo);
574 return r;
575 }
576
577 amdgpu_bo_add_to_shadow_list(*vmbo);
578
579 return 0;
580 }
581
582 /**
583 * amdgpu_vm_pt_alloc - Allocate a specific page table
584 *
585 * @adev: amdgpu_device pointer
586 * @vm: VM to allocate page tables for
587 * @cursor: Which page table to allocate
588 * @immediate: use an immediate update
589 *
590 * Make sure a specific page table or directory is allocated.
591 *
592 * Returns:
593 * 1 if page table needed to be allocated, 0 if page table was already
594 * allocated, negative errno if an error occurred.
595 */
amdgpu_vm_pt_alloc(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_vm_pt_cursor * cursor,bool immediate)596 static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
597 struct amdgpu_vm *vm,
598 struct amdgpu_vm_pt_cursor *cursor,
599 bool immediate)
600 {
601 struct amdgpu_vm_bo_base *entry = cursor->entry;
602 struct amdgpu_bo *pt_bo;
603 struct amdgpu_bo_vm *pt;
604 int r;
605
606 if (entry->bo)
607 return 0;
608
609 amdgpu_vm_eviction_unlock(vm);
610 r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt,
611 vm->root.bo->xcp_id);
612 amdgpu_vm_eviction_lock(vm);
613 if (r)
614 return r;
615
616 /* Keep a reference to the root directory to avoid
617 * freeing them up in the wrong order.
618 */
619 pt_bo = &pt->bo;
620 pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
621 amdgpu_vm_bo_base_init(entry, vm, pt_bo);
622 r = amdgpu_vm_pt_clear(adev, vm, pt, immediate);
623 if (r)
624 goto error_free_pt;
625
626 return 0;
627
628 error_free_pt:
629 amdgpu_bo_unref(&pt->shadow);
630 amdgpu_bo_unref(&pt_bo);
631 return r;
632 }
633
634 /**
635 * amdgpu_vm_pt_free - free one PD/PT
636 *
637 * @entry: PDE to free
638 */
amdgpu_vm_pt_free(struct amdgpu_vm_bo_base * entry)639 static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
640 {
641 struct amdgpu_bo *shadow;
642
643 if (!entry->bo)
644 return;
645
646 entry->bo->vm_bo = NULL;
647 shadow = amdgpu_bo_shadowed(entry->bo);
648 if (shadow) {
649 ttm_bo_set_bulk_move(&shadow->tbo, NULL);
650 amdgpu_bo_unref(&shadow);
651 }
652 ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
653
654 spin_lock(&entry->vm->status_lock);
655 list_del(&entry->vm_status);
656 spin_unlock(&entry->vm->status_lock);
657 amdgpu_bo_unref(&entry->bo);
658 }
659
amdgpu_vm_pt_free_work(struct work_struct * work)660 void amdgpu_vm_pt_free_work(struct work_struct *work)
661 {
662 struct amdgpu_vm_bo_base *entry, *next;
663 struct amdgpu_vm *vm;
664 LIST_HEAD(pt_freed);
665
666 vm = container_of(work, struct amdgpu_vm, pt_free_work);
667
668 spin_lock(&vm->status_lock);
669 list_splice_init(&vm->pt_freed, &pt_freed);
670 spin_unlock(&vm->status_lock);
671
672 /* flush_work in amdgpu_vm_fini ensure vm->root.bo is valid. */
673 amdgpu_bo_reserve(vm->root.bo, true);
674
675 list_for_each_entry_safe(entry, next, &pt_freed, vm_status)
676 amdgpu_vm_pt_free(entry);
677
678 amdgpu_bo_unreserve(vm->root.bo);
679 }
680
681 /**
682 * amdgpu_vm_pt_free_dfs - free PD/PT levels
683 *
684 * @adev: amdgpu device structure
685 * @vm: amdgpu vm structure
686 * @start: optional cursor where to start freeing PDs/PTs
687 * @unlocked: vm resv unlock status
688 *
689 * Free the page directory or page table level and all sub levels.
690 */
amdgpu_vm_pt_free_dfs(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_vm_pt_cursor * start,bool unlocked)691 static void amdgpu_vm_pt_free_dfs(struct amdgpu_device *adev,
692 struct amdgpu_vm *vm,
693 struct amdgpu_vm_pt_cursor *start,
694 bool unlocked)
695 {
696 struct amdgpu_vm_pt_cursor cursor;
697 struct amdgpu_vm_bo_base *entry;
698
699 if (unlocked) {
700 spin_lock(&vm->status_lock);
701 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
702 list_move(&entry->vm_status, &vm->pt_freed);
703
704 if (start)
705 list_move(&start->entry->vm_status, &vm->pt_freed);
706 spin_unlock(&vm->status_lock);
707 schedule_work(&vm->pt_free_work);
708 return;
709 }
710
711 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
712 amdgpu_vm_pt_free(entry);
713
714 if (start)
715 amdgpu_vm_pt_free(start->entry);
716 }
717
718 /**
719 * amdgpu_vm_pt_free_root - free root PD
720 * @adev: amdgpu device structure
721 * @vm: amdgpu vm structure
722 *
723 * Free the root page directory and everything below it.
724 */
amdgpu_vm_pt_free_root(struct amdgpu_device * adev,struct amdgpu_vm * vm)725 void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm)
726 {
727 amdgpu_vm_pt_free_dfs(adev, vm, NULL, false);
728 }
729
730 /**
731 * amdgpu_vm_pt_is_root_clean - check if a root PD is clean
732 *
733 * @adev: amdgpu_device pointer
734 * @vm: the VM to check
735 *
736 * Check all entries of the root PD, if any subsequent PDs are allocated,
737 * it means there are page table creating and filling, and is no a clean
738 * VM
739 *
740 * Returns:
741 * 0 if this VM is clean
742 */
amdgpu_vm_pt_is_root_clean(struct amdgpu_device * adev,struct amdgpu_vm * vm)743 bool amdgpu_vm_pt_is_root_clean(struct amdgpu_device *adev,
744 struct amdgpu_vm *vm)
745 {
746 enum amdgpu_vm_level root = adev->vm_manager.root_level;
747 unsigned int entries = amdgpu_vm_pt_num_entries(adev, root);
748 unsigned int i = 0;
749
750 for (i = 0; i < entries; i++) {
751 if (to_amdgpu_bo_vm(vm->root.bo)->entries[i].bo)
752 return false;
753 }
754 return true;
755 }
756
757 /**
758 * amdgpu_vm_pde_update - update a single level in the hierarchy
759 *
760 * @params: parameters for the update
761 * @entry: entry to update
762 *
763 * Makes sure the requested entry in parent is up to date.
764 */
amdgpu_vm_pde_update(struct amdgpu_vm_update_params * params,struct amdgpu_vm_bo_base * entry)765 int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
766 struct amdgpu_vm_bo_base *entry)
767 {
768 struct amdgpu_vm_bo_base *parent = amdgpu_vm_pt_parent(entry);
769 struct amdgpu_bo *bo = parent->bo, *pbo;
770 struct amdgpu_vm *vm = params->vm;
771 uint64_t pde, pt, flags;
772 unsigned int level;
773
774 for (level = 0, pbo = bo->parent; pbo; ++level)
775 pbo = pbo->parent;
776
777 level += params->adev->vm_manager.root_level;
778 amdgpu_gmc_get_pde_for_bo(entry->bo, level, &pt, &flags);
779 pde = (entry - to_amdgpu_bo_vm(parent->bo)->entries) * 8;
780 return vm->update_funcs->update(params, to_amdgpu_bo_vm(bo), pde, pt,
781 1, 0, flags);
782 }
783
784 /**
785 * amdgpu_vm_pte_update_noretry_flags - Update PTE no-retry flags
786 *
787 * @adev: amdgpu_device pointer
788 * @flags: pointer to PTE flags
789 *
790 * Update PTE no-retry flags when TF is enabled.
791 */
amdgpu_vm_pte_update_noretry_flags(struct amdgpu_device * adev,uint64_t * flags)792 static void amdgpu_vm_pte_update_noretry_flags(struct amdgpu_device *adev,
793 uint64_t *flags)
794 {
795 /*
796 * Update no-retry flags with the corresponding TF
797 * no-retry combination.
798 */
799 if ((*flags & AMDGPU_VM_NORETRY_FLAGS) == AMDGPU_VM_NORETRY_FLAGS) {
800 *flags &= ~AMDGPU_VM_NORETRY_FLAGS;
801 *flags |= adev->gmc.noretry_flags;
802 }
803 }
804
805 /*
806 * amdgpu_vm_pte_update_flags - figure out flags for PTE updates
807 *
808 * Make sure to set the right flags for the PTEs at the desired level.
809 */
amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params * params,struct amdgpu_bo_vm * pt,unsigned int level,uint64_t pe,uint64_t addr,unsigned int count,uint32_t incr,uint64_t flags)810 static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
811 struct amdgpu_bo_vm *pt,
812 unsigned int level,
813 uint64_t pe, uint64_t addr,
814 unsigned int count, uint32_t incr,
815 uint64_t flags)
816 {
817 struct amdgpu_device *adev = params->adev;
818
819 if (level != AMDGPU_VM_PTB) {
820 flags |= AMDGPU_PDE_PTE;
821 amdgpu_gmc_get_vm_pde(adev, level, &addr, &flags);
822
823 } else if (adev->asic_type >= CHIP_VEGA10 &&
824 !(flags & AMDGPU_PTE_VALID) &&
825 !(flags & AMDGPU_PTE_PRT)) {
826
827 /* Workaround for fault priority problem on GMC9 */
828 flags |= AMDGPU_PTE_EXECUTABLE;
829 }
830
831 /*
832 * Update no-retry flags to use the no-retry flag combination
833 * with TF enabled. The AMDGPU_VM_NORETRY_FLAGS flag combination
834 * does not work when TF is enabled. So, replace them with
835 * AMDGPU_VM_NORETRY_FLAGS_TF flag combination which works for
836 * all cases.
837 */
838 if (level == AMDGPU_VM_PTB)
839 amdgpu_vm_pte_update_noretry_flags(adev, &flags);
840
841 /* APUs mapping system memory may need different MTYPEs on different
842 * NUMA nodes. Only do this for contiguous ranges that can be assumed
843 * to be on the same NUMA node.
844 */
845 if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
846 adev->gmc.gmc_funcs->override_vm_pte_flags &&
847 num_possible_nodes() > 1) {
848 if (!params->pages_addr)
849 amdgpu_gmc_override_vm_pte_flags(adev, params->vm,
850 addr, &flags);
851 else
852 dev_dbg(adev->dev,
853 "override_vm_pte_flags skipped: non-contiguous\n");
854 }
855
856 params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
857 flags);
858 }
859
860 /**
861 * amdgpu_vm_pte_fragment - get fragment for PTEs
862 *
863 * @params: see amdgpu_vm_update_params definition
864 * @start: first PTE to handle
865 * @end: last PTE to handle
866 * @flags: hw mapping flags
867 * @frag: resulting fragment size
868 * @frag_end: end of this fragment
869 *
870 * Returns the first possible fragment for the start and end address.
871 */
amdgpu_vm_pte_fragment(struct amdgpu_vm_update_params * params,uint64_t start,uint64_t end,uint64_t flags,unsigned int * frag,uint64_t * frag_end)872 static void amdgpu_vm_pte_fragment(struct amdgpu_vm_update_params *params,
873 uint64_t start, uint64_t end, uint64_t flags,
874 unsigned int *frag, uint64_t *frag_end)
875 {
876 /**
877 * The MC L1 TLB supports variable sized pages, based on a fragment
878 * field in the PTE. When this field is set to a non-zero value, page
879 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
880 * flags are considered valid for all PTEs within the fragment range
881 * and corresponding mappings are assumed to be physically contiguous.
882 *
883 * The L1 TLB can store a single PTE for the whole fragment,
884 * significantly increasing the space available for translation
885 * caching. This leads to large improvements in throughput when the
886 * TLB is under pressure.
887 *
888 * The L2 TLB distributes small and large fragments into two
889 * asymmetric partitions. The large fragment cache is significantly
890 * larger. Thus, we try to use large fragments wherever possible.
891 * Userspace can support this by aligning virtual base address and
892 * allocation size to the fragment size.
893 *
894 * Starting with Vega10 the fragment size only controls the L1. The L2
895 * is now directly feed with small/huge/giant pages from the walker.
896 */
897 unsigned int max_frag;
898
899 if (params->adev->asic_type < CHIP_VEGA10)
900 max_frag = params->adev->vm_manager.fragment_size;
901 else
902 max_frag = 31;
903
904 /* system pages are non continuously */
905 if (params->pages_addr) {
906 *frag = 0;
907 *frag_end = end;
908 return;
909 }
910
911 /* This intentionally wraps around if no bit is set */
912 *frag = min_t(unsigned int, ffs(start) - 1, fls64(end - start) - 1);
913 if (*frag >= max_frag) {
914 *frag = max_frag;
915 *frag_end = end & ~((1ULL << max_frag) - 1);
916 } else {
917 *frag_end = start + (1 << *frag);
918 }
919 }
920
921 /**
922 * amdgpu_vm_ptes_update - make sure that page tables are valid
923 *
924 * @params: see amdgpu_vm_update_params definition
925 * @start: start of GPU address range
926 * @end: end of GPU address range
927 * @dst: destination address to map to, the next dst inside the function
928 * @flags: mapping flags
929 *
930 * Update the page tables in the range @start - @end.
931 *
932 * Returns:
933 * 0 for success, -EINVAL for failure.
934 */
amdgpu_vm_ptes_update(struct amdgpu_vm_update_params * params,uint64_t start,uint64_t end,uint64_t dst,uint64_t flags)935 int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
936 uint64_t start, uint64_t end,
937 uint64_t dst, uint64_t flags)
938 {
939 struct amdgpu_device *adev = params->adev;
940 struct amdgpu_vm_pt_cursor cursor;
941 uint64_t frag_start = start, frag_end;
942 unsigned int frag;
943 int r;
944
945 /* figure out the initial fragment */
946 amdgpu_vm_pte_fragment(params, frag_start, end, flags, &frag,
947 &frag_end);
948
949 /* walk over the address space and update the PTs */
950 amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
951 while (cursor.pfn < end) {
952 unsigned int shift, parent_shift, mask;
953 uint64_t incr, entry_end, pe_start;
954 struct amdgpu_bo *pt;
955
956 if (!params->unlocked) {
957 /* make sure that the page tables covering the
958 * address range are actually allocated
959 */
960 r = amdgpu_vm_pt_alloc(params->adev, params->vm,
961 &cursor, params->immediate);
962 if (r)
963 return r;
964 }
965
966 shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
967 parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
968 if (params->unlocked) {
969 /* Unlocked updates are only allowed on the leaves */
970 if (amdgpu_vm_pt_descendant(adev, &cursor))
971 continue;
972 } else if (adev->asic_type < CHIP_VEGA10 &&
973 (flags & AMDGPU_PTE_VALID)) {
974 /* No huge page support before GMC v9 */
975 if (cursor.level != AMDGPU_VM_PTB) {
976 if (!amdgpu_vm_pt_descendant(adev, &cursor))
977 return -ENOENT;
978 continue;
979 }
980 } else if (frag < shift) {
981 /* We can't use this level when the fragment size is
982 * smaller than the address shift. Go to the next
983 * child entry and try again.
984 */
985 if (amdgpu_vm_pt_descendant(adev, &cursor))
986 continue;
987 } else if (frag >= parent_shift) {
988 /* If the fragment size is even larger than the parent
989 * shift we should go up one level and check it again.
990 */
991 if (!amdgpu_vm_pt_ancestor(&cursor))
992 return -EINVAL;
993 continue;
994 }
995
996 pt = cursor.entry->bo;
997 if (!pt) {
998 /* We need all PDs and PTs for mapping something, */
999 if (flags & AMDGPU_PTE_VALID)
1000 return -ENOENT;
1001
1002 /* but unmapping something can happen at a higher
1003 * level.
1004 */
1005 if (!amdgpu_vm_pt_ancestor(&cursor))
1006 return -EINVAL;
1007
1008 pt = cursor.entry->bo;
1009 shift = parent_shift;
1010 frag_end = max(frag_end, ALIGN(frag_start + 1,
1011 1ULL << shift));
1012 }
1013
1014 /* Looks good so far, calculate parameters for the update */
1015 incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift;
1016 mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
1017 pe_start = ((cursor.pfn >> shift) & mask) * 8;
1018 entry_end = ((uint64_t)mask + 1) << shift;
1019 entry_end += cursor.pfn & ~(entry_end - 1);
1020 entry_end = min(entry_end, end);
1021
1022 do {
1023 struct amdgpu_vm *vm = params->vm;
1024 uint64_t upd_end = min(entry_end, frag_end);
1025 unsigned int nptes = (upd_end - frag_start) >> shift;
1026 uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag);
1027
1028 /* This can happen when we set higher level PDs to
1029 * silent to stop fault floods.
1030 */
1031 nptes = max(nptes, 1u);
1032
1033 trace_amdgpu_vm_update_ptes(params, frag_start, upd_end,
1034 min(nptes, 32u), dst, incr,
1035 upd_flags,
1036 vm->task_info.tgid,
1037 vm->immediate.fence_context);
1038 amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
1039 cursor.level, pe_start, dst,
1040 nptes, incr, upd_flags);
1041
1042 pe_start += nptes * 8;
1043 dst += nptes * incr;
1044
1045 frag_start = upd_end;
1046 if (frag_start >= frag_end) {
1047 /* figure out the next fragment */
1048 amdgpu_vm_pte_fragment(params, frag_start, end,
1049 flags, &frag, &frag_end);
1050 if (frag < shift)
1051 break;
1052 }
1053 } while (frag_start < entry_end);
1054
1055 if (amdgpu_vm_pt_descendant(adev, &cursor)) {
1056 /* Free all child entries.
1057 * Update the tables with the flags and addresses and free up subsequent
1058 * tables in the case of huge pages or freed up areas.
1059 * This is the maximum you can free, because all other page tables are not
1060 * completely covered by the range and so potentially still in use.
1061 */
1062 while (cursor.pfn < frag_start) {
1063 /* Make sure previous mapping is freed */
1064 if (cursor.entry->bo) {
1065 params->table_freed = true;
1066 amdgpu_vm_pt_free_dfs(adev, params->vm,
1067 &cursor,
1068 params->unlocked);
1069 }
1070 amdgpu_vm_pt_next(adev, &cursor);
1071 }
1072
1073 } else if (frag >= shift) {
1074 /* or just move on to the next on the same level. */
1075 amdgpu_vm_pt_next(adev, &cursor);
1076 }
1077 }
1078
1079 return 0;
1080 }
1081
1082 /**
1083 * amdgpu_vm_pt_map_tables - have bo of root PD cpu accessible
1084 * @adev: amdgpu device structure
1085 * @vm: amdgpu vm structure
1086 *
1087 * make root page directory and everything below it cpu accessible.
1088 */
amdgpu_vm_pt_map_tables(struct amdgpu_device * adev,struct amdgpu_vm * vm)1089 int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1090 {
1091 struct amdgpu_vm_pt_cursor cursor;
1092 struct amdgpu_vm_bo_base *entry;
1093
1094 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) {
1095
1096 struct amdgpu_bo_vm *bo;
1097 int r;
1098
1099 if (entry->bo) {
1100 bo = to_amdgpu_bo_vm(entry->bo);
1101 r = vm->update_funcs->map_table(bo);
1102 if (r)
1103 return r;
1104 }
1105 }
1106
1107 return 0;
1108 }
1109