1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6 #include <linux/bitfield.h>
7 #include <linux/highmem.h>
8
9 #include "ivpu_drv.h"
10 #include "ivpu_hw.h"
11 #include "ivpu_mmu.h"
12 #include "ivpu_mmu_context.h"
13
14 #define IVPU_MMU_VPU_ADDRESS_MASK GENMASK(47, 12)
15 #define IVPU_MMU_PGD_INDEX_MASK GENMASK(47, 39)
16 #define IVPU_MMU_PUD_INDEX_MASK GENMASK(38, 30)
17 #define IVPU_MMU_PMD_INDEX_MASK GENMASK(29, 21)
18 #define IVPU_MMU_PTE_INDEX_MASK GENMASK(20, 12)
19 #define IVPU_MMU_ENTRY_FLAGS_MASK (BIT(52) | GENMASK(11, 0))
20 #define IVPU_MMU_ENTRY_FLAG_CONT BIT(52)
21 #define IVPU_MMU_ENTRY_FLAG_NG BIT(11)
22 #define IVPU_MMU_ENTRY_FLAG_AF BIT(10)
23 #define IVPU_MMU_ENTRY_FLAG_USER BIT(6)
24 #define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2)
25 #define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE BIT(1)
26 #define IVPU_MMU_ENTRY_FLAG_VALID BIT(0)
27
28 #define IVPU_MMU_PAGE_SIZE SZ_4K
29 #define IVPU_MMU_CONT_PAGES_SIZE (IVPU_MMU_PAGE_SIZE * 16)
30 #define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE)
31 #define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE)
32 #define IVPU_MMU_PUD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PMD_MAP_SIZE)
33 #define IVPU_MMU_PGD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PUD_MAP_SIZE)
34 #define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64))
35
36 #define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000
37 #define IVPU_MMU_ENTRY_VALID (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID)
38 #define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK)
39 #define IVPU_MMU_ENTRY_MAPPED (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \
40 IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID)
41
ivpu_mmu_pgtable_init(struct ivpu_device * vdev,struct ivpu_mmu_pgtable * pgtable)42 static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
43 {
44 dma_addr_t pgd_dma;
45
46 pgtable->pgd_dma_ptr = dma_alloc_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma,
47 GFP_KERNEL);
48 if (!pgtable->pgd_dma_ptr)
49 return -ENOMEM;
50
51 pgtable->pgd_dma = pgd_dma;
52
53 return 0;
54 }
55
ivpu_mmu_pgtable_free(struct ivpu_device * vdev,u64 * cpu_addr,dma_addr_t dma_addr)56 static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr)
57 {
58 if (cpu_addr)
59 dma_free_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, cpu_addr,
60 dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK);
61 }
62
ivpu_mmu_pgtables_free(struct ivpu_device * vdev,struct ivpu_mmu_pgtable * pgtable)63 static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
64 {
65 int pgd_idx, pud_idx, pmd_idx;
66 dma_addr_t pud_dma, pmd_dma, pte_dma;
67 u64 *pud_dma_ptr, *pmd_dma_ptr, *pte_dma_ptr;
68
69 for (pgd_idx = 0; pgd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_idx) {
70 pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];
71 pud_dma = pgtable->pgd_dma_ptr[pgd_idx];
72
73 if (!pud_dma_ptr)
74 continue;
75
76 for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) {
77 pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];
78 pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx];
79
80 if (!pmd_dma_ptr)
81 continue;
82
83 for (pmd_idx = 0; pmd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_idx) {
84 pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
85 pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];
86
87 ivpu_mmu_pgtable_free(vdev, pte_dma_ptr, pte_dma);
88 }
89
90 kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);
91 ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma);
92 }
93
94 kfree(pgtable->pmd_ptrs[pgd_idx]);
95 kfree(pgtable->pte_ptrs[pgd_idx]);
96 ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma);
97 }
98
99 ivpu_mmu_pgtable_free(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);
100 }
101
102 static u64*
ivpu_mmu_ensure_pud(struct ivpu_device * vdev,struct ivpu_mmu_pgtable * pgtable,int pgd_idx)103 ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx)
104 {
105 u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];
106 dma_addr_t pud_dma;
107
108 if (pud_dma_ptr)
109 return pud_dma_ptr;
110
111 pud_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pud_dma, GFP_KERNEL);
112 if (!pud_dma_ptr)
113 return NULL;
114
115 drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]);
116 pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
117 if (!pgtable->pmd_ptrs[pgd_idx])
118 goto err_free_pud_dma_ptr;
119
120 drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]);
121 pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
122 if (!pgtable->pte_ptrs[pgd_idx])
123 goto err_free_pmd_ptrs;
124
125 pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr;
126 pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID;
127
128 return pud_dma_ptr;
129
130 err_free_pmd_ptrs:
131 kfree(pgtable->pmd_ptrs[pgd_idx]);
132
133 err_free_pud_dma_ptr:
134 ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma);
135 return NULL;
136 }
137
138 static u64*
ivpu_mmu_ensure_pmd(struct ivpu_device * vdev,struct ivpu_mmu_pgtable * pgtable,int pgd_idx,int pud_idx)139 ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx,
140 int pud_idx)
141 {
142 u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];
143 dma_addr_t pmd_dma;
144
145 if (pmd_dma_ptr)
146 return pmd_dma_ptr;
147
148 pmd_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL);
149 if (!pmd_dma_ptr)
150 return NULL;
151
152 drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]);
153 pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
154 if (!pgtable->pte_ptrs[pgd_idx][pud_idx])
155 goto err_free_pmd_dma_ptr;
156
157 pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr;
158 pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;
159
160 return pmd_dma_ptr;
161
162 err_free_pmd_dma_ptr:
163 ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma);
164 return NULL;
165 }
166
167 static u64*
ivpu_mmu_ensure_pte(struct ivpu_device * vdev,struct ivpu_mmu_pgtable * pgtable,int pgd_idx,int pud_idx,int pmd_idx)168 ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
169 int pgd_idx, int pud_idx, int pmd_idx)
170 {
171 u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
172 dma_addr_t pte_dma;
173
174 if (pte_dma_ptr)
175 return pte_dma_ptr;
176
177 pte_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL);
178 if (!pte_dma_ptr)
179 return NULL;
180
181 pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr;
182 pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;
183
184 return pte_dma_ptr;
185 }
186
187 static int
ivpu_mmu_context_map_page(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx,u64 vpu_addr,dma_addr_t dma_addr,u64 prot)188 ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
189 u64 vpu_addr, dma_addr_t dma_addr, u64 prot)
190 {
191 u64 *pte;
192 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
193 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
194 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
195 int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
196
197 /* Allocate PUD - second level page table if needed */
198 if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx))
199 return -ENOMEM;
200
201 /* Allocate PMD - third level page table if needed */
202 if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx))
203 return -ENOMEM;
204
205 /* Allocate PTE - fourth level page table if needed */
206 pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx);
207 if (!pte)
208 return -ENOMEM;
209
210 /* Update PTE */
211 pte[pte_idx] = dma_addr | prot;
212
213 return 0;
214 }
215
216 static int
ivpu_mmu_context_map_cont_64k(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx,u64 vpu_addr,dma_addr_t dma_addr,u64 prot)217 ivpu_mmu_context_map_cont_64k(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr,
218 dma_addr_t dma_addr, u64 prot)
219 {
220 size_t size = IVPU_MMU_CONT_PAGES_SIZE;
221
222 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr, size));
223 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(dma_addr, size));
224
225 prot |= IVPU_MMU_ENTRY_FLAG_CONT;
226
227 while (size) {
228 int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
229
230 if (ret)
231 return ret;
232
233 size -= IVPU_MMU_PAGE_SIZE;
234 vpu_addr += IVPU_MMU_PAGE_SIZE;
235 dma_addr += IVPU_MMU_PAGE_SIZE;
236 }
237
238 return 0;
239 }
240
ivpu_mmu_context_unmap_page(struct ivpu_mmu_context * ctx,u64 vpu_addr)241 static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr)
242 {
243 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
244 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
245 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
246 int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
247
248 /* Update PTE with dummy physical address and clear flags */
249 ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;
250 }
251
252 static void
ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context * ctx,u64 vpu_addr,size_t size)253 ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
254 {
255 struct ivpu_mmu_pgtable *pgtable = &ctx->pgtable;
256 u64 end_addr = vpu_addr + size;
257
258 /* Align to PMD entry (2 MB) */
259 vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1);
260
261 while (vpu_addr < end_addr) {
262 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
263 u64 pud_end = (pgd_idx + 1) * (u64)IVPU_MMU_PUD_MAP_SIZE;
264
265 while (vpu_addr < end_addr && vpu_addr < pud_end) {
266 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
267 u64 pmd_end = (pud_idx + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE;
268
269 while (vpu_addr < end_addr && vpu_addr < pmd_end) {
270 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
271
272 clflush_cache_range(pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx],
273 IVPU_MMU_PGTABLE_SIZE);
274 vpu_addr += IVPU_MMU_PTE_MAP_SIZE;
275 }
276 clflush_cache_range(pgtable->pmd_ptrs[pgd_idx][pud_idx],
277 IVPU_MMU_PGTABLE_SIZE);
278 }
279 clflush_cache_range(pgtable->pud_ptrs[pgd_idx], IVPU_MMU_PGTABLE_SIZE);
280 }
281 clflush_cache_range(pgtable->pgd_dma_ptr, IVPU_MMU_PGTABLE_SIZE);
282 }
283
284 static int
ivpu_mmu_context_map_pages(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx,u64 vpu_addr,dma_addr_t dma_addr,size_t size,u64 prot)285 ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
286 u64 vpu_addr, dma_addr_t dma_addr, size_t size, u64 prot)
287 {
288 int map_size;
289 int ret;
290
291 while (size) {
292 if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE &&
293 IS_ALIGNED(vpu_addr | dma_addr, IVPU_MMU_CONT_PAGES_SIZE)) {
294 ret = ivpu_mmu_context_map_cont_64k(vdev, ctx, vpu_addr, dma_addr, prot);
295 map_size = IVPU_MMU_CONT_PAGES_SIZE;
296 } else {
297 ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
298 map_size = IVPU_MMU_PAGE_SIZE;
299 }
300
301 if (ret)
302 return ret;
303
304 vpu_addr += map_size;
305 dma_addr += map_size;
306 size -= map_size;
307 }
308
309 return 0;
310 }
311
ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context * ctx,u64 vpu_addr,size_t size)312 static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
313 {
314 while (size) {
315 ivpu_mmu_context_unmap_page(ctx, vpu_addr);
316 vpu_addr += IVPU_MMU_PAGE_SIZE;
317 size -= IVPU_MMU_PAGE_SIZE;
318 }
319 }
320
321 int
ivpu_mmu_context_map_sgt(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx,u64 vpu_addr,struct sg_table * sgt,bool llc_coherent)322 ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
323 u64 vpu_addr, struct sg_table *sgt, bool llc_coherent)
324 {
325 struct scatterlist *sg;
326 int ret;
327 u64 prot;
328 u64 i;
329
330 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
331 return -EINVAL;
332
333 if (vpu_addr & ~IVPU_MMU_VPU_ADDRESS_MASK)
334 return -EINVAL;
335
336 prot = IVPU_MMU_ENTRY_MAPPED;
337 if (llc_coherent)
338 prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT;
339
340 mutex_lock(&ctx->lock);
341
342 for_each_sgtable_dma_sg(sgt, sg, i) {
343 dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset;
344 size_t size = sg_dma_len(sg) + sg->offset;
345
346 ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot);
347 if (ret) {
348 ivpu_err(vdev, "Failed to map context pages\n");
349 mutex_unlock(&ctx->lock);
350 return ret;
351 }
352 ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
353 vpu_addr += size;
354 }
355
356 mutex_unlock(&ctx->lock);
357
358 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
359 if (ret)
360 ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
361 return ret;
362 }
363
364 void
ivpu_mmu_context_unmap_sgt(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx,u64 vpu_addr,struct sg_table * sgt)365 ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
366 u64 vpu_addr, struct sg_table *sgt)
367 {
368 struct scatterlist *sg;
369 int ret;
370 u64 i;
371
372 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
373 ivpu_warn(vdev, "Unaligned vpu_addr: 0x%llx\n", vpu_addr);
374
375 mutex_lock(&ctx->lock);
376
377 for_each_sgtable_dma_sg(sgt, sg, i) {
378 size_t size = sg_dma_len(sg) + sg->offset;
379
380 ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size);
381 ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
382 vpu_addr += size;
383 }
384
385 mutex_unlock(&ctx->lock);
386
387 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
388 if (ret)
389 ivpu_warn(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
390 }
391
392 int
ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context * ctx,const struct ivpu_addr_range * range,u64 size,struct drm_mm_node * node)393 ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
394 const struct ivpu_addr_range *range,
395 u64 size, struct drm_mm_node *node)
396 {
397 lockdep_assert_held(&ctx->lock);
398
399 if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE) {
400 if (!drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0,
401 range->start, range->end, DRM_MM_INSERT_BEST))
402 return 0;
403 }
404
405 return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0,
406 range->start, range->end, DRM_MM_INSERT_BEST);
407 }
408
409 void
ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context * ctx,struct drm_mm_node * node)410 ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)
411 {
412 lockdep_assert_held(&ctx->lock);
413
414 drm_mm_remove_node(node);
415 }
416
417 static int
ivpu_mmu_context_init(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx,u32 context_id)418 ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id)
419 {
420 u64 start, end;
421 int ret;
422
423 mutex_init(&ctx->lock);
424 INIT_LIST_HEAD(&ctx->bo_list);
425
426 ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable);
427 if (ret)
428 return ret;
429
430 if (!context_id) {
431 start = vdev->hw->ranges.global.start;
432 end = vdev->hw->ranges.shave.end;
433 } else {
434 start = vdev->hw->ranges.user.start;
435 end = vdev->hw->ranges.dma.end;
436 }
437
438 drm_mm_init(&ctx->mm, start, end - start);
439 ctx->id = context_id;
440
441 return 0;
442 }
443
ivpu_mmu_context_fini(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx)444 static void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
445 {
446 if (drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd_dma_ptr))
447 return;
448
449 mutex_destroy(&ctx->lock);
450 ivpu_mmu_pgtables_free(vdev, &ctx->pgtable);
451 drm_mm_takedown(&ctx->mm);
452
453 ctx->pgtable.pgd_dma_ptr = NULL;
454 ctx->pgtable.pgd_dma = 0;
455 }
456
ivpu_mmu_global_context_init(struct ivpu_device * vdev)457 int ivpu_mmu_global_context_init(struct ivpu_device *vdev)
458 {
459 return ivpu_mmu_context_init(vdev, &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID);
460 }
461
ivpu_mmu_global_context_fini(struct ivpu_device * vdev)462 void ivpu_mmu_global_context_fini(struct ivpu_device *vdev)
463 {
464 return ivpu_mmu_context_fini(vdev, &vdev->gctx);
465 }
466
ivpu_mmu_user_context_mark_invalid(struct ivpu_device * vdev,u32 ssid)467 void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid)
468 {
469 struct ivpu_file_priv *file_priv;
470
471 xa_lock(&vdev->context_xa);
472
473 file_priv = xa_load(&vdev->context_xa, ssid);
474 if (file_priv)
475 file_priv->has_mmu_faults = true;
476
477 xa_unlock(&vdev->context_xa);
478 }
479
ivpu_mmu_user_context_init(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx,u32 ctx_id)480 int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 ctx_id)
481 {
482 int ret;
483
484 drm_WARN_ON(&vdev->drm, !ctx_id);
485
486 ret = ivpu_mmu_context_init(vdev, ctx, ctx_id);
487 if (ret) {
488 ivpu_err(vdev, "Failed to initialize context: %d\n", ret);
489 return ret;
490 }
491
492 ret = ivpu_mmu_set_pgtable(vdev, ctx_id, &ctx->pgtable);
493 if (ret) {
494 ivpu_err(vdev, "Failed to set page table: %d\n", ret);
495 goto err_context_fini;
496 }
497
498 return 0;
499
500 err_context_fini:
501 ivpu_mmu_context_fini(vdev, ctx);
502 return ret;
503 }
504
ivpu_mmu_user_context_fini(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx)505 void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
506 {
507 drm_WARN_ON(&vdev->drm, !ctx->id);
508
509 ivpu_mmu_clear_pgtable(vdev, ctx->id);
510 ivpu_mmu_context_fini(vdev, ctx);
511 }
512