1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020-2023 Intel Corporation
4  */
5 
6 #include <linux/bitfield.h>
7 #include <linux/highmem.h>
8 
9 #include "ivpu_drv.h"
10 #include "ivpu_hw.h"
11 #include "ivpu_mmu.h"
12 #include "ivpu_mmu_context.h"
13 
14 #define IVPU_MMU_PGD_INDEX_MASK          GENMASK(38, 30)
15 #define IVPU_MMU_PMD_INDEX_MASK          GENMASK(29, 21)
16 #define IVPU_MMU_PTE_INDEX_MASK          GENMASK(20, 12)
17 #define IVPU_MMU_ENTRY_FLAGS_MASK        GENMASK(11, 0)
18 #define IVPU_MMU_ENTRY_FLAG_NG           BIT(11)
19 #define IVPU_MMU_ENTRY_FLAG_AF           BIT(10)
20 #define IVPU_MMU_ENTRY_FLAG_USER         BIT(6)
21 #define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2)
22 #define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE    BIT(1)
23 #define IVPU_MMU_ENTRY_FLAG_VALID        BIT(0)
24 
25 #define IVPU_MMU_PAGE_SIZE    SZ_4K
26 #define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE)
27 #define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE)
28 #define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64))
29 
30 #define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000
31 #define IVPU_MMU_ENTRY_VALID   (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID)
32 #define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK)
33 #define IVPU_MMU_ENTRY_MAPPED  (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \
34 				IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID)
35 
36 static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
37 {
38 	dma_addr_t pgd_dma;
39 	u64 *pgd;
40 
41 	pgd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma, GFP_KERNEL);
42 	if (!pgd)
43 		return -ENOMEM;
44 
45 	pgtable->pgd = pgd;
46 	pgtable->pgd_dma = pgd_dma;
47 
48 	return 0;
49 }
50 
51 static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
52 {
53 	int pgd_index, pmd_index;
54 
55 	for (pgd_index = 0; pgd_index < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_index) {
56 		u64 **pmd_entries = pgtable->pgd_cpu_entries[pgd_index];
57 		u64 *pmd = pgtable->pgd_entries[pgd_index];
58 
59 		if (!pmd_entries)
60 			continue;
61 
62 		for (pmd_index = 0; pmd_index < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_index) {
63 			if (pmd_entries[pmd_index])
64 				dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE,
65 					    pmd_entries[pmd_index],
66 					    pmd[pmd_index] & ~IVPU_MMU_ENTRY_FLAGS_MASK);
67 		}
68 
69 		kfree(pmd_entries);
70 		dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd_entries[pgd_index],
71 			    pgtable->pgd[pgd_index] & ~IVPU_MMU_ENTRY_FLAGS_MASK);
72 	}
73 
74 	dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd,
75 		    pgtable->pgd_dma & ~IVPU_MMU_ENTRY_FLAGS_MASK);
76 }
77 
78 static u64*
79 ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, u64 pgd_index)
80 {
81 	u64 **pmd_entries;
82 	dma_addr_t pmd_dma;
83 	u64 *pmd;
84 
85 	if (pgtable->pgd_entries[pgd_index])
86 		return pgtable->pgd_entries[pgd_index];
87 
88 	pmd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL);
89 	if (!pmd)
90 		return NULL;
91 
92 	pmd_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
93 	if (!pmd_entries)
94 		goto err_free_pgd;
95 
96 	pgtable->pgd_entries[pgd_index] = pmd;
97 	pgtable->pgd_cpu_entries[pgd_index] = pmd_entries;
98 	pgtable->pgd[pgd_index] = pmd_dma | IVPU_MMU_ENTRY_VALID;
99 
100 	return pmd;
101 
102 err_free_pgd:
103 	dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pmd, pmd_dma);
104 	return NULL;
105 }
106 
107 static u64*
108 ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
109 		    int pgd_index, int pmd_index)
110 {
111 	dma_addr_t pte_dma;
112 	u64 *pte;
113 
114 	if (pgtable->pgd_cpu_entries[pgd_index][pmd_index])
115 		return pgtable->pgd_cpu_entries[pgd_index][pmd_index];
116 
117 	pte = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL);
118 	if (!pte)
119 		return NULL;
120 
121 	pgtable->pgd_cpu_entries[pgd_index][pmd_index] = pte;
122 	pgtable->pgd_entries[pgd_index][pmd_index] = pte_dma | IVPU_MMU_ENTRY_VALID;
123 
124 	return pte;
125 }
126 
127 static int
128 ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
129 			  u64 vpu_addr, dma_addr_t dma_addr, int prot)
130 {
131 	u64 *pte;
132 	int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
133 	int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
134 	int pte_index = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
135 
136 	/* Allocate PMD - second level page table if needed */
137 	if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_index))
138 		return -ENOMEM;
139 
140 	/* Allocate PTE - third level page table if needed */
141 	pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_index, pmd_index);
142 	if (!pte)
143 		return -ENOMEM;
144 
145 	/* Update PTE - third level page table with DMA address */
146 	pte[pte_index] = dma_addr | prot;
147 
148 	return 0;
149 }
150 
151 static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr)
152 {
153 	int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
154 	int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
155 	int pte_index = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
156 
157 	/* Update PTE with dummy physical address and clear flags */
158 	ctx->pgtable.pgd_cpu_entries[pgd_index][pmd_index][pte_index] = IVPU_MMU_ENTRY_INVALID;
159 }
160 
161 static void
162 ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
163 {
164 	u64 end_addr = vpu_addr + size;
165 	u64 *pgd = ctx->pgtable.pgd;
166 
167 	/* Align to PMD entry (2 MB) */
168 	vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1);
169 
170 	while (vpu_addr < end_addr) {
171 		int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
172 		u64 pmd_end = (pgd_index + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE;
173 		u64 *pmd = ctx->pgtable.pgd_entries[pgd_index];
174 
175 		while (vpu_addr < end_addr && vpu_addr < pmd_end) {
176 			int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
177 			u64 *pte = ctx->pgtable.pgd_cpu_entries[pgd_index][pmd_index];
178 
179 			clflush_cache_range(pte, IVPU_MMU_PGTABLE_SIZE);
180 			vpu_addr += IVPU_MMU_PTE_MAP_SIZE;
181 		}
182 		clflush_cache_range(pmd, IVPU_MMU_PGTABLE_SIZE);
183 	}
184 	clflush_cache_range(pgd, IVPU_MMU_PGTABLE_SIZE);
185 }
186 
187 static int
188 ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
189 			   u64 vpu_addr, dma_addr_t dma_addr, size_t size, int prot)
190 {
191 	while (size) {
192 		int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
193 
194 		if (ret)
195 			return ret;
196 
197 		vpu_addr += IVPU_MMU_PAGE_SIZE;
198 		dma_addr += IVPU_MMU_PAGE_SIZE;
199 		size -= IVPU_MMU_PAGE_SIZE;
200 	}
201 
202 	return 0;
203 }
204 
205 static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
206 {
207 	while (size) {
208 		ivpu_mmu_context_unmap_page(ctx, vpu_addr);
209 		vpu_addr += IVPU_MMU_PAGE_SIZE;
210 		size -= IVPU_MMU_PAGE_SIZE;
211 	}
212 }
213 
214 int
215 ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
216 			 u64 vpu_addr, struct sg_table *sgt,  bool llc_coherent)
217 {
218 	struct scatterlist *sg;
219 	int prot;
220 	int ret;
221 	u64 i;
222 
223 	if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
224 		return -EINVAL;
225 	/*
226 	 * VPU is only 32 bit, but DMA engine is 38 bit
227 	 * Ranges < 2 GB are reserved for VPU internal registers
228 	 * Limit range to 8 GB
229 	 */
230 	if (vpu_addr < SZ_2G || vpu_addr > SZ_8G)
231 		return -EINVAL;
232 
233 	prot = IVPU_MMU_ENTRY_MAPPED;
234 	if (llc_coherent)
235 		prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT;
236 
237 	mutex_lock(&ctx->lock);
238 
239 	for_each_sgtable_dma_sg(sgt, sg, i) {
240 		u64 dma_addr = sg_dma_address(sg) - sg->offset;
241 		size_t size = sg_dma_len(sg) + sg->offset;
242 
243 		ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot);
244 		if (ret) {
245 			ivpu_err(vdev, "Failed to map context pages\n");
246 			mutex_unlock(&ctx->lock);
247 			return ret;
248 		}
249 		ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
250 		vpu_addr += size;
251 	}
252 
253 	mutex_unlock(&ctx->lock);
254 
255 	ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
256 	if (ret)
257 		ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
258 	return ret;
259 }
260 
261 void
262 ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
263 			   u64 vpu_addr, struct sg_table *sgt)
264 {
265 	struct scatterlist *sg;
266 	int ret;
267 	u64 i;
268 
269 	if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
270 		ivpu_warn(vdev, "Unaligned vpu_addr: 0x%llx\n", vpu_addr);
271 
272 	mutex_lock(&ctx->lock);
273 
274 	for_each_sgtable_dma_sg(sgt, sg, i) {
275 		size_t size = sg_dma_len(sg) + sg->offset;
276 
277 		ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size);
278 		ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
279 		vpu_addr += size;
280 	}
281 
282 	mutex_unlock(&ctx->lock);
283 
284 	ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
285 	if (ret)
286 		ivpu_warn(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
287 }
288 
289 int
290 ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
291 				    const struct ivpu_addr_range *range,
292 				    u64 size, struct drm_mm_node *node)
293 {
294 	lockdep_assert_held(&ctx->lock);
295 
296 	return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE,
297 					  0, range->start, range->end, DRM_MM_INSERT_BEST);
298 }
299 
300 void
301 ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)
302 {
303 	lockdep_assert_held(&ctx->lock);
304 
305 	drm_mm_remove_node(node);
306 }
307 
308 static int
309 ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id)
310 {
311 	u64 start, end;
312 	int ret;
313 
314 	mutex_init(&ctx->lock);
315 	INIT_LIST_HEAD(&ctx->bo_list);
316 
317 	ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable);
318 	if (ret)
319 		return ret;
320 
321 	if (!context_id) {
322 		start = vdev->hw->ranges.global_low.start;
323 		end = vdev->hw->ranges.global_high.end;
324 	} else {
325 		start = vdev->hw->ranges.user_low.start;
326 		end = vdev->hw->ranges.user_high.end;
327 	}
328 
329 	drm_mm_init(&ctx->mm, start, end - start);
330 	ctx->id = context_id;
331 
332 	return 0;
333 }
334 
335 static void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
336 {
337 	drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd);
338 
339 	mutex_destroy(&ctx->lock);
340 	ivpu_mmu_pgtable_free(vdev, &ctx->pgtable);
341 	drm_mm_takedown(&ctx->mm);
342 }
343 
344 int ivpu_mmu_global_context_init(struct ivpu_device *vdev)
345 {
346 	return ivpu_mmu_context_init(vdev, &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID);
347 }
348 
349 void ivpu_mmu_global_context_fini(struct ivpu_device *vdev)
350 {
351 	return ivpu_mmu_context_fini(vdev, &vdev->gctx);
352 }
353 
354 void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid)
355 {
356 	struct ivpu_file_priv *file_priv;
357 
358 	xa_lock(&vdev->context_xa);
359 
360 	file_priv = xa_load(&vdev->context_xa, ssid);
361 	if (file_priv)
362 		file_priv->has_mmu_faults = true;
363 
364 	xa_unlock(&vdev->context_xa);
365 }
366 
367 int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 ctx_id)
368 {
369 	int ret;
370 
371 	drm_WARN_ON(&vdev->drm, !ctx_id);
372 
373 	ret = ivpu_mmu_context_init(vdev, ctx, ctx_id);
374 	if (ret) {
375 		ivpu_err(vdev, "Failed to initialize context: %d\n", ret);
376 		return ret;
377 	}
378 
379 	ret = ivpu_mmu_set_pgtable(vdev, ctx_id, &ctx->pgtable);
380 	if (ret) {
381 		ivpu_err(vdev, "Failed to set page table: %d\n", ret);
382 		goto err_context_fini;
383 	}
384 
385 	return 0;
386 
387 err_context_fini:
388 	ivpu_mmu_context_fini(vdev, ctx);
389 	return ret;
390 }
391 
392 void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
393 {
394 	drm_WARN_ON(&vdev->drm, !ctx->id);
395 
396 	ivpu_mmu_clear_pgtable(vdev, ctx->id);
397 	ivpu_mmu_context_fini(vdev, ctx);
398 }
399