1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2018 Intel Corporation.
4  * Copyright 2018 Google LLC.
5  *
6  * Author: Tuukka Toivonen <tuukka.toivonen@intel.com>
7  * Author: Sakari Ailus <sakari.ailus@linux.intel.com>
8  * Author: Samu Onkalo <samu.onkalo@intel.com>
9  * Author: Tomasz Figa <tfiga@chromium.org>
10  *
11  */
12 
13 #include <linux/dma-mapping.h>
14 #include <linux/iopoll.h>
15 #include <linux/pm_runtime.h>
16 #include <linux/slab.h>
17 #include <linux/vmalloc.h>
18 
19 #include <asm/set_memory.h>
20 
21 #include "ipu3-mmu.h"
22 
23 #define IPU3_PT_BITS		10
24 #define IPU3_PT_PTES		(1UL << IPU3_PT_BITS)
25 #define IPU3_PT_SIZE		(IPU3_PT_PTES << 2)
26 #define IPU3_PT_ORDER		(IPU3_PT_SIZE >> PAGE_SHIFT)
27 
28 #define IPU3_ADDR2PTE(addr)	((addr) >> IPU3_PAGE_SHIFT)
29 #define IPU3_PTE2ADDR(pte)	((phys_addr_t)(pte) << IPU3_PAGE_SHIFT)
30 
31 #define IPU3_L2PT_SHIFT		IPU3_PT_BITS
32 #define IPU3_L2PT_MASK		((1UL << IPU3_L2PT_SHIFT) - 1)
33 
34 #define IPU3_L1PT_SHIFT		IPU3_PT_BITS
35 #define IPU3_L1PT_MASK		((1UL << IPU3_L1PT_SHIFT) - 1)
36 
37 #define IPU3_MMU_ADDRESS_BITS	(IPU3_PAGE_SHIFT + \
38 				 IPU3_L2PT_SHIFT + \
39 				 IPU3_L1PT_SHIFT)
40 
41 #define IMGU_REG_BASE		0x4000
42 #define REG_TLB_INVALIDATE	(IMGU_REG_BASE + 0x300)
43 #define TLB_INVALIDATE		1
44 #define REG_L1_PHYS		(IMGU_REG_BASE + 0x304) /* 27-bit pfn */
45 #define REG_GP_HALT		(IMGU_REG_BASE + 0x5dc)
46 #define REG_GP_HALTED		(IMGU_REG_BASE + 0x5e0)
47 
48 struct imgu_mmu {
49 	struct device *dev;
50 	void __iomem *base;
51 	/* protect access to l2pts, l1pt */
52 	spinlock_t lock;
53 
54 	void *dummy_page;
55 	u32 dummy_page_pteval;
56 
57 	u32 *dummy_l2pt;
58 	u32 dummy_l2pt_pteval;
59 
60 	u32 **l2pts;
61 	u32 *l1pt;
62 
63 	struct imgu_mmu_info geometry;
64 };
65 
to_imgu_mmu(struct imgu_mmu_info * info)66 static inline struct imgu_mmu *to_imgu_mmu(struct imgu_mmu_info *info)
67 {
68 	return container_of(info, struct imgu_mmu, geometry);
69 }
70 
71 /**
72  * imgu_mmu_tlb_invalidate - invalidate translation look-aside buffer
73  * @mmu: MMU to perform the invalidate operation on
74  *
75  * This function invalidates the whole TLB. Must be called when the hardware
76  * is powered on.
77  */
imgu_mmu_tlb_invalidate(struct imgu_mmu * mmu)78 static void imgu_mmu_tlb_invalidate(struct imgu_mmu *mmu)
79 {
80 	writel(TLB_INVALIDATE, mmu->base + REG_TLB_INVALIDATE);
81 }
82 
call_if_imgu_is_powered(struct imgu_mmu * mmu,void (* func)(struct imgu_mmu * mmu))83 static void call_if_imgu_is_powered(struct imgu_mmu *mmu,
84 				    void (*func)(struct imgu_mmu *mmu))
85 {
86 	if (!pm_runtime_get_if_in_use(mmu->dev))
87 		return;
88 
89 	func(mmu);
90 	pm_runtime_put(mmu->dev);
91 }
92 
93 /**
94  * imgu_mmu_set_halt - set CIO gate halt bit
95  * @mmu: MMU to set the CIO gate bit in.
96  * @halt: Desired state of the gate bit.
97  *
98  * This function sets the CIO gate bit that controls whether external memory
99  * accesses are allowed. Must be called when the hardware is powered on.
100  */
imgu_mmu_set_halt(struct imgu_mmu * mmu,bool halt)101 static void imgu_mmu_set_halt(struct imgu_mmu *mmu, bool halt)
102 {
103 	int ret;
104 	u32 val;
105 
106 	writel(halt, mmu->base + REG_GP_HALT);
107 	ret = readl_poll_timeout(mmu->base + REG_GP_HALTED,
108 				 val, (val & 1) == halt, 1000, 100000);
109 
110 	if (ret)
111 		dev_err(mmu->dev, "failed to %s CIO gate halt\n",
112 			halt ? "set" : "clear");
113 }
114 
115 /**
116  * imgu_mmu_alloc_page_table - allocate a pre-filled page table
117  * @pteval: Value to initialize for page table entries with.
118  *
119  * Return: Pointer to allocated page table or NULL on failure.
120  */
imgu_mmu_alloc_page_table(u32 pteval)121 static u32 *imgu_mmu_alloc_page_table(u32 pteval)
122 {
123 	u32 *pt;
124 	int pte;
125 
126 	pt = (u32 *)__get_free_page(GFP_KERNEL);
127 	if (!pt)
128 		return NULL;
129 
130 	for (pte = 0; pte < IPU3_PT_PTES; pte++)
131 		pt[pte] = pteval;
132 
133 	set_memory_uc((unsigned long)pt, IPU3_PT_ORDER);
134 
135 	return pt;
136 }
137 
138 /**
139  * imgu_mmu_free_page_table - free page table
140  * @pt: Page table to free.
141  */
imgu_mmu_free_page_table(u32 * pt)142 static void imgu_mmu_free_page_table(u32 *pt)
143 {
144 	set_memory_wb((unsigned long)pt, IPU3_PT_ORDER);
145 	free_page((unsigned long)pt);
146 }
147 
148 /**
149  * address_to_pte_idx - split IOVA into L1 and L2 page table indices
150  * @iova: IOVA to split.
151  * @l1pt_idx: Output for the L1 page table index.
152  * @l2pt_idx: Output for the L2 page index.
153  */
address_to_pte_idx(unsigned long iova,u32 * l1pt_idx,u32 * l2pt_idx)154 static inline void address_to_pte_idx(unsigned long iova, u32 *l1pt_idx,
155 				      u32 *l2pt_idx)
156 {
157 	iova >>= IPU3_PAGE_SHIFT;
158 
159 	if (l2pt_idx)
160 		*l2pt_idx = iova & IPU3_L2PT_MASK;
161 
162 	iova >>= IPU3_L2PT_SHIFT;
163 
164 	if (l1pt_idx)
165 		*l1pt_idx = iova & IPU3_L1PT_MASK;
166 }
167 
imgu_mmu_get_l2pt(struct imgu_mmu * mmu,u32 l1pt_idx)168 static u32 *imgu_mmu_get_l2pt(struct imgu_mmu *mmu, u32 l1pt_idx)
169 {
170 	unsigned long flags;
171 	u32 *l2pt, *new_l2pt;
172 	u32 pteval;
173 
174 	spin_lock_irqsave(&mmu->lock, flags);
175 
176 	l2pt = mmu->l2pts[l1pt_idx];
177 	if (l2pt) {
178 		spin_unlock_irqrestore(&mmu->lock, flags);
179 		return l2pt;
180 	}
181 
182 	spin_unlock_irqrestore(&mmu->lock, flags);
183 
184 	new_l2pt = imgu_mmu_alloc_page_table(mmu->dummy_page_pteval);
185 	if (!new_l2pt)
186 		return NULL;
187 
188 	spin_lock_irqsave(&mmu->lock, flags);
189 
190 	dev_dbg(mmu->dev, "allocated page table %p for l1pt_idx %u\n",
191 		new_l2pt, l1pt_idx);
192 
193 	l2pt = mmu->l2pts[l1pt_idx];
194 	if (l2pt) {
195 		spin_unlock_irqrestore(&mmu->lock, flags);
196 		imgu_mmu_free_page_table(new_l2pt);
197 		return l2pt;
198 	}
199 
200 	l2pt = new_l2pt;
201 	mmu->l2pts[l1pt_idx] = new_l2pt;
202 
203 	pteval = IPU3_ADDR2PTE(virt_to_phys(new_l2pt));
204 	mmu->l1pt[l1pt_idx] = pteval;
205 
206 	spin_unlock_irqrestore(&mmu->lock, flags);
207 	return l2pt;
208 }
209 
__imgu_mmu_map(struct imgu_mmu * mmu,unsigned long iova,phys_addr_t paddr)210 static int __imgu_mmu_map(struct imgu_mmu *mmu, unsigned long iova,
211 			  phys_addr_t paddr)
212 {
213 	u32 l1pt_idx, l2pt_idx;
214 	unsigned long flags;
215 	u32 *l2pt;
216 
217 	if (!mmu)
218 		return -ENODEV;
219 
220 	address_to_pte_idx(iova, &l1pt_idx, &l2pt_idx);
221 
222 	l2pt = imgu_mmu_get_l2pt(mmu, l1pt_idx);
223 	if (!l2pt)
224 		return -ENOMEM;
225 
226 	spin_lock_irqsave(&mmu->lock, flags);
227 
228 	if (l2pt[l2pt_idx] != mmu->dummy_page_pteval) {
229 		spin_unlock_irqrestore(&mmu->lock, flags);
230 		return -EBUSY;
231 	}
232 
233 	l2pt[l2pt_idx] = IPU3_ADDR2PTE(paddr);
234 
235 	spin_unlock_irqrestore(&mmu->lock, flags);
236 
237 	return 0;
238 }
239 
240 /**
241  * imgu_mmu_map - map a buffer to a physical address
242  *
243  * @info: MMU mappable range
244  * @iova: the virtual address
245  * @paddr: the physical address
246  * @size: length of the mappable area
247  *
248  * The function has been adapted from iommu_map() in
249  * drivers/iommu/iommu.c .
250  */
imgu_mmu_map(struct imgu_mmu_info * info,unsigned long iova,phys_addr_t paddr,size_t size)251 int imgu_mmu_map(struct imgu_mmu_info *info, unsigned long iova,
252 		 phys_addr_t paddr, size_t size)
253 {
254 	struct imgu_mmu *mmu = to_imgu_mmu(info);
255 	int ret = 0;
256 
257 	/*
258 	 * both the virtual address and the physical one, as well as
259 	 * the size of the mapping, must be aligned (at least) to the
260 	 * size of the smallest page supported by the hardware
261 	 */
262 	if (!IS_ALIGNED(iova | paddr | size, IPU3_PAGE_SIZE)) {
263 		dev_err(mmu->dev, "unaligned: iova 0x%lx pa %pa size 0x%zx\n",
264 			iova, &paddr, size);
265 		return -EINVAL;
266 	}
267 
268 	dev_dbg(mmu->dev, "map: iova 0x%lx pa %pa size 0x%zx\n",
269 		iova, &paddr, size);
270 
271 	while (size) {
272 		dev_dbg(mmu->dev, "mapping: iova 0x%lx pa %pa\n", iova, &paddr);
273 
274 		ret = __imgu_mmu_map(mmu, iova, paddr);
275 		if (ret)
276 			break;
277 
278 		iova += IPU3_PAGE_SIZE;
279 		paddr += IPU3_PAGE_SIZE;
280 		size -= IPU3_PAGE_SIZE;
281 	}
282 
283 	call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
284 
285 	return ret;
286 }
287 
288 /**
289  * imgu_mmu_map_sg - Map a scatterlist
290  *
291  * @info: MMU mappable range
292  * @iova: the virtual address
293  * @sg: the scatterlist to map
294  * @nents: number of entries in the scatterlist
295  *
296  * The function has been adapted from default_iommu_map_sg() in
297  * drivers/iommu/iommu.c .
298  */
imgu_mmu_map_sg(struct imgu_mmu_info * info,unsigned long iova,struct scatterlist * sg,unsigned int nents)299 size_t imgu_mmu_map_sg(struct imgu_mmu_info *info, unsigned long iova,
300 		       struct scatterlist *sg, unsigned int nents)
301 {
302 	struct imgu_mmu *mmu = to_imgu_mmu(info);
303 	struct scatterlist *s;
304 	size_t s_length, mapped = 0;
305 	unsigned int i;
306 	int ret;
307 
308 	for_each_sg(sg, s, nents, i) {
309 		phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset;
310 
311 		s_length = s->length;
312 
313 		if (!IS_ALIGNED(s->offset, IPU3_PAGE_SIZE))
314 			goto out_err;
315 
316 		/* must be IPU3_PAGE_SIZE aligned to be mapped singlely */
317 		if (i == nents - 1 && !IS_ALIGNED(s->length, IPU3_PAGE_SIZE))
318 			s_length = PAGE_ALIGN(s->length);
319 
320 		ret = imgu_mmu_map(info, iova + mapped, phys, s_length);
321 		if (ret)
322 			goto out_err;
323 
324 		mapped += s_length;
325 	}
326 
327 	call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
328 
329 	return mapped;
330 
331 out_err:
332 	/* undo mappings already done */
333 	imgu_mmu_unmap(info, iova, mapped);
334 
335 	return 0;
336 }
337 
__imgu_mmu_unmap(struct imgu_mmu * mmu,unsigned long iova,size_t size)338 static size_t __imgu_mmu_unmap(struct imgu_mmu *mmu,
339 			       unsigned long iova, size_t size)
340 {
341 	u32 l1pt_idx, l2pt_idx;
342 	unsigned long flags;
343 	size_t unmap = size;
344 	u32 *l2pt;
345 
346 	if (!mmu)
347 		return 0;
348 
349 	address_to_pte_idx(iova, &l1pt_idx, &l2pt_idx);
350 
351 	spin_lock_irqsave(&mmu->lock, flags);
352 
353 	l2pt = mmu->l2pts[l1pt_idx];
354 	if (!l2pt) {
355 		spin_unlock_irqrestore(&mmu->lock, flags);
356 		return 0;
357 	}
358 
359 	if (l2pt[l2pt_idx] == mmu->dummy_page_pteval)
360 		unmap = 0;
361 
362 	l2pt[l2pt_idx] = mmu->dummy_page_pteval;
363 
364 	spin_unlock_irqrestore(&mmu->lock, flags);
365 
366 	return unmap;
367 }
368 
369 /**
370  * imgu_mmu_unmap - Unmap a buffer
371  *
372  * @info: MMU mappable range
373  * @iova: the virtual address
374  * @size: the length of the buffer
375  *
376  * The function has been adapted from iommu_unmap() in
377  * drivers/iommu/iommu.c .
378  */
imgu_mmu_unmap(struct imgu_mmu_info * info,unsigned long iova,size_t size)379 size_t imgu_mmu_unmap(struct imgu_mmu_info *info, unsigned long iova,
380 		      size_t size)
381 {
382 	struct imgu_mmu *mmu = to_imgu_mmu(info);
383 	size_t unmapped_page, unmapped = 0;
384 
385 	/*
386 	 * The virtual address, as well as the size of the mapping, must be
387 	 * aligned (at least) to the size of the smallest page supported
388 	 * by the hardware
389 	 */
390 	if (!IS_ALIGNED(iova | size, IPU3_PAGE_SIZE)) {
391 		dev_err(mmu->dev, "unaligned: iova 0x%lx size 0x%zx\n",
392 			iova, size);
393 		return -EINVAL;
394 	}
395 
396 	dev_dbg(mmu->dev, "unmap this: iova 0x%lx size 0x%zx\n", iova, size);
397 
398 	/*
399 	 * Keep iterating until we either unmap 'size' bytes (or more)
400 	 * or we hit an area that isn't mapped.
401 	 */
402 	while (unmapped < size) {
403 		unmapped_page = __imgu_mmu_unmap(mmu, iova, IPU3_PAGE_SIZE);
404 		if (!unmapped_page)
405 			break;
406 
407 		dev_dbg(mmu->dev, "unmapped: iova 0x%lx size 0x%zx\n",
408 			iova, unmapped_page);
409 
410 		iova += unmapped_page;
411 		unmapped += unmapped_page;
412 	}
413 
414 	call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
415 
416 	return unmapped;
417 }
418 
419 /**
420  * imgu_mmu_init() - initialize IPU3 MMU block
421  *
422  * @parent:	struct device parent
423  * @base:	IOMEM base of hardware registers.
424  *
425  * Return: Pointer to IPU3 MMU private data pointer or ERR_PTR() on error.
426  */
imgu_mmu_init(struct device * parent,void __iomem * base)427 struct imgu_mmu_info *imgu_mmu_init(struct device *parent, void __iomem *base)
428 {
429 	struct imgu_mmu *mmu;
430 	u32 pteval;
431 
432 	mmu = kzalloc(sizeof(*mmu), GFP_KERNEL);
433 	if (!mmu)
434 		return ERR_PTR(-ENOMEM);
435 
436 	mmu->dev = parent;
437 	mmu->base = base;
438 	spin_lock_init(&mmu->lock);
439 
440 	/* Disallow external memory access when having no valid page tables. */
441 	imgu_mmu_set_halt(mmu, true);
442 
443 	/*
444 	 * The MMU does not have a "valid" bit, so we have to use a dummy
445 	 * page for invalid entries.
446 	 */
447 	mmu->dummy_page = (void *)__get_free_page(GFP_KERNEL);
448 	if (!mmu->dummy_page)
449 		goto fail_group;
450 	pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_page));
451 	mmu->dummy_page_pteval = pteval;
452 
453 	/*
454 	 * Allocate a dummy L2 page table with all entries pointing to
455 	 * the dummy page.
456 	 */
457 	mmu->dummy_l2pt = imgu_mmu_alloc_page_table(pteval);
458 	if (!mmu->dummy_l2pt)
459 		goto fail_dummy_page;
460 	pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_l2pt));
461 	mmu->dummy_l2pt_pteval = pteval;
462 
463 	/*
464 	 * Allocate the array of L2PT CPU pointers, initialized to zero,
465 	 * which means the dummy L2PT allocated above.
466 	 */
467 	mmu->l2pts = vzalloc(IPU3_PT_PTES * sizeof(*mmu->l2pts));
468 	if (!mmu->l2pts)
469 		goto fail_l2pt;
470 
471 	/* Allocate the L1 page table. */
472 	mmu->l1pt = imgu_mmu_alloc_page_table(mmu->dummy_l2pt_pteval);
473 	if (!mmu->l1pt)
474 		goto fail_l2pts;
475 
476 	pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt));
477 	writel(pteval, mmu->base + REG_L1_PHYS);
478 	imgu_mmu_tlb_invalidate(mmu);
479 	imgu_mmu_set_halt(mmu, false);
480 
481 	mmu->geometry.aperture_start = 0;
482 	mmu->geometry.aperture_end = DMA_BIT_MASK(IPU3_MMU_ADDRESS_BITS);
483 
484 	return &mmu->geometry;
485 
486 fail_l2pts:
487 	vfree(mmu->l2pts);
488 fail_l2pt:
489 	imgu_mmu_free_page_table(mmu->dummy_l2pt);
490 fail_dummy_page:
491 	free_page((unsigned long)mmu->dummy_page);
492 fail_group:
493 	kfree(mmu);
494 
495 	return ERR_PTR(-ENOMEM);
496 }
497 
498 /**
499  * imgu_mmu_exit() - clean up IPU3 MMU block
500  *
501  * @info: MMU mappable range
502  */
imgu_mmu_exit(struct imgu_mmu_info * info)503 void imgu_mmu_exit(struct imgu_mmu_info *info)
504 {
505 	struct imgu_mmu *mmu = to_imgu_mmu(info);
506 
507 	/* We are going to free our page tables, no more memory access. */
508 	imgu_mmu_set_halt(mmu, true);
509 	imgu_mmu_tlb_invalidate(mmu);
510 
511 	imgu_mmu_free_page_table(mmu->l1pt);
512 	vfree(mmu->l2pts);
513 	imgu_mmu_free_page_table(mmu->dummy_l2pt);
514 	free_page((unsigned long)mmu->dummy_page);
515 	kfree(mmu);
516 }
517 
imgu_mmu_suspend(struct imgu_mmu_info * info)518 void imgu_mmu_suspend(struct imgu_mmu_info *info)
519 {
520 	struct imgu_mmu *mmu = to_imgu_mmu(info);
521 
522 	imgu_mmu_set_halt(mmu, true);
523 }
524 
imgu_mmu_resume(struct imgu_mmu_info * info)525 void imgu_mmu_resume(struct imgu_mmu_info *info)
526 {
527 	struct imgu_mmu *mmu = to_imgu_mmu(info);
528 	u32 pteval;
529 
530 	imgu_mmu_set_halt(mmu, true);
531 
532 	pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt));
533 	writel(pteval, mmu->base + REG_L1_PHYS);
534 
535 	imgu_mmu_tlb_invalidate(mmu);
536 	imgu_mmu_set_halt(mmu, false);
537 }
538