xref: /openbmc/linux/arch/sparc/mm/iommu.c (revision 263291fa)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * iommu.c:  IOMMU specific routines for memory management.
4  *
5  * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
6  * Copyright (C) 1995,2002 Pete Zaitcev     (zaitcev@yahoo.com)
7  * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
8  * Copyright (C) 1997,1998 Jakub Jelinek    (jj@sunsite.mff.cuni.cz)
9  */
10 
11 #include <linux/kernel.h>
12 #include <linux/init.h>
13 #include <linux/mm.h>
14 #include <linux/slab.h>
15 #include <linux/dma-map-ops.h>
16 #include <linux/of.h>
17 #include <linux/of_platform.h>
18 #include <linux/platform_device.h>
19 
20 #include <asm/io.h>
21 #include <asm/mxcc.h>
22 #include <asm/mbus.h>
23 #include <asm/cacheflush.h>
24 #include <asm/tlbflush.h>
25 #include <asm/bitext.h>
26 #include <asm/iommu.h>
27 #include <asm/dma.h>
28 
29 #include "mm_32.h"
30 
31 /*
32  * This can be sized dynamically, but we will do this
33  * only when we have a guidance about actual I/O pressures.
34  */
35 #define IOMMU_RNGE	IOMMU_RNGE_256MB
36 #define IOMMU_START	0xF0000000
37 #define IOMMU_WINSIZE	(256*1024*1024U)
38 #define IOMMU_NPTES	(IOMMU_WINSIZE/PAGE_SIZE)	/* 64K PTEs, 256KB */
39 #define IOMMU_ORDER	6				/* 4096 * (1<<6) */
40 
41 static int viking_flush;
42 /* viking.S */
43 extern void viking_flush_page(unsigned long page);
44 extern void viking_mxcc_flush_page(unsigned long page);
45 
46 /*
47  * Values precomputed according to CPU type.
48  */
49 static unsigned int ioperm_noc;		/* Consistent mapping iopte flags */
50 static pgprot_t dvma_prot;		/* Consistent mapping pte flags */
51 
52 #define IOPERM        (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
53 #define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)
54 
55 static const struct dma_map_ops sbus_iommu_dma_gflush_ops;
56 static const struct dma_map_ops sbus_iommu_dma_pflush_ops;
57 
sbus_iommu_init(struct platform_device * op)58 static void __init sbus_iommu_init(struct platform_device *op)
59 {
60 	struct iommu_struct *iommu;
61 	unsigned int impl, vers;
62 	unsigned long *bitmap;
63 	unsigned long control;
64 	unsigned long base;
65 	unsigned long tmp;
66 
67 	iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
68 	if (!iommu) {
69 		prom_printf("Unable to allocate iommu structure\n");
70 		prom_halt();
71 	}
72 
73 	iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
74 				 "iommu_regs");
75 	if (!iommu->regs) {
76 		prom_printf("Cannot map IOMMU registers\n");
77 		prom_halt();
78 	}
79 
80 	control = sbus_readl(&iommu->regs->control);
81 	impl = (control & IOMMU_CTRL_IMPL) >> 28;
82 	vers = (control & IOMMU_CTRL_VERS) >> 24;
83 	control &= ~(IOMMU_CTRL_RNGE);
84 	control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
85 	sbus_writel(control, &iommu->regs->control);
86 
87 	iommu_invalidate(iommu->regs);
88 	iommu->start = IOMMU_START;
89 	iommu->end = 0xffffffff;
90 
91 	/* Allocate IOMMU page table */
92 	/* Stupid alignment constraints give me a headache.
93 	   We need 256K or 512K or 1M or 2M area aligned to
94            its size and current gfp will fortunately give
95            it to us. */
96         tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
97 	if (!tmp) {
98 		prom_printf("Unable to allocate iommu table [0x%lx]\n",
99 			    IOMMU_NPTES * sizeof(iopte_t));
100 		prom_halt();
101 	}
102 	iommu->page_table = (iopte_t *)tmp;
103 
104 	/* Initialize new table. */
105 	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
106 	flush_cache_all();
107 	flush_tlb_all();
108 
109 	base = __pa((unsigned long)iommu->page_table) >> 4;
110 	sbus_writel(base, &iommu->regs->base);
111 	iommu_invalidate(iommu->regs);
112 
113 	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
114 	if (!bitmap) {
115 		prom_printf("Unable to allocate iommu bitmap [%d]\n",
116 			    (int)(IOMMU_NPTES>>3));
117 		prom_halt();
118 	}
119 	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
120 	/* To be coherent on HyperSparc, the page color of DVMA
121 	 * and physical addresses must match.
122 	 */
123 	if (srmmu_modtype == HyperSparc)
124 		iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
125 	else
126 		iommu->usemap.num_colors = 1;
127 
128 	printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
129 	       impl, vers, iommu->page_table,
130 	       (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
131 
132 	op->dev.archdata.iommu = iommu;
133 
134 	if (flush_page_for_dma_global)
135 		op->dev.dma_ops = &sbus_iommu_dma_gflush_ops;
136 	 else
137 		op->dev.dma_ops = &sbus_iommu_dma_pflush_ops;
138 }
139 
iommu_init(void)140 static int __init iommu_init(void)
141 {
142 	struct device_node *dp;
143 
144 	for_each_node_by_name(dp, "iommu") {
145 		struct platform_device *op = of_find_device_by_node(dp);
146 
147 		sbus_iommu_init(op);
148 		of_propagate_archdata(op);
149 	}
150 
151 	return 0;
152 }
153 
154 subsys_initcall(iommu_init);
155 
156 /* Flush the iotlb entries to ram. */
157 /* This could be better if we didn't have to flush whole pages. */
iommu_flush_iotlb(iopte_t * iopte,unsigned int niopte)158 static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
159 {
160 	unsigned long start;
161 	unsigned long end;
162 
163 	start = (unsigned long)iopte;
164 	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
165 	start &= PAGE_MASK;
166 	if (viking_mxcc_present) {
167 		while(start < end) {
168 			viking_mxcc_flush_page(start);
169 			start += PAGE_SIZE;
170 		}
171 	} else if (viking_flush) {
172 		while(start < end) {
173 			viking_flush_page(start);
174 			start += PAGE_SIZE;
175 		}
176 	} else {
177 		while(start < end) {
178 			__flush_page_to_ram(start);
179 			start += PAGE_SIZE;
180 		}
181 	}
182 }
183 
__sbus_iommu_map_page(struct device * dev,struct page * page,unsigned long offset,size_t len,bool per_page_flush)184 static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page,
185 		unsigned long offset, size_t len, bool per_page_flush)
186 {
187 	struct iommu_struct *iommu = dev->archdata.iommu;
188 	phys_addr_t paddr = page_to_phys(page) + offset;
189 	unsigned long off = paddr & ~PAGE_MASK;
190 	unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
191 	unsigned long pfn = __phys_to_pfn(paddr);
192 	unsigned int busa, busa0;
193 	iopte_t *iopte, *iopte0;
194 	int ioptex, i;
195 
196 	/* XXX So what is maxphys for us and how do drivers know it? */
197 	if (!len || len > 256 * 1024)
198 		return DMA_MAPPING_ERROR;
199 
200 	/*
201 	 * We expect unmapped highmem pages to be not in the cache.
202 	 * XXX Is this a good assumption?
203 	 * XXX What if someone else unmaps it here and races us?
204 	 */
205 	if (per_page_flush && !PageHighMem(page)) {
206 		unsigned long vaddr, p;
207 
208 		vaddr = (unsigned long)page_address(page) + offset;
209 		for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
210 			flush_page_for_dma(p);
211 	}
212 
213 	/* page color = pfn of page */
214 	ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
215 	if (ioptex < 0)
216 		panic("iommu out");
217 	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
218 	iopte0 = &iommu->page_table[ioptex];
219 
220 	busa = busa0;
221 	iopte = iopte0;
222 	for (i = 0; i < npages; i++) {
223 		iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
224 		iommu_invalidate_page(iommu->regs, busa);
225 		busa += PAGE_SIZE;
226 		iopte++;
227 		pfn++;
228 	}
229 
230 	iommu_flush_iotlb(iopte0, npages);
231 	return busa0 + off;
232 }
233 
sbus_iommu_map_page_gflush(struct device * dev,struct page * page,unsigned long offset,size_t len,enum dma_data_direction dir,unsigned long attrs)234 static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
235 		struct page *page, unsigned long offset, size_t len,
236 		enum dma_data_direction dir, unsigned long attrs)
237 {
238 	flush_page_for_dma(0);
239 	return __sbus_iommu_map_page(dev, page, offset, len, false);
240 }
241 
sbus_iommu_map_page_pflush(struct device * dev,struct page * page,unsigned long offset,size_t len,enum dma_data_direction dir,unsigned long attrs)242 static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev,
243 		struct page *page, unsigned long offset, size_t len,
244 		enum dma_data_direction dir, unsigned long attrs)
245 {
246 	return __sbus_iommu_map_page(dev, page, offset, len, true);
247 }
248 
__sbus_iommu_map_sg(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,unsigned long attrs,bool per_page_flush)249 static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
250 		int nents, enum dma_data_direction dir, unsigned long attrs,
251 		bool per_page_flush)
252 {
253 	struct scatterlist *sg;
254 	int j;
255 
256 	for_each_sg(sgl, sg, nents, j) {
257 		sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg),
258 				sg->offset, sg->length, per_page_flush);
259 		if (sg->dma_address == DMA_MAPPING_ERROR)
260 			return -EIO;
261 		sg->dma_length = sg->length;
262 	}
263 
264 	return nents;
265 }
266 
sbus_iommu_map_sg_gflush(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,unsigned long attrs)267 static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
268 		int nents, enum dma_data_direction dir, unsigned long attrs)
269 {
270 	flush_page_for_dma(0);
271 	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
272 }
273 
sbus_iommu_map_sg_pflush(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,unsigned long attrs)274 static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
275 		int nents, enum dma_data_direction dir, unsigned long attrs)
276 {
277 	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
278 }
279 
sbus_iommu_unmap_page(struct device * dev,dma_addr_t dma_addr,size_t len,enum dma_data_direction dir,unsigned long attrs)280 static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
281 		size_t len, enum dma_data_direction dir, unsigned long attrs)
282 {
283 	struct iommu_struct *iommu = dev->archdata.iommu;
284 	unsigned int busa = dma_addr & PAGE_MASK;
285 	unsigned long off = dma_addr & ~PAGE_MASK;
286 	unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
287 	unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
288 	unsigned int i;
289 
290 	BUG_ON(busa < iommu->start);
291 	for (i = 0; i < npages; i++) {
292 		iopte_val(iommu->page_table[ioptex + i]) = 0;
293 		iommu_invalidate_page(iommu->regs, busa);
294 		busa += PAGE_SIZE;
295 	}
296 	bit_map_clear(&iommu->usemap, ioptex, npages);
297 }
298 
sbus_iommu_unmap_sg(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,unsigned long attrs)299 static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
300 		int nents, enum dma_data_direction dir, unsigned long attrs)
301 {
302 	struct scatterlist *sg;
303 	int i;
304 
305 	for_each_sg(sgl, sg, nents, i) {
306 		sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir,
307 				attrs);
308 		sg->dma_address = 0x21212121;
309 	}
310 }
311 
312 #ifdef CONFIG_SBUS
sbus_iommu_alloc(struct device * dev,size_t len,dma_addr_t * dma_handle,gfp_t gfp,unsigned long attrs)313 static void *sbus_iommu_alloc(struct device *dev, size_t len,
314 		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
315 {
316 	struct iommu_struct *iommu = dev->archdata.iommu;
317 	unsigned long va, addr, page, end, ret;
318 	iopte_t *iopte = iommu->page_table;
319 	iopte_t *first;
320 	int ioptex;
321 
322 	/* XXX So what is maxphys for us and how do drivers know it? */
323 	if (!len || len > 256 * 1024)
324 		return NULL;
325 
326 	len = PAGE_ALIGN(len);
327 	va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
328 	if (va == 0)
329 		return NULL;
330 
331 	addr = ret = sparc_dma_alloc_resource(dev, len);
332 	if (!addr)
333 		goto out_free_pages;
334 
335 	BUG_ON((va & ~PAGE_MASK) != 0);
336 	BUG_ON((addr & ~PAGE_MASK) != 0);
337 	BUG_ON((len & ~PAGE_MASK) != 0);
338 
339 	/* page color = physical address */
340 	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
341 		addr >> PAGE_SHIFT);
342 	if (ioptex < 0)
343 		panic("iommu out");
344 
345 	iopte += ioptex;
346 	first = iopte;
347 	end = addr + len;
348 	while(addr < end) {
349 		page = va;
350 		{
351 			pmd_t *pmdp;
352 			pte_t *ptep;
353 
354 			if (viking_mxcc_present)
355 				viking_mxcc_flush_page(page);
356 			else if (viking_flush)
357 				viking_flush_page(page);
358 			else
359 				__flush_page_to_ram(page);
360 
361 			pmdp = pmd_off_k(addr);
362 			ptep = pte_offset_kernel(pmdp, addr);
363 
364 			set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
365 		}
366 		iopte_val(*iopte++) =
367 		    MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
368 		addr += PAGE_SIZE;
369 		va += PAGE_SIZE;
370 	}
371 	/* P3: why do we need this?
372 	 *
373 	 * DAVEM: Because there are several aspects, none of which
374 	 *        are handled by a single interface.  Some cpus are
375 	 *        completely not I/O DMA coherent, and some have
376 	 *        virtually indexed caches.  The driver DMA flushing
377 	 *        methods handle the former case, but here during
378 	 *        IOMMU page table modifications, and usage of non-cacheable
379 	 *        cpu mappings of pages potentially in the cpu caches, we have
380 	 *        to handle the latter case as well.
381 	 */
382 	flush_cache_all();
383 	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
384 	flush_tlb_all();
385 	iommu_invalidate(iommu->regs);
386 
387 	*dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
388 	return (void *)ret;
389 
390 out_free_pages:
391 	free_pages(va, get_order(len));
392 	return NULL;
393 }
394 
sbus_iommu_free(struct device * dev,size_t len,void * cpu_addr,dma_addr_t busa,unsigned long attrs)395 static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr,
396 			       dma_addr_t busa, unsigned long attrs)
397 {
398 	struct iommu_struct *iommu = dev->archdata.iommu;
399 	iopte_t *iopte = iommu->page_table;
400 	struct page *page = virt_to_page(cpu_addr);
401 	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
402 	unsigned long end;
403 
404 	if (!sparc_dma_free_resource(cpu_addr, len))
405 		return;
406 
407 	BUG_ON((busa & ~PAGE_MASK) != 0);
408 	BUG_ON((len & ~PAGE_MASK) != 0);
409 
410 	iopte += ioptex;
411 	end = busa + len;
412 	while (busa < end) {
413 		iopte_val(*iopte++) = 0;
414 		busa += PAGE_SIZE;
415 	}
416 	flush_tlb_all();
417 	iommu_invalidate(iommu->regs);
418 	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
419 
420 	__free_pages(page, get_order(len));
421 }
422 #endif
423 
424 static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
425 #ifdef CONFIG_SBUS
426 	.alloc			= sbus_iommu_alloc,
427 	.free			= sbus_iommu_free,
428 #endif
429 	.map_page		= sbus_iommu_map_page_gflush,
430 	.unmap_page		= sbus_iommu_unmap_page,
431 	.map_sg			= sbus_iommu_map_sg_gflush,
432 	.unmap_sg		= sbus_iommu_unmap_sg,
433 };
434 
435 static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
436 #ifdef CONFIG_SBUS
437 	.alloc			= sbus_iommu_alloc,
438 	.free			= sbus_iommu_free,
439 #endif
440 	.map_page		= sbus_iommu_map_page_pflush,
441 	.unmap_page		= sbus_iommu_unmap_page,
442 	.map_sg			= sbus_iommu_map_sg_pflush,
443 	.unmap_sg		= sbus_iommu_unmap_sg,
444 };
445 
ld_mmu_iommu(void)446 void __init ld_mmu_iommu(void)
447 {
448 	if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
449 		dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
450 		ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
451 	} else {
452 		dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
453 		ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
454 	}
455 }
456