1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds * iommu.c: IOMMU specific routines for memory management.
41da177e4SLinus Torvalds *
51da177e4SLinus Torvalds * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
61da177e4SLinus Torvalds * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
71da177e4SLinus Torvalds * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
81da177e4SLinus Torvalds * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
91da177e4SLinus Torvalds */
101da177e4SLinus Torvalds
111da177e4SLinus Torvalds #include <linux/kernel.h>
121da177e4SLinus Torvalds #include <linux/init.h>
131da177e4SLinus Torvalds #include <linux/mm.h>
141da177e4SLinus Torvalds #include <linux/slab.h>
150a0f0d8bSChristoph Hellwig #include <linux/dma-map-ops.h>
169dc69230SDavid S. Miller #include <linux/of.h>
17*263291faSRob Herring #include <linux/of_platform.h>
18*263291faSRob Herring #include <linux/platform_device.h>
191da177e4SLinus Torvalds
201da177e4SLinus Torvalds #include <asm/io.h>
211da177e4SLinus Torvalds #include <asm/mxcc.h>
221da177e4SLinus Torvalds #include <asm/mbus.h>
231da177e4SLinus Torvalds #include <asm/cacheflush.h>
241da177e4SLinus Torvalds #include <asm/tlbflush.h>
251da177e4SLinus Torvalds #include <asm/bitext.h>
261da177e4SLinus Torvalds #include <asm/iommu.h>
271da177e4SLinus Torvalds #include <asm/dma.h>
281da177e4SLinus Torvalds
29e8c29c83SSam Ravnborg #include "mm_32.h"
30e8c29c83SSam Ravnborg
311da177e4SLinus Torvalds /*
321da177e4SLinus Torvalds * This can be sized dynamically, but we will do this
331da177e4SLinus Torvalds * only when we have a guidance about actual I/O pressures.
341da177e4SLinus Torvalds */
351da177e4SLinus Torvalds #define IOMMU_RNGE IOMMU_RNGE_256MB
361da177e4SLinus Torvalds #define IOMMU_START 0xF0000000
371da177e4SLinus Torvalds #define IOMMU_WINSIZE (256*1024*1024U)
389a0ac1b6SAkinobu Mita #define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */
391da177e4SLinus Torvalds #define IOMMU_ORDER 6 /* 4096 * (1<<6) */
401da177e4SLinus Torvalds
411da177e4SLinus Torvalds static int viking_flush;
421da177e4SLinus Torvalds /* viking.S */
431da177e4SLinus Torvalds extern void viking_flush_page(unsigned long page);
441da177e4SLinus Torvalds extern void viking_mxcc_flush_page(unsigned long page);
451da177e4SLinus Torvalds
461da177e4SLinus Torvalds /*
471da177e4SLinus Torvalds * Values precomputed according to CPU type.
481da177e4SLinus Torvalds */
491da177e4SLinus Torvalds static unsigned int ioperm_noc; /* Consistent mapping iopte flags */
501da177e4SLinus Torvalds static pgprot_t dvma_prot; /* Consistent mapping pte flags */
511da177e4SLinus Torvalds
521da177e4SLinus Torvalds #define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
531da177e4SLinus Torvalds #define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)
541da177e4SLinus Torvalds
55255a69a9SChristoph Hellwig static const struct dma_map_ops sbus_iommu_dma_gflush_ops;
56255a69a9SChristoph Hellwig static const struct dma_map_ops sbus_iommu_dma_pflush_ops;
57255a69a9SChristoph Hellwig
sbus_iommu_init(struct platform_device * op)58cd4cd730SGrant Likely static void __init sbus_iommu_init(struct platform_device *op)
591da177e4SLinus Torvalds {
601da177e4SLinus Torvalds struct iommu_struct *iommu;
61e0039348SDavid S. Miller unsigned int impl, vers;
621da177e4SLinus Torvalds unsigned long *bitmap;
63f977ea49SSam Ravnborg unsigned long control;
64f977ea49SSam Ravnborg unsigned long base;
65e0039348SDavid S. Miller unsigned long tmp;
66e0039348SDavid S. Miller
6771cd03b0SJulia Lawall iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
681da177e4SLinus Torvalds if (!iommu) {
691da177e4SLinus Torvalds prom_printf("Unable to allocate iommu structure\n");
701da177e4SLinus Torvalds prom_halt();
711da177e4SLinus Torvalds }
72e0039348SDavid S. Miller
73046e26a8SDavid S. Miller iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
74e0039348SDavid S. Miller "iommu_regs");
751da177e4SLinus Torvalds if (!iommu->regs) {
761da177e4SLinus Torvalds prom_printf("Cannot map IOMMU registers\n");
771da177e4SLinus Torvalds prom_halt();
781da177e4SLinus Torvalds }
79f977ea49SSam Ravnborg
80f977ea49SSam Ravnborg control = sbus_readl(&iommu->regs->control);
81f977ea49SSam Ravnborg impl = (control & IOMMU_CTRL_IMPL) >> 28;
82f977ea49SSam Ravnborg vers = (control & IOMMU_CTRL_VERS) >> 24;
83f977ea49SSam Ravnborg control &= ~(IOMMU_CTRL_RNGE);
84f977ea49SSam Ravnborg control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
85f977ea49SSam Ravnborg sbus_writel(control, &iommu->regs->control);
86f977ea49SSam Ravnborg
871da177e4SLinus Torvalds iommu_invalidate(iommu->regs);
881da177e4SLinus Torvalds iommu->start = IOMMU_START;
891da177e4SLinus Torvalds iommu->end = 0xffffffff;
901da177e4SLinus Torvalds
911da177e4SLinus Torvalds /* Allocate IOMMU page table */
921da177e4SLinus Torvalds /* Stupid alignment constraints give me a headache.
931da177e4SLinus Torvalds We need 256K or 512K or 1M or 2M area aligned to
941da177e4SLinus Torvalds its size and current gfp will fortunately give
951da177e4SLinus Torvalds it to us. */
961da177e4SLinus Torvalds tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
971da177e4SLinus Torvalds if (!tmp) {
985da444aaSAkinobu Mita prom_printf("Unable to allocate iommu table [0x%lx]\n",
991da177e4SLinus Torvalds IOMMU_NPTES * sizeof(iopte_t));
1001da177e4SLinus Torvalds prom_halt();
1011da177e4SLinus Torvalds }
1021da177e4SLinus Torvalds iommu->page_table = (iopte_t *)tmp;
1031da177e4SLinus Torvalds
1041da177e4SLinus Torvalds /* Initialize new table. */
1051da177e4SLinus Torvalds memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
1061da177e4SLinus Torvalds flush_cache_all();
1071da177e4SLinus Torvalds flush_tlb_all();
108f977ea49SSam Ravnborg
109f977ea49SSam Ravnborg base = __pa((unsigned long)iommu->page_table) >> 4;
110f977ea49SSam Ravnborg sbus_writel(base, &iommu->regs->base);
1111da177e4SLinus Torvalds iommu_invalidate(iommu->regs);
1121da177e4SLinus Torvalds
1131da177e4SLinus Torvalds bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
1141da177e4SLinus Torvalds if (!bitmap) {
1151da177e4SLinus Torvalds prom_printf("Unable to allocate iommu bitmap [%d]\n",
1161da177e4SLinus Torvalds (int)(IOMMU_NPTES>>3));
1171da177e4SLinus Torvalds prom_halt();
1181da177e4SLinus Torvalds }
1191da177e4SLinus Torvalds bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
1201da177e4SLinus Torvalds /* To be coherent on HyperSparc, the page color of DVMA
1211da177e4SLinus Torvalds * and physical addresses must match.
1221da177e4SLinus Torvalds */
1231da177e4SLinus Torvalds if (srmmu_modtype == HyperSparc)
1241da177e4SLinus Torvalds iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
1251da177e4SLinus Torvalds else
1261da177e4SLinus Torvalds iommu->usemap.num_colors = 1;
1271da177e4SLinus Torvalds
128046e26a8SDavid S. Miller printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
1291da177e4SLinus Torvalds impl, vers, iommu->page_table,
1301da177e4SLinus Torvalds (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
1311da177e4SLinus Torvalds
132e0039348SDavid S. Miller op->dev.archdata.iommu = iommu;
133255a69a9SChristoph Hellwig
134255a69a9SChristoph Hellwig if (flush_page_for_dma_global)
135255a69a9SChristoph Hellwig op->dev.dma_ops = &sbus_iommu_dma_gflush_ops;
136255a69a9SChristoph Hellwig else
137255a69a9SChristoph Hellwig op->dev.dma_ops = &sbus_iommu_dma_pflush_ops;
1381da177e4SLinus Torvalds }
1391da177e4SLinus Torvalds
iommu_init(void)140046e26a8SDavid S. Miller static int __init iommu_init(void)
141046e26a8SDavid S. Miller {
142046e26a8SDavid S. Miller struct device_node *dp;
143046e26a8SDavid S. Miller
144046e26a8SDavid S. Miller for_each_node_by_name(dp, "iommu") {
145cd4cd730SGrant Likely struct platform_device *op = of_find_device_by_node(dp);
146046e26a8SDavid S. Miller
147046e26a8SDavid S. Miller sbus_iommu_init(op);
148046e26a8SDavid S. Miller of_propagate_archdata(op);
149046e26a8SDavid S. Miller }
150046e26a8SDavid S. Miller
151046e26a8SDavid S. Miller return 0;
152046e26a8SDavid S. Miller }
153046e26a8SDavid S. Miller
154046e26a8SDavid S. Miller subsys_initcall(iommu_init);
155046e26a8SDavid S. Miller
1561da177e4SLinus Torvalds /* Flush the iotlb entries to ram. */
1571da177e4SLinus Torvalds /* This could be better if we didn't have to flush whole pages. */
iommu_flush_iotlb(iopte_t * iopte,unsigned int niopte)1581da177e4SLinus Torvalds static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
1591da177e4SLinus Torvalds {
1601da177e4SLinus Torvalds unsigned long start;
1611da177e4SLinus Torvalds unsigned long end;
1621da177e4SLinus Torvalds
1633185d4d2SBob Breuer start = (unsigned long)iopte;
1641da177e4SLinus Torvalds end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
1653185d4d2SBob Breuer start &= PAGE_MASK;
1661da177e4SLinus Torvalds if (viking_mxcc_present) {
1671da177e4SLinus Torvalds while(start < end) {
1681da177e4SLinus Torvalds viking_mxcc_flush_page(start);
1691da177e4SLinus Torvalds start += PAGE_SIZE;
1701da177e4SLinus Torvalds }
1711da177e4SLinus Torvalds } else if (viking_flush) {
1721da177e4SLinus Torvalds while(start < end) {
1731da177e4SLinus Torvalds viking_flush_page(start);
1741da177e4SLinus Torvalds start += PAGE_SIZE;
1751da177e4SLinus Torvalds }
1761da177e4SLinus Torvalds } else {
1771da177e4SLinus Torvalds while(start < end) {
1781da177e4SLinus Torvalds __flush_page_to_ram(start);
1791da177e4SLinus Torvalds start += PAGE_SIZE;
1801da177e4SLinus Torvalds }
1811da177e4SLinus Torvalds }
1821da177e4SLinus Torvalds }
1831da177e4SLinus Torvalds
__sbus_iommu_map_page(struct device * dev,struct page * page,unsigned long offset,size_t len,bool per_page_flush)184376b1371SChristoph Hellwig static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page,
185376b1371SChristoph Hellwig unsigned long offset, size_t len, bool per_page_flush)
1861da177e4SLinus Torvalds {
187260489faSDavid S. Miller struct iommu_struct *iommu = dev->archdata.iommu;
188376b1371SChristoph Hellwig phys_addr_t paddr = page_to_phys(page) + offset;
189376b1371SChristoph Hellwig unsigned long off = paddr & ~PAGE_MASK;
190376b1371SChristoph Hellwig unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
191b8205942SChristoph Hellwig unsigned long pfn = __phys_to_pfn(paddr);
192376b1371SChristoph Hellwig unsigned int busa, busa0;
193376b1371SChristoph Hellwig iopte_t *iopte, *iopte0;
194376b1371SChristoph Hellwig int ioptex, i;
195376b1371SChristoph Hellwig
196376b1371SChristoph Hellwig /* XXX So what is maxphys for us and how do drivers know it? */
197376b1371SChristoph Hellwig if (!len || len > 256 * 1024)
198376b1371SChristoph Hellwig return DMA_MAPPING_ERROR;
199376b1371SChristoph Hellwig
200376b1371SChristoph Hellwig /*
201376b1371SChristoph Hellwig * We expect unmapped highmem pages to be not in the cache.
202376b1371SChristoph Hellwig * XXX Is this a good assumption?
203376b1371SChristoph Hellwig * XXX What if someone else unmaps it here and races us?
204376b1371SChristoph Hellwig */
205376b1371SChristoph Hellwig if (per_page_flush && !PageHighMem(page)) {
206376b1371SChristoph Hellwig unsigned long vaddr, p;
207376b1371SChristoph Hellwig
208376b1371SChristoph Hellwig vaddr = (unsigned long)page_address(page) + offset;
209376b1371SChristoph Hellwig for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
210376b1371SChristoph Hellwig flush_page_for_dma(p);
211376b1371SChristoph Hellwig }
2121da177e4SLinus Torvalds
2131da177e4SLinus Torvalds /* page color = pfn of page */
214b8205942SChristoph Hellwig ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
2151da177e4SLinus Torvalds if (ioptex < 0)
2161da177e4SLinus Torvalds panic("iommu out");
2171da177e4SLinus Torvalds busa0 = iommu->start + (ioptex << PAGE_SHIFT);
2181da177e4SLinus Torvalds iopte0 = &iommu->page_table[ioptex];
2191da177e4SLinus Torvalds
2201da177e4SLinus Torvalds busa = busa0;
2211da177e4SLinus Torvalds iopte = iopte0;
2221da177e4SLinus Torvalds for (i = 0; i < npages; i++) {
223b8205942SChristoph Hellwig iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
2241da177e4SLinus Torvalds iommu_invalidate_page(iommu->regs, busa);
2251da177e4SLinus Torvalds busa += PAGE_SIZE;
2261da177e4SLinus Torvalds iopte++;
227b8205942SChristoph Hellwig pfn++;
2281da177e4SLinus Torvalds }
2291da177e4SLinus Torvalds
2301da177e4SLinus Torvalds iommu_flush_iotlb(iopte0, npages);
231376b1371SChristoph Hellwig return busa0 + off;
2321da177e4SLinus Torvalds }
2331da177e4SLinus Torvalds
sbus_iommu_map_page_gflush(struct device * dev,struct page * page,unsigned long offset,size_t len,enum dma_data_direction dir,unsigned long attrs)234ce65d36fSChristoph Hellwig static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
235ce65d36fSChristoph Hellwig struct page *page, unsigned long offset, size_t len,
236ce65d36fSChristoph Hellwig enum dma_data_direction dir, unsigned long attrs)
2371da177e4SLinus Torvalds {
2381da177e4SLinus Torvalds flush_page_for_dma(0);
2398668b38cSChristoph Hellwig return __sbus_iommu_map_page(dev, page, offset, len, false);
2401da177e4SLinus Torvalds }
2411da177e4SLinus Torvalds
sbus_iommu_map_page_pflush(struct device * dev,struct page * page,unsigned long offset,size_t len,enum dma_data_direction dir,unsigned long attrs)242ce65d36fSChristoph Hellwig static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev,
243ce65d36fSChristoph Hellwig struct page *page, unsigned long offset, size_t len,
244ce65d36fSChristoph Hellwig enum dma_data_direction dir, unsigned long attrs)
2451da177e4SLinus Torvalds {
2468668b38cSChristoph Hellwig return __sbus_iommu_map_page(dev, page, offset, len, true);
247ce65d36fSChristoph Hellwig }
248ce65d36fSChristoph Hellwig
__sbus_iommu_map_sg(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,unsigned long attrs,bool per_page_flush)249ff5cbec0SChristoph Hellwig static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
250ff5cbec0SChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs,
251ff5cbec0SChristoph Hellwig bool per_page_flush)
2521da177e4SLinus Torvalds {
2536c503d0dSChristoph Hellwig struct scatterlist *sg;
254edb1f072SChristoph Hellwig int j;
2551da177e4SLinus Torvalds
2566c503d0dSChristoph Hellwig for_each_sg(sgl, sg, nents, j) {
257edb1f072SChristoph Hellwig sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg),
258edb1f072SChristoph Hellwig sg->offset, sg->length, per_page_flush);
259edb1f072SChristoph Hellwig if (sg->dma_address == DMA_MAPPING_ERROR)
260e02373fdSMartin Oliveira return -EIO;
261aa83a26aSRobert Reif sg->dma_length = sg->length;
2621da177e4SLinus Torvalds }
263ce65d36fSChristoph Hellwig
2646c503d0dSChristoph Hellwig return nents;
2651da177e4SLinus Torvalds }
2661da177e4SLinus Torvalds
sbus_iommu_map_sg_gflush(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,unsigned long attrs)267ff5cbec0SChristoph Hellwig static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
268ff5cbec0SChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs)
269ff5cbec0SChristoph Hellwig {
270ff5cbec0SChristoph Hellwig flush_page_for_dma(0);
271ff5cbec0SChristoph Hellwig return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
272ff5cbec0SChristoph Hellwig }
273ff5cbec0SChristoph Hellwig
sbus_iommu_map_sg_pflush(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,unsigned long attrs)274ff5cbec0SChristoph Hellwig static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
275ff5cbec0SChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs)
276ff5cbec0SChristoph Hellwig {
277ff5cbec0SChristoph Hellwig return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
278ff5cbec0SChristoph Hellwig }
279ff5cbec0SChristoph Hellwig
sbus_iommu_unmap_page(struct device * dev,dma_addr_t dma_addr,size_t len,enum dma_data_direction dir,unsigned long attrs)280f25b23bcSChristoph Hellwig static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
281f25b23bcSChristoph Hellwig size_t len, enum dma_data_direction dir, unsigned long attrs)
2821da177e4SLinus Torvalds {
283260489faSDavid S. Miller struct iommu_struct *iommu = dev->archdata.iommu;
284f25b23bcSChristoph Hellwig unsigned int busa = dma_addr & PAGE_MASK;
285f25b23bcSChristoph Hellwig unsigned long off = dma_addr & ~PAGE_MASK;
286f25b23bcSChristoph Hellwig unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
287f25b23bcSChristoph Hellwig unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
288f25b23bcSChristoph Hellwig unsigned int i;
2891da177e4SLinus Torvalds
2901ae61388SEric Sesterhenn BUG_ON(busa < iommu->start);
2911da177e4SLinus Torvalds for (i = 0; i < npages; i++) {
2921da177e4SLinus Torvalds iopte_val(iommu->page_table[ioptex + i]) = 0;
2931da177e4SLinus Torvalds iommu_invalidate_page(iommu->regs, busa);
2941da177e4SLinus Torvalds busa += PAGE_SIZE;
2951da177e4SLinus Torvalds }
2961da177e4SLinus Torvalds bit_map_clear(&iommu->usemap, ioptex, npages);
2971da177e4SLinus Torvalds }
2981da177e4SLinus Torvalds
sbus_iommu_unmap_sg(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,unsigned long attrs)2996c503d0dSChristoph Hellwig static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
3006c503d0dSChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs)
3011da177e4SLinus Torvalds {
3026c503d0dSChristoph Hellwig struct scatterlist *sg;
303a7fce1f7SChristoph Hellwig int i;
3041da177e4SLinus Torvalds
3056c503d0dSChristoph Hellwig for_each_sg(sgl, sg, nents, i) {
306a7fce1f7SChristoph Hellwig sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir,
307a7fce1f7SChristoph Hellwig attrs);
308aa83a26aSRobert Reif sg->dma_address = 0x21212121;
3091da177e4SLinus Torvalds }
3101da177e4SLinus Torvalds }
3111da177e4SLinus Torvalds
3121da177e4SLinus Torvalds #ifdef CONFIG_SBUS
sbus_iommu_alloc(struct device * dev,size_t len,dma_addr_t * dma_handle,gfp_t gfp,unsigned long attrs)313ce65d36fSChristoph Hellwig static void *sbus_iommu_alloc(struct device *dev, size_t len,
314ce65d36fSChristoph Hellwig dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
3151da177e4SLinus Torvalds {
3164b1c5df2SDavid S. Miller struct iommu_struct *iommu = dev->archdata.iommu;
317ce65d36fSChristoph Hellwig unsigned long va, addr, page, end, ret;
3181da177e4SLinus Torvalds iopte_t *iopte = iommu->page_table;
3191da177e4SLinus Torvalds iopte_t *first;
3201da177e4SLinus Torvalds int ioptex;
3211da177e4SLinus Torvalds
322ce65d36fSChristoph Hellwig /* XXX So what is maxphys for us and how do drivers know it? */
323ce65d36fSChristoph Hellwig if (!len || len > 256 * 1024)
324ce65d36fSChristoph Hellwig return NULL;
325ce65d36fSChristoph Hellwig
326ce65d36fSChristoph Hellwig len = PAGE_ALIGN(len);
327518a2f19SChristoph Hellwig va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
328ce65d36fSChristoph Hellwig if (va == 0)
329ce65d36fSChristoph Hellwig return NULL;
330ce65d36fSChristoph Hellwig
331ce65d36fSChristoph Hellwig addr = ret = sparc_dma_alloc_resource(dev, len);
332ce65d36fSChristoph Hellwig if (!addr)
333ce65d36fSChristoph Hellwig goto out_free_pages;
334ce65d36fSChristoph Hellwig
3351ae61388SEric Sesterhenn BUG_ON((va & ~PAGE_MASK) != 0);
3361ae61388SEric Sesterhenn BUG_ON((addr & ~PAGE_MASK) != 0);
3371ae61388SEric Sesterhenn BUG_ON((len & ~PAGE_MASK) != 0);
3381da177e4SLinus Torvalds
3391da177e4SLinus Torvalds /* page color = physical address */
3401da177e4SLinus Torvalds ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
3411da177e4SLinus Torvalds addr >> PAGE_SHIFT);
3421da177e4SLinus Torvalds if (ioptex < 0)
3431da177e4SLinus Torvalds panic("iommu out");
3441da177e4SLinus Torvalds
3451da177e4SLinus Torvalds iopte += ioptex;
3461da177e4SLinus Torvalds first = iopte;
3471da177e4SLinus Torvalds end = addr + len;
3481da177e4SLinus Torvalds while(addr < end) {
3491da177e4SLinus Torvalds page = va;
3501da177e4SLinus Torvalds {
3511da177e4SLinus Torvalds pmd_t *pmdp;
3521da177e4SLinus Torvalds pte_t *ptep;
3531da177e4SLinus Torvalds
3541da177e4SLinus Torvalds if (viking_mxcc_present)
3551da177e4SLinus Torvalds viking_mxcc_flush_page(page);
3561da177e4SLinus Torvalds else if (viking_flush)
3571da177e4SLinus Torvalds viking_flush_page(page);
3581da177e4SLinus Torvalds else
3591da177e4SLinus Torvalds __flush_page_to_ram(page);
3601da177e4SLinus Torvalds
361e05c7b1fSMike Rapoport pmdp = pmd_off_k(addr);
3627a19c361SHugh Dickins ptep = pte_offset_kernel(pmdp, addr);
3631da177e4SLinus Torvalds
3641da177e4SLinus Torvalds set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
3651da177e4SLinus Torvalds }
3661da177e4SLinus Torvalds iopte_val(*iopte++) =
3671da177e4SLinus Torvalds MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
3681da177e4SLinus Torvalds addr += PAGE_SIZE;
3691da177e4SLinus Torvalds va += PAGE_SIZE;
3701da177e4SLinus Torvalds }
3711da177e4SLinus Torvalds /* P3: why do we need this?
3721da177e4SLinus Torvalds *
3731da177e4SLinus Torvalds * DAVEM: Because there are several aspects, none of which
3741da177e4SLinus Torvalds * are handled by a single interface. Some cpus are
3751da177e4SLinus Torvalds * completely not I/O DMA coherent, and some have
3761da177e4SLinus Torvalds * virtually indexed caches. The driver DMA flushing
3771da177e4SLinus Torvalds * methods handle the former case, but here during
3781da177e4SLinus Torvalds * IOMMU page table modifications, and usage of non-cacheable
3791da177e4SLinus Torvalds * cpu mappings of pages potentially in the cpu caches, we have
3801da177e4SLinus Torvalds * to handle the latter case as well.
3811da177e4SLinus Torvalds */
3821da177e4SLinus Torvalds flush_cache_all();
3831da177e4SLinus Torvalds iommu_flush_iotlb(first, len >> PAGE_SHIFT);
3841da177e4SLinus Torvalds flush_tlb_all();
3851da177e4SLinus Torvalds iommu_invalidate(iommu->regs);
3861da177e4SLinus Torvalds
387ce65d36fSChristoph Hellwig *dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
388ce65d36fSChristoph Hellwig return (void *)ret;
389ce65d36fSChristoph Hellwig
390ce65d36fSChristoph Hellwig out_free_pages:
391ce65d36fSChristoph Hellwig free_pages(va, get_order(len));
392ce65d36fSChristoph Hellwig return NULL;
3931da177e4SLinus Torvalds }
3941da177e4SLinus Torvalds
sbus_iommu_free(struct device * dev,size_t len,void * cpu_addr,dma_addr_t busa,unsigned long attrs)395ce65d36fSChristoph Hellwig static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr,
396ce65d36fSChristoph Hellwig dma_addr_t busa, unsigned long attrs)
3971da177e4SLinus Torvalds {
3984b1c5df2SDavid S. Miller struct iommu_struct *iommu = dev->archdata.iommu;
3991da177e4SLinus Torvalds iopte_t *iopte = iommu->page_table;
400ce65d36fSChristoph Hellwig struct page *page = virt_to_page(cpu_addr);
4011da177e4SLinus Torvalds int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
402ce65d36fSChristoph Hellwig unsigned long end;
403ce65d36fSChristoph Hellwig
404ce65d36fSChristoph Hellwig if (!sparc_dma_free_resource(cpu_addr, len))
405ce65d36fSChristoph Hellwig return;
4061da177e4SLinus Torvalds
4071ae61388SEric Sesterhenn BUG_ON((busa & ~PAGE_MASK) != 0);
4081ae61388SEric Sesterhenn BUG_ON((len & ~PAGE_MASK) != 0);
4091da177e4SLinus Torvalds
4101da177e4SLinus Torvalds iopte += ioptex;
4111da177e4SLinus Torvalds end = busa + len;
4121da177e4SLinus Torvalds while (busa < end) {
4131da177e4SLinus Torvalds iopte_val(*iopte++) = 0;
4141da177e4SLinus Torvalds busa += PAGE_SIZE;
4151da177e4SLinus Torvalds }
4161da177e4SLinus Torvalds flush_tlb_all();
4171da177e4SLinus Torvalds iommu_invalidate(iommu->regs);
4181da177e4SLinus Torvalds bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
419ce65d36fSChristoph Hellwig
420ce65d36fSChristoph Hellwig __free_pages(page, get_order(len));
4211da177e4SLinus Torvalds }
4221da177e4SLinus Torvalds #endif
4231da177e4SLinus Torvalds
424ce65d36fSChristoph Hellwig static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
425d894d964SDavid S. Miller #ifdef CONFIG_SBUS
426ce65d36fSChristoph Hellwig .alloc = sbus_iommu_alloc,
427ce65d36fSChristoph Hellwig .free = sbus_iommu_free,
428d894d964SDavid S. Miller #endif
429ce65d36fSChristoph Hellwig .map_page = sbus_iommu_map_page_gflush,
430ce65d36fSChristoph Hellwig .unmap_page = sbus_iommu_unmap_page,
431ce65d36fSChristoph Hellwig .map_sg = sbus_iommu_map_sg_gflush,
432ce65d36fSChristoph Hellwig .unmap_sg = sbus_iommu_unmap_sg,
433d894d964SDavid S. Miller };
434d894d964SDavid S. Miller
435ce65d36fSChristoph Hellwig static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
436d894d964SDavid S. Miller #ifdef CONFIG_SBUS
437ce65d36fSChristoph Hellwig .alloc = sbus_iommu_alloc,
438ce65d36fSChristoph Hellwig .free = sbus_iommu_free,
439d894d964SDavid S. Miller #endif
440ce65d36fSChristoph Hellwig .map_page = sbus_iommu_map_page_pflush,
441ce65d36fSChristoph Hellwig .unmap_page = sbus_iommu_unmap_page,
442ce65d36fSChristoph Hellwig .map_sg = sbus_iommu_map_sg_pflush,
443ce65d36fSChristoph Hellwig .unmap_sg = sbus_iommu_unmap_sg,
444d894d964SDavid S. Miller };
445d894d964SDavid S. Miller
ld_mmu_iommu(void)4461da177e4SLinus Torvalds void __init ld_mmu_iommu(void)
4471da177e4SLinus Torvalds {
4481da177e4SLinus Torvalds if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
4491da177e4SLinus Torvalds dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
4501da177e4SLinus Torvalds ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
4511da177e4SLinus Torvalds } else {
4521da177e4SLinus Torvalds dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
4531da177e4SLinus Torvalds ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
4541da177e4SLinus Torvalds }
4551da177e4SLinus Torvalds }
456