1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * iommu.c: IOMMU specific routines for memory management. 4 * 5 * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) 6 * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com) 7 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) 8 * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/init.h> 13 #include <linux/mm.h> 14 #include <linux/slab.h> 15 #include <linux/dma-mapping.h> 16 #include <linux/of.h> 17 #include <linux/of_device.h> 18 19 #include <asm/pgalloc.h> 20 #include <asm/io.h> 21 #include <asm/mxcc.h> 22 #include <asm/mbus.h> 23 #include <asm/cacheflush.h> 24 #include <asm/tlbflush.h> 25 #include <asm/bitext.h> 26 #include <asm/iommu.h> 27 #include <asm/dma.h> 28 29 #include "mm_32.h" 30 31 /* 32 * This can be sized dynamically, but we will do this 33 * only when we have a guidance about actual I/O pressures. 34 */ 35 #define IOMMU_RNGE IOMMU_RNGE_256MB 36 #define IOMMU_START 0xF0000000 37 #define IOMMU_WINSIZE (256*1024*1024U) 38 #define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */ 39 #define IOMMU_ORDER 6 /* 4096 * (1<<6) */ 40 41 static int viking_flush; 42 /* viking.S */ 43 extern void viking_flush_page(unsigned long page); 44 extern void viking_mxcc_flush_page(unsigned long page); 45 46 /* 47 * Values precomputed according to CPU type. 48 */ 49 static unsigned int ioperm_noc; /* Consistent mapping iopte flags */ 50 static pgprot_t dvma_prot; /* Consistent mapping pte flags */ 51 52 #define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID) 53 #define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ) 54 55 static const struct dma_map_ops sbus_iommu_dma_gflush_ops; 56 static const struct dma_map_ops sbus_iommu_dma_pflush_ops; 57 58 static void __init sbus_iommu_init(struct platform_device *op) 59 { 60 struct iommu_struct *iommu; 61 unsigned int impl, vers; 62 unsigned long *bitmap; 63 unsigned long control; 64 unsigned long base; 65 unsigned long tmp; 66 67 iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL); 68 if (!iommu) { 69 prom_printf("Unable to allocate iommu structure\n"); 70 prom_halt(); 71 } 72 73 iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3, 74 "iommu_regs"); 75 if (!iommu->regs) { 76 prom_printf("Cannot map IOMMU registers\n"); 77 prom_halt(); 78 } 79 80 control = sbus_readl(&iommu->regs->control); 81 impl = (control & IOMMU_CTRL_IMPL) >> 28; 82 vers = (control & IOMMU_CTRL_VERS) >> 24; 83 control &= ~(IOMMU_CTRL_RNGE); 84 control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); 85 sbus_writel(control, &iommu->regs->control); 86 87 iommu_invalidate(iommu->regs); 88 iommu->start = IOMMU_START; 89 iommu->end = 0xffffffff; 90 91 /* Allocate IOMMU page table */ 92 /* Stupid alignment constraints give me a headache. 93 We need 256K or 512K or 1M or 2M area aligned to 94 its size and current gfp will fortunately give 95 it to us. */ 96 tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER); 97 if (!tmp) { 98 prom_printf("Unable to allocate iommu table [0x%lx]\n", 99 IOMMU_NPTES * sizeof(iopte_t)); 100 prom_halt(); 101 } 102 iommu->page_table = (iopte_t *)tmp; 103 104 /* Initialize new table. */ 105 memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t)); 106 flush_cache_all(); 107 flush_tlb_all(); 108 109 base = __pa((unsigned long)iommu->page_table) >> 4; 110 sbus_writel(base, &iommu->regs->base); 111 iommu_invalidate(iommu->regs); 112 113 bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL); 114 if (!bitmap) { 115 prom_printf("Unable to allocate iommu bitmap [%d]\n", 116 (int)(IOMMU_NPTES>>3)); 117 prom_halt(); 118 } 119 bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES); 120 /* To be coherent on HyperSparc, the page color of DVMA 121 * and physical addresses must match. 122 */ 123 if (srmmu_modtype == HyperSparc) 124 iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT; 125 else 126 iommu->usemap.num_colors = 1; 127 128 printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n", 129 impl, vers, iommu->page_table, 130 (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES); 131 132 op->dev.archdata.iommu = iommu; 133 134 if (flush_page_for_dma_global) 135 op->dev.dma_ops = &sbus_iommu_dma_gflush_ops; 136 else 137 op->dev.dma_ops = &sbus_iommu_dma_pflush_ops; 138 } 139 140 static int __init iommu_init(void) 141 { 142 struct device_node *dp; 143 144 for_each_node_by_name(dp, "iommu") { 145 struct platform_device *op = of_find_device_by_node(dp); 146 147 sbus_iommu_init(op); 148 of_propagate_archdata(op); 149 } 150 151 return 0; 152 } 153 154 subsys_initcall(iommu_init); 155 156 /* Flush the iotlb entries to ram. */ 157 /* This could be better if we didn't have to flush whole pages. */ 158 static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte) 159 { 160 unsigned long start; 161 unsigned long end; 162 163 start = (unsigned long)iopte; 164 end = PAGE_ALIGN(start + niopte*sizeof(iopte_t)); 165 start &= PAGE_MASK; 166 if (viking_mxcc_present) { 167 while(start < end) { 168 viking_mxcc_flush_page(start); 169 start += PAGE_SIZE; 170 } 171 } else if (viking_flush) { 172 while(start < end) { 173 viking_flush_page(start); 174 start += PAGE_SIZE; 175 } 176 } else { 177 while(start < end) { 178 __flush_page_to_ram(start); 179 start += PAGE_SIZE; 180 } 181 } 182 } 183 184 static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page, 185 unsigned long offset, size_t len, bool per_page_flush) 186 { 187 struct iommu_struct *iommu = dev->archdata.iommu; 188 phys_addr_t paddr = page_to_phys(page) + offset; 189 unsigned long off = paddr & ~PAGE_MASK; 190 unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 191 unsigned long pfn = __phys_to_pfn(paddr); 192 unsigned int busa, busa0; 193 iopte_t *iopte, *iopte0; 194 int ioptex, i; 195 196 /* XXX So what is maxphys for us and how do drivers know it? */ 197 if (!len || len > 256 * 1024) 198 return DMA_MAPPING_ERROR; 199 200 /* 201 * We expect unmapped highmem pages to be not in the cache. 202 * XXX Is this a good assumption? 203 * XXX What if someone else unmaps it here and races us? 204 */ 205 if (per_page_flush && !PageHighMem(page)) { 206 unsigned long vaddr, p; 207 208 vaddr = (unsigned long)page_address(page) + offset; 209 for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE) 210 flush_page_for_dma(p); 211 } 212 213 /* page color = pfn of page */ 214 ioptex = bit_map_string_get(&iommu->usemap, npages, pfn); 215 if (ioptex < 0) 216 panic("iommu out"); 217 busa0 = iommu->start + (ioptex << PAGE_SHIFT); 218 iopte0 = &iommu->page_table[ioptex]; 219 220 busa = busa0; 221 iopte = iopte0; 222 for (i = 0; i < npages; i++) { 223 iopte_val(*iopte) = MKIOPTE(pfn, IOPERM); 224 iommu_invalidate_page(iommu->regs, busa); 225 busa += PAGE_SIZE; 226 iopte++; 227 pfn++; 228 } 229 230 iommu_flush_iotlb(iopte0, npages); 231 return busa0 + off; 232 } 233 234 static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev, 235 struct page *page, unsigned long offset, size_t len, 236 enum dma_data_direction dir, unsigned long attrs) 237 { 238 flush_page_for_dma(0); 239 return __sbus_iommu_map_page(dev, page, offset, len, false); 240 } 241 242 static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev, 243 struct page *page, unsigned long offset, size_t len, 244 enum dma_data_direction dir, unsigned long attrs) 245 { 246 return __sbus_iommu_map_page(dev, page, offset, len, true); 247 } 248 249 static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl, 250 int nents, enum dma_data_direction dir, unsigned long attrs, 251 bool per_page_flush) 252 { 253 struct scatterlist *sg; 254 int j; 255 256 for_each_sg(sgl, sg, nents, j) { 257 sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg), 258 sg->offset, sg->length, per_page_flush); 259 if (sg->dma_address == DMA_MAPPING_ERROR) 260 return 0; 261 sg->dma_length = sg->length; 262 } 263 264 return nents; 265 } 266 267 static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl, 268 int nents, enum dma_data_direction dir, unsigned long attrs) 269 { 270 flush_page_for_dma(0); 271 return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false); 272 } 273 274 static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl, 275 int nents, enum dma_data_direction dir, unsigned long attrs) 276 { 277 return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true); 278 } 279 280 static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr, 281 size_t len, enum dma_data_direction dir, unsigned long attrs) 282 { 283 struct iommu_struct *iommu = dev->archdata.iommu; 284 unsigned int busa = dma_addr & PAGE_MASK; 285 unsigned long off = dma_addr & ~PAGE_MASK; 286 unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; 287 unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT; 288 unsigned int i; 289 290 BUG_ON(busa < iommu->start); 291 for (i = 0; i < npages; i++) { 292 iopte_val(iommu->page_table[ioptex + i]) = 0; 293 iommu_invalidate_page(iommu->regs, busa); 294 busa += PAGE_SIZE; 295 } 296 bit_map_clear(&iommu->usemap, ioptex, npages); 297 } 298 299 static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl, 300 int nents, enum dma_data_direction dir, unsigned long attrs) 301 { 302 struct scatterlist *sg; 303 int i; 304 305 for_each_sg(sgl, sg, nents, i) { 306 sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir, 307 attrs); 308 sg->dma_address = 0x21212121; 309 } 310 } 311 312 #ifdef CONFIG_SBUS 313 static void *sbus_iommu_alloc(struct device *dev, size_t len, 314 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 315 { 316 struct iommu_struct *iommu = dev->archdata.iommu; 317 unsigned long va, addr, page, end, ret; 318 iopte_t *iopte = iommu->page_table; 319 iopte_t *first; 320 int ioptex; 321 322 /* XXX So what is maxphys for us and how do drivers know it? */ 323 if (!len || len > 256 * 1024) 324 return NULL; 325 326 len = PAGE_ALIGN(len); 327 va = __get_free_pages(gfp | __GFP_ZERO, get_order(len)); 328 if (va == 0) 329 return NULL; 330 331 addr = ret = sparc_dma_alloc_resource(dev, len); 332 if (!addr) 333 goto out_free_pages; 334 335 BUG_ON((va & ~PAGE_MASK) != 0); 336 BUG_ON((addr & ~PAGE_MASK) != 0); 337 BUG_ON((len & ~PAGE_MASK) != 0); 338 339 /* page color = physical address */ 340 ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT, 341 addr >> PAGE_SHIFT); 342 if (ioptex < 0) 343 panic("iommu out"); 344 345 iopte += ioptex; 346 first = iopte; 347 end = addr + len; 348 while(addr < end) { 349 page = va; 350 { 351 pmd_t *pmdp; 352 pte_t *ptep; 353 354 if (viking_mxcc_present) 355 viking_mxcc_flush_page(page); 356 else if (viking_flush) 357 viking_flush_page(page); 358 else 359 __flush_page_to_ram(page); 360 361 pmdp = pmd_off_k(addr); 362 ptep = pte_offset_map(pmdp, addr); 363 364 set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); 365 } 366 iopte_val(*iopte++) = 367 MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc); 368 addr += PAGE_SIZE; 369 va += PAGE_SIZE; 370 } 371 /* P3: why do we need this? 372 * 373 * DAVEM: Because there are several aspects, none of which 374 * are handled by a single interface. Some cpus are 375 * completely not I/O DMA coherent, and some have 376 * virtually indexed caches. The driver DMA flushing 377 * methods handle the former case, but here during 378 * IOMMU page table modifications, and usage of non-cacheable 379 * cpu mappings of pages potentially in the cpu caches, we have 380 * to handle the latter case as well. 381 */ 382 flush_cache_all(); 383 iommu_flush_iotlb(first, len >> PAGE_SHIFT); 384 flush_tlb_all(); 385 iommu_invalidate(iommu->regs); 386 387 *dma_handle = iommu->start + (ioptex << PAGE_SHIFT); 388 return (void *)ret; 389 390 out_free_pages: 391 free_pages(va, get_order(len)); 392 return NULL; 393 } 394 395 static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr, 396 dma_addr_t busa, unsigned long attrs) 397 { 398 struct iommu_struct *iommu = dev->archdata.iommu; 399 iopte_t *iopte = iommu->page_table; 400 struct page *page = virt_to_page(cpu_addr); 401 int ioptex = (busa - iommu->start) >> PAGE_SHIFT; 402 unsigned long end; 403 404 if (!sparc_dma_free_resource(cpu_addr, len)) 405 return; 406 407 BUG_ON((busa & ~PAGE_MASK) != 0); 408 BUG_ON((len & ~PAGE_MASK) != 0); 409 410 iopte += ioptex; 411 end = busa + len; 412 while (busa < end) { 413 iopte_val(*iopte++) = 0; 414 busa += PAGE_SIZE; 415 } 416 flush_tlb_all(); 417 iommu_invalidate(iommu->regs); 418 bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT); 419 420 __free_pages(page, get_order(len)); 421 } 422 #endif 423 424 static const struct dma_map_ops sbus_iommu_dma_gflush_ops = { 425 #ifdef CONFIG_SBUS 426 .alloc = sbus_iommu_alloc, 427 .free = sbus_iommu_free, 428 #endif 429 .map_page = sbus_iommu_map_page_gflush, 430 .unmap_page = sbus_iommu_unmap_page, 431 .map_sg = sbus_iommu_map_sg_gflush, 432 .unmap_sg = sbus_iommu_unmap_sg, 433 }; 434 435 static const struct dma_map_ops sbus_iommu_dma_pflush_ops = { 436 #ifdef CONFIG_SBUS 437 .alloc = sbus_iommu_alloc, 438 .free = sbus_iommu_free, 439 #endif 440 .map_page = sbus_iommu_map_page_pflush, 441 .unmap_page = sbus_iommu_unmap_page, 442 .map_sg = sbus_iommu_map_sg_pflush, 443 .unmap_sg = sbus_iommu_unmap_sg, 444 }; 445 446 void __init ld_mmu_iommu(void) 447 { 448 if (viking_mxcc_present || srmmu_modtype == HyperSparc) { 449 dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); 450 ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID; 451 } else { 452 dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV); 453 ioperm_noc = IOPTE_WRITE | IOPTE_VALID; 454 } 455 } 456