1 /* 2 * iommu.c: IOMMU specific routines for memory management. 3 * 4 * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) 5 * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com) 6 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) 7 * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/init.h> 12 #include <linux/mm.h> 13 #include <linux/slab.h> 14 #include <linux/highmem.h> /* pte_offset_map => kmap_atomic */ 15 #include <linux/scatterlist.h> 16 #include <linux/of.h> 17 #include <linux/of_device.h> 18 19 #include <asm/pgalloc.h> 20 #include <asm/pgtable.h> 21 #include <asm/io.h> 22 #include <asm/mxcc.h> 23 #include <asm/mbus.h> 24 #include <asm/cacheflush.h> 25 #include <asm/tlbflush.h> 26 #include <asm/bitext.h> 27 #include <asm/iommu.h> 28 #include <asm/dma.h> 29 30 #include "mm_32.h" 31 32 /* 33 * This can be sized dynamically, but we will do this 34 * only when we have a guidance about actual I/O pressures. 35 */ 36 #define IOMMU_RNGE IOMMU_RNGE_256MB 37 #define IOMMU_START 0xF0000000 38 #define IOMMU_WINSIZE (256*1024*1024U) 39 #define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */ 40 #define IOMMU_ORDER 6 /* 4096 * (1<<6) */ 41 42 static int viking_flush; 43 /* viking.S */ 44 extern void viking_flush_page(unsigned long page); 45 extern void viking_mxcc_flush_page(unsigned long page); 46 47 /* 48 * Values precomputed according to CPU type. 49 */ 50 static unsigned int ioperm_noc; /* Consistent mapping iopte flags */ 51 static pgprot_t dvma_prot; /* Consistent mapping pte flags */ 52 53 #define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID) 54 #define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ) 55 56 static void __init sbus_iommu_init(struct platform_device *op) 57 { 58 struct iommu_struct *iommu; 59 unsigned int impl, vers; 60 unsigned long *bitmap; 61 unsigned long control; 62 unsigned long base; 63 unsigned long tmp; 64 65 iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL); 66 if (!iommu) { 67 prom_printf("Unable to allocate iommu structure\n"); 68 prom_halt(); 69 } 70 71 iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3, 72 "iommu_regs"); 73 if (!iommu->regs) { 74 prom_printf("Cannot map IOMMU registers\n"); 75 prom_halt(); 76 } 77 78 control = sbus_readl(&iommu->regs->control); 79 impl = (control & IOMMU_CTRL_IMPL) >> 28; 80 vers = (control & IOMMU_CTRL_VERS) >> 24; 81 control &= ~(IOMMU_CTRL_RNGE); 82 control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); 83 sbus_writel(control, &iommu->regs->control); 84 85 iommu_invalidate(iommu->regs); 86 iommu->start = IOMMU_START; 87 iommu->end = 0xffffffff; 88 89 /* Allocate IOMMU page table */ 90 /* Stupid alignment constraints give me a headache. 91 We need 256K or 512K or 1M or 2M area aligned to 92 its size and current gfp will fortunately give 93 it to us. */ 94 tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER); 95 if (!tmp) { 96 prom_printf("Unable to allocate iommu table [0x%lx]\n", 97 IOMMU_NPTES * sizeof(iopte_t)); 98 prom_halt(); 99 } 100 iommu->page_table = (iopte_t *)tmp; 101 102 /* Initialize new table. */ 103 memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t)); 104 flush_cache_all(); 105 flush_tlb_all(); 106 107 base = __pa((unsigned long)iommu->page_table) >> 4; 108 sbus_writel(base, &iommu->regs->base); 109 iommu_invalidate(iommu->regs); 110 111 bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL); 112 if (!bitmap) { 113 prom_printf("Unable to allocate iommu bitmap [%d]\n", 114 (int)(IOMMU_NPTES>>3)); 115 prom_halt(); 116 } 117 bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES); 118 /* To be coherent on HyperSparc, the page color of DVMA 119 * and physical addresses must match. 120 */ 121 if (srmmu_modtype == HyperSparc) 122 iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT; 123 else 124 iommu->usemap.num_colors = 1; 125 126 printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n", 127 impl, vers, iommu->page_table, 128 (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES); 129 130 op->dev.archdata.iommu = iommu; 131 } 132 133 static int __init iommu_init(void) 134 { 135 struct device_node *dp; 136 137 for_each_node_by_name(dp, "iommu") { 138 struct platform_device *op = of_find_device_by_node(dp); 139 140 sbus_iommu_init(op); 141 of_propagate_archdata(op); 142 } 143 144 return 0; 145 } 146 147 subsys_initcall(iommu_init); 148 149 /* Flush the iotlb entries to ram. */ 150 /* This could be better if we didn't have to flush whole pages. */ 151 static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte) 152 { 153 unsigned long start; 154 unsigned long end; 155 156 start = (unsigned long)iopte; 157 end = PAGE_ALIGN(start + niopte*sizeof(iopte_t)); 158 start &= PAGE_MASK; 159 if (viking_mxcc_present) { 160 while(start < end) { 161 viking_mxcc_flush_page(start); 162 start += PAGE_SIZE; 163 } 164 } else if (viking_flush) { 165 while(start < end) { 166 viking_flush_page(start); 167 start += PAGE_SIZE; 168 } 169 } else { 170 while(start < end) { 171 __flush_page_to_ram(start); 172 start += PAGE_SIZE; 173 } 174 } 175 } 176 177 static u32 iommu_get_one(struct device *dev, struct page *page, int npages) 178 { 179 struct iommu_struct *iommu = dev->archdata.iommu; 180 int ioptex; 181 iopte_t *iopte, *iopte0; 182 unsigned int busa, busa0; 183 int i; 184 185 /* page color = pfn of page */ 186 ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page)); 187 if (ioptex < 0) 188 panic("iommu out"); 189 busa0 = iommu->start + (ioptex << PAGE_SHIFT); 190 iopte0 = &iommu->page_table[ioptex]; 191 192 busa = busa0; 193 iopte = iopte0; 194 for (i = 0; i < npages; i++) { 195 iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM); 196 iommu_invalidate_page(iommu->regs, busa); 197 busa += PAGE_SIZE; 198 iopte++; 199 page++; 200 } 201 202 iommu_flush_iotlb(iopte0, npages); 203 204 return busa0; 205 } 206 207 static u32 iommu_get_scsi_one(struct device *dev, char *vaddr, unsigned int len) 208 { 209 unsigned long off; 210 int npages; 211 struct page *page; 212 u32 busa; 213 214 off = (unsigned long)vaddr & ~PAGE_MASK; 215 npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; 216 page = virt_to_page((unsigned long)vaddr & PAGE_MASK); 217 busa = iommu_get_one(dev, page, npages); 218 return busa + off; 219 } 220 221 static __u32 iommu_get_scsi_one_gflush(struct device *dev, char *vaddr, unsigned long len) 222 { 223 flush_page_for_dma(0); 224 return iommu_get_scsi_one(dev, vaddr, len); 225 } 226 227 static __u32 iommu_get_scsi_one_pflush(struct device *dev, char *vaddr, unsigned long len) 228 { 229 unsigned long page = ((unsigned long) vaddr) & PAGE_MASK; 230 231 while(page < ((unsigned long)(vaddr + len))) { 232 flush_page_for_dma(page); 233 page += PAGE_SIZE; 234 } 235 return iommu_get_scsi_one(dev, vaddr, len); 236 } 237 238 static void iommu_get_scsi_sgl_gflush(struct device *dev, struct scatterlist *sg, int sz) 239 { 240 int n; 241 242 flush_page_for_dma(0); 243 while (sz != 0) { 244 --sz; 245 n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; 246 sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset; 247 sg->dma_length = sg->length; 248 sg = sg_next(sg); 249 } 250 } 251 252 static void iommu_get_scsi_sgl_pflush(struct device *dev, struct scatterlist *sg, int sz) 253 { 254 unsigned long page, oldpage = 0; 255 int n, i; 256 257 while(sz != 0) { 258 --sz; 259 260 n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; 261 262 /* 263 * We expect unmapped highmem pages to be not in the cache. 264 * XXX Is this a good assumption? 265 * XXX What if someone else unmaps it here and races us? 266 */ 267 if ((page = (unsigned long) page_address(sg_page(sg))) != 0) { 268 for (i = 0; i < n; i++) { 269 if (page != oldpage) { /* Already flushed? */ 270 flush_page_for_dma(page); 271 oldpage = page; 272 } 273 page += PAGE_SIZE; 274 } 275 } 276 277 sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset; 278 sg->dma_length = sg->length; 279 sg = sg_next(sg); 280 } 281 } 282 283 static void iommu_release_one(struct device *dev, u32 busa, int npages) 284 { 285 struct iommu_struct *iommu = dev->archdata.iommu; 286 int ioptex; 287 int i; 288 289 BUG_ON(busa < iommu->start); 290 ioptex = (busa - iommu->start) >> PAGE_SHIFT; 291 for (i = 0; i < npages; i++) { 292 iopte_val(iommu->page_table[ioptex + i]) = 0; 293 iommu_invalidate_page(iommu->regs, busa); 294 busa += PAGE_SIZE; 295 } 296 bit_map_clear(&iommu->usemap, ioptex, npages); 297 } 298 299 static void iommu_release_scsi_one(struct device *dev, __u32 vaddr, unsigned long len) 300 { 301 unsigned long off; 302 int npages; 303 304 off = vaddr & ~PAGE_MASK; 305 npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; 306 iommu_release_one(dev, vaddr & PAGE_MASK, npages); 307 } 308 309 static void iommu_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz) 310 { 311 int n; 312 313 while(sz != 0) { 314 --sz; 315 316 n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; 317 iommu_release_one(dev, sg->dma_address & PAGE_MASK, n); 318 sg->dma_address = 0x21212121; 319 sg = sg_next(sg); 320 } 321 } 322 323 #ifdef CONFIG_SBUS 324 static int iommu_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, 325 unsigned long addr, int len) 326 { 327 struct iommu_struct *iommu = dev->archdata.iommu; 328 unsigned long page, end; 329 iopte_t *iopte = iommu->page_table; 330 iopte_t *first; 331 int ioptex; 332 333 BUG_ON((va & ~PAGE_MASK) != 0); 334 BUG_ON((addr & ~PAGE_MASK) != 0); 335 BUG_ON((len & ~PAGE_MASK) != 0); 336 337 /* page color = physical address */ 338 ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT, 339 addr >> PAGE_SHIFT); 340 if (ioptex < 0) 341 panic("iommu out"); 342 343 iopte += ioptex; 344 first = iopte; 345 end = addr + len; 346 while(addr < end) { 347 page = va; 348 { 349 pgd_t *pgdp; 350 pmd_t *pmdp; 351 pte_t *ptep; 352 353 if (viking_mxcc_present) 354 viking_mxcc_flush_page(page); 355 else if (viking_flush) 356 viking_flush_page(page); 357 else 358 __flush_page_to_ram(page); 359 360 pgdp = pgd_offset(&init_mm, addr); 361 pmdp = pmd_offset(pgdp, addr); 362 ptep = pte_offset_map(pmdp, addr); 363 364 set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); 365 } 366 iopte_val(*iopte++) = 367 MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc); 368 addr += PAGE_SIZE; 369 va += PAGE_SIZE; 370 } 371 /* P3: why do we need this? 372 * 373 * DAVEM: Because there are several aspects, none of which 374 * are handled by a single interface. Some cpus are 375 * completely not I/O DMA coherent, and some have 376 * virtually indexed caches. The driver DMA flushing 377 * methods handle the former case, but here during 378 * IOMMU page table modifications, and usage of non-cacheable 379 * cpu mappings of pages potentially in the cpu caches, we have 380 * to handle the latter case as well. 381 */ 382 flush_cache_all(); 383 iommu_flush_iotlb(first, len >> PAGE_SHIFT); 384 flush_tlb_all(); 385 iommu_invalidate(iommu->regs); 386 387 *pba = iommu->start + (ioptex << PAGE_SHIFT); 388 return 0; 389 } 390 391 static void iommu_unmap_dma_area(struct device *dev, unsigned long busa, int len) 392 { 393 struct iommu_struct *iommu = dev->archdata.iommu; 394 iopte_t *iopte = iommu->page_table; 395 unsigned long end; 396 int ioptex = (busa - iommu->start) >> PAGE_SHIFT; 397 398 BUG_ON((busa & ~PAGE_MASK) != 0); 399 BUG_ON((len & ~PAGE_MASK) != 0); 400 401 iopte += ioptex; 402 end = busa + len; 403 while (busa < end) { 404 iopte_val(*iopte++) = 0; 405 busa += PAGE_SIZE; 406 } 407 flush_tlb_all(); 408 iommu_invalidate(iommu->regs); 409 bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT); 410 } 411 #endif 412 413 static const struct sparc32_dma_ops iommu_dma_gflush_ops = { 414 .get_scsi_one = iommu_get_scsi_one_gflush, 415 .get_scsi_sgl = iommu_get_scsi_sgl_gflush, 416 .release_scsi_one = iommu_release_scsi_one, 417 .release_scsi_sgl = iommu_release_scsi_sgl, 418 #ifdef CONFIG_SBUS 419 .map_dma_area = iommu_map_dma_area, 420 .unmap_dma_area = iommu_unmap_dma_area, 421 #endif 422 }; 423 424 static const struct sparc32_dma_ops iommu_dma_pflush_ops = { 425 .get_scsi_one = iommu_get_scsi_one_pflush, 426 .get_scsi_sgl = iommu_get_scsi_sgl_pflush, 427 .release_scsi_one = iommu_release_scsi_one, 428 .release_scsi_sgl = iommu_release_scsi_sgl, 429 #ifdef CONFIG_SBUS 430 .map_dma_area = iommu_map_dma_area, 431 .unmap_dma_area = iommu_unmap_dma_area, 432 #endif 433 }; 434 435 void __init ld_mmu_iommu(void) 436 { 437 if (flush_page_for_dma_global) { 438 /* flush_page_for_dma flushes everything, no matter of what page is it */ 439 sparc32_dma_ops = &iommu_dma_gflush_ops; 440 } else { 441 sparc32_dma_ops = &iommu_dma_pflush_ops; 442 } 443 444 if (viking_mxcc_present || srmmu_modtype == HyperSparc) { 445 dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); 446 ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID; 447 } else { 448 dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV); 449 ioperm_noc = IOPTE_WRITE | IOPTE_VALID; 450 } 451 } 452