1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 ** PARISC 1.1 Dynamic DMA mapping support. 4 ** This implementation is for PA-RISC platforms that do not support 5 ** I/O TLBs (aka DMA address translation hardware). 6 ** See Documentation/DMA-API-HOWTO.txt for interface definitions. 7 ** 8 ** (c) Copyright 1999,2000 Hewlett-Packard Company 9 ** (c) Copyright 2000 Grant Grundler 10 ** (c) Copyright 2000 Philipp Rumpf <prumpf@tux.org> 11 ** (c) Copyright 2000 John Marvin 12 ** 13 ** "leveraged" from 2.3.47: arch/ia64/kernel/pci-dma.c. 14 ** (I assume it's from David Mosberger-Tang but there was no Copyright) 15 ** 16 ** AFAIK, all PA7100LC and PA7300LC platforms can use this code. 17 ** 18 ** - ggg 19 */ 20 21 #include <linux/init.h> 22 #include <linux/gfp.h> 23 #include <linux/mm.h> 24 #include <linux/pci.h> 25 #include <linux/proc_fs.h> 26 #include <linux/seq_file.h> 27 #include <linux/string.h> 28 #include <linux/types.h> 29 #include <linux/scatterlist.h> 30 #include <linux/export.h> 31 32 #include <asm/cacheflush.h> 33 #include <asm/dma.h> /* for DMA_CHUNK_SIZE */ 34 #include <asm/io.h> 35 #include <asm/page.h> /* get_order */ 36 #include <asm/pgalloc.h> 37 #include <linux/uaccess.h> 38 #include <asm/tlbflush.h> /* for purge_tlb_*() macros */ 39 40 static struct proc_dir_entry * proc_gsc_root __read_mostly = NULL; 41 static unsigned long pcxl_used_bytes __read_mostly = 0; 42 static unsigned long pcxl_used_pages __read_mostly = 0; 43 44 extern unsigned long pcxl_dma_start; /* Start of pcxl dma mapping area */ 45 static DEFINE_SPINLOCK(pcxl_res_lock); 46 static char *pcxl_res_map; 47 static int pcxl_res_hint; 48 static int pcxl_res_size; 49 50 #ifdef DEBUG_PCXL_RESOURCE 51 #define DBG_RES(x...) printk(x) 52 #else 53 #define DBG_RES(x...) 54 #endif 55 56 57 /* 58 ** Dump a hex representation of the resource map. 59 */ 60 61 #ifdef DUMP_RESMAP 62 static 63 void dump_resmap(void) 64 { 65 u_long *res_ptr = (unsigned long *)pcxl_res_map; 66 u_long i = 0; 67 68 printk("res_map: "); 69 for(; i < (pcxl_res_size / sizeof(unsigned long)); ++i, ++res_ptr) 70 printk("%08lx ", *res_ptr); 71 72 printk("\n"); 73 } 74 #else 75 static inline void dump_resmap(void) {;} 76 #endif 77 78 static inline int map_pte_uncached(pte_t * pte, 79 unsigned long vaddr, 80 unsigned long size, unsigned long *paddr_ptr) 81 { 82 unsigned long end; 83 unsigned long orig_vaddr = vaddr; 84 85 vaddr &= ~PMD_MASK; 86 end = vaddr + size; 87 if (end > PMD_SIZE) 88 end = PMD_SIZE; 89 do { 90 unsigned long flags; 91 92 if (!pte_none(*pte)) 93 printk(KERN_ERR "map_pte_uncached: page already exists\n"); 94 purge_tlb_start(flags); 95 set_pte(pte, __mk_pte(*paddr_ptr, PAGE_KERNEL_UNC)); 96 pdtlb_kernel(orig_vaddr); 97 purge_tlb_end(flags); 98 vaddr += PAGE_SIZE; 99 orig_vaddr += PAGE_SIZE; 100 (*paddr_ptr) += PAGE_SIZE; 101 pte++; 102 } while (vaddr < end); 103 return 0; 104 } 105 106 static inline int map_pmd_uncached(pmd_t * pmd, unsigned long vaddr, 107 unsigned long size, unsigned long *paddr_ptr) 108 { 109 unsigned long end; 110 unsigned long orig_vaddr = vaddr; 111 112 vaddr &= ~PGDIR_MASK; 113 end = vaddr + size; 114 if (end > PGDIR_SIZE) 115 end = PGDIR_SIZE; 116 do { 117 pte_t * pte = pte_alloc_kernel(pmd, vaddr); 118 if (!pte) 119 return -ENOMEM; 120 if (map_pte_uncached(pte, orig_vaddr, end - vaddr, paddr_ptr)) 121 return -ENOMEM; 122 vaddr = (vaddr + PMD_SIZE) & PMD_MASK; 123 orig_vaddr += PMD_SIZE; 124 pmd++; 125 } while (vaddr < end); 126 return 0; 127 } 128 129 static inline int map_uncached_pages(unsigned long vaddr, unsigned long size, 130 unsigned long paddr) 131 { 132 pgd_t * dir; 133 unsigned long end = vaddr + size; 134 135 dir = pgd_offset_k(vaddr); 136 do { 137 pmd_t *pmd; 138 139 pmd = pmd_alloc(NULL, dir, vaddr); 140 if (!pmd) 141 return -ENOMEM; 142 if (map_pmd_uncached(pmd, vaddr, end - vaddr, &paddr)) 143 return -ENOMEM; 144 vaddr = vaddr + PGDIR_SIZE; 145 dir++; 146 } while (vaddr && (vaddr < end)); 147 return 0; 148 } 149 150 static inline void unmap_uncached_pte(pmd_t * pmd, unsigned long vaddr, 151 unsigned long size) 152 { 153 pte_t * pte; 154 unsigned long end; 155 unsigned long orig_vaddr = vaddr; 156 157 if (pmd_none(*pmd)) 158 return; 159 if (pmd_bad(*pmd)) { 160 pmd_ERROR(*pmd); 161 pmd_clear(pmd); 162 return; 163 } 164 pte = pte_offset_map(pmd, vaddr); 165 vaddr &= ~PMD_MASK; 166 end = vaddr + size; 167 if (end > PMD_SIZE) 168 end = PMD_SIZE; 169 do { 170 unsigned long flags; 171 pte_t page = *pte; 172 173 pte_clear(&init_mm, vaddr, pte); 174 purge_tlb_start(flags); 175 pdtlb_kernel(orig_vaddr); 176 purge_tlb_end(flags); 177 vaddr += PAGE_SIZE; 178 orig_vaddr += PAGE_SIZE; 179 pte++; 180 if (pte_none(page) || pte_present(page)) 181 continue; 182 printk(KERN_CRIT "Whee.. Swapped out page in kernel page table\n"); 183 } while (vaddr < end); 184 } 185 186 static inline void unmap_uncached_pmd(pgd_t * dir, unsigned long vaddr, 187 unsigned long size) 188 { 189 pmd_t * pmd; 190 unsigned long end; 191 unsigned long orig_vaddr = vaddr; 192 193 if (pgd_none(*dir)) 194 return; 195 if (pgd_bad(*dir)) { 196 pgd_ERROR(*dir); 197 pgd_clear(dir); 198 return; 199 } 200 pmd = pmd_offset(dir, vaddr); 201 vaddr &= ~PGDIR_MASK; 202 end = vaddr + size; 203 if (end > PGDIR_SIZE) 204 end = PGDIR_SIZE; 205 do { 206 unmap_uncached_pte(pmd, orig_vaddr, end - vaddr); 207 vaddr = (vaddr + PMD_SIZE) & PMD_MASK; 208 orig_vaddr += PMD_SIZE; 209 pmd++; 210 } while (vaddr < end); 211 } 212 213 static void unmap_uncached_pages(unsigned long vaddr, unsigned long size) 214 { 215 pgd_t * dir; 216 unsigned long end = vaddr + size; 217 218 dir = pgd_offset_k(vaddr); 219 do { 220 unmap_uncached_pmd(dir, vaddr, end - vaddr); 221 vaddr = vaddr + PGDIR_SIZE; 222 dir++; 223 } while (vaddr && (vaddr < end)); 224 } 225 226 #define PCXL_SEARCH_LOOP(idx, mask, size) \ 227 for(; res_ptr < res_end; ++res_ptr) \ 228 { \ 229 if(0 == ((*res_ptr) & mask)) { \ 230 *res_ptr |= mask; \ 231 idx = (int)((u_long)res_ptr - (u_long)pcxl_res_map); \ 232 pcxl_res_hint = idx + (size >> 3); \ 233 goto resource_found; \ 234 } \ 235 } 236 237 #define PCXL_FIND_FREE_MAPPING(idx, mask, size) { \ 238 u##size *res_ptr = (u##size *)&(pcxl_res_map[pcxl_res_hint & ~((size >> 3) - 1)]); \ 239 u##size *res_end = (u##size *)&pcxl_res_map[pcxl_res_size]; \ 240 PCXL_SEARCH_LOOP(idx, mask, size); \ 241 res_ptr = (u##size *)&pcxl_res_map[0]; \ 242 PCXL_SEARCH_LOOP(idx, mask, size); \ 243 } 244 245 unsigned long 246 pcxl_alloc_range(size_t size) 247 { 248 int res_idx; 249 u_long mask, flags; 250 unsigned int pages_needed = size >> PAGE_SHIFT; 251 252 mask = (u_long) -1L; 253 mask >>= BITS_PER_LONG - pages_needed; 254 255 DBG_RES("pcxl_alloc_range() size: %d pages_needed %d pages_mask 0x%08lx\n", 256 size, pages_needed, mask); 257 258 spin_lock_irqsave(&pcxl_res_lock, flags); 259 260 if(pages_needed <= 8) { 261 PCXL_FIND_FREE_MAPPING(res_idx, mask, 8); 262 } else if(pages_needed <= 16) { 263 PCXL_FIND_FREE_MAPPING(res_idx, mask, 16); 264 } else if(pages_needed <= 32) { 265 PCXL_FIND_FREE_MAPPING(res_idx, mask, 32); 266 } else { 267 panic("%s: pcxl_alloc_range() Too many pages to map.\n", 268 __FILE__); 269 } 270 271 dump_resmap(); 272 panic("%s: pcxl_alloc_range() out of dma mapping resources\n", 273 __FILE__); 274 275 resource_found: 276 277 DBG_RES("pcxl_alloc_range() res_idx %d mask 0x%08lx res_hint: %d\n", 278 res_idx, mask, pcxl_res_hint); 279 280 pcxl_used_pages += pages_needed; 281 pcxl_used_bytes += ((pages_needed >> 3) ? (pages_needed >> 3) : 1); 282 283 spin_unlock_irqrestore(&pcxl_res_lock, flags); 284 285 dump_resmap(); 286 287 /* 288 ** return the corresponding vaddr in the pcxl dma map 289 */ 290 return (pcxl_dma_start + (res_idx << (PAGE_SHIFT + 3))); 291 } 292 293 #define PCXL_FREE_MAPPINGS(idx, m, size) \ 294 u##size *res_ptr = (u##size *)&(pcxl_res_map[(idx) + (((size >> 3) - 1) & (~((size >> 3) - 1)))]); \ 295 /* BUG_ON((*res_ptr & m) != m); */ \ 296 *res_ptr &= ~m; 297 298 /* 299 ** clear bits in the pcxl resource map 300 */ 301 static void 302 pcxl_free_range(unsigned long vaddr, size_t size) 303 { 304 u_long mask, flags; 305 unsigned int res_idx = (vaddr - pcxl_dma_start) >> (PAGE_SHIFT + 3); 306 unsigned int pages_mapped = size >> PAGE_SHIFT; 307 308 mask = (u_long) -1L; 309 mask >>= BITS_PER_LONG - pages_mapped; 310 311 DBG_RES("pcxl_free_range() res_idx: %d size: %d pages_mapped %d mask 0x%08lx\n", 312 res_idx, size, pages_mapped, mask); 313 314 spin_lock_irqsave(&pcxl_res_lock, flags); 315 316 if(pages_mapped <= 8) { 317 PCXL_FREE_MAPPINGS(res_idx, mask, 8); 318 } else if(pages_mapped <= 16) { 319 PCXL_FREE_MAPPINGS(res_idx, mask, 16); 320 } else if(pages_mapped <= 32) { 321 PCXL_FREE_MAPPINGS(res_idx, mask, 32); 322 } else { 323 panic("%s: pcxl_free_range() Too many pages to unmap.\n", 324 __FILE__); 325 } 326 327 pcxl_used_pages -= (pages_mapped ? pages_mapped : 1); 328 pcxl_used_bytes -= ((pages_mapped >> 3) ? (pages_mapped >> 3) : 1); 329 330 spin_unlock_irqrestore(&pcxl_res_lock, flags); 331 332 dump_resmap(); 333 } 334 335 static int proc_pcxl_dma_show(struct seq_file *m, void *v) 336 { 337 #if 0 338 u_long i = 0; 339 unsigned long *res_ptr = (u_long *)pcxl_res_map; 340 #endif 341 unsigned long total_pages = pcxl_res_size << 3; /* 8 bits per byte */ 342 343 seq_printf(m, "\nDMA Mapping Area size : %d bytes (%ld pages)\n", 344 PCXL_DMA_MAP_SIZE, total_pages); 345 346 seq_printf(m, "Resource bitmap : %d bytes\n", pcxl_res_size); 347 348 seq_puts(m, " total: free: used: % used:\n"); 349 seq_printf(m, "blocks %8d %8ld %8ld %8ld%%\n", pcxl_res_size, 350 pcxl_res_size - pcxl_used_bytes, pcxl_used_bytes, 351 (pcxl_used_bytes * 100) / pcxl_res_size); 352 353 seq_printf(m, "pages %8ld %8ld %8ld %8ld%%\n", total_pages, 354 total_pages - pcxl_used_pages, pcxl_used_pages, 355 (pcxl_used_pages * 100 / total_pages)); 356 357 #if 0 358 seq_puts(m, "\nResource bitmap:"); 359 360 for(; i < (pcxl_res_size / sizeof(u_long)); ++i, ++res_ptr) { 361 if ((i & 7) == 0) 362 seq_puts(m,"\n "); 363 seq_printf(m, "%s %08lx", buf, *res_ptr); 364 } 365 #endif 366 seq_putc(m, '\n'); 367 return 0; 368 } 369 370 static int __init 371 pcxl_dma_init(void) 372 { 373 if (pcxl_dma_start == 0) 374 return 0; 375 376 pcxl_res_size = PCXL_DMA_MAP_SIZE >> (PAGE_SHIFT + 3); 377 pcxl_res_hint = 0; 378 pcxl_res_map = (char *)__get_free_pages(GFP_KERNEL, 379 get_order(pcxl_res_size)); 380 memset(pcxl_res_map, 0, pcxl_res_size); 381 proc_gsc_root = proc_mkdir("gsc", NULL); 382 if (!proc_gsc_root) 383 printk(KERN_WARNING 384 "pcxl_dma_init: Unable to create gsc /proc dir entry\n"); 385 else { 386 struct proc_dir_entry* ent; 387 ent = proc_create_single("pcxl_dma", 0, proc_gsc_root, 388 proc_pcxl_dma_show); 389 if (!ent) 390 printk(KERN_WARNING 391 "pci-dma.c: Unable to create pcxl_dma /proc entry.\n"); 392 } 393 return 0; 394 } 395 396 __initcall(pcxl_dma_init); 397 398 static void *pa11_dma_alloc(struct device *dev, size_t size, 399 dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) 400 { 401 unsigned long vaddr; 402 unsigned long paddr; 403 int order; 404 405 order = get_order(size); 406 size = 1 << (order + PAGE_SHIFT); 407 vaddr = pcxl_alloc_range(size); 408 paddr = __get_free_pages(flag, order); 409 flush_kernel_dcache_range(paddr, size); 410 paddr = __pa(paddr); 411 map_uncached_pages(vaddr, size, paddr); 412 *dma_handle = (dma_addr_t) paddr; 413 414 #if 0 415 /* This probably isn't needed to support EISA cards. 416 ** ISA cards will certainly only support 24-bit DMA addressing. 417 ** Not clear if we can, want, or need to support ISA. 418 */ 419 if (!dev || *dev->coherent_dma_mask < 0xffffffff) 420 gfp |= GFP_DMA; 421 #endif 422 return (void *)vaddr; 423 } 424 425 static void pa11_dma_free(struct device *dev, size_t size, void *vaddr, 426 dma_addr_t dma_handle, unsigned long attrs) 427 { 428 int order; 429 430 order = get_order(size); 431 size = 1 << (order + PAGE_SHIFT); 432 unmap_uncached_pages((unsigned long)vaddr, size); 433 pcxl_free_range((unsigned long)vaddr, size); 434 free_pages((unsigned long)__va(dma_handle), order); 435 } 436 437 static dma_addr_t pa11_dma_map_page(struct device *dev, struct page *page, 438 unsigned long offset, size_t size, 439 enum dma_data_direction direction, unsigned long attrs) 440 { 441 void *addr = page_address(page) + offset; 442 BUG_ON(direction == DMA_NONE); 443 444 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 445 flush_kernel_dcache_range((unsigned long) addr, size); 446 447 return virt_to_phys(addr); 448 } 449 450 static void pa11_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, 451 size_t size, enum dma_data_direction direction, 452 unsigned long attrs) 453 { 454 BUG_ON(direction == DMA_NONE); 455 456 if (attrs & DMA_ATTR_SKIP_CPU_SYNC) 457 return; 458 459 if (direction == DMA_TO_DEVICE) 460 return; 461 462 /* 463 * For PCI_DMA_FROMDEVICE this flush is not necessary for the 464 * simple map/unmap case. However, it IS necessary if if 465 * pci_dma_sync_single_* has been called and the buffer reused. 466 */ 467 468 flush_kernel_dcache_range((unsigned long) phys_to_virt(dma_handle), size); 469 } 470 471 static int pa11_dma_map_sg(struct device *dev, struct scatterlist *sglist, 472 int nents, enum dma_data_direction direction, 473 unsigned long attrs) 474 { 475 int i; 476 struct scatterlist *sg; 477 478 BUG_ON(direction == DMA_NONE); 479 480 for_each_sg(sglist, sg, nents, i) { 481 unsigned long vaddr = (unsigned long)sg_virt(sg); 482 483 sg_dma_address(sg) = (dma_addr_t) virt_to_phys(vaddr); 484 sg_dma_len(sg) = sg->length; 485 486 if (attrs & DMA_ATTR_SKIP_CPU_SYNC) 487 continue; 488 489 flush_kernel_dcache_range(vaddr, sg->length); 490 } 491 return nents; 492 } 493 494 static void pa11_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, 495 int nents, enum dma_data_direction direction, 496 unsigned long attrs) 497 { 498 int i; 499 struct scatterlist *sg; 500 501 BUG_ON(direction == DMA_NONE); 502 503 if (attrs & DMA_ATTR_SKIP_CPU_SYNC) 504 return; 505 506 if (direction == DMA_TO_DEVICE) 507 return; 508 509 /* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ 510 511 for_each_sg(sglist, sg, nents, i) 512 flush_kernel_vmap_range(sg_virt(sg), sg->length); 513 } 514 515 static void pa11_dma_sync_single_for_cpu(struct device *dev, 516 dma_addr_t dma_handle, size_t size, 517 enum dma_data_direction direction) 518 { 519 BUG_ON(direction == DMA_NONE); 520 521 flush_kernel_dcache_range((unsigned long) phys_to_virt(dma_handle), 522 size); 523 } 524 525 static void pa11_dma_sync_single_for_device(struct device *dev, 526 dma_addr_t dma_handle, size_t size, 527 enum dma_data_direction direction) 528 { 529 BUG_ON(direction == DMA_NONE); 530 531 flush_kernel_dcache_range((unsigned long) phys_to_virt(dma_handle), 532 size); 533 } 534 535 static void pa11_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction) 536 { 537 int i; 538 struct scatterlist *sg; 539 540 /* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ 541 542 for_each_sg(sglist, sg, nents, i) 543 flush_kernel_vmap_range(sg_virt(sg), sg->length); 544 } 545 546 static void pa11_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction) 547 { 548 int i; 549 struct scatterlist *sg; 550 551 /* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ 552 553 for_each_sg(sglist, sg, nents, i) 554 flush_kernel_vmap_range(sg_virt(sg), sg->length); 555 } 556 557 static void pa11_dma_cache_sync(struct device *dev, void *vaddr, size_t size, 558 enum dma_data_direction direction) 559 { 560 flush_kernel_dcache_range((unsigned long)vaddr, size); 561 } 562 563 const struct dma_map_ops pcxl_dma_ops = { 564 .alloc = pa11_dma_alloc, 565 .free = pa11_dma_free, 566 .map_page = pa11_dma_map_page, 567 .unmap_page = pa11_dma_unmap_page, 568 .map_sg = pa11_dma_map_sg, 569 .unmap_sg = pa11_dma_unmap_sg, 570 .sync_single_for_cpu = pa11_dma_sync_single_for_cpu, 571 .sync_single_for_device = pa11_dma_sync_single_for_device, 572 .sync_sg_for_cpu = pa11_dma_sync_sg_for_cpu, 573 .sync_sg_for_device = pa11_dma_sync_sg_for_device, 574 .cache_sync = pa11_dma_cache_sync, 575 }; 576 577 static void *pcx_dma_alloc(struct device *dev, size_t size, 578 dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) 579 { 580 void *addr; 581 582 if ((attrs & DMA_ATTR_NON_CONSISTENT) == 0) 583 return NULL; 584 585 addr = (void *)__get_free_pages(flag, get_order(size)); 586 if (addr) 587 *dma_handle = (dma_addr_t)virt_to_phys(addr); 588 589 return addr; 590 } 591 592 static void pcx_dma_free(struct device *dev, size_t size, void *vaddr, 593 dma_addr_t iova, unsigned long attrs) 594 { 595 free_pages((unsigned long)vaddr, get_order(size)); 596 return; 597 } 598 599 const struct dma_map_ops pcx_dma_ops = { 600 .alloc = pcx_dma_alloc, 601 .free = pcx_dma_free, 602 .map_page = pa11_dma_map_page, 603 .unmap_page = pa11_dma_unmap_page, 604 .map_sg = pa11_dma_map_sg, 605 .unmap_sg = pa11_dma_unmap_sg, 606 .sync_single_for_cpu = pa11_dma_sync_single_for_cpu, 607 .sync_single_for_device = pa11_dma_sync_single_for_device, 608 .sync_sg_for_cpu = pa11_dma_sync_sg_for_cpu, 609 .sync_sg_for_device = pa11_dma_sync_sg_for_device, 610 .cache_sync = pa11_dma_cache_sync, 611 }; 612