1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Mips Jazz DMA controller support 4 * Copyright (C) 1995, 1996 by Andreas Busse 5 * 6 * NOTE: Some of the argument checking could be removed when 7 * things have settled down. Also, instead of returning 0xffffffff 8 * on failure of vdma_alloc() one could leave page #0 unused 9 * and return the more usual NULL pointer as logical address. 10 */ 11 #include <linux/kernel.h> 12 #include <linux/init.h> 13 #include <linux/export.h> 14 #include <linux/errno.h> 15 #include <linux/mm.h> 16 #include <linux/memblock.h> 17 #include <linux/spinlock.h> 18 #include <linux/gfp.h> 19 #include <linux/dma-direct.h> 20 #include <linux/dma-noncoherent.h> 21 #include <asm/mipsregs.h> 22 #include <asm/jazz.h> 23 #include <asm/io.h> 24 #include <linux/uaccess.h> 25 #include <asm/dma.h> 26 #include <asm/jazzdma.h> 27 #include <asm/pgtable.h> 28 29 /* 30 * Set this to one to enable additional vdma debug code. 31 */ 32 #define CONF_DEBUG_VDMA 0 33 34 static VDMA_PGTBL_ENTRY *pgtbl; 35 36 static DEFINE_SPINLOCK(vdma_lock); 37 38 /* 39 * Debug stuff 40 */ 41 #define vdma_debug ((CONF_DEBUG_VDMA) ? debuglvl : 0) 42 43 static int debuglvl = 3; 44 45 /* 46 * Initialize the pagetable with a one-to-one mapping of 47 * the first 16 Mbytes of main memory and declare all 48 * entries to be unused. Using this method will at least 49 * allow some early device driver operations to work. 50 */ 51 static inline void vdma_pgtbl_init(void) 52 { 53 unsigned long paddr = 0; 54 int i; 55 56 for (i = 0; i < VDMA_PGTBL_ENTRIES; i++) { 57 pgtbl[i].frame = paddr; 58 pgtbl[i].owner = VDMA_PAGE_EMPTY; 59 paddr += VDMA_PAGESIZE; 60 } 61 } 62 63 /* 64 * Initialize the Jazz R4030 dma controller 65 */ 66 static int __init vdma_init(void) 67 { 68 /* 69 * Allocate 32k of memory for DMA page tables. This needs to be page 70 * aligned and should be uncached to avoid cache flushing after every 71 * update. 72 */ 73 pgtbl = (VDMA_PGTBL_ENTRY *)__get_free_pages(GFP_KERNEL | GFP_DMA, 74 get_order(VDMA_PGTBL_SIZE)); 75 BUG_ON(!pgtbl); 76 dma_cache_wback_inv((unsigned long)pgtbl, VDMA_PGTBL_SIZE); 77 pgtbl = (VDMA_PGTBL_ENTRY *)CKSEG1ADDR((unsigned long)pgtbl); 78 79 /* 80 * Clear the R4030 translation table 81 */ 82 vdma_pgtbl_init(); 83 84 r4030_write_reg32(JAZZ_R4030_TRSTBL_BASE, 85 CPHYSADDR((unsigned long)pgtbl)); 86 r4030_write_reg32(JAZZ_R4030_TRSTBL_LIM, VDMA_PGTBL_SIZE); 87 r4030_write_reg32(JAZZ_R4030_TRSTBL_INV, 0); 88 89 printk(KERN_INFO "VDMA: R4030 DMA pagetables initialized.\n"); 90 return 0; 91 } 92 arch_initcall(vdma_init); 93 94 /* 95 * Allocate DMA pagetables using a simple first-fit algorithm 96 */ 97 unsigned long vdma_alloc(unsigned long paddr, unsigned long size) 98 { 99 int first, last, pages, frame, i; 100 unsigned long laddr, flags; 101 102 /* check arguments */ 103 104 if (paddr > 0x1fffffff) { 105 if (vdma_debug) 106 printk("vdma_alloc: Invalid physical address: %08lx\n", 107 paddr); 108 return DMA_MAPPING_ERROR; /* invalid physical address */ 109 } 110 if (size > 0x400000 || size == 0) { 111 if (vdma_debug) 112 printk("vdma_alloc: Invalid size: %08lx\n", size); 113 return DMA_MAPPING_ERROR; /* invalid physical address */ 114 } 115 116 spin_lock_irqsave(&vdma_lock, flags); 117 /* 118 * Find free chunk 119 */ 120 pages = VDMA_PAGE(paddr + size) - VDMA_PAGE(paddr) + 1; 121 first = 0; 122 while (1) { 123 while (pgtbl[first].owner != VDMA_PAGE_EMPTY && 124 first < VDMA_PGTBL_ENTRIES) first++; 125 if (first + pages > VDMA_PGTBL_ENTRIES) { /* nothing free */ 126 spin_unlock_irqrestore(&vdma_lock, flags); 127 return DMA_MAPPING_ERROR; 128 } 129 130 last = first + 1; 131 while (pgtbl[last].owner == VDMA_PAGE_EMPTY 132 && last - first < pages) 133 last++; 134 135 if (last - first == pages) 136 break; /* found */ 137 first = last + 1; 138 } 139 140 /* 141 * Mark pages as allocated 142 */ 143 laddr = (first << 12) + (paddr & (VDMA_PAGESIZE - 1)); 144 frame = paddr & ~(VDMA_PAGESIZE - 1); 145 146 for (i = first; i < last; i++) { 147 pgtbl[i].frame = frame; 148 pgtbl[i].owner = laddr; 149 frame += VDMA_PAGESIZE; 150 } 151 152 /* 153 * Update translation table and return logical start address 154 */ 155 r4030_write_reg32(JAZZ_R4030_TRSTBL_INV, 0); 156 157 if (vdma_debug > 1) 158 printk("vdma_alloc: Allocated %d pages starting from %08lx\n", 159 pages, laddr); 160 161 if (vdma_debug > 2) { 162 printk("LADDR: "); 163 for (i = first; i < last; i++) 164 printk("%08x ", i << 12); 165 printk("\nPADDR: "); 166 for (i = first; i < last; i++) 167 printk("%08x ", pgtbl[i].frame); 168 printk("\nOWNER: "); 169 for (i = first; i < last; i++) 170 printk("%08x ", pgtbl[i].owner); 171 printk("\n"); 172 } 173 174 spin_unlock_irqrestore(&vdma_lock, flags); 175 176 return laddr; 177 } 178 179 EXPORT_SYMBOL(vdma_alloc); 180 181 /* 182 * Free previously allocated dma translation pages 183 * Note that this does NOT change the translation table, 184 * it just marks the free'd pages as unused! 185 */ 186 int vdma_free(unsigned long laddr) 187 { 188 int i; 189 190 i = laddr >> 12; 191 192 if (pgtbl[i].owner != laddr) { 193 printk 194 ("vdma_free: trying to free other's dma pages, laddr=%8lx\n", 195 laddr); 196 return -1; 197 } 198 199 while (i < VDMA_PGTBL_ENTRIES && pgtbl[i].owner == laddr) { 200 pgtbl[i].owner = VDMA_PAGE_EMPTY; 201 i++; 202 } 203 204 if (vdma_debug > 1) 205 printk("vdma_free: freed %ld pages starting from %08lx\n", 206 i - (laddr >> 12), laddr); 207 208 return 0; 209 } 210 211 EXPORT_SYMBOL(vdma_free); 212 213 /* 214 * Map certain page(s) to another physical address. 215 * Caller must have allocated the page(s) before. 216 */ 217 int vdma_remap(unsigned long laddr, unsigned long paddr, unsigned long size) 218 { 219 int first, pages; 220 221 if (laddr > 0xffffff) { 222 if (vdma_debug) 223 printk 224 ("vdma_map: Invalid logical address: %08lx\n", 225 laddr); 226 return -EINVAL; /* invalid logical address */ 227 } 228 if (paddr > 0x1fffffff) { 229 if (vdma_debug) 230 printk 231 ("vdma_map: Invalid physical address: %08lx\n", 232 paddr); 233 return -EINVAL; /* invalid physical address */ 234 } 235 236 pages = (((paddr & (VDMA_PAGESIZE - 1)) + size) >> 12) + 1; 237 first = laddr >> 12; 238 if (vdma_debug) 239 printk("vdma_remap: first=%x, pages=%x\n", first, pages); 240 if (first + pages > VDMA_PGTBL_ENTRIES) { 241 if (vdma_debug) 242 printk("vdma_alloc: Invalid size: %08lx\n", size); 243 return -EINVAL; 244 } 245 246 paddr &= ~(VDMA_PAGESIZE - 1); 247 while (pages > 0 && first < VDMA_PGTBL_ENTRIES) { 248 if (pgtbl[first].owner != laddr) { 249 if (vdma_debug) 250 printk("Trying to remap other's pages.\n"); 251 return -EPERM; /* not owner */ 252 } 253 pgtbl[first].frame = paddr; 254 paddr += VDMA_PAGESIZE; 255 first++; 256 pages--; 257 } 258 259 /* 260 * Update translation table 261 */ 262 r4030_write_reg32(JAZZ_R4030_TRSTBL_INV, 0); 263 264 if (vdma_debug > 2) { 265 int i; 266 pages = (((paddr & (VDMA_PAGESIZE - 1)) + size) >> 12) + 1; 267 first = laddr >> 12; 268 printk("LADDR: "); 269 for (i = first; i < first + pages; i++) 270 printk("%08x ", i << 12); 271 printk("\nPADDR: "); 272 for (i = first; i < first + pages; i++) 273 printk("%08x ", pgtbl[i].frame); 274 printk("\nOWNER: "); 275 for (i = first; i < first + pages; i++) 276 printk("%08x ", pgtbl[i].owner); 277 printk("\n"); 278 } 279 280 return 0; 281 } 282 283 /* 284 * Translate a physical address to a logical address. 285 * This will return the logical address of the first 286 * match. 287 */ 288 unsigned long vdma_phys2log(unsigned long paddr) 289 { 290 int i; 291 int frame; 292 293 frame = paddr & ~(VDMA_PAGESIZE - 1); 294 295 for (i = 0; i < VDMA_PGTBL_ENTRIES; i++) { 296 if (pgtbl[i].frame == frame) 297 break; 298 } 299 300 if (i == VDMA_PGTBL_ENTRIES) 301 return ~0UL; 302 303 return (i << 12) + (paddr & (VDMA_PAGESIZE - 1)); 304 } 305 306 EXPORT_SYMBOL(vdma_phys2log); 307 308 /* 309 * Translate a logical DMA address to a physical address 310 */ 311 unsigned long vdma_log2phys(unsigned long laddr) 312 { 313 return pgtbl[laddr >> 12].frame + (laddr & (VDMA_PAGESIZE - 1)); 314 } 315 316 EXPORT_SYMBOL(vdma_log2phys); 317 318 /* 319 * Print DMA statistics 320 */ 321 void vdma_stats(void) 322 { 323 int i; 324 325 printk("vdma_stats: CONFIG: %08x\n", 326 r4030_read_reg32(JAZZ_R4030_CONFIG)); 327 printk("R4030 translation table base: %08x\n", 328 r4030_read_reg32(JAZZ_R4030_TRSTBL_BASE)); 329 printk("R4030 translation table limit: %08x\n", 330 r4030_read_reg32(JAZZ_R4030_TRSTBL_LIM)); 331 printk("vdma_stats: INV_ADDR: %08x\n", 332 r4030_read_reg32(JAZZ_R4030_INV_ADDR)); 333 printk("vdma_stats: R_FAIL_ADDR: %08x\n", 334 r4030_read_reg32(JAZZ_R4030_R_FAIL_ADDR)); 335 printk("vdma_stats: M_FAIL_ADDR: %08x\n", 336 r4030_read_reg32(JAZZ_R4030_M_FAIL_ADDR)); 337 printk("vdma_stats: IRQ_SOURCE: %08x\n", 338 r4030_read_reg32(JAZZ_R4030_IRQ_SOURCE)); 339 printk("vdma_stats: I386_ERROR: %08x\n", 340 r4030_read_reg32(JAZZ_R4030_I386_ERROR)); 341 printk("vdma_chnl_modes: "); 342 for (i = 0; i < 8; i++) 343 printk("%04x ", 344 (unsigned) r4030_read_reg32(JAZZ_R4030_CHNL_MODE + 345 (i << 5))); 346 printk("\n"); 347 printk("vdma_chnl_enables: "); 348 for (i = 0; i < 8; i++) 349 printk("%04x ", 350 (unsigned) r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 351 (i << 5))); 352 printk("\n"); 353 } 354 355 /* 356 * DMA transfer functions 357 */ 358 359 /* 360 * Enable a DMA channel. Also clear any error conditions. 361 */ 362 void vdma_enable(int channel) 363 { 364 int status; 365 366 if (vdma_debug) 367 printk("vdma_enable: channel %d\n", channel); 368 369 /* 370 * Check error conditions first 371 */ 372 status = r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5)); 373 if (status & 0x400) 374 printk("VDMA: Channel %d: Address error!\n", channel); 375 if (status & 0x200) 376 printk("VDMA: Channel %d: Memory error!\n", channel); 377 378 /* 379 * Clear all interrupt flags 380 */ 381 r4030_write_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5), 382 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 383 (channel << 5)) | R4030_TC_INTR 384 | R4030_MEM_INTR | R4030_ADDR_INTR); 385 386 /* 387 * Enable the desired channel 388 */ 389 r4030_write_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5), 390 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 391 (channel << 5)) | 392 R4030_CHNL_ENABLE); 393 } 394 395 EXPORT_SYMBOL(vdma_enable); 396 397 /* 398 * Disable a DMA channel 399 */ 400 void vdma_disable(int channel) 401 { 402 if (vdma_debug) { 403 int status = 404 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 405 (channel << 5)); 406 407 printk("vdma_disable: channel %d\n", channel); 408 printk("VDMA: channel %d status: %04x (%s) mode: " 409 "%02x addr: %06x count: %06x\n", 410 channel, status, 411 ((status & 0x600) ? "ERROR" : "OK"), 412 (unsigned) r4030_read_reg32(JAZZ_R4030_CHNL_MODE + 413 (channel << 5)), 414 (unsigned) r4030_read_reg32(JAZZ_R4030_CHNL_ADDR + 415 (channel << 5)), 416 (unsigned) r4030_read_reg32(JAZZ_R4030_CHNL_COUNT + 417 (channel << 5))); 418 } 419 420 r4030_write_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5), 421 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 422 (channel << 5)) & 423 ~R4030_CHNL_ENABLE); 424 425 /* 426 * After disabling a DMA channel a remote bus register should be 427 * read to ensure that the current DMA acknowledge cycle is completed. 428 */ 429 *((volatile unsigned int *) JAZZ_DUMMY_DEVICE); 430 } 431 432 EXPORT_SYMBOL(vdma_disable); 433 434 /* 435 * Set DMA mode. This function accepts the mode values used 436 * to set a PC-style DMA controller. For the SCSI and FDC 437 * channels, we also set the default modes each time we're 438 * called. 439 * NOTE: The FAST and BURST dma modes are supported by the 440 * R4030 Rev. 2 and PICA chipsets only. I leave them disabled 441 * for now. 442 */ 443 void vdma_set_mode(int channel, int mode) 444 { 445 if (vdma_debug) 446 printk("vdma_set_mode: channel %d, mode 0x%x\n", channel, 447 mode); 448 449 switch (channel) { 450 case JAZZ_SCSI_DMA: /* scsi */ 451 r4030_write_reg32(JAZZ_R4030_CHNL_MODE + (channel << 5), 452 /* R4030_MODE_FAST | */ 453 /* R4030_MODE_BURST | */ 454 R4030_MODE_INTR_EN | 455 R4030_MODE_WIDTH_16 | 456 R4030_MODE_ATIME_80); 457 break; 458 459 case JAZZ_FLOPPY_DMA: /* floppy */ 460 r4030_write_reg32(JAZZ_R4030_CHNL_MODE + (channel << 5), 461 /* R4030_MODE_FAST | */ 462 /* R4030_MODE_BURST | */ 463 R4030_MODE_INTR_EN | 464 R4030_MODE_WIDTH_8 | 465 R4030_MODE_ATIME_120); 466 break; 467 468 case JAZZ_AUDIOL_DMA: 469 case JAZZ_AUDIOR_DMA: 470 printk("VDMA: Audio DMA not supported yet.\n"); 471 break; 472 473 default: 474 printk 475 ("VDMA: vdma_set_mode() called with unsupported channel %d!\n", 476 channel); 477 } 478 479 switch (mode) { 480 case DMA_MODE_READ: 481 r4030_write_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5), 482 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 483 (channel << 5)) & 484 ~R4030_CHNL_WRITE); 485 break; 486 487 case DMA_MODE_WRITE: 488 r4030_write_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5), 489 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 490 (channel << 5)) | 491 R4030_CHNL_WRITE); 492 break; 493 494 default: 495 printk 496 ("VDMA: vdma_set_mode() called with unknown dma mode 0x%x\n", 497 mode); 498 } 499 } 500 501 EXPORT_SYMBOL(vdma_set_mode); 502 503 /* 504 * Set Transfer Address 505 */ 506 void vdma_set_addr(int channel, long addr) 507 { 508 if (vdma_debug) 509 printk("vdma_set_addr: channel %d, addr %lx\n", channel, 510 addr); 511 512 r4030_write_reg32(JAZZ_R4030_CHNL_ADDR + (channel << 5), addr); 513 } 514 515 EXPORT_SYMBOL(vdma_set_addr); 516 517 /* 518 * Set Transfer Count 519 */ 520 void vdma_set_count(int channel, int count) 521 { 522 if (vdma_debug) 523 printk("vdma_set_count: channel %d, count %08x\n", channel, 524 (unsigned) count); 525 526 r4030_write_reg32(JAZZ_R4030_CHNL_COUNT + (channel << 5), count); 527 } 528 529 EXPORT_SYMBOL(vdma_set_count); 530 531 /* 532 * Get Residual 533 */ 534 int vdma_get_residue(int channel) 535 { 536 int residual; 537 538 residual = r4030_read_reg32(JAZZ_R4030_CHNL_COUNT + (channel << 5)); 539 540 if (vdma_debug) 541 printk("vdma_get_residual: channel %d: residual=%d\n", 542 channel, residual); 543 544 return residual; 545 } 546 547 /* 548 * Get DMA channel enable register 549 */ 550 int vdma_get_enable(int channel) 551 { 552 int enable; 553 554 enable = r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5)); 555 556 if (vdma_debug) 557 printk("vdma_get_enable: channel %d: enable=%d\n", channel, 558 enable); 559 560 return enable; 561 } 562 563 static void *jazz_dma_alloc(struct device *dev, size_t size, 564 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 565 { 566 void *ret; 567 568 ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); 569 if (!ret) 570 return NULL; 571 572 *dma_handle = vdma_alloc(virt_to_phys(ret), size); 573 if (*dma_handle == DMA_MAPPING_ERROR) { 574 dma_direct_free_pages(dev, size, ret, *dma_handle, attrs); 575 return NULL; 576 } 577 578 return ret; 579 } 580 581 static void jazz_dma_free(struct device *dev, size_t size, void *vaddr, 582 dma_addr_t dma_handle, unsigned long attrs) 583 { 584 vdma_free(dma_handle); 585 dma_direct_free_pages(dev, size, vaddr, dma_handle, attrs); 586 } 587 588 static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page, 589 unsigned long offset, size_t size, enum dma_data_direction dir, 590 unsigned long attrs) 591 { 592 phys_addr_t phys = page_to_phys(page) + offset; 593 594 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 595 arch_sync_dma_for_device(phys, size, dir); 596 return vdma_alloc(phys, size); 597 } 598 599 static void jazz_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, 600 size_t size, enum dma_data_direction dir, unsigned long attrs) 601 { 602 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 603 arch_sync_dma_for_cpu(vdma_log2phys(dma_addr), size, dir); 604 vdma_free(dma_addr); 605 } 606 607 static int jazz_dma_map_sg(struct device *dev, struct scatterlist *sglist, 608 int nents, enum dma_data_direction dir, unsigned long attrs) 609 { 610 int i; 611 struct scatterlist *sg; 612 613 for_each_sg(sglist, sg, nents, i) { 614 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 615 arch_sync_dma_for_device(sg_phys(sg), sg->length, 616 dir); 617 sg->dma_address = vdma_alloc(sg_phys(sg), sg->length); 618 if (sg->dma_address == DMA_MAPPING_ERROR) 619 return 0; 620 sg_dma_len(sg) = sg->length; 621 } 622 623 return nents; 624 } 625 626 static void jazz_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, 627 int nents, enum dma_data_direction dir, unsigned long attrs) 628 { 629 int i; 630 struct scatterlist *sg; 631 632 for_each_sg(sglist, sg, nents, i) { 633 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 634 arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); 635 vdma_free(sg->dma_address); 636 } 637 } 638 639 static void jazz_dma_sync_single_for_device(struct device *dev, 640 dma_addr_t addr, size_t size, enum dma_data_direction dir) 641 { 642 arch_sync_dma_for_device(vdma_log2phys(addr), size, dir); 643 } 644 645 static void jazz_dma_sync_single_for_cpu(struct device *dev, 646 dma_addr_t addr, size_t size, enum dma_data_direction dir) 647 { 648 arch_sync_dma_for_cpu(vdma_log2phys(addr), size, dir); 649 } 650 651 static void jazz_dma_sync_sg_for_device(struct device *dev, 652 struct scatterlist *sgl, int nents, enum dma_data_direction dir) 653 { 654 struct scatterlist *sg; 655 int i; 656 657 for_each_sg(sgl, sg, nents, i) 658 arch_sync_dma_for_device(sg_phys(sg), sg->length, dir); 659 } 660 661 static void jazz_dma_sync_sg_for_cpu(struct device *dev, 662 struct scatterlist *sgl, int nents, enum dma_data_direction dir) 663 { 664 struct scatterlist *sg; 665 int i; 666 667 for_each_sg(sgl, sg, nents, i) 668 arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); 669 } 670 671 const struct dma_map_ops jazz_dma_ops = { 672 .alloc = jazz_dma_alloc, 673 .free = jazz_dma_free, 674 .map_page = jazz_dma_map_page, 675 .unmap_page = jazz_dma_unmap_page, 676 .map_sg = jazz_dma_map_sg, 677 .unmap_sg = jazz_dma_unmap_sg, 678 .sync_single_for_cpu = jazz_dma_sync_single_for_cpu, 679 .sync_single_for_device = jazz_dma_sync_single_for_device, 680 .sync_sg_for_cpu = jazz_dma_sync_sg_for_cpu, 681 .sync_sg_for_device = jazz_dma_sync_sg_for_device, 682 .dma_supported = dma_direct_supported, 683 .cache_sync = arch_dma_cache_sync, 684 .mmap = dma_common_mmap, 685 .get_sgtable = dma_common_get_sgtable, 686 }; 687 EXPORT_SYMBOL(jazz_dma_ops); 688