1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Mips Jazz DMA controller support 4 * Copyright (C) 1995, 1996 by Andreas Busse 5 * 6 * NOTE: Some of the argument checking could be removed when 7 * things have settled down. Also, instead of returning 0xffffffff 8 * on failure of vdma_alloc() one could leave page #0 unused 9 * and return the more usual NULL pointer as logical address. 10 */ 11 #include <linux/kernel.h> 12 #include <linux/init.h> 13 #include <linux/export.h> 14 #include <linux/errno.h> 15 #include <linux/mm.h> 16 #include <linux/memblock.h> 17 #include <linux/spinlock.h> 18 #include <linux/gfp.h> 19 #include <linux/dma-direct.h> 20 #include <linux/dma-noncoherent.h> 21 #include <asm/mipsregs.h> 22 #include <asm/jazz.h> 23 #include <asm/io.h> 24 #include <linux/uaccess.h> 25 #include <asm/dma.h> 26 #include <asm/jazzdma.h> 27 #include <asm/pgtable.h> 28 29 /* 30 * Set this to one to enable additional vdma debug code. 31 */ 32 #define CONF_DEBUG_VDMA 0 33 34 static VDMA_PGTBL_ENTRY *pgtbl; 35 36 static DEFINE_SPINLOCK(vdma_lock); 37 38 /* 39 * Debug stuff 40 */ 41 #define vdma_debug ((CONF_DEBUG_VDMA) ? debuglvl : 0) 42 43 static int debuglvl = 3; 44 45 /* 46 * Initialize the pagetable with a one-to-one mapping of 47 * the first 16 Mbytes of main memory and declare all 48 * entries to be unused. Using this method will at least 49 * allow some early device driver operations to work. 50 */ 51 static inline void vdma_pgtbl_init(void) 52 { 53 unsigned long paddr = 0; 54 int i; 55 56 for (i = 0; i < VDMA_PGTBL_ENTRIES; i++) { 57 pgtbl[i].frame = paddr; 58 pgtbl[i].owner = VDMA_PAGE_EMPTY; 59 paddr += VDMA_PAGESIZE; 60 } 61 } 62 63 /* 64 * Initialize the Jazz R4030 dma controller 65 */ 66 static int __init vdma_init(void) 67 { 68 /* 69 * Allocate 32k of memory for DMA page tables. This needs to be page 70 * aligned and should be uncached to avoid cache flushing after every 71 * update. 72 */ 73 pgtbl = (VDMA_PGTBL_ENTRY *)__get_free_pages(GFP_KERNEL | GFP_DMA, 74 get_order(VDMA_PGTBL_SIZE)); 75 BUG_ON(!pgtbl); 76 dma_cache_wback_inv((unsigned long)pgtbl, VDMA_PGTBL_SIZE); 77 pgtbl = (VDMA_PGTBL_ENTRY *)KSEG1ADDR(pgtbl); 78 79 /* 80 * Clear the R4030 translation table 81 */ 82 vdma_pgtbl_init(); 83 84 r4030_write_reg32(JAZZ_R4030_TRSTBL_BASE, CPHYSADDR(pgtbl)); 85 r4030_write_reg32(JAZZ_R4030_TRSTBL_LIM, VDMA_PGTBL_SIZE); 86 r4030_write_reg32(JAZZ_R4030_TRSTBL_INV, 0); 87 88 printk(KERN_INFO "VDMA: R4030 DMA pagetables initialized.\n"); 89 return 0; 90 } 91 arch_initcall(vdma_init); 92 93 /* 94 * Allocate DMA pagetables using a simple first-fit algorithm 95 */ 96 unsigned long vdma_alloc(unsigned long paddr, unsigned long size) 97 { 98 int first, last, pages, frame, i; 99 unsigned long laddr, flags; 100 101 /* check arguments */ 102 103 if (paddr > 0x1fffffff) { 104 if (vdma_debug) 105 printk("vdma_alloc: Invalid physical address: %08lx\n", 106 paddr); 107 return VDMA_ERROR; /* invalid physical address */ 108 } 109 if (size > 0x400000 || size == 0) { 110 if (vdma_debug) 111 printk("vdma_alloc: Invalid size: %08lx\n", size); 112 return VDMA_ERROR; /* invalid physical address */ 113 } 114 115 spin_lock_irqsave(&vdma_lock, flags); 116 /* 117 * Find free chunk 118 */ 119 pages = VDMA_PAGE(paddr + size) - VDMA_PAGE(paddr) + 1; 120 first = 0; 121 while (1) { 122 while (pgtbl[first].owner != VDMA_PAGE_EMPTY && 123 first < VDMA_PGTBL_ENTRIES) first++; 124 if (first + pages > VDMA_PGTBL_ENTRIES) { /* nothing free */ 125 spin_unlock_irqrestore(&vdma_lock, flags); 126 return VDMA_ERROR; 127 } 128 129 last = first + 1; 130 while (pgtbl[last].owner == VDMA_PAGE_EMPTY 131 && last - first < pages) 132 last++; 133 134 if (last - first == pages) 135 break; /* found */ 136 first = last + 1; 137 } 138 139 /* 140 * Mark pages as allocated 141 */ 142 laddr = (first << 12) + (paddr & (VDMA_PAGESIZE - 1)); 143 frame = paddr & ~(VDMA_PAGESIZE - 1); 144 145 for (i = first; i < last; i++) { 146 pgtbl[i].frame = frame; 147 pgtbl[i].owner = laddr; 148 frame += VDMA_PAGESIZE; 149 } 150 151 /* 152 * Update translation table and return logical start address 153 */ 154 r4030_write_reg32(JAZZ_R4030_TRSTBL_INV, 0); 155 156 if (vdma_debug > 1) 157 printk("vdma_alloc: Allocated %d pages starting from %08lx\n", 158 pages, laddr); 159 160 if (vdma_debug > 2) { 161 printk("LADDR: "); 162 for (i = first; i < last; i++) 163 printk("%08x ", i << 12); 164 printk("\nPADDR: "); 165 for (i = first; i < last; i++) 166 printk("%08x ", pgtbl[i].frame); 167 printk("\nOWNER: "); 168 for (i = first; i < last; i++) 169 printk("%08x ", pgtbl[i].owner); 170 printk("\n"); 171 } 172 173 spin_unlock_irqrestore(&vdma_lock, flags); 174 175 return laddr; 176 } 177 178 EXPORT_SYMBOL(vdma_alloc); 179 180 /* 181 * Free previously allocated dma translation pages 182 * Note that this does NOT change the translation table, 183 * it just marks the free'd pages as unused! 184 */ 185 int vdma_free(unsigned long laddr) 186 { 187 int i; 188 189 i = laddr >> 12; 190 191 if (pgtbl[i].owner != laddr) { 192 printk 193 ("vdma_free: trying to free other's dma pages, laddr=%8lx\n", 194 laddr); 195 return -1; 196 } 197 198 while (i < VDMA_PGTBL_ENTRIES && pgtbl[i].owner == laddr) { 199 pgtbl[i].owner = VDMA_PAGE_EMPTY; 200 i++; 201 } 202 203 if (vdma_debug > 1) 204 printk("vdma_free: freed %ld pages starting from %08lx\n", 205 i - (laddr >> 12), laddr); 206 207 return 0; 208 } 209 210 EXPORT_SYMBOL(vdma_free); 211 212 /* 213 * Map certain page(s) to another physical address. 214 * Caller must have allocated the page(s) before. 215 */ 216 int vdma_remap(unsigned long laddr, unsigned long paddr, unsigned long size) 217 { 218 int first, pages; 219 220 if (laddr > 0xffffff) { 221 if (vdma_debug) 222 printk 223 ("vdma_map: Invalid logical address: %08lx\n", 224 laddr); 225 return -EINVAL; /* invalid logical address */ 226 } 227 if (paddr > 0x1fffffff) { 228 if (vdma_debug) 229 printk 230 ("vdma_map: Invalid physical address: %08lx\n", 231 paddr); 232 return -EINVAL; /* invalid physical address */ 233 } 234 235 pages = (((paddr & (VDMA_PAGESIZE - 1)) + size) >> 12) + 1; 236 first = laddr >> 12; 237 if (vdma_debug) 238 printk("vdma_remap: first=%x, pages=%x\n", first, pages); 239 if (first + pages > VDMA_PGTBL_ENTRIES) { 240 if (vdma_debug) 241 printk("vdma_alloc: Invalid size: %08lx\n", size); 242 return -EINVAL; 243 } 244 245 paddr &= ~(VDMA_PAGESIZE - 1); 246 while (pages > 0 && first < VDMA_PGTBL_ENTRIES) { 247 if (pgtbl[first].owner != laddr) { 248 if (vdma_debug) 249 printk("Trying to remap other's pages.\n"); 250 return -EPERM; /* not owner */ 251 } 252 pgtbl[first].frame = paddr; 253 paddr += VDMA_PAGESIZE; 254 first++; 255 pages--; 256 } 257 258 /* 259 * Update translation table 260 */ 261 r4030_write_reg32(JAZZ_R4030_TRSTBL_INV, 0); 262 263 if (vdma_debug > 2) { 264 int i; 265 pages = (((paddr & (VDMA_PAGESIZE - 1)) + size) >> 12) + 1; 266 first = laddr >> 12; 267 printk("LADDR: "); 268 for (i = first; i < first + pages; i++) 269 printk("%08x ", i << 12); 270 printk("\nPADDR: "); 271 for (i = first; i < first + pages; i++) 272 printk("%08x ", pgtbl[i].frame); 273 printk("\nOWNER: "); 274 for (i = first; i < first + pages; i++) 275 printk("%08x ", pgtbl[i].owner); 276 printk("\n"); 277 } 278 279 return 0; 280 } 281 282 /* 283 * Translate a physical address to a logical address. 284 * This will return the logical address of the first 285 * match. 286 */ 287 unsigned long vdma_phys2log(unsigned long paddr) 288 { 289 int i; 290 int frame; 291 292 frame = paddr & ~(VDMA_PAGESIZE - 1); 293 294 for (i = 0; i < VDMA_PGTBL_ENTRIES; i++) { 295 if (pgtbl[i].frame == frame) 296 break; 297 } 298 299 if (i == VDMA_PGTBL_ENTRIES) 300 return ~0UL; 301 302 return (i << 12) + (paddr & (VDMA_PAGESIZE - 1)); 303 } 304 305 EXPORT_SYMBOL(vdma_phys2log); 306 307 /* 308 * Translate a logical DMA address to a physical address 309 */ 310 unsigned long vdma_log2phys(unsigned long laddr) 311 { 312 return pgtbl[laddr >> 12].frame + (laddr & (VDMA_PAGESIZE - 1)); 313 } 314 315 EXPORT_SYMBOL(vdma_log2phys); 316 317 /* 318 * Print DMA statistics 319 */ 320 void vdma_stats(void) 321 { 322 int i; 323 324 printk("vdma_stats: CONFIG: %08x\n", 325 r4030_read_reg32(JAZZ_R4030_CONFIG)); 326 printk("R4030 translation table base: %08x\n", 327 r4030_read_reg32(JAZZ_R4030_TRSTBL_BASE)); 328 printk("R4030 translation table limit: %08x\n", 329 r4030_read_reg32(JAZZ_R4030_TRSTBL_LIM)); 330 printk("vdma_stats: INV_ADDR: %08x\n", 331 r4030_read_reg32(JAZZ_R4030_INV_ADDR)); 332 printk("vdma_stats: R_FAIL_ADDR: %08x\n", 333 r4030_read_reg32(JAZZ_R4030_R_FAIL_ADDR)); 334 printk("vdma_stats: M_FAIL_ADDR: %08x\n", 335 r4030_read_reg32(JAZZ_R4030_M_FAIL_ADDR)); 336 printk("vdma_stats: IRQ_SOURCE: %08x\n", 337 r4030_read_reg32(JAZZ_R4030_IRQ_SOURCE)); 338 printk("vdma_stats: I386_ERROR: %08x\n", 339 r4030_read_reg32(JAZZ_R4030_I386_ERROR)); 340 printk("vdma_chnl_modes: "); 341 for (i = 0; i < 8; i++) 342 printk("%04x ", 343 (unsigned) r4030_read_reg32(JAZZ_R4030_CHNL_MODE + 344 (i << 5))); 345 printk("\n"); 346 printk("vdma_chnl_enables: "); 347 for (i = 0; i < 8; i++) 348 printk("%04x ", 349 (unsigned) r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 350 (i << 5))); 351 printk("\n"); 352 } 353 354 /* 355 * DMA transfer functions 356 */ 357 358 /* 359 * Enable a DMA channel. Also clear any error conditions. 360 */ 361 void vdma_enable(int channel) 362 { 363 int status; 364 365 if (vdma_debug) 366 printk("vdma_enable: channel %d\n", channel); 367 368 /* 369 * Check error conditions first 370 */ 371 status = r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5)); 372 if (status & 0x400) 373 printk("VDMA: Channel %d: Address error!\n", channel); 374 if (status & 0x200) 375 printk("VDMA: Channel %d: Memory error!\n", channel); 376 377 /* 378 * Clear all interrupt flags 379 */ 380 r4030_write_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5), 381 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 382 (channel << 5)) | R4030_TC_INTR 383 | R4030_MEM_INTR | R4030_ADDR_INTR); 384 385 /* 386 * Enable the desired channel 387 */ 388 r4030_write_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5), 389 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 390 (channel << 5)) | 391 R4030_CHNL_ENABLE); 392 } 393 394 EXPORT_SYMBOL(vdma_enable); 395 396 /* 397 * Disable a DMA channel 398 */ 399 void vdma_disable(int channel) 400 { 401 if (vdma_debug) { 402 int status = 403 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 404 (channel << 5)); 405 406 printk("vdma_disable: channel %d\n", channel); 407 printk("VDMA: channel %d status: %04x (%s) mode: " 408 "%02x addr: %06x count: %06x\n", 409 channel, status, 410 ((status & 0x600) ? "ERROR" : "OK"), 411 (unsigned) r4030_read_reg32(JAZZ_R4030_CHNL_MODE + 412 (channel << 5)), 413 (unsigned) r4030_read_reg32(JAZZ_R4030_CHNL_ADDR + 414 (channel << 5)), 415 (unsigned) r4030_read_reg32(JAZZ_R4030_CHNL_COUNT + 416 (channel << 5))); 417 } 418 419 r4030_write_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5), 420 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 421 (channel << 5)) & 422 ~R4030_CHNL_ENABLE); 423 424 /* 425 * After disabling a DMA channel a remote bus register should be 426 * read to ensure that the current DMA acknowledge cycle is completed. 427 */ 428 *((volatile unsigned int *) JAZZ_DUMMY_DEVICE); 429 } 430 431 EXPORT_SYMBOL(vdma_disable); 432 433 /* 434 * Set DMA mode. This function accepts the mode values used 435 * to set a PC-style DMA controller. For the SCSI and FDC 436 * channels, we also set the default modes each time we're 437 * called. 438 * NOTE: The FAST and BURST dma modes are supported by the 439 * R4030 Rev. 2 and PICA chipsets only. I leave them disabled 440 * for now. 441 */ 442 void vdma_set_mode(int channel, int mode) 443 { 444 if (vdma_debug) 445 printk("vdma_set_mode: channel %d, mode 0x%x\n", channel, 446 mode); 447 448 switch (channel) { 449 case JAZZ_SCSI_DMA: /* scsi */ 450 r4030_write_reg32(JAZZ_R4030_CHNL_MODE + (channel << 5), 451 /* R4030_MODE_FAST | */ 452 /* R4030_MODE_BURST | */ 453 R4030_MODE_INTR_EN | 454 R4030_MODE_WIDTH_16 | 455 R4030_MODE_ATIME_80); 456 break; 457 458 case JAZZ_FLOPPY_DMA: /* floppy */ 459 r4030_write_reg32(JAZZ_R4030_CHNL_MODE + (channel << 5), 460 /* R4030_MODE_FAST | */ 461 /* R4030_MODE_BURST | */ 462 R4030_MODE_INTR_EN | 463 R4030_MODE_WIDTH_8 | 464 R4030_MODE_ATIME_120); 465 break; 466 467 case JAZZ_AUDIOL_DMA: 468 case JAZZ_AUDIOR_DMA: 469 printk("VDMA: Audio DMA not supported yet.\n"); 470 break; 471 472 default: 473 printk 474 ("VDMA: vdma_set_mode() called with unsupported channel %d!\n", 475 channel); 476 } 477 478 switch (mode) { 479 case DMA_MODE_READ: 480 r4030_write_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5), 481 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 482 (channel << 5)) & 483 ~R4030_CHNL_WRITE); 484 break; 485 486 case DMA_MODE_WRITE: 487 r4030_write_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5), 488 r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + 489 (channel << 5)) | 490 R4030_CHNL_WRITE); 491 break; 492 493 default: 494 printk 495 ("VDMA: vdma_set_mode() called with unknown dma mode 0x%x\n", 496 mode); 497 } 498 } 499 500 EXPORT_SYMBOL(vdma_set_mode); 501 502 /* 503 * Set Transfer Address 504 */ 505 void vdma_set_addr(int channel, long addr) 506 { 507 if (vdma_debug) 508 printk("vdma_set_addr: channel %d, addr %lx\n", channel, 509 addr); 510 511 r4030_write_reg32(JAZZ_R4030_CHNL_ADDR + (channel << 5), addr); 512 } 513 514 EXPORT_SYMBOL(vdma_set_addr); 515 516 /* 517 * Set Transfer Count 518 */ 519 void vdma_set_count(int channel, int count) 520 { 521 if (vdma_debug) 522 printk("vdma_set_count: channel %d, count %08x\n", channel, 523 (unsigned) count); 524 525 r4030_write_reg32(JAZZ_R4030_CHNL_COUNT + (channel << 5), count); 526 } 527 528 EXPORT_SYMBOL(vdma_set_count); 529 530 /* 531 * Get Residual 532 */ 533 int vdma_get_residue(int channel) 534 { 535 int residual; 536 537 residual = r4030_read_reg32(JAZZ_R4030_CHNL_COUNT + (channel << 5)); 538 539 if (vdma_debug) 540 printk("vdma_get_residual: channel %d: residual=%d\n", 541 channel, residual); 542 543 return residual; 544 } 545 546 /* 547 * Get DMA channel enable register 548 */ 549 int vdma_get_enable(int channel) 550 { 551 int enable; 552 553 enable = r4030_read_reg32(JAZZ_R4030_CHNL_ENABLE + (channel << 5)); 554 555 if (vdma_debug) 556 printk("vdma_get_enable: channel %d: enable=%d\n", channel, 557 enable); 558 559 return enable; 560 } 561 562 static void *jazz_dma_alloc(struct device *dev, size_t size, 563 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 564 { 565 void *ret; 566 567 ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); 568 if (!ret) 569 return NULL; 570 571 *dma_handle = vdma_alloc(virt_to_phys(ret), size); 572 if (*dma_handle == VDMA_ERROR) { 573 dma_direct_free_pages(dev, size, ret, *dma_handle, attrs); 574 return NULL; 575 } 576 577 if (!(attrs & DMA_ATTR_NON_CONSISTENT)) { 578 dma_cache_wback_inv((unsigned long)ret, size); 579 ret = (void *)UNCAC_ADDR(ret); 580 } 581 return ret; 582 } 583 584 static void jazz_dma_free(struct device *dev, size_t size, void *vaddr, 585 dma_addr_t dma_handle, unsigned long attrs) 586 { 587 vdma_free(dma_handle); 588 if (!(attrs & DMA_ATTR_NON_CONSISTENT)) 589 vaddr = (void *)CAC_ADDR((unsigned long)vaddr); 590 dma_direct_free_pages(dev, size, vaddr, dma_handle, attrs); 591 } 592 593 static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page, 594 unsigned long offset, size_t size, enum dma_data_direction dir, 595 unsigned long attrs) 596 { 597 phys_addr_t phys = page_to_phys(page) + offset; 598 599 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 600 arch_sync_dma_for_device(dev, phys, size, dir); 601 return vdma_alloc(phys, size); 602 } 603 604 static void jazz_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, 605 size_t size, enum dma_data_direction dir, unsigned long attrs) 606 { 607 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 608 arch_sync_dma_for_cpu(dev, vdma_log2phys(dma_addr), size, dir); 609 vdma_free(dma_addr); 610 } 611 612 static int jazz_dma_map_sg(struct device *dev, struct scatterlist *sglist, 613 int nents, enum dma_data_direction dir, unsigned long attrs) 614 { 615 int i; 616 struct scatterlist *sg; 617 618 for_each_sg(sglist, sg, nents, i) { 619 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 620 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, 621 dir); 622 sg->dma_address = vdma_alloc(sg_phys(sg), sg->length); 623 if (sg->dma_address == VDMA_ERROR) 624 return 0; 625 sg_dma_len(sg) = sg->length; 626 } 627 628 return nents; 629 } 630 631 static void jazz_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, 632 int nents, enum dma_data_direction dir, unsigned long attrs) 633 { 634 int i; 635 struct scatterlist *sg; 636 637 for_each_sg(sglist, sg, nents, i) { 638 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 639 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, 640 dir); 641 vdma_free(sg->dma_address); 642 } 643 } 644 645 static void jazz_dma_sync_single_for_device(struct device *dev, 646 dma_addr_t addr, size_t size, enum dma_data_direction dir) 647 { 648 arch_sync_dma_for_device(dev, vdma_log2phys(addr), size, dir); 649 } 650 651 static void jazz_dma_sync_single_for_cpu(struct device *dev, 652 dma_addr_t addr, size_t size, enum dma_data_direction dir) 653 { 654 arch_sync_dma_for_cpu(dev, vdma_log2phys(addr), size, dir); 655 } 656 657 static void jazz_dma_sync_sg_for_device(struct device *dev, 658 struct scatterlist *sgl, int nents, enum dma_data_direction dir) 659 { 660 struct scatterlist *sg; 661 int i; 662 663 for_each_sg(sgl, sg, nents, i) 664 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); 665 } 666 667 static void jazz_dma_sync_sg_for_cpu(struct device *dev, 668 struct scatterlist *sgl, int nents, enum dma_data_direction dir) 669 { 670 struct scatterlist *sg; 671 int i; 672 673 for_each_sg(sgl, sg, nents, i) 674 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); 675 } 676 677 static int jazz_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 678 { 679 return dma_addr == VDMA_ERROR; 680 } 681 682 const struct dma_map_ops jazz_dma_ops = { 683 .alloc = jazz_dma_alloc, 684 .free = jazz_dma_free, 685 .map_page = jazz_dma_map_page, 686 .unmap_page = jazz_dma_unmap_page, 687 .map_sg = jazz_dma_map_sg, 688 .unmap_sg = jazz_dma_unmap_sg, 689 .sync_single_for_cpu = jazz_dma_sync_single_for_cpu, 690 .sync_single_for_device = jazz_dma_sync_single_for_device, 691 .sync_sg_for_cpu = jazz_dma_sync_sg_for_cpu, 692 .sync_sg_for_device = jazz_dma_sync_sg_for_device, 693 .dma_supported = dma_direct_supported, 694 .cache_sync = arch_dma_cache_sync, 695 .mapping_error = jazz_dma_mapping_error, 696 }; 697 EXPORT_SYMBOL(jazz_dma_ops); 698