1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/delay.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <linux/err.h> 13 #include <linux/init.h> 14 #include <linux/interrupt.h> 15 #include <linux/list.h> 16 #include <linux/platform_device.h> 17 #include <linux/slab.h> 18 #include <linux/spinlock.h> 19 #include <linux/sys_soc.h> 20 #include <linux/of.h> 21 #include <linux/of_dma.h> 22 #include <linux/of_device.h> 23 #include <linux/of_irq.h> 24 #include <linux/workqueue.h> 25 #include <linux/completion.h> 26 #include <linux/soc/ti/k3-ringacc.h> 27 #include <linux/soc/ti/ti_sci_protocol.h> 28 #include <linux/soc/ti/ti_sci_inta_msi.h> 29 #include <linux/dma/ti-cppi5.h> 30 31 #include "../virt-dma.h" 32 #include "k3-udma.h" 33 #include "k3-psil-priv.h" 34 35 struct udma_static_tr { 36 u8 elsize; /* RPSTR0 */ 37 u16 elcnt; /* RPSTR0 */ 38 u16 bstcnt; /* RPSTR1 */ 39 }; 40 41 #define K3_UDMA_MAX_RFLOWS 1024 42 #define K3_UDMA_DEFAULT_RING_SIZE 16 43 44 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */ 45 #define UDMA_RFLOW_SRCTAG_NONE 0 46 #define UDMA_RFLOW_SRCTAG_CFG_TAG 1 47 #define UDMA_RFLOW_SRCTAG_FLOW_ID 2 48 #define UDMA_RFLOW_SRCTAG_SRC_TAG 4 49 50 #define UDMA_RFLOW_DSTTAG_NONE 0 51 #define UDMA_RFLOW_DSTTAG_CFG_TAG 1 52 #define UDMA_RFLOW_DSTTAG_FLOW_ID 2 53 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO 4 54 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI 5 55 56 struct udma_chan; 57 58 enum udma_mmr { 59 MMR_GCFG = 0, 60 MMR_RCHANRT, 61 MMR_TCHANRT, 62 MMR_LAST, 63 }; 64 65 static const char * const mmr_names[] = { "gcfg", "rchanrt", "tchanrt" }; 66 67 struct udma_tchan { 68 void __iomem *reg_rt; 69 70 int id; 71 struct k3_ring *t_ring; /* Transmit ring */ 72 struct k3_ring *tc_ring; /* Transmit Completion ring */ 73 }; 74 75 struct udma_rflow { 76 int id; 77 struct k3_ring *fd_ring; /* Free Descriptor ring */ 78 struct k3_ring *r_ring; /* Receive ring */ 79 }; 80 81 struct udma_rchan { 82 void __iomem *reg_rt; 83 84 int id; 85 }; 86 87 #define UDMA_FLAG_PDMA_ACC32 BIT(0) 88 #define UDMA_FLAG_PDMA_BURST BIT(1) 89 90 struct udma_match_data { 91 u32 psil_base; 92 bool enable_memcpy_support; 93 u32 flags; 94 u32 statictr_z_mask; 95 }; 96 97 struct udma_soc_data { 98 u32 rchan_oes_offset; 99 }; 100 101 struct udma_hwdesc { 102 size_t cppi5_desc_size; 103 void *cppi5_desc_vaddr; 104 dma_addr_t cppi5_desc_paddr; 105 106 /* TR descriptor internal pointers */ 107 void *tr_req_base; 108 struct cppi5_tr_resp_t *tr_resp_base; 109 }; 110 111 struct udma_rx_flush { 112 struct udma_hwdesc hwdescs[2]; 113 114 size_t buffer_size; 115 void *buffer_vaddr; 116 dma_addr_t buffer_paddr; 117 }; 118 119 struct udma_dev { 120 struct dma_device ddev; 121 struct device *dev; 122 void __iomem *mmrs[MMR_LAST]; 123 const struct udma_match_data *match_data; 124 const struct udma_soc_data *soc_data; 125 126 u8 tpl_levels; 127 u32 tpl_start_idx[3]; 128 129 size_t desc_align; /* alignment to use for descriptors */ 130 131 struct udma_tisci_rm tisci_rm; 132 133 struct k3_ringacc *ringacc; 134 135 struct work_struct purge_work; 136 struct list_head desc_to_purge; 137 spinlock_t lock; 138 139 struct udma_rx_flush rx_flush; 140 141 int tchan_cnt; 142 int echan_cnt; 143 int rchan_cnt; 144 int rflow_cnt; 145 unsigned long *tchan_map; 146 unsigned long *rchan_map; 147 unsigned long *rflow_gp_map; 148 unsigned long *rflow_gp_map_allocated; 149 unsigned long *rflow_in_use; 150 151 struct udma_tchan *tchans; 152 struct udma_rchan *rchans; 153 struct udma_rflow *rflows; 154 155 struct udma_chan *channels; 156 u32 psil_base; 157 u32 atype; 158 }; 159 160 struct udma_desc { 161 struct virt_dma_desc vd; 162 163 bool terminated; 164 165 enum dma_transfer_direction dir; 166 167 struct udma_static_tr static_tr; 168 u32 residue; 169 170 unsigned int sglen; 171 unsigned int desc_idx; /* Only used for cyclic in packet mode */ 172 unsigned int tr_idx; 173 174 u32 metadata_size; 175 void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */ 176 177 unsigned int hwdesc_count; 178 struct udma_hwdesc hwdesc[]; 179 }; 180 181 enum udma_chan_state { 182 UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */ 183 UDMA_CHAN_IS_ACTIVE, /* Normal operation */ 184 UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */ 185 }; 186 187 struct udma_tx_drain { 188 struct delayed_work work; 189 ktime_t tstamp; 190 u32 residue; 191 }; 192 193 struct udma_chan_config { 194 bool pkt_mode; /* TR or packet */ 195 bool needs_epib; /* EPIB is needed for the communication or not */ 196 u32 psd_size; /* size of Protocol Specific Data */ 197 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */ 198 u32 hdesc_size; /* Size of a packet descriptor in packet mode */ 199 bool notdpkt; /* Suppress sending TDC packet */ 200 int remote_thread_id; 201 u32 atype; 202 u32 src_thread; 203 u32 dst_thread; 204 enum psil_endpoint_type ep_type; 205 bool enable_acc32; 206 bool enable_burst; 207 enum udma_tp_level channel_tpl; /* Channel Throughput Level */ 208 209 enum dma_transfer_direction dir; 210 }; 211 212 struct udma_chan { 213 struct virt_dma_chan vc; 214 struct dma_slave_config cfg; 215 struct udma_dev *ud; 216 struct udma_desc *desc; 217 struct udma_desc *terminated_desc; 218 struct udma_static_tr static_tr; 219 char *name; 220 221 struct udma_tchan *tchan; 222 struct udma_rchan *rchan; 223 struct udma_rflow *rflow; 224 225 bool psil_paired; 226 227 int irq_num_ring; 228 int irq_num_udma; 229 230 bool cyclic; 231 bool paused; 232 233 enum udma_chan_state state; 234 struct completion teardown_completed; 235 236 struct udma_tx_drain tx_drain; 237 238 u32 bcnt; /* number of bytes completed since the start of the channel */ 239 240 /* Channel configuration parameters */ 241 struct udma_chan_config config; 242 243 /* dmapool for packet mode descriptors */ 244 bool use_dma_pool; 245 struct dma_pool *hdesc_pool; 246 247 u32 id; 248 }; 249 250 static inline struct udma_dev *to_udma_dev(struct dma_device *d) 251 { 252 return container_of(d, struct udma_dev, ddev); 253 } 254 255 static inline struct udma_chan *to_udma_chan(struct dma_chan *c) 256 { 257 return container_of(c, struct udma_chan, vc.chan); 258 } 259 260 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t) 261 { 262 return container_of(t, struct udma_desc, vd.tx); 263 } 264 265 /* Generic register access functions */ 266 static inline u32 udma_read(void __iomem *base, int reg) 267 { 268 return readl(base + reg); 269 } 270 271 static inline void udma_write(void __iomem *base, int reg, u32 val) 272 { 273 writel(val, base + reg); 274 } 275 276 static inline void udma_update_bits(void __iomem *base, int reg, 277 u32 mask, u32 val) 278 { 279 u32 tmp, orig; 280 281 orig = readl(base + reg); 282 tmp = orig & ~mask; 283 tmp |= (val & mask); 284 285 if (tmp != orig) 286 writel(tmp, base + reg); 287 } 288 289 /* TCHANRT */ 290 static inline u32 udma_tchanrt_read(struct udma_chan *uc, int reg) 291 { 292 if (!uc->tchan) 293 return 0; 294 return udma_read(uc->tchan->reg_rt, reg); 295 } 296 297 static inline void udma_tchanrt_write(struct udma_chan *uc, int reg, u32 val) 298 { 299 if (!uc->tchan) 300 return; 301 udma_write(uc->tchan->reg_rt, reg, val); 302 } 303 304 static inline void udma_tchanrt_update_bits(struct udma_chan *uc, int reg, 305 u32 mask, u32 val) 306 { 307 if (!uc->tchan) 308 return; 309 udma_update_bits(uc->tchan->reg_rt, reg, mask, val); 310 } 311 312 /* RCHANRT */ 313 static inline u32 udma_rchanrt_read(struct udma_chan *uc, int reg) 314 { 315 if (!uc->rchan) 316 return 0; 317 return udma_read(uc->rchan->reg_rt, reg); 318 } 319 320 static inline void udma_rchanrt_write(struct udma_chan *uc, int reg, u32 val) 321 { 322 if (!uc->rchan) 323 return; 324 udma_write(uc->rchan->reg_rt, reg, val); 325 } 326 327 static inline void udma_rchanrt_update_bits(struct udma_chan *uc, int reg, 328 u32 mask, u32 val) 329 { 330 if (!uc->rchan) 331 return; 332 udma_update_bits(uc->rchan->reg_rt, reg, mask, val); 333 } 334 335 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) 336 { 337 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 338 339 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 340 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci, 341 tisci_rm->tisci_navss_dev_id, 342 src_thread, dst_thread); 343 } 344 345 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread, 346 u32 dst_thread) 347 { 348 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 349 350 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 351 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci, 352 tisci_rm->tisci_navss_dev_id, 353 src_thread, dst_thread); 354 } 355 356 static void udma_reset_uchan(struct udma_chan *uc) 357 { 358 memset(&uc->config, 0, sizeof(uc->config)); 359 uc->config.remote_thread_id = -1; 360 uc->state = UDMA_CHAN_IS_IDLE; 361 } 362 363 static void udma_dump_chan_stdata(struct udma_chan *uc) 364 { 365 struct device *dev = uc->ud->dev; 366 u32 offset; 367 int i; 368 369 if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) { 370 dev_dbg(dev, "TCHAN State data:\n"); 371 for (i = 0; i < 32; i++) { 372 offset = UDMA_CHAN_RT_STDATA_REG + i * 4; 373 dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i, 374 udma_tchanrt_read(uc, offset)); 375 } 376 } 377 378 if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) { 379 dev_dbg(dev, "RCHAN State data:\n"); 380 for (i = 0; i < 32; i++) { 381 offset = UDMA_CHAN_RT_STDATA_REG + i * 4; 382 dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i, 383 udma_rchanrt_read(uc, offset)); 384 } 385 } 386 } 387 388 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d, 389 int idx) 390 { 391 return d->hwdesc[idx].cppi5_desc_paddr; 392 } 393 394 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx) 395 { 396 return d->hwdesc[idx].cppi5_desc_vaddr; 397 } 398 399 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc, 400 dma_addr_t paddr) 401 { 402 struct udma_desc *d = uc->terminated_desc; 403 404 if (d) { 405 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 406 d->desc_idx); 407 408 if (desc_paddr != paddr) 409 d = NULL; 410 } 411 412 if (!d) { 413 d = uc->desc; 414 if (d) { 415 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 416 d->desc_idx); 417 418 if (desc_paddr != paddr) 419 d = NULL; 420 } 421 } 422 423 return d; 424 } 425 426 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d) 427 { 428 if (uc->use_dma_pool) { 429 int i; 430 431 for (i = 0; i < d->hwdesc_count; i++) { 432 if (!d->hwdesc[i].cppi5_desc_vaddr) 433 continue; 434 435 dma_pool_free(uc->hdesc_pool, 436 d->hwdesc[i].cppi5_desc_vaddr, 437 d->hwdesc[i].cppi5_desc_paddr); 438 439 d->hwdesc[i].cppi5_desc_vaddr = NULL; 440 } 441 } else if (d->hwdesc[0].cppi5_desc_vaddr) { 442 struct udma_dev *ud = uc->ud; 443 444 dma_free_coherent(ud->dev, d->hwdesc[0].cppi5_desc_size, 445 d->hwdesc[0].cppi5_desc_vaddr, 446 d->hwdesc[0].cppi5_desc_paddr); 447 448 d->hwdesc[0].cppi5_desc_vaddr = NULL; 449 } 450 } 451 452 static void udma_purge_desc_work(struct work_struct *work) 453 { 454 struct udma_dev *ud = container_of(work, typeof(*ud), purge_work); 455 struct virt_dma_desc *vd, *_vd; 456 unsigned long flags; 457 LIST_HEAD(head); 458 459 spin_lock_irqsave(&ud->lock, flags); 460 list_splice_tail_init(&ud->desc_to_purge, &head); 461 spin_unlock_irqrestore(&ud->lock, flags); 462 463 list_for_each_entry_safe(vd, _vd, &head, node) { 464 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 465 struct udma_desc *d = to_udma_desc(&vd->tx); 466 467 udma_free_hwdesc(uc, d); 468 list_del(&vd->node); 469 kfree(d); 470 } 471 472 /* If more to purge, schedule the work again */ 473 if (!list_empty(&ud->desc_to_purge)) 474 schedule_work(&ud->purge_work); 475 } 476 477 static void udma_desc_free(struct virt_dma_desc *vd) 478 { 479 struct udma_dev *ud = to_udma_dev(vd->tx.chan->device); 480 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 481 struct udma_desc *d = to_udma_desc(&vd->tx); 482 unsigned long flags; 483 484 if (uc->terminated_desc == d) 485 uc->terminated_desc = NULL; 486 487 if (uc->use_dma_pool) { 488 udma_free_hwdesc(uc, d); 489 kfree(d); 490 return; 491 } 492 493 spin_lock_irqsave(&ud->lock, flags); 494 list_add_tail(&vd->node, &ud->desc_to_purge); 495 spin_unlock_irqrestore(&ud->lock, flags); 496 497 schedule_work(&ud->purge_work); 498 } 499 500 static bool udma_is_chan_running(struct udma_chan *uc) 501 { 502 u32 trt_ctl = 0; 503 u32 rrt_ctl = 0; 504 505 if (uc->tchan) 506 trt_ctl = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); 507 if (uc->rchan) 508 rrt_ctl = udma_rchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); 509 510 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN) 511 return true; 512 513 return false; 514 } 515 516 static bool udma_is_chan_paused(struct udma_chan *uc) 517 { 518 u32 val, pause_mask; 519 520 switch (uc->config.dir) { 521 case DMA_DEV_TO_MEM: 522 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG); 523 pause_mask = UDMA_PEER_RT_EN_PAUSE; 524 break; 525 case DMA_MEM_TO_DEV: 526 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG); 527 pause_mask = UDMA_PEER_RT_EN_PAUSE; 528 break; 529 case DMA_MEM_TO_MEM: 530 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); 531 pause_mask = UDMA_CHAN_RT_CTL_PAUSE; 532 break; 533 default: 534 return false; 535 } 536 537 if (val & pause_mask) 538 return true; 539 540 return false; 541 } 542 543 static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc) 544 { 545 return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr; 546 } 547 548 static int udma_push_to_ring(struct udma_chan *uc, int idx) 549 { 550 struct udma_desc *d = uc->desc; 551 struct k3_ring *ring = NULL; 552 dma_addr_t paddr; 553 554 switch (uc->config.dir) { 555 case DMA_DEV_TO_MEM: 556 ring = uc->rflow->fd_ring; 557 break; 558 case DMA_MEM_TO_DEV: 559 case DMA_MEM_TO_MEM: 560 ring = uc->tchan->t_ring; 561 break; 562 default: 563 return -EINVAL; 564 } 565 566 /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */ 567 if (idx == -1) { 568 paddr = udma_get_rx_flush_hwdesc_paddr(uc); 569 } else { 570 paddr = udma_curr_cppi5_desc_paddr(d, idx); 571 572 wmb(); /* Ensure that writes are not moved over this point */ 573 } 574 575 return k3_ringacc_ring_push(ring, &paddr); 576 } 577 578 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr) 579 { 580 if (uc->config.dir != DMA_DEV_TO_MEM) 581 return false; 582 583 if (addr == udma_get_rx_flush_hwdesc_paddr(uc)) 584 return true; 585 586 return false; 587 } 588 589 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) 590 { 591 struct k3_ring *ring = NULL; 592 int ret; 593 594 switch (uc->config.dir) { 595 case DMA_DEV_TO_MEM: 596 ring = uc->rflow->r_ring; 597 break; 598 case DMA_MEM_TO_DEV: 599 case DMA_MEM_TO_MEM: 600 ring = uc->tchan->tc_ring; 601 break; 602 default: 603 return -ENOENT; 604 } 605 606 ret = k3_ringacc_ring_pop(ring, addr); 607 if (ret) 608 return ret; 609 610 rmb(); /* Ensure that reads are not moved before this point */ 611 612 /* Teardown completion */ 613 if (cppi5_desc_is_tdcm(*addr)) 614 return 0; 615 616 /* Check for flush descriptor */ 617 if (udma_desc_is_rx_flush(uc, *addr)) 618 return -ENOENT; 619 620 return 0; 621 } 622 623 static void udma_reset_rings(struct udma_chan *uc) 624 { 625 struct k3_ring *ring1 = NULL; 626 struct k3_ring *ring2 = NULL; 627 628 switch (uc->config.dir) { 629 case DMA_DEV_TO_MEM: 630 if (uc->rchan) { 631 ring1 = uc->rflow->fd_ring; 632 ring2 = uc->rflow->r_ring; 633 } 634 break; 635 case DMA_MEM_TO_DEV: 636 case DMA_MEM_TO_MEM: 637 if (uc->tchan) { 638 ring1 = uc->tchan->t_ring; 639 ring2 = uc->tchan->tc_ring; 640 } 641 break; 642 default: 643 break; 644 } 645 646 if (ring1) 647 k3_ringacc_ring_reset_dma(ring1, 648 k3_ringacc_ring_get_occ(ring1)); 649 if (ring2) 650 k3_ringacc_ring_reset(ring2); 651 652 /* make sure we are not leaking memory by stalled descriptor */ 653 if (uc->terminated_desc) { 654 udma_desc_free(&uc->terminated_desc->vd); 655 uc->terminated_desc = NULL; 656 } 657 } 658 659 static void udma_reset_counters(struct udma_chan *uc) 660 { 661 u32 val; 662 663 if (uc->tchan) { 664 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 665 udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); 666 667 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); 668 udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); 669 670 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG); 671 udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val); 672 673 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); 674 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); 675 } 676 677 if (uc->rchan) { 678 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 679 udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); 680 681 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); 682 udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); 683 684 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG); 685 udma_rchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val); 686 687 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); 688 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); 689 } 690 691 uc->bcnt = 0; 692 } 693 694 static int udma_reset_chan(struct udma_chan *uc, bool hard) 695 { 696 switch (uc->config.dir) { 697 case DMA_DEV_TO_MEM: 698 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0); 699 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 700 break; 701 case DMA_MEM_TO_DEV: 702 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 703 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0); 704 break; 705 case DMA_MEM_TO_MEM: 706 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 707 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 708 break; 709 default: 710 return -EINVAL; 711 } 712 713 /* Reset all counters */ 714 udma_reset_counters(uc); 715 716 /* Hard reset: re-initialize the channel to reset */ 717 if (hard) { 718 struct udma_chan_config ucc_backup; 719 int ret; 720 721 memcpy(&ucc_backup, &uc->config, sizeof(uc->config)); 722 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan); 723 724 /* restore the channel configuration */ 725 memcpy(&uc->config, &ucc_backup, sizeof(uc->config)); 726 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan); 727 if (ret) 728 return ret; 729 730 /* 731 * Setting forced teardown after forced reset helps recovering 732 * the rchan. 733 */ 734 if (uc->config.dir == DMA_DEV_TO_MEM) 735 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 736 UDMA_CHAN_RT_CTL_EN | 737 UDMA_CHAN_RT_CTL_TDOWN | 738 UDMA_CHAN_RT_CTL_FTDOWN); 739 } 740 uc->state = UDMA_CHAN_IS_IDLE; 741 742 return 0; 743 } 744 745 static void udma_start_desc(struct udma_chan *uc) 746 { 747 struct udma_chan_config *ucc = &uc->config; 748 749 if (ucc->pkt_mode && (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) { 750 int i; 751 752 /* Push all descriptors to ring for packet mode cyclic or RX */ 753 for (i = 0; i < uc->desc->sglen; i++) 754 udma_push_to_ring(uc, i); 755 } else { 756 udma_push_to_ring(uc, 0); 757 } 758 } 759 760 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc) 761 { 762 /* Only PDMAs have staticTR */ 763 if (uc->config.ep_type == PSIL_EP_NATIVE) 764 return false; 765 766 /* Check if the staticTR configuration has changed for TX */ 767 if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr))) 768 return true; 769 770 return false; 771 } 772 773 static int udma_start(struct udma_chan *uc) 774 { 775 struct virt_dma_desc *vd = vchan_next_desc(&uc->vc); 776 777 if (!vd) { 778 uc->desc = NULL; 779 return -ENOENT; 780 } 781 782 list_del(&vd->node); 783 784 uc->desc = to_udma_desc(&vd->tx); 785 786 /* Channel is already running and does not need reconfiguration */ 787 if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) { 788 udma_start_desc(uc); 789 goto out; 790 } 791 792 /* Make sure that we clear the teardown bit, if it is set */ 793 udma_reset_chan(uc, false); 794 795 /* Push descriptors before we start the channel */ 796 udma_start_desc(uc); 797 798 switch (uc->desc->dir) { 799 case DMA_DEV_TO_MEM: 800 /* Config remote TR */ 801 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 802 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 803 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 804 const struct udma_match_data *match_data = 805 uc->ud->match_data; 806 807 if (uc->config.enable_acc32) 808 val |= PDMA_STATIC_TR_XY_ACC32; 809 if (uc->config.enable_burst) 810 val |= PDMA_STATIC_TR_XY_BURST; 811 812 udma_rchanrt_write(uc, 813 UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG, 814 val); 815 816 udma_rchanrt_write(uc, 817 UDMA_CHAN_RT_PEER_STATIC_TR_Z_REG, 818 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt, 819 match_data->statictr_z_mask)); 820 821 /* save the current staticTR configuration */ 822 memcpy(&uc->static_tr, &uc->desc->static_tr, 823 sizeof(uc->static_tr)); 824 } 825 826 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 827 UDMA_CHAN_RT_CTL_EN); 828 829 /* Enable remote */ 830 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 831 UDMA_PEER_RT_EN_ENABLE); 832 833 break; 834 case DMA_MEM_TO_DEV: 835 /* Config remote TR */ 836 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 837 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 838 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 839 840 if (uc->config.enable_acc32) 841 val |= PDMA_STATIC_TR_XY_ACC32; 842 if (uc->config.enable_burst) 843 val |= PDMA_STATIC_TR_XY_BURST; 844 845 udma_tchanrt_write(uc, 846 UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG, 847 val); 848 849 /* save the current staticTR configuration */ 850 memcpy(&uc->static_tr, &uc->desc->static_tr, 851 sizeof(uc->static_tr)); 852 } 853 854 /* Enable remote */ 855 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 856 UDMA_PEER_RT_EN_ENABLE); 857 858 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 859 UDMA_CHAN_RT_CTL_EN); 860 861 break; 862 case DMA_MEM_TO_MEM: 863 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 864 UDMA_CHAN_RT_CTL_EN); 865 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 866 UDMA_CHAN_RT_CTL_EN); 867 868 break; 869 default: 870 return -EINVAL; 871 } 872 873 uc->state = UDMA_CHAN_IS_ACTIVE; 874 out: 875 876 return 0; 877 } 878 879 static int udma_stop(struct udma_chan *uc) 880 { 881 enum udma_chan_state old_state = uc->state; 882 883 uc->state = UDMA_CHAN_IS_TERMINATING; 884 reinit_completion(&uc->teardown_completed); 885 886 switch (uc->config.dir) { 887 case DMA_DEV_TO_MEM: 888 if (!uc->cyclic && !uc->desc) 889 udma_push_to_ring(uc, -1); 890 891 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 892 UDMA_PEER_RT_EN_ENABLE | 893 UDMA_PEER_RT_EN_TEARDOWN); 894 break; 895 case DMA_MEM_TO_DEV: 896 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 897 UDMA_PEER_RT_EN_ENABLE | 898 UDMA_PEER_RT_EN_FLUSH); 899 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 900 UDMA_CHAN_RT_CTL_EN | 901 UDMA_CHAN_RT_CTL_TDOWN); 902 break; 903 case DMA_MEM_TO_MEM: 904 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 905 UDMA_CHAN_RT_CTL_EN | 906 UDMA_CHAN_RT_CTL_TDOWN); 907 break; 908 default: 909 uc->state = old_state; 910 complete_all(&uc->teardown_completed); 911 return -EINVAL; 912 } 913 914 return 0; 915 } 916 917 static void udma_cyclic_packet_elapsed(struct udma_chan *uc) 918 { 919 struct udma_desc *d = uc->desc; 920 struct cppi5_host_desc_t *h_desc; 921 922 h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr; 923 cppi5_hdesc_reset_to_original(h_desc); 924 udma_push_to_ring(uc, d->desc_idx); 925 d->desc_idx = (d->desc_idx + 1) % d->sglen; 926 } 927 928 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d) 929 { 930 struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr; 931 932 memcpy(d->metadata, h_desc->epib, d->metadata_size); 933 } 934 935 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) 936 { 937 u32 peer_bcnt, bcnt; 938 939 /* Only TX towards PDMA is affected */ 940 if (uc->config.ep_type == PSIL_EP_NATIVE || 941 uc->config.dir != DMA_MEM_TO_DEV) 942 return true; 943 944 peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); 945 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 946 947 /* Transfer is incomplete, store current residue and time stamp */ 948 if (peer_bcnt < bcnt) { 949 uc->tx_drain.residue = bcnt - peer_bcnt; 950 uc->tx_drain.tstamp = ktime_get(); 951 return false; 952 } 953 954 return true; 955 } 956 957 static void udma_check_tx_completion(struct work_struct *work) 958 { 959 struct udma_chan *uc = container_of(work, typeof(*uc), 960 tx_drain.work.work); 961 bool desc_done = true; 962 u32 residue_diff; 963 ktime_t time_diff; 964 unsigned long delay; 965 966 while (1) { 967 if (uc->desc) { 968 /* Get previous residue and time stamp */ 969 residue_diff = uc->tx_drain.residue; 970 time_diff = uc->tx_drain.tstamp; 971 /* 972 * Get current residue and time stamp or see if 973 * transfer is complete 974 */ 975 desc_done = udma_is_desc_really_done(uc, uc->desc); 976 } 977 978 if (!desc_done) { 979 /* 980 * Find the time delta and residue delta w.r.t 981 * previous poll 982 */ 983 time_diff = ktime_sub(uc->tx_drain.tstamp, 984 time_diff) + 1; 985 residue_diff -= uc->tx_drain.residue; 986 if (residue_diff) { 987 /* 988 * Try to guess when we should check 989 * next time by calculating rate at 990 * which data is being drained at the 991 * peer device 992 */ 993 delay = (time_diff / residue_diff) * 994 uc->tx_drain.residue; 995 } else { 996 /* No progress, check again in 1 second */ 997 schedule_delayed_work(&uc->tx_drain.work, HZ); 998 break; 999 } 1000 1001 usleep_range(ktime_to_us(delay), 1002 ktime_to_us(delay) + 10); 1003 continue; 1004 } 1005 1006 if (uc->desc) { 1007 struct udma_desc *d = uc->desc; 1008 1009 uc->bcnt += d->residue; 1010 udma_start(uc); 1011 vchan_cookie_complete(&d->vd); 1012 break; 1013 } 1014 1015 break; 1016 } 1017 } 1018 1019 static irqreturn_t udma_ring_irq_handler(int irq, void *data) 1020 { 1021 struct udma_chan *uc = data; 1022 struct udma_desc *d; 1023 unsigned long flags; 1024 dma_addr_t paddr = 0; 1025 1026 if (udma_pop_from_ring(uc, &paddr) || !paddr) 1027 return IRQ_HANDLED; 1028 1029 spin_lock_irqsave(&uc->vc.lock, flags); 1030 1031 /* Teardown completion message */ 1032 if (cppi5_desc_is_tdcm(paddr)) { 1033 complete_all(&uc->teardown_completed); 1034 1035 if (uc->terminated_desc) { 1036 udma_desc_free(&uc->terminated_desc->vd); 1037 uc->terminated_desc = NULL; 1038 } 1039 1040 if (!uc->desc) 1041 udma_start(uc); 1042 1043 goto out; 1044 } 1045 1046 d = udma_udma_desc_from_paddr(uc, paddr); 1047 1048 if (d) { 1049 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 1050 d->desc_idx); 1051 if (desc_paddr != paddr) { 1052 dev_err(uc->ud->dev, "not matching descriptors!\n"); 1053 goto out; 1054 } 1055 1056 if (d == uc->desc) { 1057 /* active descriptor */ 1058 if (uc->cyclic) { 1059 udma_cyclic_packet_elapsed(uc); 1060 vchan_cyclic_callback(&d->vd); 1061 } else { 1062 if (udma_is_desc_really_done(uc, d)) { 1063 uc->bcnt += d->residue; 1064 udma_start(uc); 1065 vchan_cookie_complete(&d->vd); 1066 } else { 1067 schedule_delayed_work(&uc->tx_drain.work, 1068 0); 1069 } 1070 } 1071 } else { 1072 /* 1073 * terminated descriptor, mark the descriptor as 1074 * completed to update the channel's cookie marker 1075 */ 1076 dma_cookie_complete(&d->vd.tx); 1077 } 1078 } 1079 out: 1080 spin_unlock_irqrestore(&uc->vc.lock, flags); 1081 1082 return IRQ_HANDLED; 1083 } 1084 1085 static irqreturn_t udma_udma_irq_handler(int irq, void *data) 1086 { 1087 struct udma_chan *uc = data; 1088 struct udma_desc *d; 1089 unsigned long flags; 1090 1091 spin_lock_irqsave(&uc->vc.lock, flags); 1092 d = uc->desc; 1093 if (d) { 1094 d->tr_idx = (d->tr_idx + 1) % d->sglen; 1095 1096 if (uc->cyclic) { 1097 vchan_cyclic_callback(&d->vd); 1098 } else { 1099 /* TODO: figure out the real amount of data */ 1100 uc->bcnt += d->residue; 1101 udma_start(uc); 1102 vchan_cookie_complete(&d->vd); 1103 } 1104 } 1105 1106 spin_unlock_irqrestore(&uc->vc.lock, flags); 1107 1108 return IRQ_HANDLED; 1109 } 1110 1111 /** 1112 * __udma_alloc_gp_rflow_range - alloc range of GP RX flows 1113 * @ud: UDMA device 1114 * @from: Start the search from this flow id number 1115 * @cnt: Number of consecutive flow ids to allocate 1116 * 1117 * Allocate range of RX flow ids for future use, those flows can be requested 1118 * only using explicit flow id number. if @from is set to -1 it will try to find 1119 * first free range. if @from is positive value it will force allocation only 1120 * of the specified range of flows. 1121 * 1122 * Returns -ENOMEM if can't find free range. 1123 * -EEXIST if requested range is busy. 1124 * -EINVAL if wrong input values passed. 1125 * Returns flow id on success. 1126 */ 1127 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1128 { 1129 int start, tmp_from; 1130 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS); 1131 1132 tmp_from = from; 1133 if (tmp_from < 0) 1134 tmp_from = ud->rchan_cnt; 1135 /* default flows can't be allocated and accessible only by id */ 1136 if (tmp_from < ud->rchan_cnt) 1137 return -EINVAL; 1138 1139 if (tmp_from + cnt > ud->rflow_cnt) 1140 return -EINVAL; 1141 1142 bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated, 1143 ud->rflow_cnt); 1144 1145 start = bitmap_find_next_zero_area(tmp, 1146 ud->rflow_cnt, 1147 tmp_from, cnt, 0); 1148 if (start >= ud->rflow_cnt) 1149 return -ENOMEM; 1150 1151 if (from >= 0 && start != from) 1152 return -EEXIST; 1153 1154 bitmap_set(ud->rflow_gp_map_allocated, start, cnt); 1155 return start; 1156 } 1157 1158 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1159 { 1160 if (from < ud->rchan_cnt) 1161 return -EINVAL; 1162 if (from + cnt > ud->rflow_cnt) 1163 return -EINVAL; 1164 1165 bitmap_clear(ud->rflow_gp_map_allocated, from, cnt); 1166 return 0; 1167 } 1168 1169 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id) 1170 { 1171 /* 1172 * Attempt to request rflow by ID can be made for any rflow 1173 * if not in use with assumption that caller knows what's doing. 1174 * TI-SCI FW will perform additional permission check ant way, it's 1175 * safe 1176 */ 1177 1178 if (id < 0 || id >= ud->rflow_cnt) 1179 return ERR_PTR(-ENOENT); 1180 1181 if (test_bit(id, ud->rflow_in_use)) 1182 return ERR_PTR(-ENOENT); 1183 1184 /* GP rflow has to be allocated first */ 1185 if (!test_bit(id, ud->rflow_gp_map) && 1186 !test_bit(id, ud->rflow_gp_map_allocated)) 1187 return ERR_PTR(-EINVAL); 1188 1189 dev_dbg(ud->dev, "get rflow%d\n", id); 1190 set_bit(id, ud->rflow_in_use); 1191 return &ud->rflows[id]; 1192 } 1193 1194 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow) 1195 { 1196 if (!test_bit(rflow->id, ud->rflow_in_use)) { 1197 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id); 1198 return; 1199 } 1200 1201 dev_dbg(ud->dev, "put rflow%d\n", rflow->id); 1202 clear_bit(rflow->id, ud->rflow_in_use); 1203 } 1204 1205 #define UDMA_RESERVE_RESOURCE(res) \ 1206 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ 1207 enum udma_tp_level tpl, \ 1208 int id) \ 1209 { \ 1210 if (id >= 0) { \ 1211 if (test_bit(id, ud->res##_map)) { \ 1212 dev_err(ud->dev, "res##%d is in use\n", id); \ 1213 return ERR_PTR(-ENOENT); \ 1214 } \ 1215 } else { \ 1216 int start; \ 1217 \ 1218 if (tpl >= ud->tpl_levels) \ 1219 tpl = ud->tpl_levels - 1; \ 1220 \ 1221 start = ud->tpl_start_idx[tpl]; \ 1222 \ 1223 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \ 1224 start); \ 1225 if (id == ud->res##_cnt) { \ 1226 return ERR_PTR(-ENOENT); \ 1227 } \ 1228 } \ 1229 \ 1230 set_bit(id, ud->res##_map); \ 1231 return &ud->res##s[id]; \ 1232 } 1233 1234 UDMA_RESERVE_RESOURCE(tchan); 1235 UDMA_RESERVE_RESOURCE(rchan); 1236 1237 static int udma_get_tchan(struct udma_chan *uc) 1238 { 1239 struct udma_dev *ud = uc->ud; 1240 1241 if (uc->tchan) { 1242 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n", 1243 uc->id, uc->tchan->id); 1244 return 0; 1245 } 1246 1247 uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, -1); 1248 1249 return PTR_ERR_OR_ZERO(uc->tchan); 1250 } 1251 1252 static int udma_get_rchan(struct udma_chan *uc) 1253 { 1254 struct udma_dev *ud = uc->ud; 1255 1256 if (uc->rchan) { 1257 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n", 1258 uc->id, uc->rchan->id); 1259 return 0; 1260 } 1261 1262 uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, -1); 1263 1264 return PTR_ERR_OR_ZERO(uc->rchan); 1265 } 1266 1267 static int udma_get_chan_pair(struct udma_chan *uc) 1268 { 1269 struct udma_dev *ud = uc->ud; 1270 int chan_id, end; 1271 1272 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) { 1273 dev_info(ud->dev, "chan%d: already have %d pair allocated\n", 1274 uc->id, uc->tchan->id); 1275 return 0; 1276 } 1277 1278 if (uc->tchan) { 1279 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n", 1280 uc->id, uc->tchan->id); 1281 return -EBUSY; 1282 } else if (uc->rchan) { 1283 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n", 1284 uc->id, uc->rchan->id); 1285 return -EBUSY; 1286 } 1287 1288 /* Can be optimized, but let's have it like this for now */ 1289 end = min(ud->tchan_cnt, ud->rchan_cnt); 1290 /* Try to use the highest TPL channel pair for MEM_TO_MEM channels */ 1291 chan_id = ud->tpl_start_idx[ud->tpl_levels - 1]; 1292 for (; chan_id < end; chan_id++) { 1293 if (!test_bit(chan_id, ud->tchan_map) && 1294 !test_bit(chan_id, ud->rchan_map)) 1295 break; 1296 } 1297 1298 if (chan_id == end) 1299 return -ENOENT; 1300 1301 set_bit(chan_id, ud->tchan_map); 1302 set_bit(chan_id, ud->rchan_map); 1303 uc->tchan = &ud->tchans[chan_id]; 1304 uc->rchan = &ud->rchans[chan_id]; 1305 1306 return 0; 1307 } 1308 1309 static int udma_get_rflow(struct udma_chan *uc, int flow_id) 1310 { 1311 struct udma_dev *ud = uc->ud; 1312 1313 if (!uc->rchan) { 1314 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id); 1315 return -EINVAL; 1316 } 1317 1318 if (uc->rflow) { 1319 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n", 1320 uc->id, uc->rflow->id); 1321 return 0; 1322 } 1323 1324 uc->rflow = __udma_get_rflow(ud, flow_id); 1325 1326 return PTR_ERR_OR_ZERO(uc->rflow); 1327 } 1328 1329 static void udma_put_rchan(struct udma_chan *uc) 1330 { 1331 struct udma_dev *ud = uc->ud; 1332 1333 if (uc->rchan) { 1334 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id, 1335 uc->rchan->id); 1336 clear_bit(uc->rchan->id, ud->rchan_map); 1337 uc->rchan = NULL; 1338 } 1339 } 1340 1341 static void udma_put_tchan(struct udma_chan *uc) 1342 { 1343 struct udma_dev *ud = uc->ud; 1344 1345 if (uc->tchan) { 1346 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id, 1347 uc->tchan->id); 1348 clear_bit(uc->tchan->id, ud->tchan_map); 1349 uc->tchan = NULL; 1350 } 1351 } 1352 1353 static void udma_put_rflow(struct udma_chan *uc) 1354 { 1355 struct udma_dev *ud = uc->ud; 1356 1357 if (uc->rflow) { 1358 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id, 1359 uc->rflow->id); 1360 __udma_put_rflow(ud, uc->rflow); 1361 uc->rflow = NULL; 1362 } 1363 } 1364 1365 static void udma_free_tx_resources(struct udma_chan *uc) 1366 { 1367 if (!uc->tchan) 1368 return; 1369 1370 k3_ringacc_ring_free(uc->tchan->t_ring); 1371 k3_ringacc_ring_free(uc->tchan->tc_ring); 1372 uc->tchan->t_ring = NULL; 1373 uc->tchan->tc_ring = NULL; 1374 1375 udma_put_tchan(uc); 1376 } 1377 1378 static int udma_alloc_tx_resources(struct udma_chan *uc) 1379 { 1380 struct k3_ring_cfg ring_cfg; 1381 struct udma_dev *ud = uc->ud; 1382 int ret; 1383 1384 ret = udma_get_tchan(uc); 1385 if (ret) 1386 return ret; 1387 1388 ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->tchan->id, -1, 1389 &uc->tchan->t_ring, 1390 &uc->tchan->tc_ring); 1391 if (ret) { 1392 ret = -EBUSY; 1393 goto err_ring; 1394 } 1395 1396 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1397 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1398 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1399 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1400 1401 ret = k3_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg); 1402 ret |= k3_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg); 1403 1404 if (ret) 1405 goto err_ringcfg; 1406 1407 return 0; 1408 1409 err_ringcfg: 1410 k3_ringacc_ring_free(uc->tchan->tc_ring); 1411 uc->tchan->tc_ring = NULL; 1412 k3_ringacc_ring_free(uc->tchan->t_ring); 1413 uc->tchan->t_ring = NULL; 1414 err_ring: 1415 udma_put_tchan(uc); 1416 1417 return ret; 1418 } 1419 1420 static void udma_free_rx_resources(struct udma_chan *uc) 1421 { 1422 if (!uc->rchan) 1423 return; 1424 1425 if (uc->rflow) { 1426 struct udma_rflow *rflow = uc->rflow; 1427 1428 k3_ringacc_ring_free(rflow->fd_ring); 1429 k3_ringacc_ring_free(rflow->r_ring); 1430 rflow->fd_ring = NULL; 1431 rflow->r_ring = NULL; 1432 1433 udma_put_rflow(uc); 1434 } 1435 1436 udma_put_rchan(uc); 1437 } 1438 1439 static int udma_alloc_rx_resources(struct udma_chan *uc) 1440 { 1441 struct udma_dev *ud = uc->ud; 1442 struct k3_ring_cfg ring_cfg; 1443 struct udma_rflow *rflow; 1444 int fd_ring_id; 1445 int ret; 1446 1447 ret = udma_get_rchan(uc); 1448 if (ret) 1449 return ret; 1450 1451 /* For MEM_TO_MEM we don't need rflow or rings */ 1452 if (uc->config.dir == DMA_MEM_TO_MEM) 1453 return 0; 1454 1455 ret = udma_get_rflow(uc, uc->rchan->id); 1456 if (ret) { 1457 ret = -EBUSY; 1458 goto err_rflow; 1459 } 1460 1461 rflow = uc->rflow; 1462 fd_ring_id = ud->tchan_cnt + ud->echan_cnt + uc->rchan->id; 1463 ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1, 1464 &rflow->fd_ring, &rflow->r_ring); 1465 if (ret) { 1466 ret = -EBUSY; 1467 goto err_ring; 1468 } 1469 1470 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1471 1472 if (uc->config.pkt_mode) 1473 ring_cfg.size = SG_MAX_SEGMENTS; 1474 else 1475 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1476 1477 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1478 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1479 1480 ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg); 1481 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1482 ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg); 1483 1484 if (ret) 1485 goto err_ringcfg; 1486 1487 return 0; 1488 1489 err_ringcfg: 1490 k3_ringacc_ring_free(rflow->r_ring); 1491 rflow->r_ring = NULL; 1492 k3_ringacc_ring_free(rflow->fd_ring); 1493 rflow->fd_ring = NULL; 1494 err_ring: 1495 udma_put_rflow(uc); 1496 err_rflow: 1497 udma_put_rchan(uc); 1498 1499 return ret; 1500 } 1501 1502 #define TISCI_TCHAN_VALID_PARAMS ( \ 1503 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1504 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \ 1505 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \ 1506 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1507 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \ 1508 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1509 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1510 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1511 1512 #define TISCI_RCHAN_VALID_PARAMS ( \ 1513 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1514 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1515 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1516 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1517 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \ 1518 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \ 1519 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \ 1520 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \ 1521 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1522 1523 static int udma_tisci_m2m_channel_config(struct udma_chan *uc) 1524 { 1525 struct udma_dev *ud = uc->ud; 1526 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1527 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1528 struct udma_tchan *tchan = uc->tchan; 1529 struct udma_rchan *rchan = uc->rchan; 1530 int ret = 0; 1531 1532 /* Non synchronized - mem to mem type of transfer */ 1533 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1534 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1535 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1536 1537 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; 1538 req_tx.nav_id = tisci_rm->tisci_dev_id; 1539 req_tx.index = tchan->id; 1540 req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1541 req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1542 req_tx.txcq_qnum = tc_ring; 1543 req_tx.tx_atype = ud->atype; 1544 1545 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1546 if (ret) { 1547 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1548 return ret; 1549 } 1550 1551 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; 1552 req_rx.nav_id = tisci_rm->tisci_dev_id; 1553 req_rx.index = rchan->id; 1554 req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1555 req_rx.rxcq_qnum = tc_ring; 1556 req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1557 req_rx.rx_atype = ud->atype; 1558 1559 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1560 if (ret) 1561 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret); 1562 1563 return ret; 1564 } 1565 1566 static int udma_tisci_tx_channel_config(struct udma_chan *uc) 1567 { 1568 struct udma_dev *ud = uc->ud; 1569 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1570 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1571 struct udma_tchan *tchan = uc->tchan; 1572 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1573 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1574 u32 mode, fetch_size; 1575 int ret = 0; 1576 1577 if (uc->config.pkt_mode) { 1578 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1579 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1580 uc->config.psd_size, 0); 1581 } else { 1582 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1583 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1584 } 1585 1586 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; 1587 req_tx.nav_id = tisci_rm->tisci_dev_id; 1588 req_tx.index = tchan->id; 1589 req_tx.tx_chan_type = mode; 1590 req_tx.tx_supr_tdpkt = uc->config.notdpkt; 1591 req_tx.tx_fetch_size = fetch_size >> 2; 1592 req_tx.txcq_qnum = tc_ring; 1593 req_tx.tx_atype = uc->config.atype; 1594 1595 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1596 if (ret) 1597 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1598 1599 return ret; 1600 } 1601 1602 static int udma_tisci_rx_channel_config(struct udma_chan *uc) 1603 { 1604 struct udma_dev *ud = uc->ud; 1605 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1606 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1607 struct udma_rchan *rchan = uc->rchan; 1608 int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring); 1609 int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring); 1610 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1611 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; 1612 u32 mode, fetch_size; 1613 int ret = 0; 1614 1615 if (uc->config.pkt_mode) { 1616 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1617 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1618 uc->config.psd_size, 0); 1619 } else { 1620 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1621 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1622 } 1623 1624 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; 1625 req_rx.nav_id = tisci_rm->tisci_dev_id; 1626 req_rx.index = rchan->id; 1627 req_rx.rx_fetch_size = fetch_size >> 2; 1628 req_rx.rxcq_qnum = rx_ring; 1629 req_rx.rx_chan_type = mode; 1630 req_rx.rx_atype = uc->config.atype; 1631 1632 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1633 if (ret) { 1634 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); 1635 return ret; 1636 } 1637 1638 flow_req.valid_params = 1639 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 1640 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 1641 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 1642 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 1643 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1644 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 1645 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 1646 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 1647 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 1648 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1649 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1650 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1651 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1652 1653 flow_req.nav_id = tisci_rm->tisci_dev_id; 1654 flow_req.flow_index = rchan->id; 1655 1656 if (uc->config.needs_epib) 1657 flow_req.rx_einfo_present = 1; 1658 else 1659 flow_req.rx_einfo_present = 0; 1660 if (uc->config.psd_size) 1661 flow_req.rx_psinfo_present = 1; 1662 else 1663 flow_req.rx_psinfo_present = 0; 1664 flow_req.rx_error_handling = 1; 1665 flow_req.rx_dest_qnum = rx_ring; 1666 flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE; 1667 flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG; 1668 flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI; 1669 flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO; 1670 flow_req.rx_fdq0_sz0_qnum = fd_ring; 1671 flow_req.rx_fdq1_qnum = fd_ring; 1672 flow_req.rx_fdq2_qnum = fd_ring; 1673 flow_req.rx_fdq3_qnum = fd_ring; 1674 1675 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req); 1676 1677 if (ret) 1678 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret); 1679 1680 return 0; 1681 } 1682 1683 static int udma_alloc_chan_resources(struct dma_chan *chan) 1684 { 1685 struct udma_chan *uc = to_udma_chan(chan); 1686 struct udma_dev *ud = to_udma_dev(chan->device); 1687 const struct udma_soc_data *soc_data = ud->soc_data; 1688 struct k3_ring *irq_ring; 1689 u32 irq_udma_idx; 1690 int ret; 1691 1692 if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) { 1693 uc->use_dma_pool = true; 1694 /* in case of MEM_TO_MEM we have maximum of two TRs */ 1695 if (uc->config.dir == DMA_MEM_TO_MEM) { 1696 uc->config.hdesc_size = cppi5_trdesc_calc_size( 1697 sizeof(struct cppi5_tr_type15_t), 2); 1698 uc->config.pkt_mode = false; 1699 } 1700 } 1701 1702 if (uc->use_dma_pool) { 1703 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, 1704 uc->config.hdesc_size, 1705 ud->desc_align, 1706 0); 1707 if (!uc->hdesc_pool) { 1708 dev_err(ud->ddev.dev, 1709 "Descriptor pool allocation failed\n"); 1710 uc->use_dma_pool = false; 1711 ret = -ENOMEM; 1712 goto err_cleanup; 1713 } 1714 } 1715 1716 /* 1717 * Make sure that the completion is in a known state: 1718 * No teardown, the channel is idle 1719 */ 1720 reinit_completion(&uc->teardown_completed); 1721 complete_all(&uc->teardown_completed); 1722 uc->state = UDMA_CHAN_IS_IDLE; 1723 1724 switch (uc->config.dir) { 1725 case DMA_MEM_TO_MEM: 1726 /* Non synchronized - mem to mem type of transfer */ 1727 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, 1728 uc->id); 1729 1730 ret = udma_get_chan_pair(uc); 1731 if (ret) 1732 goto err_cleanup; 1733 1734 ret = udma_alloc_tx_resources(uc); 1735 if (ret) { 1736 udma_put_rchan(uc); 1737 goto err_cleanup; 1738 } 1739 1740 ret = udma_alloc_rx_resources(uc); 1741 if (ret) { 1742 udma_free_tx_resources(uc); 1743 goto err_cleanup; 1744 } 1745 1746 uc->config.src_thread = ud->psil_base + uc->tchan->id; 1747 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 1748 K3_PSIL_DST_THREAD_ID_OFFSET; 1749 1750 irq_ring = uc->tchan->tc_ring; 1751 irq_udma_idx = uc->tchan->id; 1752 1753 ret = udma_tisci_m2m_channel_config(uc); 1754 break; 1755 case DMA_MEM_TO_DEV: 1756 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 1757 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 1758 uc->id); 1759 1760 ret = udma_alloc_tx_resources(uc); 1761 if (ret) 1762 goto err_cleanup; 1763 1764 uc->config.src_thread = ud->psil_base + uc->tchan->id; 1765 uc->config.dst_thread = uc->config.remote_thread_id; 1766 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 1767 1768 irq_ring = uc->tchan->tc_ring; 1769 irq_udma_idx = uc->tchan->id; 1770 1771 ret = udma_tisci_tx_channel_config(uc); 1772 break; 1773 case DMA_DEV_TO_MEM: 1774 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 1775 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 1776 uc->id); 1777 1778 ret = udma_alloc_rx_resources(uc); 1779 if (ret) 1780 goto err_cleanup; 1781 1782 uc->config.src_thread = uc->config.remote_thread_id; 1783 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 1784 K3_PSIL_DST_THREAD_ID_OFFSET; 1785 1786 irq_ring = uc->rflow->r_ring; 1787 irq_udma_idx = soc_data->rchan_oes_offset + uc->rchan->id; 1788 1789 ret = udma_tisci_rx_channel_config(uc); 1790 break; 1791 default: 1792 /* Can not happen */ 1793 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 1794 __func__, uc->id, uc->config.dir); 1795 ret = -EINVAL; 1796 goto err_cleanup; 1797 1798 } 1799 1800 /* check if the channel configuration was successful */ 1801 if (ret) 1802 goto err_res_free; 1803 1804 if (udma_is_chan_running(uc)) { 1805 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 1806 udma_reset_chan(uc, false); 1807 if (udma_is_chan_running(uc)) { 1808 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 1809 ret = -EBUSY; 1810 goto err_res_free; 1811 } 1812 } 1813 1814 /* PSI-L pairing */ 1815 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); 1816 if (ret) { 1817 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 1818 uc->config.src_thread, uc->config.dst_thread); 1819 goto err_res_free; 1820 } 1821 1822 uc->psil_paired = true; 1823 1824 uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring); 1825 if (uc->irq_num_ring <= 0) { 1826 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 1827 k3_ringacc_get_ring_id(irq_ring)); 1828 ret = -EINVAL; 1829 goto err_psi_free; 1830 } 1831 1832 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 1833 IRQF_TRIGGER_HIGH, uc->name, uc); 1834 if (ret) { 1835 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 1836 goto err_irq_free; 1837 } 1838 1839 /* Event from UDMA (TR events) only needed for slave TR mode channels */ 1840 if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) { 1841 uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, 1842 irq_udma_idx); 1843 if (uc->irq_num_udma <= 0) { 1844 dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", 1845 irq_udma_idx); 1846 free_irq(uc->irq_num_ring, uc); 1847 ret = -EINVAL; 1848 goto err_irq_free; 1849 } 1850 1851 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0, 1852 uc->name, uc); 1853 if (ret) { 1854 dev_err(ud->dev, "chan%d: UDMA irq request failed\n", 1855 uc->id); 1856 free_irq(uc->irq_num_ring, uc); 1857 goto err_irq_free; 1858 } 1859 } else { 1860 uc->irq_num_udma = 0; 1861 } 1862 1863 udma_reset_rings(uc); 1864 1865 return 0; 1866 1867 err_irq_free: 1868 uc->irq_num_ring = 0; 1869 uc->irq_num_udma = 0; 1870 err_psi_free: 1871 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); 1872 uc->psil_paired = false; 1873 err_res_free: 1874 udma_free_tx_resources(uc); 1875 udma_free_rx_resources(uc); 1876 err_cleanup: 1877 udma_reset_uchan(uc); 1878 1879 if (uc->use_dma_pool) { 1880 dma_pool_destroy(uc->hdesc_pool); 1881 uc->use_dma_pool = false; 1882 } 1883 1884 return ret; 1885 } 1886 1887 static int udma_slave_config(struct dma_chan *chan, 1888 struct dma_slave_config *cfg) 1889 { 1890 struct udma_chan *uc = to_udma_chan(chan); 1891 1892 memcpy(&uc->cfg, cfg, sizeof(uc->cfg)); 1893 1894 return 0; 1895 } 1896 1897 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc, 1898 size_t tr_size, int tr_count, 1899 enum dma_transfer_direction dir) 1900 { 1901 struct udma_hwdesc *hwdesc; 1902 struct cppi5_desc_hdr_t *tr_desc; 1903 struct udma_desc *d; 1904 u32 reload_count = 0; 1905 u32 ring_id; 1906 1907 switch (tr_size) { 1908 case 16: 1909 case 32: 1910 case 64: 1911 case 128: 1912 break; 1913 default: 1914 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size); 1915 return NULL; 1916 } 1917 1918 /* We have only one descriptor containing multiple TRs */ 1919 d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT); 1920 if (!d) 1921 return NULL; 1922 1923 d->sglen = tr_count; 1924 1925 d->hwdesc_count = 1; 1926 hwdesc = &d->hwdesc[0]; 1927 1928 /* Allocate memory for DMA ring descriptor */ 1929 if (uc->use_dma_pool) { 1930 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 1931 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 1932 GFP_NOWAIT, 1933 &hwdesc->cppi5_desc_paddr); 1934 } else { 1935 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1936 tr_count); 1937 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 1938 uc->ud->desc_align); 1939 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev, 1940 hwdesc->cppi5_desc_size, 1941 &hwdesc->cppi5_desc_paddr, 1942 GFP_NOWAIT); 1943 } 1944 1945 if (!hwdesc->cppi5_desc_vaddr) { 1946 kfree(d); 1947 return NULL; 1948 } 1949 1950 /* Start of the TR req records */ 1951 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 1952 /* Start address of the TR response array */ 1953 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count; 1954 1955 tr_desc = hwdesc->cppi5_desc_vaddr; 1956 1957 if (uc->cyclic) 1958 reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE; 1959 1960 if (dir == DMA_DEV_TO_MEM) 1961 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 1962 else 1963 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 1964 1965 cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count); 1966 cppi5_desc_set_pktids(tr_desc, uc->id, 1967 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 1968 cppi5_desc_set_retpolicy(tr_desc, 0, ring_id); 1969 1970 return d; 1971 } 1972 1973 /** 1974 * udma_get_tr_counters - calculate TR counters for a given length 1975 * @len: Length of the trasnfer 1976 * @align_to: Preferred alignment 1977 * @tr0_cnt0: First TR icnt0 1978 * @tr0_cnt1: First TR icnt1 1979 * @tr1_cnt0: Second (if used) TR icnt0 1980 * 1981 * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated 1982 * For len >= SZ_64K two TRs are used in a simple way: 1983 * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1) 1984 * Second TR: the remaining length (tr1_cnt0) 1985 * 1986 * Returns the number of TRs the length needs (1 or 2) 1987 * -EINVAL if the length can not be supported 1988 */ 1989 static int udma_get_tr_counters(size_t len, unsigned long align_to, 1990 u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0) 1991 { 1992 if (len < SZ_64K) { 1993 *tr0_cnt0 = len; 1994 *tr0_cnt1 = 1; 1995 1996 return 1; 1997 } 1998 1999 if (align_to > 3) 2000 align_to = 3; 2001 2002 realign: 2003 *tr0_cnt0 = SZ_64K - BIT(align_to); 2004 if (len / *tr0_cnt0 >= SZ_64K) { 2005 if (align_to) { 2006 align_to--; 2007 goto realign; 2008 } 2009 return -EINVAL; 2010 } 2011 2012 *tr0_cnt1 = len / *tr0_cnt0; 2013 *tr1_cnt0 = len % *tr0_cnt0; 2014 2015 return 2; 2016 } 2017 2018 static struct udma_desc * 2019 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, 2020 unsigned int sglen, enum dma_transfer_direction dir, 2021 unsigned long tx_flags, void *context) 2022 { 2023 struct scatterlist *sgent; 2024 struct udma_desc *d; 2025 struct cppi5_tr_type1_t *tr_req = NULL; 2026 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2027 unsigned int i; 2028 size_t tr_size; 2029 int num_tr = 0; 2030 int tr_idx = 0; 2031 2032 /* estimate the number of TRs we will need */ 2033 for_each_sg(sgl, sgent, sglen, i) { 2034 if (sg_dma_len(sgent) < SZ_64K) 2035 num_tr++; 2036 else 2037 num_tr += 2; 2038 } 2039 2040 /* Now allocate and setup the descriptor. */ 2041 tr_size = sizeof(struct cppi5_tr_type1_t); 2042 d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir); 2043 if (!d) 2044 return NULL; 2045 2046 d->sglen = sglen; 2047 2048 tr_req = d->hwdesc[0].tr_req_base; 2049 for_each_sg(sgl, sgent, sglen, i) { 2050 dma_addr_t sg_addr = sg_dma_address(sgent); 2051 2052 num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr), 2053 &tr0_cnt0, &tr0_cnt1, &tr1_cnt0); 2054 if (num_tr < 0) { 2055 dev_err(uc->ud->dev, "size %u is not supported\n", 2056 sg_dma_len(sgent)); 2057 udma_free_hwdesc(uc, d); 2058 kfree(d); 2059 return NULL; 2060 } 2061 2062 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false, 2063 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2064 cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT); 2065 2066 tr_req[tr_idx].addr = sg_addr; 2067 tr_req[tr_idx].icnt0 = tr0_cnt0; 2068 tr_req[tr_idx].icnt1 = tr0_cnt1; 2069 tr_req[tr_idx].dim1 = tr0_cnt0; 2070 tr_idx++; 2071 2072 if (num_tr == 2) { 2073 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 2074 false, false, 2075 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2076 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2077 CPPI5_TR_CSF_SUPR_EVT); 2078 2079 tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0; 2080 tr_req[tr_idx].icnt0 = tr1_cnt0; 2081 tr_req[tr_idx].icnt1 = 1; 2082 tr_req[tr_idx].dim1 = tr1_cnt0; 2083 tr_idx++; 2084 } 2085 2086 d->residue += sg_dma_len(sgent); 2087 } 2088 2089 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, 2090 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); 2091 2092 return d; 2093 } 2094 2095 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d, 2096 enum dma_slave_buswidth dev_width, 2097 u16 elcnt) 2098 { 2099 if (uc->config.ep_type != PSIL_EP_PDMA_XY) 2100 return 0; 2101 2102 /* Bus width translates to the element size (ES) */ 2103 switch (dev_width) { 2104 case DMA_SLAVE_BUSWIDTH_1_BYTE: 2105 d->static_tr.elsize = 0; 2106 break; 2107 case DMA_SLAVE_BUSWIDTH_2_BYTES: 2108 d->static_tr.elsize = 1; 2109 break; 2110 case DMA_SLAVE_BUSWIDTH_3_BYTES: 2111 d->static_tr.elsize = 2; 2112 break; 2113 case DMA_SLAVE_BUSWIDTH_4_BYTES: 2114 d->static_tr.elsize = 3; 2115 break; 2116 case DMA_SLAVE_BUSWIDTH_8_BYTES: 2117 d->static_tr.elsize = 4; 2118 break; 2119 default: /* not reached */ 2120 return -EINVAL; 2121 } 2122 2123 d->static_tr.elcnt = elcnt; 2124 2125 /* 2126 * PDMA must to close the packet when the channel is in packet mode. 2127 * For TR mode when the channel is not cyclic we also need PDMA to close 2128 * the packet otherwise the transfer will stall because PDMA holds on 2129 * the data it has received from the peripheral. 2130 */ 2131 if (uc->config.pkt_mode || !uc->cyclic) { 2132 unsigned int div = dev_width * elcnt; 2133 2134 if (uc->cyclic) 2135 d->static_tr.bstcnt = d->residue / d->sglen / div; 2136 else 2137 d->static_tr.bstcnt = d->residue / div; 2138 2139 if (uc->config.dir == DMA_DEV_TO_MEM && 2140 d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) 2141 return -EINVAL; 2142 } else { 2143 d->static_tr.bstcnt = 0; 2144 } 2145 2146 return 0; 2147 } 2148 2149 static struct udma_desc * 2150 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, 2151 unsigned int sglen, enum dma_transfer_direction dir, 2152 unsigned long tx_flags, void *context) 2153 { 2154 struct scatterlist *sgent; 2155 struct cppi5_host_desc_t *h_desc = NULL; 2156 struct udma_desc *d; 2157 u32 ring_id; 2158 unsigned int i; 2159 2160 d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT); 2161 if (!d) 2162 return NULL; 2163 2164 d->sglen = sglen; 2165 d->hwdesc_count = sglen; 2166 2167 if (dir == DMA_DEV_TO_MEM) 2168 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2169 else 2170 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2171 2172 for_each_sg(sgl, sgent, sglen, i) { 2173 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 2174 dma_addr_t sg_addr = sg_dma_address(sgent); 2175 struct cppi5_host_desc_t *desc; 2176 size_t sg_len = sg_dma_len(sgent); 2177 2178 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2179 GFP_NOWAIT, 2180 &hwdesc->cppi5_desc_paddr); 2181 if (!hwdesc->cppi5_desc_vaddr) { 2182 dev_err(uc->ud->dev, 2183 "descriptor%d allocation failed\n", i); 2184 2185 udma_free_hwdesc(uc, d); 2186 kfree(d); 2187 return NULL; 2188 } 2189 2190 d->residue += sg_len; 2191 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2192 desc = hwdesc->cppi5_desc_vaddr; 2193 2194 if (i == 0) { 2195 cppi5_hdesc_init(desc, 0, 0); 2196 /* Flow and Packed ID */ 2197 cppi5_desc_set_pktids(&desc->hdr, uc->id, 2198 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2199 cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id); 2200 } else { 2201 cppi5_hdesc_reset_hbdesc(desc); 2202 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff); 2203 } 2204 2205 /* attach the sg buffer to the descriptor */ 2206 cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len); 2207 2208 /* Attach link as host buffer descriptor */ 2209 if (h_desc) 2210 cppi5_hdesc_link_hbdesc(h_desc, 2211 hwdesc->cppi5_desc_paddr); 2212 2213 if (dir == DMA_MEM_TO_DEV) 2214 h_desc = desc; 2215 } 2216 2217 if (d->residue >= SZ_4M) { 2218 dev_err(uc->ud->dev, 2219 "%s: Transfer size %u is over the supported 4M range\n", 2220 __func__, d->residue); 2221 udma_free_hwdesc(uc, d); 2222 kfree(d); 2223 return NULL; 2224 } 2225 2226 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2227 cppi5_hdesc_set_pktlen(h_desc, d->residue); 2228 2229 return d; 2230 } 2231 2232 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc, 2233 void *data, size_t len) 2234 { 2235 struct udma_desc *d = to_udma_desc(desc); 2236 struct udma_chan *uc = to_udma_chan(desc->chan); 2237 struct cppi5_host_desc_t *h_desc; 2238 u32 psd_size = len; 2239 u32 flags = 0; 2240 2241 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2242 return -ENOTSUPP; 2243 2244 if (!data || len > uc->config.metadata_size) 2245 return -EINVAL; 2246 2247 if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE) 2248 return -EINVAL; 2249 2250 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2251 if (d->dir == DMA_MEM_TO_DEV) 2252 memcpy(h_desc->epib, data, len); 2253 2254 if (uc->config.needs_epib) 2255 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 2256 2257 d->metadata = data; 2258 d->metadata_size = len; 2259 if (uc->config.needs_epib) 2260 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 2261 2262 cppi5_hdesc_update_flags(h_desc, flags); 2263 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 2264 2265 return 0; 2266 } 2267 2268 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc, 2269 size_t *payload_len, size_t *max_len) 2270 { 2271 struct udma_desc *d = to_udma_desc(desc); 2272 struct udma_chan *uc = to_udma_chan(desc->chan); 2273 struct cppi5_host_desc_t *h_desc; 2274 2275 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2276 return ERR_PTR(-ENOTSUPP); 2277 2278 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2279 2280 *max_len = uc->config.metadata_size; 2281 2282 *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ? 2283 CPPI5_INFO0_HDESC_EPIB_SIZE : 0; 2284 *payload_len += cppi5_hdesc_get_psdata_size(h_desc); 2285 2286 return h_desc->epib; 2287 } 2288 2289 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc, 2290 size_t payload_len) 2291 { 2292 struct udma_desc *d = to_udma_desc(desc); 2293 struct udma_chan *uc = to_udma_chan(desc->chan); 2294 struct cppi5_host_desc_t *h_desc; 2295 u32 psd_size = payload_len; 2296 u32 flags = 0; 2297 2298 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2299 return -ENOTSUPP; 2300 2301 if (payload_len > uc->config.metadata_size) 2302 return -EINVAL; 2303 2304 if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE) 2305 return -EINVAL; 2306 2307 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2308 2309 if (uc->config.needs_epib) { 2310 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 2311 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 2312 } 2313 2314 cppi5_hdesc_update_flags(h_desc, flags); 2315 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 2316 2317 return 0; 2318 } 2319 2320 static struct dma_descriptor_metadata_ops metadata_ops = { 2321 .attach = udma_attach_metadata, 2322 .get_ptr = udma_get_metadata_ptr, 2323 .set_len = udma_set_metadata_len, 2324 }; 2325 2326 static struct dma_async_tx_descriptor * 2327 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 2328 unsigned int sglen, enum dma_transfer_direction dir, 2329 unsigned long tx_flags, void *context) 2330 { 2331 struct udma_chan *uc = to_udma_chan(chan); 2332 enum dma_slave_buswidth dev_width; 2333 struct udma_desc *d; 2334 u32 burst; 2335 2336 if (dir != uc->config.dir) { 2337 dev_err(chan->device->dev, 2338 "%s: chan%d is for %s, not supporting %s\n", 2339 __func__, uc->id, 2340 dmaengine_get_direction_text(uc->config.dir), 2341 dmaengine_get_direction_text(dir)); 2342 return NULL; 2343 } 2344 2345 if (dir == DMA_DEV_TO_MEM) { 2346 dev_width = uc->cfg.src_addr_width; 2347 burst = uc->cfg.src_maxburst; 2348 } else if (dir == DMA_MEM_TO_DEV) { 2349 dev_width = uc->cfg.dst_addr_width; 2350 burst = uc->cfg.dst_maxburst; 2351 } else { 2352 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 2353 return NULL; 2354 } 2355 2356 if (!burst) 2357 burst = 1; 2358 2359 if (uc->config.pkt_mode) 2360 d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags, 2361 context); 2362 else 2363 d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags, 2364 context); 2365 2366 if (!d) 2367 return NULL; 2368 2369 d->dir = dir; 2370 d->desc_idx = 0; 2371 d->tr_idx = 0; 2372 2373 /* static TR for remote PDMA */ 2374 if (udma_configure_statictr(uc, d, dev_width, burst)) { 2375 dev_err(uc->ud->dev, 2376 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 2377 __func__, d->static_tr.bstcnt); 2378 2379 udma_free_hwdesc(uc, d); 2380 kfree(d); 2381 return NULL; 2382 } 2383 2384 if (uc->config.metadata_size) 2385 d->vd.tx.metadata_ops = &metadata_ops; 2386 2387 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 2388 } 2389 2390 static struct udma_desc * 2391 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, 2392 size_t buf_len, size_t period_len, 2393 enum dma_transfer_direction dir, unsigned long flags) 2394 { 2395 struct udma_desc *d; 2396 size_t tr_size, period_addr; 2397 struct cppi5_tr_type1_t *tr_req; 2398 unsigned int periods = buf_len / period_len; 2399 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2400 unsigned int i; 2401 int num_tr; 2402 2403 num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0, 2404 &tr0_cnt1, &tr1_cnt0); 2405 if (num_tr < 0) { 2406 dev_err(uc->ud->dev, "size %zu is not supported\n", 2407 period_len); 2408 return NULL; 2409 } 2410 2411 /* Now allocate and setup the descriptor. */ 2412 tr_size = sizeof(struct cppi5_tr_type1_t); 2413 d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir); 2414 if (!d) 2415 return NULL; 2416 2417 tr_req = d->hwdesc[0].tr_req_base; 2418 period_addr = buf_addr; 2419 for (i = 0; i < periods; i++) { 2420 int tr_idx = i * num_tr; 2421 2422 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false, 2423 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2424 2425 tr_req[tr_idx].addr = period_addr; 2426 tr_req[tr_idx].icnt0 = tr0_cnt0; 2427 tr_req[tr_idx].icnt1 = tr0_cnt1; 2428 tr_req[tr_idx].dim1 = tr0_cnt0; 2429 2430 if (num_tr == 2) { 2431 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2432 CPPI5_TR_CSF_SUPR_EVT); 2433 tr_idx++; 2434 2435 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 2436 false, false, 2437 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2438 2439 tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0; 2440 tr_req[tr_idx].icnt0 = tr1_cnt0; 2441 tr_req[tr_idx].icnt1 = 1; 2442 tr_req[tr_idx].dim1 = tr1_cnt0; 2443 } 2444 2445 if (!(flags & DMA_PREP_INTERRUPT)) 2446 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2447 CPPI5_TR_CSF_SUPR_EVT); 2448 2449 period_addr += period_len; 2450 } 2451 2452 return d; 2453 } 2454 2455 static struct udma_desc * 2456 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr, 2457 size_t buf_len, size_t period_len, 2458 enum dma_transfer_direction dir, unsigned long flags) 2459 { 2460 struct udma_desc *d; 2461 u32 ring_id; 2462 int i; 2463 int periods = buf_len / period_len; 2464 2465 if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1)) 2466 return NULL; 2467 2468 if (period_len >= SZ_4M) 2469 return NULL; 2470 2471 d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT); 2472 if (!d) 2473 return NULL; 2474 2475 d->hwdesc_count = periods; 2476 2477 /* TODO: re-check this... */ 2478 if (dir == DMA_DEV_TO_MEM) 2479 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2480 else 2481 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2482 2483 for (i = 0; i < periods; i++) { 2484 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 2485 dma_addr_t period_addr = buf_addr + (period_len * i); 2486 struct cppi5_host_desc_t *h_desc; 2487 2488 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2489 GFP_NOWAIT, 2490 &hwdesc->cppi5_desc_paddr); 2491 if (!hwdesc->cppi5_desc_vaddr) { 2492 dev_err(uc->ud->dev, 2493 "descriptor%d allocation failed\n", i); 2494 2495 udma_free_hwdesc(uc, d); 2496 kfree(d); 2497 return NULL; 2498 } 2499 2500 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2501 h_desc = hwdesc->cppi5_desc_vaddr; 2502 2503 cppi5_hdesc_init(h_desc, 0, 0); 2504 cppi5_hdesc_set_pktlen(h_desc, period_len); 2505 2506 /* Flow and Packed ID */ 2507 cppi5_desc_set_pktids(&h_desc->hdr, uc->id, 2508 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2509 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id); 2510 2511 /* attach each period to a new descriptor */ 2512 cppi5_hdesc_attach_buf(h_desc, 2513 period_addr, period_len, 2514 period_addr, period_len); 2515 } 2516 2517 return d; 2518 } 2519 2520 static struct dma_async_tx_descriptor * 2521 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 2522 size_t period_len, enum dma_transfer_direction dir, 2523 unsigned long flags) 2524 { 2525 struct udma_chan *uc = to_udma_chan(chan); 2526 enum dma_slave_buswidth dev_width; 2527 struct udma_desc *d; 2528 u32 burst; 2529 2530 if (dir != uc->config.dir) { 2531 dev_err(chan->device->dev, 2532 "%s: chan%d is for %s, not supporting %s\n", 2533 __func__, uc->id, 2534 dmaengine_get_direction_text(uc->config.dir), 2535 dmaengine_get_direction_text(dir)); 2536 return NULL; 2537 } 2538 2539 uc->cyclic = true; 2540 2541 if (dir == DMA_DEV_TO_MEM) { 2542 dev_width = uc->cfg.src_addr_width; 2543 burst = uc->cfg.src_maxburst; 2544 } else if (dir == DMA_MEM_TO_DEV) { 2545 dev_width = uc->cfg.dst_addr_width; 2546 burst = uc->cfg.dst_maxburst; 2547 } else { 2548 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 2549 return NULL; 2550 } 2551 2552 if (!burst) 2553 burst = 1; 2554 2555 if (uc->config.pkt_mode) 2556 d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len, 2557 dir, flags); 2558 else 2559 d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len, 2560 dir, flags); 2561 2562 if (!d) 2563 return NULL; 2564 2565 d->sglen = buf_len / period_len; 2566 2567 d->dir = dir; 2568 d->residue = buf_len; 2569 2570 /* static TR for remote PDMA */ 2571 if (udma_configure_statictr(uc, d, dev_width, burst)) { 2572 dev_err(uc->ud->dev, 2573 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 2574 __func__, d->static_tr.bstcnt); 2575 2576 udma_free_hwdesc(uc, d); 2577 kfree(d); 2578 return NULL; 2579 } 2580 2581 if (uc->config.metadata_size) 2582 d->vd.tx.metadata_ops = &metadata_ops; 2583 2584 return vchan_tx_prep(&uc->vc, &d->vd, flags); 2585 } 2586 2587 static struct dma_async_tx_descriptor * 2588 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 2589 size_t len, unsigned long tx_flags) 2590 { 2591 struct udma_chan *uc = to_udma_chan(chan); 2592 struct udma_desc *d; 2593 struct cppi5_tr_type15_t *tr_req; 2594 int num_tr; 2595 size_t tr_size = sizeof(struct cppi5_tr_type15_t); 2596 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2597 2598 if (uc->config.dir != DMA_MEM_TO_MEM) { 2599 dev_err(chan->device->dev, 2600 "%s: chan%d is for %s, not supporting %s\n", 2601 __func__, uc->id, 2602 dmaengine_get_direction_text(uc->config.dir), 2603 dmaengine_get_direction_text(DMA_MEM_TO_MEM)); 2604 return NULL; 2605 } 2606 2607 num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0, 2608 &tr0_cnt1, &tr1_cnt0); 2609 if (num_tr < 0) { 2610 dev_err(uc->ud->dev, "size %zu is not supported\n", 2611 len); 2612 return NULL; 2613 } 2614 2615 d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM); 2616 if (!d) 2617 return NULL; 2618 2619 d->dir = DMA_MEM_TO_MEM; 2620 d->desc_idx = 0; 2621 d->tr_idx = 0; 2622 d->residue = len; 2623 2624 tr_req = d->hwdesc[0].tr_req_base; 2625 2626 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true, 2627 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2628 cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT); 2629 2630 tr_req[0].addr = src; 2631 tr_req[0].icnt0 = tr0_cnt0; 2632 tr_req[0].icnt1 = tr0_cnt1; 2633 tr_req[0].icnt2 = 1; 2634 tr_req[0].icnt3 = 1; 2635 tr_req[0].dim1 = tr0_cnt0; 2636 2637 tr_req[0].daddr = dest; 2638 tr_req[0].dicnt0 = tr0_cnt0; 2639 tr_req[0].dicnt1 = tr0_cnt1; 2640 tr_req[0].dicnt2 = 1; 2641 tr_req[0].dicnt3 = 1; 2642 tr_req[0].ddim1 = tr0_cnt0; 2643 2644 if (num_tr == 2) { 2645 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true, 2646 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2647 cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT); 2648 2649 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0; 2650 tr_req[1].icnt0 = tr1_cnt0; 2651 tr_req[1].icnt1 = 1; 2652 tr_req[1].icnt2 = 1; 2653 tr_req[1].icnt3 = 1; 2654 2655 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0; 2656 tr_req[1].dicnt0 = tr1_cnt0; 2657 tr_req[1].dicnt1 = 1; 2658 tr_req[1].dicnt2 = 1; 2659 tr_req[1].dicnt3 = 1; 2660 } 2661 2662 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, 2663 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); 2664 2665 if (uc->config.metadata_size) 2666 d->vd.tx.metadata_ops = &metadata_ops; 2667 2668 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 2669 } 2670 2671 static void udma_issue_pending(struct dma_chan *chan) 2672 { 2673 struct udma_chan *uc = to_udma_chan(chan); 2674 unsigned long flags; 2675 2676 spin_lock_irqsave(&uc->vc.lock, flags); 2677 2678 /* If we have something pending and no active descriptor, then */ 2679 if (vchan_issue_pending(&uc->vc) && !uc->desc) { 2680 /* 2681 * start a descriptor if the channel is NOT [marked as 2682 * terminating _and_ it is still running (teardown has not 2683 * completed yet)]. 2684 */ 2685 if (!(uc->state == UDMA_CHAN_IS_TERMINATING && 2686 udma_is_chan_running(uc))) 2687 udma_start(uc); 2688 } 2689 2690 spin_unlock_irqrestore(&uc->vc.lock, flags); 2691 } 2692 2693 static enum dma_status udma_tx_status(struct dma_chan *chan, 2694 dma_cookie_t cookie, 2695 struct dma_tx_state *txstate) 2696 { 2697 struct udma_chan *uc = to_udma_chan(chan); 2698 enum dma_status ret; 2699 unsigned long flags; 2700 2701 spin_lock_irqsave(&uc->vc.lock, flags); 2702 2703 ret = dma_cookie_status(chan, cookie, txstate); 2704 2705 if (!udma_is_chan_running(uc)) 2706 ret = DMA_COMPLETE; 2707 2708 if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc)) 2709 ret = DMA_PAUSED; 2710 2711 if (ret == DMA_COMPLETE || !txstate) 2712 goto out; 2713 2714 if (uc->desc && uc->desc->vd.tx.cookie == cookie) { 2715 u32 peer_bcnt = 0; 2716 u32 bcnt = 0; 2717 u32 residue = uc->desc->residue; 2718 u32 delay = 0; 2719 2720 if (uc->desc->dir == DMA_MEM_TO_DEV) { 2721 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); 2722 2723 if (uc->config.ep_type != PSIL_EP_NATIVE) { 2724 peer_bcnt = udma_tchanrt_read(uc, 2725 UDMA_CHAN_RT_PEER_BCNT_REG); 2726 2727 if (bcnt > peer_bcnt) 2728 delay = bcnt - peer_bcnt; 2729 } 2730 } else if (uc->desc->dir == DMA_DEV_TO_MEM) { 2731 bcnt = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 2732 2733 if (uc->config.ep_type != PSIL_EP_NATIVE) { 2734 peer_bcnt = udma_rchanrt_read(uc, 2735 UDMA_CHAN_RT_PEER_BCNT_REG); 2736 2737 if (peer_bcnt > bcnt) 2738 delay = peer_bcnt - bcnt; 2739 } 2740 } else { 2741 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 2742 } 2743 2744 bcnt -= uc->bcnt; 2745 if (bcnt && !(bcnt % uc->desc->residue)) 2746 residue = 0; 2747 else 2748 residue -= bcnt % uc->desc->residue; 2749 2750 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) { 2751 ret = DMA_COMPLETE; 2752 delay = 0; 2753 } 2754 2755 dma_set_residue(txstate, residue); 2756 dma_set_in_flight_bytes(txstate, delay); 2757 2758 } else { 2759 ret = DMA_COMPLETE; 2760 } 2761 2762 out: 2763 spin_unlock_irqrestore(&uc->vc.lock, flags); 2764 return ret; 2765 } 2766 2767 static int udma_pause(struct dma_chan *chan) 2768 { 2769 struct udma_chan *uc = to_udma_chan(chan); 2770 2771 /* pause the channel */ 2772 switch (uc->config.dir) { 2773 case DMA_DEV_TO_MEM: 2774 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 2775 UDMA_PEER_RT_EN_PAUSE, 2776 UDMA_PEER_RT_EN_PAUSE); 2777 break; 2778 case DMA_MEM_TO_DEV: 2779 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 2780 UDMA_PEER_RT_EN_PAUSE, 2781 UDMA_PEER_RT_EN_PAUSE); 2782 break; 2783 case DMA_MEM_TO_MEM: 2784 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG, 2785 UDMA_CHAN_RT_CTL_PAUSE, 2786 UDMA_CHAN_RT_CTL_PAUSE); 2787 break; 2788 default: 2789 return -EINVAL; 2790 } 2791 2792 return 0; 2793 } 2794 2795 static int udma_resume(struct dma_chan *chan) 2796 { 2797 struct udma_chan *uc = to_udma_chan(chan); 2798 2799 /* resume the channel */ 2800 switch (uc->config.dir) { 2801 case DMA_DEV_TO_MEM: 2802 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 2803 UDMA_PEER_RT_EN_PAUSE, 0); 2804 2805 break; 2806 case DMA_MEM_TO_DEV: 2807 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 2808 UDMA_PEER_RT_EN_PAUSE, 0); 2809 break; 2810 case DMA_MEM_TO_MEM: 2811 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG, 2812 UDMA_CHAN_RT_CTL_PAUSE, 0); 2813 break; 2814 default: 2815 return -EINVAL; 2816 } 2817 2818 return 0; 2819 } 2820 2821 static int udma_terminate_all(struct dma_chan *chan) 2822 { 2823 struct udma_chan *uc = to_udma_chan(chan); 2824 unsigned long flags; 2825 LIST_HEAD(head); 2826 2827 spin_lock_irqsave(&uc->vc.lock, flags); 2828 2829 if (udma_is_chan_running(uc)) 2830 udma_stop(uc); 2831 2832 if (uc->desc) { 2833 uc->terminated_desc = uc->desc; 2834 uc->desc = NULL; 2835 uc->terminated_desc->terminated = true; 2836 cancel_delayed_work(&uc->tx_drain.work); 2837 } 2838 2839 uc->paused = false; 2840 2841 vchan_get_all_descriptors(&uc->vc, &head); 2842 spin_unlock_irqrestore(&uc->vc.lock, flags); 2843 vchan_dma_desc_free_list(&uc->vc, &head); 2844 2845 return 0; 2846 } 2847 2848 static void udma_synchronize(struct dma_chan *chan) 2849 { 2850 struct udma_chan *uc = to_udma_chan(chan); 2851 unsigned long timeout = msecs_to_jiffies(1000); 2852 2853 vchan_synchronize(&uc->vc); 2854 2855 if (uc->state == UDMA_CHAN_IS_TERMINATING) { 2856 timeout = wait_for_completion_timeout(&uc->teardown_completed, 2857 timeout); 2858 if (!timeout) { 2859 dev_warn(uc->ud->dev, "chan%d teardown timeout!\n", 2860 uc->id); 2861 udma_dump_chan_stdata(uc); 2862 udma_reset_chan(uc, true); 2863 } 2864 } 2865 2866 udma_reset_chan(uc, false); 2867 if (udma_is_chan_running(uc)) 2868 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id); 2869 2870 cancel_delayed_work_sync(&uc->tx_drain.work); 2871 udma_reset_rings(uc); 2872 } 2873 2874 static void udma_desc_pre_callback(struct virt_dma_chan *vc, 2875 struct virt_dma_desc *vd, 2876 struct dmaengine_result *result) 2877 { 2878 struct udma_chan *uc = to_udma_chan(&vc->chan); 2879 struct udma_desc *d; 2880 2881 if (!vd) 2882 return; 2883 2884 d = to_udma_desc(&vd->tx); 2885 2886 if (d->metadata_size) 2887 udma_fetch_epib(uc, d); 2888 2889 /* Provide residue information for the client */ 2890 if (result) { 2891 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx); 2892 2893 if (cppi5_desc_get_type(desc_vaddr) == 2894 CPPI5_INFO0_DESC_TYPE_VAL_HOST) { 2895 result->residue = d->residue - 2896 cppi5_hdesc_get_pktlen(desc_vaddr); 2897 if (result->residue) 2898 result->result = DMA_TRANS_ABORTED; 2899 else 2900 result->result = DMA_TRANS_NOERROR; 2901 } else { 2902 result->residue = 0; 2903 result->result = DMA_TRANS_NOERROR; 2904 } 2905 } 2906 } 2907 2908 /* 2909 * This tasklet handles the completion of a DMA descriptor by 2910 * calling its callback and freeing it. 2911 */ 2912 static void udma_vchan_complete(struct tasklet_struct *t) 2913 { 2914 struct virt_dma_chan *vc = from_tasklet(vc, t, task); 2915 struct virt_dma_desc *vd, *_vd; 2916 struct dmaengine_desc_callback cb; 2917 LIST_HEAD(head); 2918 2919 spin_lock_irq(&vc->lock); 2920 list_splice_tail_init(&vc->desc_completed, &head); 2921 vd = vc->cyclic; 2922 if (vd) { 2923 vc->cyclic = NULL; 2924 dmaengine_desc_get_callback(&vd->tx, &cb); 2925 } else { 2926 memset(&cb, 0, sizeof(cb)); 2927 } 2928 spin_unlock_irq(&vc->lock); 2929 2930 udma_desc_pre_callback(vc, vd, NULL); 2931 dmaengine_desc_callback_invoke(&cb, NULL); 2932 2933 list_for_each_entry_safe(vd, _vd, &head, node) { 2934 struct dmaengine_result result; 2935 2936 dmaengine_desc_get_callback(&vd->tx, &cb); 2937 2938 list_del(&vd->node); 2939 2940 udma_desc_pre_callback(vc, vd, &result); 2941 dmaengine_desc_callback_invoke(&cb, &result); 2942 2943 vchan_vdesc_fini(vd); 2944 } 2945 } 2946 2947 static void udma_free_chan_resources(struct dma_chan *chan) 2948 { 2949 struct udma_chan *uc = to_udma_chan(chan); 2950 struct udma_dev *ud = to_udma_dev(chan->device); 2951 2952 udma_terminate_all(chan); 2953 if (uc->terminated_desc) { 2954 udma_reset_chan(uc, false); 2955 udma_reset_rings(uc); 2956 } 2957 2958 cancel_delayed_work_sync(&uc->tx_drain.work); 2959 2960 if (uc->irq_num_ring > 0) { 2961 free_irq(uc->irq_num_ring, uc); 2962 2963 uc->irq_num_ring = 0; 2964 } 2965 if (uc->irq_num_udma > 0) { 2966 free_irq(uc->irq_num_udma, uc); 2967 2968 uc->irq_num_udma = 0; 2969 } 2970 2971 /* Release PSI-L pairing */ 2972 if (uc->psil_paired) { 2973 navss_psil_unpair(ud, uc->config.src_thread, 2974 uc->config.dst_thread); 2975 uc->psil_paired = false; 2976 } 2977 2978 vchan_free_chan_resources(&uc->vc); 2979 tasklet_kill(&uc->vc.task); 2980 2981 udma_free_tx_resources(uc); 2982 udma_free_rx_resources(uc); 2983 udma_reset_uchan(uc); 2984 2985 if (uc->use_dma_pool) { 2986 dma_pool_destroy(uc->hdesc_pool); 2987 uc->use_dma_pool = false; 2988 } 2989 } 2990 2991 static struct platform_driver udma_driver; 2992 2993 struct udma_filter_param { 2994 int remote_thread_id; 2995 u32 atype; 2996 }; 2997 2998 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) 2999 { 3000 struct udma_chan_config *ucc; 3001 struct psil_endpoint_config *ep_config; 3002 struct udma_filter_param *filter_param; 3003 struct udma_chan *uc; 3004 struct udma_dev *ud; 3005 3006 if (chan->device->dev->driver != &udma_driver.driver) 3007 return false; 3008 3009 uc = to_udma_chan(chan); 3010 ucc = &uc->config; 3011 ud = uc->ud; 3012 filter_param = param; 3013 3014 if (filter_param->atype > 2) { 3015 dev_err(ud->dev, "Invalid channel atype: %u\n", 3016 filter_param->atype); 3017 return false; 3018 } 3019 3020 ucc->remote_thread_id = filter_param->remote_thread_id; 3021 ucc->atype = filter_param->atype; 3022 3023 if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) 3024 ucc->dir = DMA_MEM_TO_DEV; 3025 else 3026 ucc->dir = DMA_DEV_TO_MEM; 3027 3028 ep_config = psil_get_ep_config(ucc->remote_thread_id); 3029 if (IS_ERR(ep_config)) { 3030 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n", 3031 ucc->remote_thread_id); 3032 ucc->dir = DMA_MEM_TO_MEM; 3033 ucc->remote_thread_id = -1; 3034 ucc->atype = 0; 3035 return false; 3036 } 3037 3038 ucc->pkt_mode = ep_config->pkt_mode; 3039 ucc->channel_tpl = ep_config->channel_tpl; 3040 ucc->notdpkt = ep_config->notdpkt; 3041 ucc->ep_type = ep_config->ep_type; 3042 3043 if (ucc->ep_type != PSIL_EP_NATIVE) { 3044 const struct udma_match_data *match_data = ud->match_data; 3045 3046 if (match_data->flags & UDMA_FLAG_PDMA_ACC32) 3047 ucc->enable_acc32 = ep_config->pdma_acc32; 3048 if (match_data->flags & UDMA_FLAG_PDMA_BURST) 3049 ucc->enable_burst = ep_config->pdma_burst; 3050 } 3051 3052 ucc->needs_epib = ep_config->needs_epib; 3053 ucc->psd_size = ep_config->psd_size; 3054 ucc->metadata_size = 3055 (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) + 3056 ucc->psd_size; 3057 3058 if (ucc->pkt_mode) 3059 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 3060 ucc->metadata_size, ud->desc_align); 3061 3062 dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id, 3063 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir)); 3064 3065 return true; 3066 } 3067 3068 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, 3069 struct of_dma *ofdma) 3070 { 3071 struct udma_dev *ud = ofdma->of_dma_data; 3072 dma_cap_mask_t mask = ud->ddev.cap_mask; 3073 struct udma_filter_param filter_param; 3074 struct dma_chan *chan; 3075 3076 if (dma_spec->args_count != 1 && dma_spec->args_count != 2) 3077 return NULL; 3078 3079 filter_param.remote_thread_id = dma_spec->args[0]; 3080 if (dma_spec->args_count == 2) 3081 filter_param.atype = dma_spec->args[1]; 3082 else 3083 filter_param.atype = 0; 3084 3085 chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, 3086 ofdma->of_node); 3087 if (!chan) { 3088 dev_err(ud->dev, "get channel fail in %s.\n", __func__); 3089 return ERR_PTR(-EINVAL); 3090 } 3091 3092 return chan; 3093 } 3094 3095 static struct udma_match_data am654_main_data = { 3096 .psil_base = 0x1000, 3097 .enable_memcpy_support = true, 3098 .statictr_z_mask = GENMASK(11, 0), 3099 }; 3100 3101 static struct udma_match_data am654_mcu_data = { 3102 .psil_base = 0x6000, 3103 .enable_memcpy_support = false, 3104 .statictr_z_mask = GENMASK(11, 0), 3105 }; 3106 3107 static struct udma_match_data j721e_main_data = { 3108 .psil_base = 0x1000, 3109 .enable_memcpy_support = true, 3110 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, 3111 .statictr_z_mask = GENMASK(23, 0), 3112 }; 3113 3114 static struct udma_match_data j721e_mcu_data = { 3115 .psil_base = 0x6000, 3116 .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ 3117 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, 3118 .statictr_z_mask = GENMASK(23, 0), 3119 }; 3120 3121 static const struct of_device_id udma_of_match[] = { 3122 { 3123 .compatible = "ti,am654-navss-main-udmap", 3124 .data = &am654_main_data, 3125 }, 3126 { 3127 .compatible = "ti,am654-navss-mcu-udmap", 3128 .data = &am654_mcu_data, 3129 }, { 3130 .compatible = "ti,j721e-navss-main-udmap", 3131 .data = &j721e_main_data, 3132 }, { 3133 .compatible = "ti,j721e-navss-mcu-udmap", 3134 .data = &j721e_mcu_data, 3135 }, 3136 { /* Sentinel */ }, 3137 }; 3138 3139 static struct udma_soc_data am654_soc_data = { 3140 .rchan_oes_offset = 0x200, 3141 }; 3142 3143 static struct udma_soc_data j721e_soc_data = { 3144 .rchan_oes_offset = 0x400, 3145 }; 3146 3147 static struct udma_soc_data j7200_soc_data = { 3148 .rchan_oes_offset = 0x80, 3149 }; 3150 3151 static const struct soc_device_attribute k3_soc_devices[] = { 3152 { .family = "AM65X", .data = &am654_soc_data }, 3153 { .family = "J721E", .data = &j721e_soc_data }, 3154 { .family = "J7200", .data = &j7200_soc_data }, 3155 { /* sentinel */ } 3156 }; 3157 3158 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) 3159 { 3160 int i; 3161 3162 for (i = 0; i < MMR_LAST; i++) { 3163 ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]); 3164 if (IS_ERR(ud->mmrs[i])) 3165 return PTR_ERR(ud->mmrs[i]); 3166 } 3167 3168 return 0; 3169 } 3170 3171 static int udma_setup_resources(struct udma_dev *ud) 3172 { 3173 struct device *dev = ud->dev; 3174 int ch_count, ret, i, j; 3175 u32 cap2, cap3; 3176 struct ti_sci_resource_desc *rm_desc; 3177 struct ti_sci_resource *rm_res, irq_res; 3178 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 3179 static const char * const range_names[] = { "ti,sci-rm-range-tchan", 3180 "ti,sci-rm-range-rchan", 3181 "ti,sci-rm-range-rflow" }; 3182 3183 cap2 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(2)); 3184 cap3 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(3)); 3185 3186 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3); 3187 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2); 3188 ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2); 3189 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); 3190 ch_count = ud->tchan_cnt + ud->rchan_cnt; 3191 3192 /* Set up the throughput level start indexes */ 3193 if (of_device_is_compatible(dev->of_node, 3194 "ti,am654-navss-main-udmap")) { 3195 ud->tpl_levels = 2; 3196 ud->tpl_start_idx[0] = 8; 3197 } else if (of_device_is_compatible(dev->of_node, 3198 "ti,am654-navss-mcu-udmap")) { 3199 ud->tpl_levels = 2; 3200 ud->tpl_start_idx[0] = 2; 3201 } else if (UDMA_CAP3_UCHAN_CNT(cap3)) { 3202 ud->tpl_levels = 3; 3203 ud->tpl_start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); 3204 ud->tpl_start_idx[0] = ud->tpl_start_idx[1] + 3205 UDMA_CAP3_HCHAN_CNT(cap3); 3206 } else if (UDMA_CAP3_HCHAN_CNT(cap3)) { 3207 ud->tpl_levels = 2; 3208 ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); 3209 } else { 3210 ud->tpl_levels = 1; 3211 } 3212 3213 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 3214 sizeof(unsigned long), GFP_KERNEL); 3215 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 3216 GFP_KERNEL); 3217 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 3218 sizeof(unsigned long), GFP_KERNEL); 3219 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 3220 GFP_KERNEL); 3221 ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt), 3222 sizeof(unsigned long), 3223 GFP_KERNEL); 3224 ud->rflow_gp_map_allocated = devm_kcalloc(dev, 3225 BITS_TO_LONGS(ud->rflow_cnt), 3226 sizeof(unsigned long), 3227 GFP_KERNEL); 3228 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), 3229 sizeof(unsigned long), 3230 GFP_KERNEL); 3231 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), 3232 GFP_KERNEL); 3233 3234 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map || 3235 !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans || 3236 !ud->rflows || !ud->rflow_in_use) 3237 return -ENOMEM; 3238 3239 /* 3240 * RX flows with the same Ids as RX channels are reserved to be used 3241 * as default flows if remote HW can't generate flow_ids. Those 3242 * RX flows can be requested only explicitly by id. 3243 */ 3244 bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt); 3245 3246 /* by default no GP rflows are assigned to Linux */ 3247 bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt); 3248 3249 /* Get resource ranges from tisci */ 3250 for (i = 0; i < RM_RANGE_LAST; i++) 3251 tisci_rm->rm_ranges[i] = 3252 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 3253 tisci_rm->tisci_dev_id, 3254 (char *)range_names[i]); 3255 3256 /* tchan ranges */ 3257 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 3258 if (IS_ERR(rm_res)) { 3259 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 3260 } else { 3261 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 3262 for (i = 0; i < rm_res->sets; i++) { 3263 rm_desc = &rm_res->desc[i]; 3264 bitmap_clear(ud->tchan_map, rm_desc->start, 3265 rm_desc->num); 3266 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n", 3267 rm_desc->start, rm_desc->num); 3268 } 3269 } 3270 irq_res.sets = rm_res->sets; 3271 3272 /* rchan and matching default flow ranges */ 3273 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 3274 if (IS_ERR(rm_res)) { 3275 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 3276 } else { 3277 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 3278 for (i = 0; i < rm_res->sets; i++) { 3279 rm_desc = &rm_res->desc[i]; 3280 bitmap_clear(ud->rchan_map, rm_desc->start, 3281 rm_desc->num); 3282 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n", 3283 rm_desc->start, rm_desc->num); 3284 } 3285 } 3286 3287 irq_res.sets += rm_res->sets; 3288 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 3289 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 3290 for (i = 0; i < rm_res->sets; i++) { 3291 irq_res.desc[i].start = rm_res->desc[i].start; 3292 irq_res.desc[i].num = rm_res->desc[i].num; 3293 } 3294 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 3295 for (j = 0; j < rm_res->sets; j++, i++) { 3296 irq_res.desc[i].start = rm_res->desc[j].start + 3297 ud->soc_data->rchan_oes_offset; 3298 irq_res.desc[i].num = rm_res->desc[j].num; 3299 } 3300 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 3301 kfree(irq_res.desc); 3302 if (ret) { 3303 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 3304 return ret; 3305 } 3306 3307 /* GP rflow ranges */ 3308 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 3309 if (IS_ERR(rm_res)) { 3310 /* all gp flows are assigned exclusively to Linux */ 3311 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt, 3312 ud->rflow_cnt - ud->rchan_cnt); 3313 } else { 3314 for (i = 0; i < rm_res->sets; i++) { 3315 rm_desc = &rm_res->desc[i]; 3316 bitmap_clear(ud->rflow_gp_map, rm_desc->start, 3317 rm_desc->num); 3318 dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n", 3319 rm_desc->start, rm_desc->num); 3320 } 3321 } 3322 3323 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt); 3324 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt); 3325 if (!ch_count) 3326 return -ENODEV; 3327 3328 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels), 3329 GFP_KERNEL); 3330 if (!ud->channels) 3331 return -ENOMEM; 3332 3333 dev_info(dev, "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", 3334 ch_count, 3335 ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), 3336 ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt), 3337 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map, 3338 ud->rflow_cnt)); 3339 3340 return ch_count; 3341 } 3342 3343 static int udma_setup_rx_flush(struct udma_dev *ud) 3344 { 3345 struct udma_rx_flush *rx_flush = &ud->rx_flush; 3346 struct cppi5_desc_hdr_t *tr_desc; 3347 struct cppi5_tr_type1_t *tr_req; 3348 struct cppi5_host_desc_t *desc; 3349 struct device *dev = ud->dev; 3350 struct udma_hwdesc *hwdesc; 3351 size_t tr_size; 3352 3353 /* Allocate 1K buffer for discarded data on RX channel teardown */ 3354 rx_flush->buffer_size = SZ_1K; 3355 rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size, 3356 GFP_KERNEL); 3357 if (!rx_flush->buffer_vaddr) 3358 return -ENOMEM; 3359 3360 rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr, 3361 rx_flush->buffer_size, 3362 DMA_TO_DEVICE); 3363 if (dma_mapping_error(dev, rx_flush->buffer_paddr)) 3364 return -ENOMEM; 3365 3366 /* Set up descriptor to be used for TR mode */ 3367 hwdesc = &rx_flush->hwdescs[0]; 3368 tr_size = sizeof(struct cppi5_tr_type1_t); 3369 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1); 3370 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 3371 ud->desc_align); 3372 3373 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 3374 GFP_KERNEL); 3375 if (!hwdesc->cppi5_desc_vaddr) 3376 return -ENOMEM; 3377 3378 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 3379 hwdesc->cppi5_desc_size, 3380 DMA_TO_DEVICE); 3381 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 3382 return -ENOMEM; 3383 3384 /* Start of the TR req records */ 3385 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 3386 /* Start address of the TR response array */ 3387 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size; 3388 3389 tr_desc = hwdesc->cppi5_desc_vaddr; 3390 cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0); 3391 cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3392 cppi5_desc_set_retpolicy(tr_desc, 0, 0); 3393 3394 tr_req = hwdesc->tr_req_base; 3395 cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false, 3396 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3397 cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT); 3398 3399 tr_req->addr = rx_flush->buffer_paddr; 3400 tr_req->icnt0 = rx_flush->buffer_size; 3401 tr_req->icnt1 = 1; 3402 3403 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, 3404 hwdesc->cppi5_desc_size, DMA_TO_DEVICE); 3405 3406 /* Set up descriptor to be used for packet mode */ 3407 hwdesc = &rx_flush->hwdescs[1]; 3408 hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 3409 CPPI5_INFO0_HDESC_EPIB_SIZE + 3410 CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE, 3411 ud->desc_align); 3412 3413 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 3414 GFP_KERNEL); 3415 if (!hwdesc->cppi5_desc_vaddr) 3416 return -ENOMEM; 3417 3418 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 3419 hwdesc->cppi5_desc_size, 3420 DMA_TO_DEVICE); 3421 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 3422 return -ENOMEM; 3423 3424 desc = hwdesc->cppi5_desc_vaddr; 3425 cppi5_hdesc_init(desc, 0, 0); 3426 cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3427 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0); 3428 3429 cppi5_hdesc_attach_buf(desc, 3430 rx_flush->buffer_paddr, rx_flush->buffer_size, 3431 rx_flush->buffer_paddr, rx_flush->buffer_size); 3432 3433 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, 3434 hwdesc->cppi5_desc_size, DMA_TO_DEVICE); 3435 return 0; 3436 } 3437 3438 #ifdef CONFIG_DEBUG_FS 3439 static void udma_dbg_summary_show_chan(struct seq_file *s, 3440 struct dma_chan *chan) 3441 { 3442 struct udma_chan *uc = to_udma_chan(chan); 3443 struct udma_chan_config *ucc = &uc->config; 3444 3445 seq_printf(s, " %-13s| %s", dma_chan_name(chan), 3446 chan->dbg_client_name ?: "in-use"); 3447 seq_printf(s, " (%s, ", dmaengine_get_direction_text(uc->config.dir)); 3448 3449 switch (uc->config.dir) { 3450 case DMA_MEM_TO_MEM: 3451 seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id, 3452 ucc->src_thread, ucc->dst_thread); 3453 break; 3454 case DMA_DEV_TO_MEM: 3455 seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id, 3456 ucc->src_thread, ucc->dst_thread); 3457 break; 3458 case DMA_MEM_TO_DEV: 3459 seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id, 3460 ucc->src_thread, ucc->dst_thread); 3461 break; 3462 default: 3463 seq_printf(s, ")\n"); 3464 return; 3465 } 3466 3467 if (ucc->ep_type == PSIL_EP_NATIVE) { 3468 seq_printf(s, "PSI-L Native"); 3469 if (ucc->metadata_size) { 3470 seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : ""); 3471 if (ucc->psd_size) 3472 seq_printf(s, " PSDsize:%u", ucc->psd_size); 3473 seq_printf(s, " ]"); 3474 } 3475 } else { 3476 seq_printf(s, "PDMA"); 3477 if (ucc->enable_acc32 || ucc->enable_burst) 3478 seq_printf(s, "[%s%s ]", 3479 ucc->enable_acc32 ? " ACC32" : "", 3480 ucc->enable_burst ? " BURST" : ""); 3481 } 3482 3483 seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode"); 3484 } 3485 3486 static void udma_dbg_summary_show(struct seq_file *s, 3487 struct dma_device *dma_dev) 3488 { 3489 struct dma_chan *chan; 3490 3491 list_for_each_entry(chan, &dma_dev->channels, device_node) { 3492 if (chan->client_count) 3493 udma_dbg_summary_show_chan(s, chan); 3494 } 3495 } 3496 #endif /* CONFIG_DEBUG_FS */ 3497 3498 #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 3499 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 3500 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ 3501 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 3502 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 3503 3504 static int udma_probe(struct platform_device *pdev) 3505 { 3506 struct device_node *navss_node = pdev->dev.parent->of_node; 3507 const struct soc_device_attribute *soc; 3508 struct device *dev = &pdev->dev; 3509 struct udma_dev *ud; 3510 const struct of_device_id *match; 3511 int i, ret; 3512 int ch_count; 3513 3514 ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48)); 3515 if (ret) 3516 dev_err(dev, "failed to set dma mask stuff\n"); 3517 3518 ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL); 3519 if (!ud) 3520 return -ENOMEM; 3521 3522 ret = udma_get_mmrs(pdev, ud); 3523 if (ret) 3524 return ret; 3525 3526 ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci"); 3527 if (IS_ERR(ud->tisci_rm.tisci)) 3528 return PTR_ERR(ud->tisci_rm.tisci); 3529 3530 ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id", 3531 &ud->tisci_rm.tisci_dev_id); 3532 if (ret) { 3533 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret); 3534 return ret; 3535 } 3536 pdev->id = ud->tisci_rm.tisci_dev_id; 3537 3538 ret = of_property_read_u32(navss_node, "ti,sci-dev-id", 3539 &ud->tisci_rm.tisci_navss_dev_id); 3540 if (ret) { 3541 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret); 3542 return ret; 3543 } 3544 3545 ret = of_property_read_u32(dev->of_node, "ti,udma-atype", &ud->atype); 3546 if (!ret && ud->atype > 2) { 3547 dev_err(dev, "Invalid atype: %u\n", ud->atype); 3548 return -EINVAL; 3549 } 3550 3551 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; 3552 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; 3553 3554 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc"); 3555 if (IS_ERR(ud->ringacc)) 3556 return PTR_ERR(ud->ringacc); 3557 3558 dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 3559 DOMAIN_BUS_TI_SCI_INTA_MSI); 3560 if (!dev->msi_domain) { 3561 dev_err(dev, "Failed to get MSI domain\n"); 3562 return -EPROBE_DEFER; 3563 } 3564 3565 match = of_match_node(udma_of_match, dev->of_node); 3566 if (!match) { 3567 dev_err(dev, "No compatible match found\n"); 3568 return -ENODEV; 3569 } 3570 ud->match_data = match->data; 3571 3572 soc = soc_device_match(k3_soc_devices); 3573 if (!soc) { 3574 dev_err(dev, "No compatible SoC found\n"); 3575 return -ENODEV; 3576 } 3577 ud->soc_data = soc->data; 3578 3579 dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask); 3580 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); 3581 3582 ud->ddev.device_alloc_chan_resources = udma_alloc_chan_resources; 3583 ud->ddev.device_config = udma_slave_config; 3584 ud->ddev.device_prep_slave_sg = udma_prep_slave_sg; 3585 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; 3586 ud->ddev.device_issue_pending = udma_issue_pending; 3587 ud->ddev.device_tx_status = udma_tx_status; 3588 ud->ddev.device_pause = udma_pause; 3589 ud->ddev.device_resume = udma_resume; 3590 ud->ddev.device_terminate_all = udma_terminate_all; 3591 ud->ddev.device_synchronize = udma_synchronize; 3592 #ifdef CONFIG_DEBUG_FS 3593 ud->ddev.dbg_summary_show = udma_dbg_summary_show; 3594 #endif 3595 3596 ud->ddev.device_free_chan_resources = udma_free_chan_resources; 3597 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; 3598 ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; 3599 ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 3600 ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 3601 ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES; 3602 ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | 3603 DESC_METADATA_ENGINE; 3604 if (ud->match_data->enable_memcpy_support) { 3605 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask); 3606 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy; 3607 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM); 3608 } 3609 3610 ud->ddev.dev = dev; 3611 ud->dev = dev; 3612 ud->psil_base = ud->match_data->psil_base; 3613 3614 INIT_LIST_HEAD(&ud->ddev.channels); 3615 INIT_LIST_HEAD(&ud->desc_to_purge); 3616 3617 ch_count = udma_setup_resources(ud); 3618 if (ch_count <= 0) 3619 return ch_count; 3620 3621 spin_lock_init(&ud->lock); 3622 INIT_WORK(&ud->purge_work, udma_purge_desc_work); 3623 3624 ud->desc_align = 64; 3625 if (ud->desc_align < dma_get_cache_alignment()) 3626 ud->desc_align = dma_get_cache_alignment(); 3627 3628 ret = udma_setup_rx_flush(ud); 3629 if (ret) 3630 return ret; 3631 3632 for (i = 0; i < ud->tchan_cnt; i++) { 3633 struct udma_tchan *tchan = &ud->tchans[i]; 3634 3635 tchan->id = i; 3636 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000; 3637 } 3638 3639 for (i = 0; i < ud->rchan_cnt; i++) { 3640 struct udma_rchan *rchan = &ud->rchans[i]; 3641 3642 rchan->id = i; 3643 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000; 3644 } 3645 3646 for (i = 0; i < ud->rflow_cnt; i++) { 3647 struct udma_rflow *rflow = &ud->rflows[i]; 3648 3649 rflow->id = i; 3650 } 3651 3652 for (i = 0; i < ch_count; i++) { 3653 struct udma_chan *uc = &ud->channels[i]; 3654 3655 uc->ud = ud; 3656 uc->vc.desc_free = udma_desc_free; 3657 uc->id = i; 3658 uc->tchan = NULL; 3659 uc->rchan = NULL; 3660 uc->config.remote_thread_id = -1; 3661 uc->config.dir = DMA_MEM_TO_MEM; 3662 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", 3663 dev_name(dev), i); 3664 3665 vchan_init(&uc->vc, &ud->ddev); 3666 /* Use custom vchan completion handling */ 3667 tasklet_setup(&uc->vc.task, udma_vchan_complete); 3668 init_completion(&uc->teardown_completed); 3669 INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion); 3670 } 3671 3672 ret = dma_async_device_register(&ud->ddev); 3673 if (ret) { 3674 dev_err(dev, "failed to register slave DMA engine: %d\n", ret); 3675 return ret; 3676 } 3677 3678 platform_set_drvdata(pdev, ud); 3679 3680 ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud); 3681 if (ret) { 3682 dev_err(dev, "failed to register of_dma controller\n"); 3683 dma_async_device_unregister(&ud->ddev); 3684 } 3685 3686 return ret; 3687 } 3688 3689 static struct platform_driver udma_driver = { 3690 .driver = { 3691 .name = "ti-udma", 3692 .of_match_table = udma_of_match, 3693 .suppress_bind_attrs = true, 3694 }, 3695 .probe = udma_probe, 3696 }; 3697 builtin_platform_driver(udma_driver); 3698 3699 /* Private interfaces to UDMA */ 3700 #include "k3-udma-private.c" 3701