1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/delay.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <linux/err.h> 13 #include <linux/init.h> 14 #include <linux/interrupt.h> 15 #include <linux/list.h> 16 #include <linux/platform_device.h> 17 #include <linux/slab.h> 18 #include <linux/spinlock.h> 19 #include <linux/of.h> 20 #include <linux/of_dma.h> 21 #include <linux/of_device.h> 22 #include <linux/of_irq.h> 23 #include <linux/workqueue.h> 24 #include <linux/completion.h> 25 #include <linux/soc/ti/k3-ringacc.h> 26 #include <linux/soc/ti/ti_sci_protocol.h> 27 #include <linux/soc/ti/ti_sci_inta_msi.h> 28 #include <linux/dma/ti-cppi5.h> 29 30 #include "../virt-dma.h" 31 #include "k3-udma.h" 32 #include "k3-psil-priv.h" 33 34 struct udma_static_tr { 35 u8 elsize; /* RPSTR0 */ 36 u16 elcnt; /* RPSTR0 */ 37 u16 bstcnt; /* RPSTR1 */ 38 }; 39 40 #define K3_UDMA_MAX_RFLOWS 1024 41 #define K3_UDMA_DEFAULT_RING_SIZE 16 42 43 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */ 44 #define UDMA_RFLOW_SRCTAG_NONE 0 45 #define UDMA_RFLOW_SRCTAG_CFG_TAG 1 46 #define UDMA_RFLOW_SRCTAG_FLOW_ID 2 47 #define UDMA_RFLOW_SRCTAG_SRC_TAG 4 48 49 #define UDMA_RFLOW_DSTTAG_NONE 0 50 #define UDMA_RFLOW_DSTTAG_CFG_TAG 1 51 #define UDMA_RFLOW_DSTTAG_FLOW_ID 2 52 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO 4 53 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI 5 54 55 struct udma_chan; 56 57 enum udma_mmr { 58 MMR_GCFG = 0, 59 MMR_RCHANRT, 60 MMR_TCHANRT, 61 MMR_LAST, 62 }; 63 64 static const char * const mmr_names[] = { "gcfg", "rchanrt", "tchanrt" }; 65 66 struct udma_tchan { 67 void __iomem *reg_rt; 68 69 int id; 70 struct k3_ring *t_ring; /* Transmit ring */ 71 struct k3_ring *tc_ring; /* Transmit Completion ring */ 72 }; 73 74 struct udma_rflow { 75 int id; 76 struct k3_ring *fd_ring; /* Free Descriptor ring */ 77 struct k3_ring *r_ring; /* Receive ring */ 78 }; 79 80 struct udma_rchan { 81 void __iomem *reg_rt; 82 83 int id; 84 }; 85 86 #define UDMA_FLAG_PDMA_ACC32 BIT(0) 87 #define UDMA_FLAG_PDMA_BURST BIT(1) 88 89 struct udma_match_data { 90 u32 psil_base; 91 bool enable_memcpy_support; 92 u32 flags; 93 u32 statictr_z_mask; 94 u32 rchan_oes_offset; 95 96 u8 tpl_levels; 97 u32 level_start_idx[]; 98 }; 99 100 struct udma_hwdesc { 101 size_t cppi5_desc_size; 102 void *cppi5_desc_vaddr; 103 dma_addr_t cppi5_desc_paddr; 104 105 /* TR descriptor internal pointers */ 106 void *tr_req_base; 107 struct cppi5_tr_resp_t *tr_resp_base; 108 }; 109 110 struct udma_rx_flush { 111 struct udma_hwdesc hwdescs[2]; 112 113 size_t buffer_size; 114 void *buffer_vaddr; 115 dma_addr_t buffer_paddr; 116 }; 117 118 struct udma_dev { 119 struct dma_device ddev; 120 struct device *dev; 121 void __iomem *mmrs[MMR_LAST]; 122 const struct udma_match_data *match_data; 123 124 size_t desc_align; /* alignment to use for descriptors */ 125 126 struct udma_tisci_rm tisci_rm; 127 128 struct k3_ringacc *ringacc; 129 130 struct work_struct purge_work; 131 struct list_head desc_to_purge; 132 spinlock_t lock; 133 134 struct udma_rx_flush rx_flush; 135 136 int tchan_cnt; 137 int echan_cnt; 138 int rchan_cnt; 139 int rflow_cnt; 140 unsigned long *tchan_map; 141 unsigned long *rchan_map; 142 unsigned long *rflow_gp_map; 143 unsigned long *rflow_gp_map_allocated; 144 unsigned long *rflow_in_use; 145 146 struct udma_tchan *tchans; 147 struct udma_rchan *rchans; 148 struct udma_rflow *rflows; 149 150 struct udma_chan *channels; 151 u32 psil_base; 152 u32 atype; 153 }; 154 155 struct udma_desc { 156 struct virt_dma_desc vd; 157 158 bool terminated; 159 160 enum dma_transfer_direction dir; 161 162 struct udma_static_tr static_tr; 163 u32 residue; 164 165 unsigned int sglen; 166 unsigned int desc_idx; /* Only used for cyclic in packet mode */ 167 unsigned int tr_idx; 168 169 u32 metadata_size; 170 void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */ 171 172 unsigned int hwdesc_count; 173 struct udma_hwdesc hwdesc[0]; 174 }; 175 176 enum udma_chan_state { 177 UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */ 178 UDMA_CHAN_IS_ACTIVE, /* Normal operation */ 179 UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */ 180 }; 181 182 struct udma_tx_drain { 183 struct delayed_work work; 184 ktime_t tstamp; 185 u32 residue; 186 }; 187 188 struct udma_chan_config { 189 bool pkt_mode; /* TR or packet */ 190 bool needs_epib; /* EPIB is needed for the communication or not */ 191 u32 psd_size; /* size of Protocol Specific Data */ 192 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */ 193 u32 hdesc_size; /* Size of a packet descriptor in packet mode */ 194 bool notdpkt; /* Suppress sending TDC packet */ 195 int remote_thread_id; 196 u32 atype; 197 u32 src_thread; 198 u32 dst_thread; 199 enum psil_endpoint_type ep_type; 200 bool enable_acc32; 201 bool enable_burst; 202 enum udma_tp_level channel_tpl; /* Channel Throughput Level */ 203 204 enum dma_transfer_direction dir; 205 }; 206 207 struct udma_chan { 208 struct virt_dma_chan vc; 209 struct dma_slave_config cfg; 210 struct udma_dev *ud; 211 struct udma_desc *desc; 212 struct udma_desc *terminated_desc; 213 struct udma_static_tr static_tr; 214 char *name; 215 216 struct udma_tchan *tchan; 217 struct udma_rchan *rchan; 218 struct udma_rflow *rflow; 219 220 bool psil_paired; 221 222 int irq_num_ring; 223 int irq_num_udma; 224 225 bool cyclic; 226 bool paused; 227 228 enum udma_chan_state state; 229 struct completion teardown_completed; 230 231 struct udma_tx_drain tx_drain; 232 233 u32 bcnt; /* number of bytes completed since the start of the channel */ 234 u32 in_ring_cnt; /* number of descriptors in flight */ 235 236 /* Channel configuration parameters */ 237 struct udma_chan_config config; 238 239 /* dmapool for packet mode descriptors */ 240 bool use_dma_pool; 241 struct dma_pool *hdesc_pool; 242 243 u32 id; 244 }; 245 246 static inline struct udma_dev *to_udma_dev(struct dma_device *d) 247 { 248 return container_of(d, struct udma_dev, ddev); 249 } 250 251 static inline struct udma_chan *to_udma_chan(struct dma_chan *c) 252 { 253 return container_of(c, struct udma_chan, vc.chan); 254 } 255 256 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t) 257 { 258 return container_of(t, struct udma_desc, vd.tx); 259 } 260 261 /* Generic register access functions */ 262 static inline u32 udma_read(void __iomem *base, int reg) 263 { 264 return readl(base + reg); 265 } 266 267 static inline void udma_write(void __iomem *base, int reg, u32 val) 268 { 269 writel(val, base + reg); 270 } 271 272 static inline void udma_update_bits(void __iomem *base, int reg, 273 u32 mask, u32 val) 274 { 275 u32 tmp, orig; 276 277 orig = readl(base + reg); 278 tmp = orig & ~mask; 279 tmp |= (val & mask); 280 281 if (tmp != orig) 282 writel(tmp, base + reg); 283 } 284 285 /* TCHANRT */ 286 static inline u32 udma_tchanrt_read(struct udma_tchan *tchan, int reg) 287 { 288 if (!tchan) 289 return 0; 290 return udma_read(tchan->reg_rt, reg); 291 } 292 293 static inline void udma_tchanrt_write(struct udma_tchan *tchan, int reg, 294 u32 val) 295 { 296 if (!tchan) 297 return; 298 udma_write(tchan->reg_rt, reg, val); 299 } 300 301 static inline void udma_tchanrt_update_bits(struct udma_tchan *tchan, int reg, 302 u32 mask, u32 val) 303 { 304 if (!tchan) 305 return; 306 udma_update_bits(tchan->reg_rt, reg, mask, val); 307 } 308 309 /* RCHANRT */ 310 static inline u32 udma_rchanrt_read(struct udma_rchan *rchan, int reg) 311 { 312 if (!rchan) 313 return 0; 314 return udma_read(rchan->reg_rt, reg); 315 } 316 317 static inline void udma_rchanrt_write(struct udma_rchan *rchan, int reg, 318 u32 val) 319 { 320 if (!rchan) 321 return; 322 udma_write(rchan->reg_rt, reg, val); 323 } 324 325 static inline void udma_rchanrt_update_bits(struct udma_rchan *rchan, int reg, 326 u32 mask, u32 val) 327 { 328 if (!rchan) 329 return; 330 udma_update_bits(rchan->reg_rt, reg, mask, val); 331 } 332 333 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) 334 { 335 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 336 337 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 338 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci, 339 tisci_rm->tisci_navss_dev_id, 340 src_thread, dst_thread); 341 } 342 343 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread, 344 u32 dst_thread) 345 { 346 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 347 348 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 349 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci, 350 tisci_rm->tisci_navss_dev_id, 351 src_thread, dst_thread); 352 } 353 354 static void udma_reset_uchan(struct udma_chan *uc) 355 { 356 memset(&uc->config, 0, sizeof(uc->config)); 357 uc->config.remote_thread_id = -1; 358 uc->state = UDMA_CHAN_IS_IDLE; 359 } 360 361 static void udma_dump_chan_stdata(struct udma_chan *uc) 362 { 363 struct device *dev = uc->ud->dev; 364 u32 offset; 365 int i; 366 367 if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) { 368 dev_dbg(dev, "TCHAN State data:\n"); 369 for (i = 0; i < 32; i++) { 370 offset = UDMA_TCHAN_RT_STDATA_REG + i * 4; 371 dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i, 372 udma_tchanrt_read(uc->tchan, offset)); 373 } 374 } 375 376 if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) { 377 dev_dbg(dev, "RCHAN State data:\n"); 378 for (i = 0; i < 32; i++) { 379 offset = UDMA_RCHAN_RT_STDATA_REG + i * 4; 380 dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i, 381 udma_rchanrt_read(uc->rchan, offset)); 382 } 383 } 384 } 385 386 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d, 387 int idx) 388 { 389 return d->hwdesc[idx].cppi5_desc_paddr; 390 } 391 392 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx) 393 { 394 return d->hwdesc[idx].cppi5_desc_vaddr; 395 } 396 397 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc, 398 dma_addr_t paddr) 399 { 400 struct udma_desc *d = uc->terminated_desc; 401 402 if (d) { 403 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 404 d->desc_idx); 405 406 if (desc_paddr != paddr) 407 d = NULL; 408 } 409 410 if (!d) { 411 d = uc->desc; 412 if (d) { 413 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 414 d->desc_idx); 415 416 if (desc_paddr != paddr) 417 d = NULL; 418 } 419 } 420 421 return d; 422 } 423 424 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d) 425 { 426 if (uc->use_dma_pool) { 427 int i; 428 429 for (i = 0; i < d->hwdesc_count; i++) { 430 if (!d->hwdesc[i].cppi5_desc_vaddr) 431 continue; 432 433 dma_pool_free(uc->hdesc_pool, 434 d->hwdesc[i].cppi5_desc_vaddr, 435 d->hwdesc[i].cppi5_desc_paddr); 436 437 d->hwdesc[i].cppi5_desc_vaddr = NULL; 438 } 439 } else if (d->hwdesc[0].cppi5_desc_vaddr) { 440 struct udma_dev *ud = uc->ud; 441 442 dma_free_coherent(ud->dev, d->hwdesc[0].cppi5_desc_size, 443 d->hwdesc[0].cppi5_desc_vaddr, 444 d->hwdesc[0].cppi5_desc_paddr); 445 446 d->hwdesc[0].cppi5_desc_vaddr = NULL; 447 } 448 } 449 450 static void udma_purge_desc_work(struct work_struct *work) 451 { 452 struct udma_dev *ud = container_of(work, typeof(*ud), purge_work); 453 struct virt_dma_desc *vd, *_vd; 454 unsigned long flags; 455 LIST_HEAD(head); 456 457 spin_lock_irqsave(&ud->lock, flags); 458 list_splice_tail_init(&ud->desc_to_purge, &head); 459 spin_unlock_irqrestore(&ud->lock, flags); 460 461 list_for_each_entry_safe(vd, _vd, &head, node) { 462 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 463 struct udma_desc *d = to_udma_desc(&vd->tx); 464 465 udma_free_hwdesc(uc, d); 466 list_del(&vd->node); 467 kfree(d); 468 } 469 470 /* If more to purge, schedule the work again */ 471 if (!list_empty(&ud->desc_to_purge)) 472 schedule_work(&ud->purge_work); 473 } 474 475 static void udma_desc_free(struct virt_dma_desc *vd) 476 { 477 struct udma_dev *ud = to_udma_dev(vd->tx.chan->device); 478 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 479 struct udma_desc *d = to_udma_desc(&vd->tx); 480 unsigned long flags; 481 482 if (uc->terminated_desc == d) 483 uc->terminated_desc = NULL; 484 485 if (uc->use_dma_pool) { 486 udma_free_hwdesc(uc, d); 487 kfree(d); 488 return; 489 } 490 491 spin_lock_irqsave(&ud->lock, flags); 492 list_add_tail(&vd->node, &ud->desc_to_purge); 493 spin_unlock_irqrestore(&ud->lock, flags); 494 495 schedule_work(&ud->purge_work); 496 } 497 498 static bool udma_is_chan_running(struct udma_chan *uc) 499 { 500 u32 trt_ctl = 0; 501 u32 rrt_ctl = 0; 502 503 if (uc->tchan) 504 trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG); 505 if (uc->rchan) 506 rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG); 507 508 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN) 509 return true; 510 511 return false; 512 } 513 514 static bool udma_is_chan_paused(struct udma_chan *uc) 515 { 516 u32 val, pause_mask; 517 518 switch (uc->config.dir) { 519 case DMA_DEV_TO_MEM: 520 val = udma_rchanrt_read(uc->rchan, 521 UDMA_RCHAN_RT_PEER_RT_EN_REG); 522 pause_mask = UDMA_PEER_RT_EN_PAUSE; 523 break; 524 case DMA_MEM_TO_DEV: 525 val = udma_tchanrt_read(uc->tchan, 526 UDMA_TCHAN_RT_PEER_RT_EN_REG); 527 pause_mask = UDMA_PEER_RT_EN_PAUSE; 528 break; 529 case DMA_MEM_TO_MEM: 530 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG); 531 pause_mask = UDMA_CHAN_RT_CTL_PAUSE; 532 break; 533 default: 534 return false; 535 } 536 537 if (val & pause_mask) 538 return true; 539 540 return false; 541 } 542 543 static void udma_sync_for_device(struct udma_chan *uc, int idx) 544 { 545 struct udma_desc *d = uc->desc; 546 547 if (uc->cyclic && uc->config.pkt_mode) { 548 dma_sync_single_for_device(uc->ud->dev, 549 d->hwdesc[idx].cppi5_desc_paddr, 550 d->hwdesc[idx].cppi5_desc_size, 551 DMA_TO_DEVICE); 552 } else { 553 int i; 554 555 for (i = 0; i < d->hwdesc_count; i++) { 556 if (!d->hwdesc[i].cppi5_desc_vaddr) 557 continue; 558 559 dma_sync_single_for_device(uc->ud->dev, 560 d->hwdesc[i].cppi5_desc_paddr, 561 d->hwdesc[i].cppi5_desc_size, 562 DMA_TO_DEVICE); 563 } 564 } 565 } 566 567 static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc) 568 { 569 return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr; 570 } 571 572 static int udma_push_to_ring(struct udma_chan *uc, int idx) 573 { 574 struct udma_desc *d = uc->desc; 575 struct k3_ring *ring = NULL; 576 dma_addr_t paddr; 577 int ret; 578 579 switch (uc->config.dir) { 580 case DMA_DEV_TO_MEM: 581 ring = uc->rflow->fd_ring; 582 break; 583 case DMA_MEM_TO_DEV: 584 case DMA_MEM_TO_MEM: 585 ring = uc->tchan->t_ring; 586 break; 587 default: 588 return -EINVAL; 589 } 590 591 /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */ 592 if (idx == -1) { 593 paddr = udma_get_rx_flush_hwdesc_paddr(uc); 594 } else { 595 paddr = udma_curr_cppi5_desc_paddr(d, idx); 596 597 wmb(); /* Ensure that writes are not moved over this point */ 598 udma_sync_for_device(uc, idx); 599 } 600 601 ret = k3_ringacc_ring_push(ring, &paddr); 602 if (!ret) 603 uc->in_ring_cnt++; 604 605 return ret; 606 } 607 608 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr) 609 { 610 if (uc->config.dir != DMA_DEV_TO_MEM) 611 return false; 612 613 if (addr == udma_get_rx_flush_hwdesc_paddr(uc)) 614 return true; 615 616 return false; 617 } 618 619 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) 620 { 621 struct k3_ring *ring = NULL; 622 int ret = -ENOENT; 623 624 switch (uc->config.dir) { 625 case DMA_DEV_TO_MEM: 626 ring = uc->rflow->r_ring; 627 break; 628 case DMA_MEM_TO_DEV: 629 case DMA_MEM_TO_MEM: 630 ring = uc->tchan->tc_ring; 631 break; 632 default: 633 break; 634 } 635 636 if (ring && k3_ringacc_ring_get_occ(ring)) { 637 struct udma_desc *d = NULL; 638 639 ret = k3_ringacc_ring_pop(ring, addr); 640 if (ret) 641 return ret; 642 643 /* Teardown completion */ 644 if (cppi5_desc_is_tdcm(*addr)) 645 return ret; 646 647 /* Check for flush descriptor */ 648 if (udma_desc_is_rx_flush(uc, *addr)) 649 return -ENOENT; 650 651 d = udma_udma_desc_from_paddr(uc, *addr); 652 653 if (d) 654 dma_sync_single_for_cpu(uc->ud->dev, *addr, 655 d->hwdesc[0].cppi5_desc_size, 656 DMA_FROM_DEVICE); 657 rmb(); /* Ensure that reads are not moved before this point */ 658 659 if (!ret) 660 uc->in_ring_cnt--; 661 } 662 663 return ret; 664 } 665 666 static void udma_reset_rings(struct udma_chan *uc) 667 { 668 struct k3_ring *ring1 = NULL; 669 struct k3_ring *ring2 = NULL; 670 671 switch (uc->config.dir) { 672 case DMA_DEV_TO_MEM: 673 if (uc->rchan) { 674 ring1 = uc->rflow->fd_ring; 675 ring2 = uc->rflow->r_ring; 676 } 677 break; 678 case DMA_MEM_TO_DEV: 679 case DMA_MEM_TO_MEM: 680 if (uc->tchan) { 681 ring1 = uc->tchan->t_ring; 682 ring2 = uc->tchan->tc_ring; 683 } 684 break; 685 default: 686 break; 687 } 688 689 if (ring1) 690 k3_ringacc_ring_reset_dma(ring1, 691 k3_ringacc_ring_get_occ(ring1)); 692 if (ring2) 693 k3_ringacc_ring_reset(ring2); 694 695 /* make sure we are not leaking memory by stalled descriptor */ 696 if (uc->terminated_desc) { 697 udma_desc_free(&uc->terminated_desc->vd); 698 uc->terminated_desc = NULL; 699 } 700 701 uc->in_ring_cnt = 0; 702 } 703 704 static void udma_reset_counters(struct udma_chan *uc) 705 { 706 u32 val; 707 708 if (uc->tchan) { 709 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); 710 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_BCNT_REG, val); 711 712 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG); 713 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG, val); 714 715 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PCNT_REG); 716 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PCNT_REG, val); 717 718 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); 719 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG, val); 720 } 721 722 if (uc->rchan) { 723 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_BCNT_REG); 724 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_BCNT_REG, val); 725 726 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG); 727 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG, val); 728 729 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PCNT_REG); 730 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PCNT_REG, val); 731 732 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG); 733 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG, val); 734 } 735 736 uc->bcnt = 0; 737 } 738 739 static int udma_reset_chan(struct udma_chan *uc, bool hard) 740 { 741 switch (uc->config.dir) { 742 case DMA_DEV_TO_MEM: 743 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 0); 744 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0); 745 break; 746 case DMA_MEM_TO_DEV: 747 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0); 748 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 0); 749 break; 750 case DMA_MEM_TO_MEM: 751 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0); 752 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0); 753 break; 754 default: 755 return -EINVAL; 756 } 757 758 /* Reset all counters */ 759 udma_reset_counters(uc); 760 761 /* Hard reset: re-initialize the channel to reset */ 762 if (hard) { 763 struct udma_chan_config ucc_backup; 764 int ret; 765 766 memcpy(&ucc_backup, &uc->config, sizeof(uc->config)); 767 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan); 768 769 /* restore the channel configuration */ 770 memcpy(&uc->config, &ucc_backup, sizeof(uc->config)); 771 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan); 772 if (ret) 773 return ret; 774 775 /* 776 * Setting forced teardown after forced reset helps recovering 777 * the rchan. 778 */ 779 if (uc->config.dir == DMA_DEV_TO_MEM) 780 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 781 UDMA_CHAN_RT_CTL_EN | 782 UDMA_CHAN_RT_CTL_TDOWN | 783 UDMA_CHAN_RT_CTL_FTDOWN); 784 } 785 uc->state = UDMA_CHAN_IS_IDLE; 786 787 return 0; 788 } 789 790 static void udma_start_desc(struct udma_chan *uc) 791 { 792 struct udma_chan_config *ucc = &uc->config; 793 794 if (ucc->pkt_mode && (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) { 795 int i; 796 797 /* Push all descriptors to ring for packet mode cyclic or RX */ 798 for (i = 0; i < uc->desc->sglen; i++) 799 udma_push_to_ring(uc, i); 800 } else { 801 udma_push_to_ring(uc, 0); 802 } 803 } 804 805 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc) 806 { 807 /* Only PDMAs have staticTR */ 808 if (uc->config.ep_type == PSIL_EP_NATIVE) 809 return false; 810 811 /* Check if the staticTR configuration has changed for TX */ 812 if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr))) 813 return true; 814 815 return false; 816 } 817 818 static int udma_start(struct udma_chan *uc) 819 { 820 struct virt_dma_desc *vd = vchan_next_desc(&uc->vc); 821 822 if (!vd) { 823 uc->desc = NULL; 824 return -ENOENT; 825 } 826 827 list_del(&vd->node); 828 829 uc->desc = to_udma_desc(&vd->tx); 830 831 /* Channel is already running and does not need reconfiguration */ 832 if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) { 833 udma_start_desc(uc); 834 goto out; 835 } 836 837 /* Make sure that we clear the teardown bit, if it is set */ 838 udma_reset_chan(uc, false); 839 840 /* Push descriptors before we start the channel */ 841 udma_start_desc(uc); 842 843 switch (uc->desc->dir) { 844 case DMA_DEV_TO_MEM: 845 /* Config remote TR */ 846 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 847 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 848 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 849 const struct udma_match_data *match_data = 850 uc->ud->match_data; 851 852 if (uc->config.enable_acc32) 853 val |= PDMA_STATIC_TR_XY_ACC32; 854 if (uc->config.enable_burst) 855 val |= PDMA_STATIC_TR_XY_BURST; 856 857 udma_rchanrt_write(uc->rchan, 858 UDMA_RCHAN_RT_PEER_STATIC_TR_XY_REG, val); 859 860 udma_rchanrt_write(uc->rchan, 861 UDMA_RCHAN_RT_PEER_STATIC_TR_Z_REG, 862 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt, 863 match_data->statictr_z_mask)); 864 865 /* save the current staticTR configuration */ 866 memcpy(&uc->static_tr, &uc->desc->static_tr, 867 sizeof(uc->static_tr)); 868 } 869 870 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 871 UDMA_CHAN_RT_CTL_EN); 872 873 /* Enable remote */ 874 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 875 UDMA_PEER_RT_EN_ENABLE); 876 877 break; 878 case DMA_MEM_TO_DEV: 879 /* Config remote TR */ 880 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 881 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 882 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 883 884 if (uc->config.enable_acc32) 885 val |= PDMA_STATIC_TR_XY_ACC32; 886 if (uc->config.enable_burst) 887 val |= PDMA_STATIC_TR_XY_BURST; 888 889 udma_tchanrt_write(uc->tchan, 890 UDMA_TCHAN_RT_PEER_STATIC_TR_XY_REG, val); 891 892 /* save the current staticTR configuration */ 893 memcpy(&uc->static_tr, &uc->desc->static_tr, 894 sizeof(uc->static_tr)); 895 } 896 897 /* Enable remote */ 898 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 899 UDMA_PEER_RT_EN_ENABLE); 900 901 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 902 UDMA_CHAN_RT_CTL_EN); 903 904 break; 905 case DMA_MEM_TO_MEM: 906 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 907 UDMA_CHAN_RT_CTL_EN); 908 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 909 UDMA_CHAN_RT_CTL_EN); 910 911 break; 912 default: 913 return -EINVAL; 914 } 915 916 uc->state = UDMA_CHAN_IS_ACTIVE; 917 out: 918 919 return 0; 920 } 921 922 static int udma_stop(struct udma_chan *uc) 923 { 924 enum udma_chan_state old_state = uc->state; 925 926 uc->state = UDMA_CHAN_IS_TERMINATING; 927 reinit_completion(&uc->teardown_completed); 928 929 switch (uc->config.dir) { 930 case DMA_DEV_TO_MEM: 931 if (!uc->cyclic && !uc->desc) 932 udma_push_to_ring(uc, -1); 933 934 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 935 UDMA_PEER_RT_EN_ENABLE | 936 UDMA_PEER_RT_EN_TEARDOWN); 937 break; 938 case DMA_MEM_TO_DEV: 939 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 940 UDMA_PEER_RT_EN_ENABLE | 941 UDMA_PEER_RT_EN_FLUSH); 942 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 943 UDMA_CHAN_RT_CTL_EN | 944 UDMA_CHAN_RT_CTL_TDOWN); 945 break; 946 case DMA_MEM_TO_MEM: 947 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 948 UDMA_CHAN_RT_CTL_EN | 949 UDMA_CHAN_RT_CTL_TDOWN); 950 break; 951 default: 952 uc->state = old_state; 953 complete_all(&uc->teardown_completed); 954 return -EINVAL; 955 } 956 957 return 0; 958 } 959 960 static void udma_cyclic_packet_elapsed(struct udma_chan *uc) 961 { 962 struct udma_desc *d = uc->desc; 963 struct cppi5_host_desc_t *h_desc; 964 965 h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr; 966 cppi5_hdesc_reset_to_original(h_desc); 967 udma_push_to_ring(uc, d->desc_idx); 968 d->desc_idx = (d->desc_idx + 1) % d->sglen; 969 } 970 971 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d) 972 { 973 struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr; 974 975 memcpy(d->metadata, h_desc->epib, d->metadata_size); 976 } 977 978 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) 979 { 980 u32 peer_bcnt, bcnt; 981 982 /* Only TX towards PDMA is affected */ 983 if (uc->config.ep_type == PSIL_EP_NATIVE || 984 uc->config.dir != DMA_MEM_TO_DEV) 985 return true; 986 987 peer_bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); 988 bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); 989 990 /* Transfer is incomplete, store current residue and time stamp */ 991 if (peer_bcnt < bcnt) { 992 uc->tx_drain.residue = bcnt - peer_bcnt; 993 uc->tx_drain.tstamp = ktime_get(); 994 return false; 995 } 996 997 return true; 998 } 999 1000 static void udma_check_tx_completion(struct work_struct *work) 1001 { 1002 struct udma_chan *uc = container_of(work, typeof(*uc), 1003 tx_drain.work.work); 1004 bool desc_done = true; 1005 u32 residue_diff; 1006 ktime_t time_diff; 1007 unsigned long delay; 1008 1009 while (1) { 1010 if (uc->desc) { 1011 /* Get previous residue and time stamp */ 1012 residue_diff = uc->tx_drain.residue; 1013 time_diff = uc->tx_drain.tstamp; 1014 /* 1015 * Get current residue and time stamp or see if 1016 * transfer is complete 1017 */ 1018 desc_done = udma_is_desc_really_done(uc, uc->desc); 1019 } 1020 1021 if (!desc_done) { 1022 /* 1023 * Find the time delta and residue delta w.r.t 1024 * previous poll 1025 */ 1026 time_diff = ktime_sub(uc->tx_drain.tstamp, 1027 time_diff) + 1; 1028 residue_diff -= uc->tx_drain.residue; 1029 if (residue_diff) { 1030 /* 1031 * Try to guess when we should check 1032 * next time by calculating rate at 1033 * which data is being drained at the 1034 * peer device 1035 */ 1036 delay = (time_diff / residue_diff) * 1037 uc->tx_drain.residue; 1038 } else { 1039 /* No progress, check again in 1 second */ 1040 schedule_delayed_work(&uc->tx_drain.work, HZ); 1041 break; 1042 } 1043 1044 usleep_range(ktime_to_us(delay), 1045 ktime_to_us(delay) + 10); 1046 continue; 1047 } 1048 1049 if (uc->desc) { 1050 struct udma_desc *d = uc->desc; 1051 1052 uc->bcnt += d->residue; 1053 udma_start(uc); 1054 vchan_cookie_complete(&d->vd); 1055 break; 1056 } 1057 1058 break; 1059 } 1060 } 1061 1062 static irqreturn_t udma_ring_irq_handler(int irq, void *data) 1063 { 1064 struct udma_chan *uc = data; 1065 struct udma_desc *d; 1066 unsigned long flags; 1067 dma_addr_t paddr = 0; 1068 1069 if (udma_pop_from_ring(uc, &paddr) || !paddr) 1070 return IRQ_HANDLED; 1071 1072 spin_lock_irqsave(&uc->vc.lock, flags); 1073 1074 /* Teardown completion message */ 1075 if (cppi5_desc_is_tdcm(paddr)) { 1076 /* Compensate our internal pop/push counter */ 1077 uc->in_ring_cnt++; 1078 1079 complete_all(&uc->teardown_completed); 1080 1081 if (uc->terminated_desc) { 1082 udma_desc_free(&uc->terminated_desc->vd); 1083 uc->terminated_desc = NULL; 1084 } 1085 1086 if (!uc->desc) 1087 udma_start(uc); 1088 1089 goto out; 1090 } 1091 1092 d = udma_udma_desc_from_paddr(uc, paddr); 1093 1094 if (d) { 1095 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 1096 d->desc_idx); 1097 if (desc_paddr != paddr) { 1098 dev_err(uc->ud->dev, "not matching descriptors!\n"); 1099 goto out; 1100 } 1101 1102 if (d == uc->desc) { 1103 /* active descriptor */ 1104 if (uc->cyclic) { 1105 udma_cyclic_packet_elapsed(uc); 1106 vchan_cyclic_callback(&d->vd); 1107 } else { 1108 if (udma_is_desc_really_done(uc, d)) { 1109 uc->bcnt += d->residue; 1110 udma_start(uc); 1111 vchan_cookie_complete(&d->vd); 1112 } else { 1113 schedule_delayed_work(&uc->tx_drain.work, 1114 0); 1115 } 1116 } 1117 } else { 1118 /* 1119 * terminated descriptor, mark the descriptor as 1120 * completed to update the channel's cookie marker 1121 */ 1122 dma_cookie_complete(&d->vd.tx); 1123 } 1124 } 1125 out: 1126 spin_unlock_irqrestore(&uc->vc.lock, flags); 1127 1128 return IRQ_HANDLED; 1129 } 1130 1131 static irqreturn_t udma_udma_irq_handler(int irq, void *data) 1132 { 1133 struct udma_chan *uc = data; 1134 struct udma_desc *d; 1135 unsigned long flags; 1136 1137 spin_lock_irqsave(&uc->vc.lock, flags); 1138 d = uc->desc; 1139 if (d) { 1140 d->tr_idx = (d->tr_idx + 1) % d->sglen; 1141 1142 if (uc->cyclic) { 1143 vchan_cyclic_callback(&d->vd); 1144 } else { 1145 /* TODO: figure out the real amount of data */ 1146 uc->bcnt += d->residue; 1147 udma_start(uc); 1148 vchan_cookie_complete(&d->vd); 1149 } 1150 } 1151 1152 spin_unlock_irqrestore(&uc->vc.lock, flags); 1153 1154 return IRQ_HANDLED; 1155 } 1156 1157 /** 1158 * __udma_alloc_gp_rflow_range - alloc range of GP RX flows 1159 * @ud: UDMA device 1160 * @from: Start the search from this flow id number 1161 * @cnt: Number of consecutive flow ids to allocate 1162 * 1163 * Allocate range of RX flow ids for future use, those flows can be requested 1164 * only using explicit flow id number. if @from is set to -1 it will try to find 1165 * first free range. if @from is positive value it will force allocation only 1166 * of the specified range of flows. 1167 * 1168 * Returns -ENOMEM if can't find free range. 1169 * -EEXIST if requested range is busy. 1170 * -EINVAL if wrong input values passed. 1171 * Returns flow id on success. 1172 */ 1173 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1174 { 1175 int start, tmp_from; 1176 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS); 1177 1178 tmp_from = from; 1179 if (tmp_from < 0) 1180 tmp_from = ud->rchan_cnt; 1181 /* default flows can't be allocated and accessible only by id */ 1182 if (tmp_from < ud->rchan_cnt) 1183 return -EINVAL; 1184 1185 if (tmp_from + cnt > ud->rflow_cnt) 1186 return -EINVAL; 1187 1188 bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated, 1189 ud->rflow_cnt); 1190 1191 start = bitmap_find_next_zero_area(tmp, 1192 ud->rflow_cnt, 1193 tmp_from, cnt, 0); 1194 if (start >= ud->rflow_cnt) 1195 return -ENOMEM; 1196 1197 if (from >= 0 && start != from) 1198 return -EEXIST; 1199 1200 bitmap_set(ud->rflow_gp_map_allocated, start, cnt); 1201 return start; 1202 } 1203 1204 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1205 { 1206 if (from < ud->rchan_cnt) 1207 return -EINVAL; 1208 if (from + cnt > ud->rflow_cnt) 1209 return -EINVAL; 1210 1211 bitmap_clear(ud->rflow_gp_map_allocated, from, cnt); 1212 return 0; 1213 } 1214 1215 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id) 1216 { 1217 /* 1218 * Attempt to request rflow by ID can be made for any rflow 1219 * if not in use with assumption that caller knows what's doing. 1220 * TI-SCI FW will perform additional permission check ant way, it's 1221 * safe 1222 */ 1223 1224 if (id < 0 || id >= ud->rflow_cnt) 1225 return ERR_PTR(-ENOENT); 1226 1227 if (test_bit(id, ud->rflow_in_use)) 1228 return ERR_PTR(-ENOENT); 1229 1230 /* GP rflow has to be allocated first */ 1231 if (!test_bit(id, ud->rflow_gp_map) && 1232 !test_bit(id, ud->rflow_gp_map_allocated)) 1233 return ERR_PTR(-EINVAL); 1234 1235 dev_dbg(ud->dev, "get rflow%d\n", id); 1236 set_bit(id, ud->rflow_in_use); 1237 return &ud->rflows[id]; 1238 } 1239 1240 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow) 1241 { 1242 if (!test_bit(rflow->id, ud->rflow_in_use)) { 1243 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id); 1244 return; 1245 } 1246 1247 dev_dbg(ud->dev, "put rflow%d\n", rflow->id); 1248 clear_bit(rflow->id, ud->rflow_in_use); 1249 } 1250 1251 #define UDMA_RESERVE_RESOURCE(res) \ 1252 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ 1253 enum udma_tp_level tpl, \ 1254 int id) \ 1255 { \ 1256 if (id >= 0) { \ 1257 if (test_bit(id, ud->res##_map)) { \ 1258 dev_err(ud->dev, "res##%d is in use\n", id); \ 1259 return ERR_PTR(-ENOENT); \ 1260 } \ 1261 } else { \ 1262 int start; \ 1263 \ 1264 if (tpl >= ud->match_data->tpl_levels) \ 1265 tpl = ud->match_data->tpl_levels - 1; \ 1266 \ 1267 start = ud->match_data->level_start_idx[tpl]; \ 1268 \ 1269 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \ 1270 start); \ 1271 if (id == ud->res##_cnt) { \ 1272 return ERR_PTR(-ENOENT); \ 1273 } \ 1274 } \ 1275 \ 1276 set_bit(id, ud->res##_map); \ 1277 return &ud->res##s[id]; \ 1278 } 1279 1280 UDMA_RESERVE_RESOURCE(tchan); 1281 UDMA_RESERVE_RESOURCE(rchan); 1282 1283 static int udma_get_tchan(struct udma_chan *uc) 1284 { 1285 struct udma_dev *ud = uc->ud; 1286 1287 if (uc->tchan) { 1288 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n", 1289 uc->id, uc->tchan->id); 1290 return 0; 1291 } 1292 1293 uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, -1); 1294 if (IS_ERR(uc->tchan)) 1295 return PTR_ERR(uc->tchan); 1296 1297 return 0; 1298 } 1299 1300 static int udma_get_rchan(struct udma_chan *uc) 1301 { 1302 struct udma_dev *ud = uc->ud; 1303 1304 if (uc->rchan) { 1305 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n", 1306 uc->id, uc->rchan->id); 1307 return 0; 1308 } 1309 1310 uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, -1); 1311 if (IS_ERR(uc->rchan)) 1312 return PTR_ERR(uc->rchan); 1313 1314 return 0; 1315 } 1316 1317 static int udma_get_chan_pair(struct udma_chan *uc) 1318 { 1319 struct udma_dev *ud = uc->ud; 1320 const struct udma_match_data *match_data = ud->match_data; 1321 int chan_id, end; 1322 1323 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) { 1324 dev_info(ud->dev, "chan%d: already have %d pair allocated\n", 1325 uc->id, uc->tchan->id); 1326 return 0; 1327 } 1328 1329 if (uc->tchan) { 1330 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n", 1331 uc->id, uc->tchan->id); 1332 return -EBUSY; 1333 } else if (uc->rchan) { 1334 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n", 1335 uc->id, uc->rchan->id); 1336 return -EBUSY; 1337 } 1338 1339 /* Can be optimized, but let's have it like this for now */ 1340 end = min(ud->tchan_cnt, ud->rchan_cnt); 1341 /* Try to use the highest TPL channel pair for MEM_TO_MEM channels */ 1342 chan_id = match_data->level_start_idx[match_data->tpl_levels - 1]; 1343 for (; chan_id < end; chan_id++) { 1344 if (!test_bit(chan_id, ud->tchan_map) && 1345 !test_bit(chan_id, ud->rchan_map)) 1346 break; 1347 } 1348 1349 if (chan_id == end) 1350 return -ENOENT; 1351 1352 set_bit(chan_id, ud->tchan_map); 1353 set_bit(chan_id, ud->rchan_map); 1354 uc->tchan = &ud->tchans[chan_id]; 1355 uc->rchan = &ud->rchans[chan_id]; 1356 1357 return 0; 1358 } 1359 1360 static int udma_get_rflow(struct udma_chan *uc, int flow_id) 1361 { 1362 struct udma_dev *ud = uc->ud; 1363 1364 if (!uc->rchan) { 1365 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id); 1366 return -EINVAL; 1367 } 1368 1369 if (uc->rflow) { 1370 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n", 1371 uc->id, uc->rflow->id); 1372 return 0; 1373 } 1374 1375 uc->rflow = __udma_get_rflow(ud, flow_id); 1376 if (IS_ERR(uc->rflow)) 1377 return PTR_ERR(uc->rflow); 1378 1379 return 0; 1380 } 1381 1382 static void udma_put_rchan(struct udma_chan *uc) 1383 { 1384 struct udma_dev *ud = uc->ud; 1385 1386 if (uc->rchan) { 1387 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id, 1388 uc->rchan->id); 1389 clear_bit(uc->rchan->id, ud->rchan_map); 1390 uc->rchan = NULL; 1391 } 1392 } 1393 1394 static void udma_put_tchan(struct udma_chan *uc) 1395 { 1396 struct udma_dev *ud = uc->ud; 1397 1398 if (uc->tchan) { 1399 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id, 1400 uc->tchan->id); 1401 clear_bit(uc->tchan->id, ud->tchan_map); 1402 uc->tchan = NULL; 1403 } 1404 } 1405 1406 static void udma_put_rflow(struct udma_chan *uc) 1407 { 1408 struct udma_dev *ud = uc->ud; 1409 1410 if (uc->rflow) { 1411 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id, 1412 uc->rflow->id); 1413 __udma_put_rflow(ud, uc->rflow); 1414 uc->rflow = NULL; 1415 } 1416 } 1417 1418 static void udma_free_tx_resources(struct udma_chan *uc) 1419 { 1420 if (!uc->tchan) 1421 return; 1422 1423 k3_ringacc_ring_free(uc->tchan->t_ring); 1424 k3_ringacc_ring_free(uc->tchan->tc_ring); 1425 uc->tchan->t_ring = NULL; 1426 uc->tchan->tc_ring = NULL; 1427 1428 udma_put_tchan(uc); 1429 } 1430 1431 static int udma_alloc_tx_resources(struct udma_chan *uc) 1432 { 1433 struct k3_ring_cfg ring_cfg; 1434 struct udma_dev *ud = uc->ud; 1435 int ret; 1436 1437 ret = udma_get_tchan(uc); 1438 if (ret) 1439 return ret; 1440 1441 uc->tchan->t_ring = k3_ringacc_request_ring(ud->ringacc, 1442 uc->tchan->id, 0); 1443 if (!uc->tchan->t_ring) { 1444 ret = -EBUSY; 1445 goto err_tx_ring; 1446 } 1447 1448 uc->tchan->tc_ring = k3_ringacc_request_ring(ud->ringacc, -1, 0); 1449 if (!uc->tchan->tc_ring) { 1450 ret = -EBUSY; 1451 goto err_txc_ring; 1452 } 1453 1454 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1455 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1456 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1457 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1458 1459 ret = k3_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg); 1460 ret |= k3_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg); 1461 1462 if (ret) 1463 goto err_ringcfg; 1464 1465 return 0; 1466 1467 err_ringcfg: 1468 k3_ringacc_ring_free(uc->tchan->tc_ring); 1469 uc->tchan->tc_ring = NULL; 1470 err_txc_ring: 1471 k3_ringacc_ring_free(uc->tchan->t_ring); 1472 uc->tchan->t_ring = NULL; 1473 err_tx_ring: 1474 udma_put_tchan(uc); 1475 1476 return ret; 1477 } 1478 1479 static void udma_free_rx_resources(struct udma_chan *uc) 1480 { 1481 if (!uc->rchan) 1482 return; 1483 1484 if (uc->rflow) { 1485 struct udma_rflow *rflow = uc->rflow; 1486 1487 k3_ringacc_ring_free(rflow->fd_ring); 1488 k3_ringacc_ring_free(rflow->r_ring); 1489 rflow->fd_ring = NULL; 1490 rflow->r_ring = NULL; 1491 1492 udma_put_rflow(uc); 1493 } 1494 1495 udma_put_rchan(uc); 1496 } 1497 1498 static int udma_alloc_rx_resources(struct udma_chan *uc) 1499 { 1500 struct udma_dev *ud = uc->ud; 1501 struct k3_ring_cfg ring_cfg; 1502 struct udma_rflow *rflow; 1503 int fd_ring_id; 1504 int ret; 1505 1506 ret = udma_get_rchan(uc); 1507 if (ret) 1508 return ret; 1509 1510 /* For MEM_TO_MEM we don't need rflow or rings */ 1511 if (uc->config.dir == DMA_MEM_TO_MEM) 1512 return 0; 1513 1514 ret = udma_get_rflow(uc, uc->rchan->id); 1515 if (ret) { 1516 ret = -EBUSY; 1517 goto err_rflow; 1518 } 1519 1520 rflow = uc->rflow; 1521 fd_ring_id = ud->tchan_cnt + ud->echan_cnt + uc->rchan->id; 1522 rflow->fd_ring = k3_ringacc_request_ring(ud->ringacc, fd_ring_id, 0); 1523 if (!rflow->fd_ring) { 1524 ret = -EBUSY; 1525 goto err_rx_ring; 1526 } 1527 1528 rflow->r_ring = k3_ringacc_request_ring(ud->ringacc, -1, 0); 1529 if (!rflow->r_ring) { 1530 ret = -EBUSY; 1531 goto err_rxc_ring; 1532 } 1533 1534 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1535 1536 if (uc->config.pkt_mode) 1537 ring_cfg.size = SG_MAX_SEGMENTS; 1538 else 1539 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1540 1541 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1542 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1543 1544 ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg); 1545 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1546 ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg); 1547 1548 if (ret) 1549 goto err_ringcfg; 1550 1551 return 0; 1552 1553 err_ringcfg: 1554 k3_ringacc_ring_free(rflow->r_ring); 1555 rflow->r_ring = NULL; 1556 err_rxc_ring: 1557 k3_ringacc_ring_free(rflow->fd_ring); 1558 rflow->fd_ring = NULL; 1559 err_rx_ring: 1560 udma_put_rflow(uc); 1561 err_rflow: 1562 udma_put_rchan(uc); 1563 1564 return ret; 1565 } 1566 1567 #define TISCI_TCHAN_VALID_PARAMS ( \ 1568 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1569 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \ 1570 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \ 1571 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1572 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \ 1573 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1574 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1575 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1576 1577 #define TISCI_RCHAN_VALID_PARAMS ( \ 1578 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1579 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1580 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1581 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1582 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \ 1583 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \ 1584 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \ 1585 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \ 1586 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1587 1588 static int udma_tisci_m2m_channel_config(struct udma_chan *uc) 1589 { 1590 struct udma_dev *ud = uc->ud; 1591 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1592 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1593 struct udma_tchan *tchan = uc->tchan; 1594 struct udma_rchan *rchan = uc->rchan; 1595 int ret = 0; 1596 1597 /* Non synchronized - mem to mem type of transfer */ 1598 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1599 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1600 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1601 1602 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; 1603 req_tx.nav_id = tisci_rm->tisci_dev_id; 1604 req_tx.index = tchan->id; 1605 req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1606 req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1607 req_tx.txcq_qnum = tc_ring; 1608 req_tx.tx_atype = ud->atype; 1609 1610 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1611 if (ret) { 1612 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1613 return ret; 1614 } 1615 1616 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; 1617 req_rx.nav_id = tisci_rm->tisci_dev_id; 1618 req_rx.index = rchan->id; 1619 req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1620 req_rx.rxcq_qnum = tc_ring; 1621 req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1622 req_rx.rx_atype = ud->atype; 1623 1624 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1625 if (ret) 1626 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret); 1627 1628 return ret; 1629 } 1630 1631 static int udma_tisci_tx_channel_config(struct udma_chan *uc) 1632 { 1633 struct udma_dev *ud = uc->ud; 1634 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1635 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1636 struct udma_tchan *tchan = uc->tchan; 1637 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1638 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1639 u32 mode, fetch_size; 1640 int ret = 0; 1641 1642 if (uc->config.pkt_mode) { 1643 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1644 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1645 uc->config.psd_size, 0); 1646 } else { 1647 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1648 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1649 } 1650 1651 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; 1652 req_tx.nav_id = tisci_rm->tisci_dev_id; 1653 req_tx.index = tchan->id; 1654 req_tx.tx_chan_type = mode; 1655 req_tx.tx_supr_tdpkt = uc->config.notdpkt; 1656 req_tx.tx_fetch_size = fetch_size >> 2; 1657 req_tx.txcq_qnum = tc_ring; 1658 req_tx.tx_atype = uc->config.atype; 1659 1660 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1661 if (ret) 1662 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1663 1664 return ret; 1665 } 1666 1667 static int udma_tisci_rx_channel_config(struct udma_chan *uc) 1668 { 1669 struct udma_dev *ud = uc->ud; 1670 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1671 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1672 struct udma_rchan *rchan = uc->rchan; 1673 int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring); 1674 int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring); 1675 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1676 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; 1677 u32 mode, fetch_size; 1678 int ret = 0; 1679 1680 if (uc->config.pkt_mode) { 1681 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1682 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1683 uc->config.psd_size, 0); 1684 } else { 1685 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1686 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1687 } 1688 1689 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; 1690 req_rx.nav_id = tisci_rm->tisci_dev_id; 1691 req_rx.index = rchan->id; 1692 req_rx.rx_fetch_size = fetch_size >> 2; 1693 req_rx.rxcq_qnum = rx_ring; 1694 req_rx.rx_chan_type = mode; 1695 req_rx.rx_atype = uc->config.atype; 1696 1697 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1698 if (ret) { 1699 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); 1700 return ret; 1701 } 1702 1703 flow_req.valid_params = 1704 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 1705 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 1706 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 1707 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 1708 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1709 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 1710 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 1711 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 1712 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 1713 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1714 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1715 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1716 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1717 1718 flow_req.nav_id = tisci_rm->tisci_dev_id; 1719 flow_req.flow_index = rchan->id; 1720 1721 if (uc->config.needs_epib) 1722 flow_req.rx_einfo_present = 1; 1723 else 1724 flow_req.rx_einfo_present = 0; 1725 if (uc->config.psd_size) 1726 flow_req.rx_psinfo_present = 1; 1727 else 1728 flow_req.rx_psinfo_present = 0; 1729 flow_req.rx_error_handling = 1; 1730 flow_req.rx_dest_qnum = rx_ring; 1731 flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE; 1732 flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG; 1733 flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI; 1734 flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO; 1735 flow_req.rx_fdq0_sz0_qnum = fd_ring; 1736 flow_req.rx_fdq1_qnum = fd_ring; 1737 flow_req.rx_fdq2_qnum = fd_ring; 1738 flow_req.rx_fdq3_qnum = fd_ring; 1739 1740 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req); 1741 1742 if (ret) 1743 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret); 1744 1745 return 0; 1746 } 1747 1748 static int udma_alloc_chan_resources(struct dma_chan *chan) 1749 { 1750 struct udma_chan *uc = to_udma_chan(chan); 1751 struct udma_dev *ud = to_udma_dev(chan->device); 1752 const struct udma_match_data *match_data = ud->match_data; 1753 struct k3_ring *irq_ring; 1754 u32 irq_udma_idx; 1755 int ret; 1756 1757 if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) { 1758 uc->use_dma_pool = true; 1759 /* in case of MEM_TO_MEM we have maximum of two TRs */ 1760 if (uc->config.dir == DMA_MEM_TO_MEM) { 1761 uc->config.hdesc_size = cppi5_trdesc_calc_size( 1762 sizeof(struct cppi5_tr_type15_t), 2); 1763 uc->config.pkt_mode = false; 1764 } 1765 } 1766 1767 if (uc->use_dma_pool) { 1768 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, 1769 uc->config.hdesc_size, 1770 ud->desc_align, 1771 0); 1772 if (!uc->hdesc_pool) { 1773 dev_err(ud->ddev.dev, 1774 "Descriptor pool allocation failed\n"); 1775 uc->use_dma_pool = false; 1776 return -ENOMEM; 1777 } 1778 } 1779 1780 /* 1781 * Make sure that the completion is in a known state: 1782 * No teardown, the channel is idle 1783 */ 1784 reinit_completion(&uc->teardown_completed); 1785 complete_all(&uc->teardown_completed); 1786 uc->state = UDMA_CHAN_IS_IDLE; 1787 1788 switch (uc->config.dir) { 1789 case DMA_MEM_TO_MEM: 1790 /* Non synchronized - mem to mem type of transfer */ 1791 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, 1792 uc->id); 1793 1794 ret = udma_get_chan_pair(uc); 1795 if (ret) 1796 return ret; 1797 1798 ret = udma_alloc_tx_resources(uc); 1799 if (ret) 1800 return ret; 1801 1802 ret = udma_alloc_rx_resources(uc); 1803 if (ret) { 1804 udma_free_tx_resources(uc); 1805 return ret; 1806 } 1807 1808 uc->config.src_thread = ud->psil_base + uc->tchan->id; 1809 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 1810 K3_PSIL_DST_THREAD_ID_OFFSET; 1811 1812 irq_ring = uc->tchan->tc_ring; 1813 irq_udma_idx = uc->tchan->id; 1814 1815 ret = udma_tisci_m2m_channel_config(uc); 1816 break; 1817 case DMA_MEM_TO_DEV: 1818 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 1819 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 1820 uc->id); 1821 1822 ret = udma_alloc_tx_resources(uc); 1823 if (ret) { 1824 uc->config.remote_thread_id = -1; 1825 return ret; 1826 } 1827 1828 uc->config.src_thread = ud->psil_base + uc->tchan->id; 1829 uc->config.dst_thread = uc->config.remote_thread_id; 1830 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 1831 1832 irq_ring = uc->tchan->tc_ring; 1833 irq_udma_idx = uc->tchan->id; 1834 1835 ret = udma_tisci_tx_channel_config(uc); 1836 break; 1837 case DMA_DEV_TO_MEM: 1838 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 1839 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 1840 uc->id); 1841 1842 ret = udma_alloc_rx_resources(uc); 1843 if (ret) { 1844 uc->config.remote_thread_id = -1; 1845 return ret; 1846 } 1847 1848 uc->config.src_thread = uc->config.remote_thread_id; 1849 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 1850 K3_PSIL_DST_THREAD_ID_OFFSET; 1851 1852 irq_ring = uc->rflow->r_ring; 1853 irq_udma_idx = match_data->rchan_oes_offset + uc->rchan->id; 1854 1855 ret = udma_tisci_rx_channel_config(uc); 1856 break; 1857 default: 1858 /* Can not happen */ 1859 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 1860 __func__, uc->id, uc->config.dir); 1861 return -EINVAL; 1862 } 1863 1864 /* check if the channel configuration was successful */ 1865 if (ret) 1866 goto err_res_free; 1867 1868 if (udma_is_chan_running(uc)) { 1869 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 1870 udma_stop(uc); 1871 if (udma_is_chan_running(uc)) { 1872 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 1873 goto err_res_free; 1874 } 1875 } 1876 1877 /* PSI-L pairing */ 1878 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); 1879 if (ret) { 1880 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 1881 uc->config.src_thread, uc->config.dst_thread); 1882 goto err_res_free; 1883 } 1884 1885 uc->psil_paired = true; 1886 1887 uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring); 1888 if (uc->irq_num_ring <= 0) { 1889 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 1890 k3_ringacc_get_ring_id(irq_ring)); 1891 ret = -EINVAL; 1892 goto err_psi_free; 1893 } 1894 1895 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 1896 IRQF_TRIGGER_HIGH, uc->name, uc); 1897 if (ret) { 1898 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 1899 goto err_irq_free; 1900 } 1901 1902 /* Event from UDMA (TR events) only needed for slave TR mode channels */ 1903 if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) { 1904 uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, 1905 irq_udma_idx); 1906 if (uc->irq_num_udma <= 0) { 1907 dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", 1908 irq_udma_idx); 1909 free_irq(uc->irq_num_ring, uc); 1910 ret = -EINVAL; 1911 goto err_irq_free; 1912 } 1913 1914 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0, 1915 uc->name, uc); 1916 if (ret) { 1917 dev_err(ud->dev, "chan%d: UDMA irq request failed\n", 1918 uc->id); 1919 free_irq(uc->irq_num_ring, uc); 1920 goto err_irq_free; 1921 } 1922 } else { 1923 uc->irq_num_udma = 0; 1924 } 1925 1926 udma_reset_rings(uc); 1927 1928 INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work, 1929 udma_check_tx_completion); 1930 return 0; 1931 1932 err_irq_free: 1933 uc->irq_num_ring = 0; 1934 uc->irq_num_udma = 0; 1935 err_psi_free: 1936 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); 1937 uc->psil_paired = false; 1938 err_res_free: 1939 udma_free_tx_resources(uc); 1940 udma_free_rx_resources(uc); 1941 1942 udma_reset_uchan(uc); 1943 1944 if (uc->use_dma_pool) { 1945 dma_pool_destroy(uc->hdesc_pool); 1946 uc->use_dma_pool = false; 1947 } 1948 1949 return ret; 1950 } 1951 1952 static int udma_slave_config(struct dma_chan *chan, 1953 struct dma_slave_config *cfg) 1954 { 1955 struct udma_chan *uc = to_udma_chan(chan); 1956 1957 memcpy(&uc->cfg, cfg, sizeof(uc->cfg)); 1958 1959 return 0; 1960 } 1961 1962 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc, 1963 size_t tr_size, int tr_count, 1964 enum dma_transfer_direction dir) 1965 { 1966 struct udma_hwdesc *hwdesc; 1967 struct cppi5_desc_hdr_t *tr_desc; 1968 struct udma_desc *d; 1969 u32 reload_count = 0; 1970 u32 ring_id; 1971 1972 switch (tr_size) { 1973 case 16: 1974 case 32: 1975 case 64: 1976 case 128: 1977 break; 1978 default: 1979 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size); 1980 return NULL; 1981 } 1982 1983 /* We have only one descriptor containing multiple TRs */ 1984 d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT); 1985 if (!d) 1986 return NULL; 1987 1988 d->sglen = tr_count; 1989 1990 d->hwdesc_count = 1; 1991 hwdesc = &d->hwdesc[0]; 1992 1993 /* Allocate memory for DMA ring descriptor */ 1994 if (uc->use_dma_pool) { 1995 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 1996 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 1997 GFP_NOWAIT, 1998 &hwdesc->cppi5_desc_paddr); 1999 } else { 2000 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 2001 tr_count); 2002 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 2003 uc->ud->desc_align); 2004 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev, 2005 hwdesc->cppi5_desc_size, 2006 &hwdesc->cppi5_desc_paddr, 2007 GFP_NOWAIT); 2008 } 2009 2010 if (!hwdesc->cppi5_desc_vaddr) { 2011 kfree(d); 2012 return NULL; 2013 } 2014 2015 /* Start of the TR req records */ 2016 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 2017 /* Start address of the TR response array */ 2018 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count; 2019 2020 tr_desc = hwdesc->cppi5_desc_vaddr; 2021 2022 if (uc->cyclic) 2023 reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE; 2024 2025 if (dir == DMA_DEV_TO_MEM) 2026 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2027 else 2028 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2029 2030 cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count); 2031 cppi5_desc_set_pktids(tr_desc, uc->id, 2032 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2033 cppi5_desc_set_retpolicy(tr_desc, 0, ring_id); 2034 2035 return d; 2036 } 2037 2038 /** 2039 * udma_get_tr_counters - calculate TR counters for a given length 2040 * @len: Length of the trasnfer 2041 * @align_to: Preferred alignment 2042 * @tr0_cnt0: First TR icnt0 2043 * @tr0_cnt1: First TR icnt1 2044 * @tr1_cnt0: Second (if used) TR icnt0 2045 * 2046 * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated 2047 * For len >= SZ_64K two TRs are used in a simple way: 2048 * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1) 2049 * Second TR: the remaining length (tr1_cnt0) 2050 * 2051 * Returns the number of TRs the length needs (1 or 2) 2052 * -EINVAL if the length can not be supported 2053 */ 2054 static int udma_get_tr_counters(size_t len, unsigned long align_to, 2055 u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0) 2056 { 2057 if (len < SZ_64K) { 2058 *tr0_cnt0 = len; 2059 *tr0_cnt1 = 1; 2060 2061 return 1; 2062 } 2063 2064 if (align_to > 3) 2065 align_to = 3; 2066 2067 realign: 2068 *tr0_cnt0 = SZ_64K - BIT(align_to); 2069 if (len / *tr0_cnt0 >= SZ_64K) { 2070 if (align_to) { 2071 align_to--; 2072 goto realign; 2073 } 2074 return -EINVAL; 2075 } 2076 2077 *tr0_cnt1 = len / *tr0_cnt0; 2078 *tr1_cnt0 = len % *tr0_cnt0; 2079 2080 return 2; 2081 } 2082 2083 static struct udma_desc * 2084 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, 2085 unsigned int sglen, enum dma_transfer_direction dir, 2086 unsigned long tx_flags, void *context) 2087 { 2088 struct scatterlist *sgent; 2089 struct udma_desc *d; 2090 struct cppi5_tr_type1_t *tr_req = NULL; 2091 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2092 unsigned int i; 2093 size_t tr_size; 2094 int num_tr = 0; 2095 int tr_idx = 0; 2096 2097 if (!is_slave_direction(dir)) { 2098 dev_err(uc->ud->dev, "Only slave cyclic is supported\n"); 2099 return NULL; 2100 } 2101 2102 /* estimate the number of TRs we will need */ 2103 for_each_sg(sgl, sgent, sglen, i) { 2104 if (sg_dma_len(sgent) < SZ_64K) 2105 num_tr++; 2106 else 2107 num_tr += 2; 2108 } 2109 2110 /* Now allocate and setup the descriptor. */ 2111 tr_size = sizeof(struct cppi5_tr_type1_t); 2112 d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir); 2113 if (!d) 2114 return NULL; 2115 2116 d->sglen = sglen; 2117 2118 tr_req = d->hwdesc[0].tr_req_base; 2119 for_each_sg(sgl, sgent, sglen, i) { 2120 dma_addr_t sg_addr = sg_dma_address(sgent); 2121 2122 num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr), 2123 &tr0_cnt0, &tr0_cnt1, &tr1_cnt0); 2124 if (num_tr < 0) { 2125 dev_err(uc->ud->dev, "size %u is not supported\n", 2126 sg_dma_len(sgent)); 2127 udma_free_hwdesc(uc, d); 2128 kfree(d); 2129 return NULL; 2130 } 2131 2132 cppi5_tr_init(&tr_req[i].flags, CPPI5_TR_TYPE1, false, false, 2133 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2134 cppi5_tr_csf_set(&tr_req[i].flags, CPPI5_TR_CSF_SUPR_EVT); 2135 2136 tr_req[tr_idx].addr = sg_addr; 2137 tr_req[tr_idx].icnt0 = tr0_cnt0; 2138 tr_req[tr_idx].icnt1 = tr0_cnt1; 2139 tr_req[tr_idx].dim1 = tr0_cnt0; 2140 tr_idx++; 2141 2142 if (num_tr == 2) { 2143 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 2144 false, false, 2145 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2146 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2147 CPPI5_TR_CSF_SUPR_EVT); 2148 2149 tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0; 2150 tr_req[tr_idx].icnt0 = tr1_cnt0; 2151 tr_req[tr_idx].icnt1 = 1; 2152 tr_req[tr_idx].dim1 = tr1_cnt0; 2153 tr_idx++; 2154 } 2155 2156 d->residue += sg_dma_len(sgent); 2157 } 2158 2159 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, CPPI5_TR_CSF_EOP); 2160 2161 return d; 2162 } 2163 2164 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d, 2165 enum dma_slave_buswidth dev_width, 2166 u16 elcnt) 2167 { 2168 if (uc->config.ep_type != PSIL_EP_PDMA_XY) 2169 return 0; 2170 2171 /* Bus width translates to the element size (ES) */ 2172 switch (dev_width) { 2173 case DMA_SLAVE_BUSWIDTH_1_BYTE: 2174 d->static_tr.elsize = 0; 2175 break; 2176 case DMA_SLAVE_BUSWIDTH_2_BYTES: 2177 d->static_tr.elsize = 1; 2178 break; 2179 case DMA_SLAVE_BUSWIDTH_3_BYTES: 2180 d->static_tr.elsize = 2; 2181 break; 2182 case DMA_SLAVE_BUSWIDTH_4_BYTES: 2183 d->static_tr.elsize = 3; 2184 break; 2185 case DMA_SLAVE_BUSWIDTH_8_BYTES: 2186 d->static_tr.elsize = 4; 2187 break; 2188 default: /* not reached */ 2189 return -EINVAL; 2190 } 2191 2192 d->static_tr.elcnt = elcnt; 2193 2194 /* 2195 * PDMA must to close the packet when the channel is in packet mode. 2196 * For TR mode when the channel is not cyclic we also need PDMA to close 2197 * the packet otherwise the transfer will stall because PDMA holds on 2198 * the data it has received from the peripheral. 2199 */ 2200 if (uc->config.pkt_mode || !uc->cyclic) { 2201 unsigned int div = dev_width * elcnt; 2202 2203 if (uc->cyclic) 2204 d->static_tr.bstcnt = d->residue / d->sglen / div; 2205 else 2206 d->static_tr.bstcnt = d->residue / div; 2207 2208 if (uc->config.dir == DMA_DEV_TO_MEM && 2209 d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) 2210 return -EINVAL; 2211 } else { 2212 d->static_tr.bstcnt = 0; 2213 } 2214 2215 return 0; 2216 } 2217 2218 static struct udma_desc * 2219 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, 2220 unsigned int sglen, enum dma_transfer_direction dir, 2221 unsigned long tx_flags, void *context) 2222 { 2223 struct scatterlist *sgent; 2224 struct cppi5_host_desc_t *h_desc = NULL; 2225 struct udma_desc *d; 2226 u32 ring_id; 2227 unsigned int i; 2228 2229 d = kzalloc(sizeof(*d) + sglen * sizeof(d->hwdesc[0]), GFP_NOWAIT); 2230 if (!d) 2231 return NULL; 2232 2233 d->sglen = sglen; 2234 d->hwdesc_count = sglen; 2235 2236 if (dir == DMA_DEV_TO_MEM) 2237 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2238 else 2239 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2240 2241 for_each_sg(sgl, sgent, sglen, i) { 2242 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 2243 dma_addr_t sg_addr = sg_dma_address(sgent); 2244 struct cppi5_host_desc_t *desc; 2245 size_t sg_len = sg_dma_len(sgent); 2246 2247 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2248 GFP_NOWAIT, 2249 &hwdesc->cppi5_desc_paddr); 2250 if (!hwdesc->cppi5_desc_vaddr) { 2251 dev_err(uc->ud->dev, 2252 "descriptor%d allocation failed\n", i); 2253 2254 udma_free_hwdesc(uc, d); 2255 kfree(d); 2256 return NULL; 2257 } 2258 2259 d->residue += sg_len; 2260 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2261 desc = hwdesc->cppi5_desc_vaddr; 2262 2263 if (i == 0) { 2264 cppi5_hdesc_init(desc, 0, 0); 2265 /* Flow and Packed ID */ 2266 cppi5_desc_set_pktids(&desc->hdr, uc->id, 2267 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2268 cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id); 2269 } else { 2270 cppi5_hdesc_reset_hbdesc(desc); 2271 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff); 2272 } 2273 2274 /* attach the sg buffer to the descriptor */ 2275 cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len); 2276 2277 /* Attach link as host buffer descriptor */ 2278 if (h_desc) 2279 cppi5_hdesc_link_hbdesc(h_desc, 2280 hwdesc->cppi5_desc_paddr); 2281 2282 if (dir == DMA_MEM_TO_DEV) 2283 h_desc = desc; 2284 } 2285 2286 if (d->residue >= SZ_4M) { 2287 dev_err(uc->ud->dev, 2288 "%s: Transfer size %u is over the supported 4M range\n", 2289 __func__, d->residue); 2290 udma_free_hwdesc(uc, d); 2291 kfree(d); 2292 return NULL; 2293 } 2294 2295 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2296 cppi5_hdesc_set_pktlen(h_desc, d->residue); 2297 2298 return d; 2299 } 2300 2301 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc, 2302 void *data, size_t len) 2303 { 2304 struct udma_desc *d = to_udma_desc(desc); 2305 struct udma_chan *uc = to_udma_chan(desc->chan); 2306 struct cppi5_host_desc_t *h_desc; 2307 u32 psd_size = len; 2308 u32 flags = 0; 2309 2310 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2311 return -ENOTSUPP; 2312 2313 if (!data || len > uc->config.metadata_size) 2314 return -EINVAL; 2315 2316 if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE) 2317 return -EINVAL; 2318 2319 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2320 if (d->dir == DMA_MEM_TO_DEV) 2321 memcpy(h_desc->epib, data, len); 2322 2323 if (uc->config.needs_epib) 2324 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 2325 2326 d->metadata = data; 2327 d->metadata_size = len; 2328 if (uc->config.needs_epib) 2329 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 2330 2331 cppi5_hdesc_update_flags(h_desc, flags); 2332 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 2333 2334 return 0; 2335 } 2336 2337 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc, 2338 size_t *payload_len, size_t *max_len) 2339 { 2340 struct udma_desc *d = to_udma_desc(desc); 2341 struct udma_chan *uc = to_udma_chan(desc->chan); 2342 struct cppi5_host_desc_t *h_desc; 2343 2344 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2345 return ERR_PTR(-ENOTSUPP); 2346 2347 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2348 2349 *max_len = uc->config.metadata_size; 2350 2351 *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ? 2352 CPPI5_INFO0_HDESC_EPIB_SIZE : 0; 2353 *payload_len += cppi5_hdesc_get_psdata_size(h_desc); 2354 2355 return h_desc->epib; 2356 } 2357 2358 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc, 2359 size_t payload_len) 2360 { 2361 struct udma_desc *d = to_udma_desc(desc); 2362 struct udma_chan *uc = to_udma_chan(desc->chan); 2363 struct cppi5_host_desc_t *h_desc; 2364 u32 psd_size = payload_len; 2365 u32 flags = 0; 2366 2367 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2368 return -ENOTSUPP; 2369 2370 if (payload_len > uc->config.metadata_size) 2371 return -EINVAL; 2372 2373 if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE) 2374 return -EINVAL; 2375 2376 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2377 2378 if (uc->config.needs_epib) { 2379 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 2380 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 2381 } 2382 2383 cppi5_hdesc_update_flags(h_desc, flags); 2384 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 2385 2386 return 0; 2387 } 2388 2389 static struct dma_descriptor_metadata_ops metadata_ops = { 2390 .attach = udma_attach_metadata, 2391 .get_ptr = udma_get_metadata_ptr, 2392 .set_len = udma_set_metadata_len, 2393 }; 2394 2395 static struct dma_async_tx_descriptor * 2396 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 2397 unsigned int sglen, enum dma_transfer_direction dir, 2398 unsigned long tx_flags, void *context) 2399 { 2400 struct udma_chan *uc = to_udma_chan(chan); 2401 enum dma_slave_buswidth dev_width; 2402 struct udma_desc *d; 2403 u32 burst; 2404 2405 if (dir != uc->config.dir) { 2406 dev_err(chan->device->dev, 2407 "%s: chan%d is for %s, not supporting %s\n", 2408 __func__, uc->id, 2409 dmaengine_get_direction_text(uc->config.dir), 2410 dmaengine_get_direction_text(dir)); 2411 return NULL; 2412 } 2413 2414 if (dir == DMA_DEV_TO_MEM) { 2415 dev_width = uc->cfg.src_addr_width; 2416 burst = uc->cfg.src_maxburst; 2417 } else if (dir == DMA_MEM_TO_DEV) { 2418 dev_width = uc->cfg.dst_addr_width; 2419 burst = uc->cfg.dst_maxburst; 2420 } else { 2421 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 2422 return NULL; 2423 } 2424 2425 if (!burst) 2426 burst = 1; 2427 2428 if (uc->config.pkt_mode) 2429 d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags, 2430 context); 2431 else 2432 d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags, 2433 context); 2434 2435 if (!d) 2436 return NULL; 2437 2438 d->dir = dir; 2439 d->desc_idx = 0; 2440 d->tr_idx = 0; 2441 2442 /* static TR for remote PDMA */ 2443 if (udma_configure_statictr(uc, d, dev_width, burst)) { 2444 dev_err(uc->ud->dev, 2445 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 2446 __func__, d->static_tr.bstcnt); 2447 2448 udma_free_hwdesc(uc, d); 2449 kfree(d); 2450 return NULL; 2451 } 2452 2453 if (uc->config.metadata_size) 2454 d->vd.tx.metadata_ops = &metadata_ops; 2455 2456 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 2457 } 2458 2459 static struct udma_desc * 2460 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, 2461 size_t buf_len, size_t period_len, 2462 enum dma_transfer_direction dir, unsigned long flags) 2463 { 2464 struct udma_desc *d; 2465 size_t tr_size, period_addr; 2466 struct cppi5_tr_type1_t *tr_req; 2467 unsigned int periods = buf_len / period_len; 2468 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2469 unsigned int i; 2470 int num_tr; 2471 2472 if (!is_slave_direction(dir)) { 2473 dev_err(uc->ud->dev, "Only slave cyclic is supported\n"); 2474 return NULL; 2475 } 2476 2477 num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0, 2478 &tr0_cnt1, &tr1_cnt0); 2479 if (num_tr < 0) { 2480 dev_err(uc->ud->dev, "size %zu is not supported\n", 2481 period_len); 2482 return NULL; 2483 } 2484 2485 /* Now allocate and setup the descriptor. */ 2486 tr_size = sizeof(struct cppi5_tr_type1_t); 2487 d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir); 2488 if (!d) 2489 return NULL; 2490 2491 tr_req = d->hwdesc[0].tr_req_base; 2492 period_addr = buf_addr; 2493 for (i = 0; i < periods; i++) { 2494 int tr_idx = i * num_tr; 2495 2496 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false, 2497 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2498 2499 tr_req[tr_idx].addr = period_addr; 2500 tr_req[tr_idx].icnt0 = tr0_cnt0; 2501 tr_req[tr_idx].icnt1 = tr0_cnt1; 2502 tr_req[tr_idx].dim1 = tr0_cnt0; 2503 2504 if (num_tr == 2) { 2505 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2506 CPPI5_TR_CSF_SUPR_EVT); 2507 tr_idx++; 2508 2509 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 2510 false, false, 2511 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2512 2513 tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0; 2514 tr_req[tr_idx].icnt0 = tr1_cnt0; 2515 tr_req[tr_idx].icnt1 = 1; 2516 tr_req[tr_idx].dim1 = tr1_cnt0; 2517 } 2518 2519 if (!(flags & DMA_PREP_INTERRUPT)) 2520 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2521 CPPI5_TR_CSF_SUPR_EVT); 2522 2523 period_addr += period_len; 2524 } 2525 2526 return d; 2527 } 2528 2529 static struct udma_desc * 2530 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr, 2531 size_t buf_len, size_t period_len, 2532 enum dma_transfer_direction dir, unsigned long flags) 2533 { 2534 struct udma_desc *d; 2535 u32 ring_id; 2536 int i; 2537 int periods = buf_len / period_len; 2538 2539 if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1)) 2540 return NULL; 2541 2542 if (period_len >= SZ_4M) 2543 return NULL; 2544 2545 d = kzalloc(sizeof(*d) + periods * sizeof(d->hwdesc[0]), GFP_NOWAIT); 2546 if (!d) 2547 return NULL; 2548 2549 d->hwdesc_count = periods; 2550 2551 /* TODO: re-check this... */ 2552 if (dir == DMA_DEV_TO_MEM) 2553 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2554 else 2555 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2556 2557 for (i = 0; i < periods; i++) { 2558 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 2559 dma_addr_t period_addr = buf_addr + (period_len * i); 2560 struct cppi5_host_desc_t *h_desc; 2561 2562 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2563 GFP_NOWAIT, 2564 &hwdesc->cppi5_desc_paddr); 2565 if (!hwdesc->cppi5_desc_vaddr) { 2566 dev_err(uc->ud->dev, 2567 "descriptor%d allocation failed\n", i); 2568 2569 udma_free_hwdesc(uc, d); 2570 kfree(d); 2571 return NULL; 2572 } 2573 2574 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2575 h_desc = hwdesc->cppi5_desc_vaddr; 2576 2577 cppi5_hdesc_init(h_desc, 0, 0); 2578 cppi5_hdesc_set_pktlen(h_desc, period_len); 2579 2580 /* Flow and Packed ID */ 2581 cppi5_desc_set_pktids(&h_desc->hdr, uc->id, 2582 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2583 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id); 2584 2585 /* attach each period to a new descriptor */ 2586 cppi5_hdesc_attach_buf(h_desc, 2587 period_addr, period_len, 2588 period_addr, period_len); 2589 } 2590 2591 return d; 2592 } 2593 2594 static struct dma_async_tx_descriptor * 2595 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 2596 size_t period_len, enum dma_transfer_direction dir, 2597 unsigned long flags) 2598 { 2599 struct udma_chan *uc = to_udma_chan(chan); 2600 enum dma_slave_buswidth dev_width; 2601 struct udma_desc *d; 2602 u32 burst; 2603 2604 if (dir != uc->config.dir) { 2605 dev_err(chan->device->dev, 2606 "%s: chan%d is for %s, not supporting %s\n", 2607 __func__, uc->id, 2608 dmaengine_get_direction_text(uc->config.dir), 2609 dmaengine_get_direction_text(dir)); 2610 return NULL; 2611 } 2612 2613 uc->cyclic = true; 2614 2615 if (dir == DMA_DEV_TO_MEM) { 2616 dev_width = uc->cfg.src_addr_width; 2617 burst = uc->cfg.src_maxburst; 2618 } else if (dir == DMA_MEM_TO_DEV) { 2619 dev_width = uc->cfg.dst_addr_width; 2620 burst = uc->cfg.dst_maxburst; 2621 } else { 2622 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 2623 return NULL; 2624 } 2625 2626 if (!burst) 2627 burst = 1; 2628 2629 if (uc->config.pkt_mode) 2630 d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len, 2631 dir, flags); 2632 else 2633 d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len, 2634 dir, flags); 2635 2636 if (!d) 2637 return NULL; 2638 2639 d->sglen = buf_len / period_len; 2640 2641 d->dir = dir; 2642 d->residue = buf_len; 2643 2644 /* static TR for remote PDMA */ 2645 if (udma_configure_statictr(uc, d, dev_width, burst)) { 2646 dev_err(uc->ud->dev, 2647 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 2648 __func__, d->static_tr.bstcnt); 2649 2650 udma_free_hwdesc(uc, d); 2651 kfree(d); 2652 return NULL; 2653 } 2654 2655 if (uc->config.metadata_size) 2656 d->vd.tx.metadata_ops = &metadata_ops; 2657 2658 return vchan_tx_prep(&uc->vc, &d->vd, flags); 2659 } 2660 2661 static struct dma_async_tx_descriptor * 2662 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 2663 size_t len, unsigned long tx_flags) 2664 { 2665 struct udma_chan *uc = to_udma_chan(chan); 2666 struct udma_desc *d; 2667 struct cppi5_tr_type15_t *tr_req; 2668 int num_tr; 2669 size_t tr_size = sizeof(struct cppi5_tr_type15_t); 2670 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2671 2672 if (uc->config.dir != DMA_MEM_TO_MEM) { 2673 dev_err(chan->device->dev, 2674 "%s: chan%d is for %s, not supporting %s\n", 2675 __func__, uc->id, 2676 dmaengine_get_direction_text(uc->config.dir), 2677 dmaengine_get_direction_text(DMA_MEM_TO_MEM)); 2678 return NULL; 2679 } 2680 2681 num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0, 2682 &tr0_cnt1, &tr1_cnt0); 2683 if (num_tr < 0) { 2684 dev_err(uc->ud->dev, "size %zu is not supported\n", 2685 len); 2686 return NULL; 2687 } 2688 2689 d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM); 2690 if (!d) 2691 return NULL; 2692 2693 d->dir = DMA_MEM_TO_MEM; 2694 d->desc_idx = 0; 2695 d->tr_idx = 0; 2696 d->residue = len; 2697 2698 tr_req = d->hwdesc[0].tr_req_base; 2699 2700 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true, 2701 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2702 cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT); 2703 2704 tr_req[0].addr = src; 2705 tr_req[0].icnt0 = tr0_cnt0; 2706 tr_req[0].icnt1 = tr0_cnt1; 2707 tr_req[0].icnt2 = 1; 2708 tr_req[0].icnt3 = 1; 2709 tr_req[0].dim1 = tr0_cnt0; 2710 2711 tr_req[0].daddr = dest; 2712 tr_req[0].dicnt0 = tr0_cnt0; 2713 tr_req[0].dicnt1 = tr0_cnt1; 2714 tr_req[0].dicnt2 = 1; 2715 tr_req[0].dicnt3 = 1; 2716 tr_req[0].ddim1 = tr0_cnt0; 2717 2718 if (num_tr == 2) { 2719 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true, 2720 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2721 cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT); 2722 2723 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0; 2724 tr_req[1].icnt0 = tr1_cnt0; 2725 tr_req[1].icnt1 = 1; 2726 tr_req[1].icnt2 = 1; 2727 tr_req[1].icnt3 = 1; 2728 2729 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0; 2730 tr_req[1].dicnt0 = tr1_cnt0; 2731 tr_req[1].dicnt1 = 1; 2732 tr_req[1].dicnt2 = 1; 2733 tr_req[1].dicnt3 = 1; 2734 } 2735 2736 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, CPPI5_TR_CSF_EOP); 2737 2738 if (uc->config.metadata_size) 2739 d->vd.tx.metadata_ops = &metadata_ops; 2740 2741 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 2742 } 2743 2744 static void udma_issue_pending(struct dma_chan *chan) 2745 { 2746 struct udma_chan *uc = to_udma_chan(chan); 2747 unsigned long flags; 2748 2749 spin_lock_irqsave(&uc->vc.lock, flags); 2750 2751 /* If we have something pending and no active descriptor, then */ 2752 if (vchan_issue_pending(&uc->vc) && !uc->desc) { 2753 /* 2754 * start a descriptor if the channel is NOT [marked as 2755 * terminating _and_ it is still running (teardown has not 2756 * completed yet)]. 2757 */ 2758 if (!(uc->state == UDMA_CHAN_IS_TERMINATING && 2759 udma_is_chan_running(uc))) 2760 udma_start(uc); 2761 } 2762 2763 spin_unlock_irqrestore(&uc->vc.lock, flags); 2764 } 2765 2766 static enum dma_status udma_tx_status(struct dma_chan *chan, 2767 dma_cookie_t cookie, 2768 struct dma_tx_state *txstate) 2769 { 2770 struct udma_chan *uc = to_udma_chan(chan); 2771 enum dma_status ret; 2772 unsigned long flags; 2773 2774 spin_lock_irqsave(&uc->vc.lock, flags); 2775 2776 ret = dma_cookie_status(chan, cookie, txstate); 2777 2778 if (!udma_is_chan_running(uc)) 2779 ret = DMA_COMPLETE; 2780 2781 if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc)) 2782 ret = DMA_PAUSED; 2783 2784 if (ret == DMA_COMPLETE || !txstate) 2785 goto out; 2786 2787 if (uc->desc && uc->desc->vd.tx.cookie == cookie) { 2788 u32 peer_bcnt = 0; 2789 u32 bcnt = 0; 2790 u32 residue = uc->desc->residue; 2791 u32 delay = 0; 2792 2793 if (uc->desc->dir == DMA_MEM_TO_DEV) { 2794 bcnt = udma_tchanrt_read(uc->tchan, 2795 UDMA_TCHAN_RT_SBCNT_REG); 2796 2797 if (uc->config.ep_type != PSIL_EP_NATIVE) { 2798 peer_bcnt = udma_tchanrt_read(uc->tchan, 2799 UDMA_TCHAN_RT_PEER_BCNT_REG); 2800 2801 if (bcnt > peer_bcnt) 2802 delay = bcnt - peer_bcnt; 2803 } 2804 } else if (uc->desc->dir == DMA_DEV_TO_MEM) { 2805 bcnt = udma_rchanrt_read(uc->rchan, 2806 UDMA_RCHAN_RT_BCNT_REG); 2807 2808 if (uc->config.ep_type != PSIL_EP_NATIVE) { 2809 peer_bcnt = udma_rchanrt_read(uc->rchan, 2810 UDMA_RCHAN_RT_PEER_BCNT_REG); 2811 2812 if (peer_bcnt > bcnt) 2813 delay = peer_bcnt - bcnt; 2814 } 2815 } else { 2816 bcnt = udma_tchanrt_read(uc->tchan, 2817 UDMA_TCHAN_RT_BCNT_REG); 2818 } 2819 2820 bcnt -= uc->bcnt; 2821 if (bcnt && !(bcnt % uc->desc->residue)) 2822 residue = 0; 2823 else 2824 residue -= bcnt % uc->desc->residue; 2825 2826 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) { 2827 ret = DMA_COMPLETE; 2828 delay = 0; 2829 } 2830 2831 dma_set_residue(txstate, residue); 2832 dma_set_in_flight_bytes(txstate, delay); 2833 2834 } else { 2835 ret = DMA_COMPLETE; 2836 } 2837 2838 out: 2839 spin_unlock_irqrestore(&uc->vc.lock, flags); 2840 return ret; 2841 } 2842 2843 static int udma_pause(struct dma_chan *chan) 2844 { 2845 struct udma_chan *uc = to_udma_chan(chan); 2846 2847 /* pause the channel */ 2848 switch (uc->config.dir) { 2849 case DMA_DEV_TO_MEM: 2850 udma_rchanrt_update_bits(uc->rchan, 2851 UDMA_RCHAN_RT_PEER_RT_EN_REG, 2852 UDMA_PEER_RT_EN_PAUSE, 2853 UDMA_PEER_RT_EN_PAUSE); 2854 break; 2855 case DMA_MEM_TO_DEV: 2856 udma_tchanrt_update_bits(uc->tchan, 2857 UDMA_TCHAN_RT_PEER_RT_EN_REG, 2858 UDMA_PEER_RT_EN_PAUSE, 2859 UDMA_PEER_RT_EN_PAUSE); 2860 break; 2861 case DMA_MEM_TO_MEM: 2862 udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 2863 UDMA_CHAN_RT_CTL_PAUSE, 2864 UDMA_CHAN_RT_CTL_PAUSE); 2865 break; 2866 default: 2867 return -EINVAL; 2868 } 2869 2870 return 0; 2871 } 2872 2873 static int udma_resume(struct dma_chan *chan) 2874 { 2875 struct udma_chan *uc = to_udma_chan(chan); 2876 2877 /* resume the channel */ 2878 switch (uc->config.dir) { 2879 case DMA_DEV_TO_MEM: 2880 udma_rchanrt_update_bits(uc->rchan, 2881 UDMA_RCHAN_RT_PEER_RT_EN_REG, 2882 UDMA_PEER_RT_EN_PAUSE, 0); 2883 2884 break; 2885 case DMA_MEM_TO_DEV: 2886 udma_tchanrt_update_bits(uc->tchan, 2887 UDMA_TCHAN_RT_PEER_RT_EN_REG, 2888 UDMA_PEER_RT_EN_PAUSE, 0); 2889 break; 2890 case DMA_MEM_TO_MEM: 2891 udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 2892 UDMA_CHAN_RT_CTL_PAUSE, 0); 2893 break; 2894 default: 2895 return -EINVAL; 2896 } 2897 2898 return 0; 2899 } 2900 2901 static int udma_terminate_all(struct dma_chan *chan) 2902 { 2903 struct udma_chan *uc = to_udma_chan(chan); 2904 unsigned long flags; 2905 LIST_HEAD(head); 2906 2907 spin_lock_irqsave(&uc->vc.lock, flags); 2908 2909 if (udma_is_chan_running(uc)) 2910 udma_stop(uc); 2911 2912 if (uc->desc) { 2913 uc->terminated_desc = uc->desc; 2914 uc->desc = NULL; 2915 uc->terminated_desc->terminated = true; 2916 cancel_delayed_work(&uc->tx_drain.work); 2917 } 2918 2919 uc->paused = false; 2920 2921 vchan_get_all_descriptors(&uc->vc, &head); 2922 spin_unlock_irqrestore(&uc->vc.lock, flags); 2923 vchan_dma_desc_free_list(&uc->vc, &head); 2924 2925 return 0; 2926 } 2927 2928 static void udma_synchronize(struct dma_chan *chan) 2929 { 2930 struct udma_chan *uc = to_udma_chan(chan); 2931 unsigned long timeout = msecs_to_jiffies(1000); 2932 2933 vchan_synchronize(&uc->vc); 2934 2935 if (uc->state == UDMA_CHAN_IS_TERMINATING) { 2936 timeout = wait_for_completion_timeout(&uc->teardown_completed, 2937 timeout); 2938 if (!timeout) { 2939 dev_warn(uc->ud->dev, "chan%d teardown timeout!\n", 2940 uc->id); 2941 udma_dump_chan_stdata(uc); 2942 udma_reset_chan(uc, true); 2943 } 2944 } 2945 2946 udma_reset_chan(uc, false); 2947 if (udma_is_chan_running(uc)) 2948 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id); 2949 2950 cancel_delayed_work_sync(&uc->tx_drain.work); 2951 udma_reset_rings(uc); 2952 } 2953 2954 static void udma_desc_pre_callback(struct virt_dma_chan *vc, 2955 struct virt_dma_desc *vd, 2956 struct dmaengine_result *result) 2957 { 2958 struct udma_chan *uc = to_udma_chan(&vc->chan); 2959 struct udma_desc *d; 2960 2961 if (!vd) 2962 return; 2963 2964 d = to_udma_desc(&vd->tx); 2965 2966 if (d->metadata_size) 2967 udma_fetch_epib(uc, d); 2968 2969 /* Provide residue information for the client */ 2970 if (result) { 2971 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx); 2972 2973 if (cppi5_desc_get_type(desc_vaddr) == 2974 CPPI5_INFO0_DESC_TYPE_VAL_HOST) { 2975 result->residue = d->residue - 2976 cppi5_hdesc_get_pktlen(desc_vaddr); 2977 if (result->residue) 2978 result->result = DMA_TRANS_ABORTED; 2979 else 2980 result->result = DMA_TRANS_NOERROR; 2981 } else { 2982 result->residue = 0; 2983 result->result = DMA_TRANS_NOERROR; 2984 } 2985 } 2986 } 2987 2988 /* 2989 * This tasklet handles the completion of a DMA descriptor by 2990 * calling its callback and freeing it. 2991 */ 2992 static void udma_vchan_complete(unsigned long arg) 2993 { 2994 struct virt_dma_chan *vc = (struct virt_dma_chan *)arg; 2995 struct virt_dma_desc *vd, *_vd; 2996 struct dmaengine_desc_callback cb; 2997 LIST_HEAD(head); 2998 2999 spin_lock_irq(&vc->lock); 3000 list_splice_tail_init(&vc->desc_completed, &head); 3001 vd = vc->cyclic; 3002 if (vd) { 3003 vc->cyclic = NULL; 3004 dmaengine_desc_get_callback(&vd->tx, &cb); 3005 } else { 3006 memset(&cb, 0, sizeof(cb)); 3007 } 3008 spin_unlock_irq(&vc->lock); 3009 3010 udma_desc_pre_callback(vc, vd, NULL); 3011 dmaengine_desc_callback_invoke(&cb, NULL); 3012 3013 list_for_each_entry_safe(vd, _vd, &head, node) { 3014 struct dmaengine_result result; 3015 3016 dmaengine_desc_get_callback(&vd->tx, &cb); 3017 3018 list_del(&vd->node); 3019 3020 udma_desc_pre_callback(vc, vd, &result); 3021 dmaengine_desc_callback_invoke(&cb, &result); 3022 3023 vchan_vdesc_fini(vd); 3024 } 3025 } 3026 3027 static void udma_free_chan_resources(struct dma_chan *chan) 3028 { 3029 struct udma_chan *uc = to_udma_chan(chan); 3030 struct udma_dev *ud = to_udma_dev(chan->device); 3031 3032 udma_terminate_all(chan); 3033 if (uc->terminated_desc) { 3034 udma_reset_chan(uc, false); 3035 udma_reset_rings(uc); 3036 } 3037 3038 cancel_delayed_work_sync(&uc->tx_drain.work); 3039 destroy_delayed_work_on_stack(&uc->tx_drain.work); 3040 3041 if (uc->irq_num_ring > 0) { 3042 free_irq(uc->irq_num_ring, uc); 3043 3044 uc->irq_num_ring = 0; 3045 } 3046 if (uc->irq_num_udma > 0) { 3047 free_irq(uc->irq_num_udma, uc); 3048 3049 uc->irq_num_udma = 0; 3050 } 3051 3052 /* Release PSI-L pairing */ 3053 if (uc->psil_paired) { 3054 navss_psil_unpair(ud, uc->config.src_thread, 3055 uc->config.dst_thread); 3056 uc->psil_paired = false; 3057 } 3058 3059 vchan_free_chan_resources(&uc->vc); 3060 tasklet_kill(&uc->vc.task); 3061 3062 udma_free_tx_resources(uc); 3063 udma_free_rx_resources(uc); 3064 udma_reset_uchan(uc); 3065 3066 if (uc->use_dma_pool) { 3067 dma_pool_destroy(uc->hdesc_pool); 3068 uc->use_dma_pool = false; 3069 } 3070 } 3071 3072 static struct platform_driver udma_driver; 3073 3074 struct udma_filter_param { 3075 int remote_thread_id; 3076 u32 atype; 3077 }; 3078 3079 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) 3080 { 3081 struct udma_chan_config *ucc; 3082 struct psil_endpoint_config *ep_config; 3083 struct udma_filter_param *filter_param; 3084 struct udma_chan *uc; 3085 struct udma_dev *ud; 3086 3087 if (chan->device->dev->driver != &udma_driver.driver) 3088 return false; 3089 3090 uc = to_udma_chan(chan); 3091 ucc = &uc->config; 3092 ud = uc->ud; 3093 filter_param = param; 3094 3095 if (filter_param->atype > 2) { 3096 dev_err(ud->dev, "Invalid channel atype: %u\n", 3097 filter_param->atype); 3098 return false; 3099 } 3100 3101 ucc->remote_thread_id = filter_param->remote_thread_id; 3102 ucc->atype = filter_param->atype; 3103 3104 if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) 3105 ucc->dir = DMA_MEM_TO_DEV; 3106 else 3107 ucc->dir = DMA_DEV_TO_MEM; 3108 3109 ep_config = psil_get_ep_config(ucc->remote_thread_id); 3110 if (IS_ERR(ep_config)) { 3111 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n", 3112 ucc->remote_thread_id); 3113 ucc->dir = DMA_MEM_TO_MEM; 3114 ucc->remote_thread_id = -1; 3115 ucc->atype = 0; 3116 return false; 3117 } 3118 3119 ucc->pkt_mode = ep_config->pkt_mode; 3120 ucc->channel_tpl = ep_config->channel_tpl; 3121 ucc->notdpkt = ep_config->notdpkt; 3122 ucc->ep_type = ep_config->ep_type; 3123 3124 if (ucc->ep_type != PSIL_EP_NATIVE) { 3125 const struct udma_match_data *match_data = ud->match_data; 3126 3127 if (match_data->flags & UDMA_FLAG_PDMA_ACC32) 3128 ucc->enable_acc32 = ep_config->pdma_acc32; 3129 if (match_data->flags & UDMA_FLAG_PDMA_BURST) 3130 ucc->enable_burst = ep_config->pdma_burst; 3131 } 3132 3133 ucc->needs_epib = ep_config->needs_epib; 3134 ucc->psd_size = ep_config->psd_size; 3135 ucc->metadata_size = 3136 (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) + 3137 ucc->psd_size; 3138 3139 if (ucc->pkt_mode) 3140 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 3141 ucc->metadata_size, ud->desc_align); 3142 3143 dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id, 3144 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir)); 3145 3146 return true; 3147 } 3148 3149 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, 3150 struct of_dma *ofdma) 3151 { 3152 struct udma_dev *ud = ofdma->of_dma_data; 3153 dma_cap_mask_t mask = ud->ddev.cap_mask; 3154 struct udma_filter_param filter_param; 3155 struct dma_chan *chan; 3156 3157 if (dma_spec->args_count != 1 && dma_spec->args_count != 2) 3158 return NULL; 3159 3160 filter_param.remote_thread_id = dma_spec->args[0]; 3161 if (dma_spec->args_count == 2) 3162 filter_param.atype = dma_spec->args[1]; 3163 else 3164 filter_param.atype = 0; 3165 3166 chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, 3167 ofdma->of_node); 3168 if (!chan) { 3169 dev_err(ud->dev, "get channel fail in %s.\n", __func__); 3170 return ERR_PTR(-EINVAL); 3171 } 3172 3173 return chan; 3174 } 3175 3176 static struct udma_match_data am654_main_data = { 3177 .psil_base = 0x1000, 3178 .enable_memcpy_support = true, 3179 .statictr_z_mask = GENMASK(11, 0), 3180 .rchan_oes_offset = 0x2000, 3181 .tpl_levels = 2, 3182 .level_start_idx = { 3183 [0] = 8, /* Normal channels */ 3184 [1] = 0, /* High Throughput channels */ 3185 }, 3186 }; 3187 3188 static struct udma_match_data am654_mcu_data = { 3189 .psil_base = 0x6000, 3190 .enable_memcpy_support = true, /* TEST: DMA domains */ 3191 .statictr_z_mask = GENMASK(11, 0), 3192 .rchan_oes_offset = 0x2000, 3193 .tpl_levels = 2, 3194 .level_start_idx = { 3195 [0] = 2, /* Normal channels */ 3196 [1] = 0, /* High Throughput channels */ 3197 }, 3198 }; 3199 3200 static struct udma_match_data j721e_main_data = { 3201 .psil_base = 0x1000, 3202 .enable_memcpy_support = true, 3203 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, 3204 .statictr_z_mask = GENMASK(23, 0), 3205 .rchan_oes_offset = 0x400, 3206 .tpl_levels = 3, 3207 .level_start_idx = { 3208 [0] = 16, /* Normal channels */ 3209 [1] = 4, /* High Throughput channels */ 3210 [2] = 0, /* Ultra High Throughput channels */ 3211 }, 3212 }; 3213 3214 static struct udma_match_data j721e_mcu_data = { 3215 .psil_base = 0x6000, 3216 .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ 3217 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, 3218 .statictr_z_mask = GENMASK(23, 0), 3219 .rchan_oes_offset = 0x400, 3220 .tpl_levels = 2, 3221 .level_start_idx = { 3222 [0] = 2, /* Normal channels */ 3223 [1] = 0, /* High Throughput channels */ 3224 }, 3225 }; 3226 3227 static const struct of_device_id udma_of_match[] = { 3228 { 3229 .compatible = "ti,am654-navss-main-udmap", 3230 .data = &am654_main_data, 3231 }, 3232 { 3233 .compatible = "ti,am654-navss-mcu-udmap", 3234 .data = &am654_mcu_data, 3235 }, { 3236 .compatible = "ti,j721e-navss-main-udmap", 3237 .data = &j721e_main_data, 3238 }, { 3239 .compatible = "ti,j721e-navss-mcu-udmap", 3240 .data = &j721e_mcu_data, 3241 }, 3242 { /* Sentinel */ }, 3243 }; 3244 3245 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) 3246 { 3247 struct resource *res; 3248 int i; 3249 3250 for (i = 0; i < MMR_LAST; i++) { 3251 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 3252 mmr_names[i]); 3253 ud->mmrs[i] = devm_ioremap_resource(&pdev->dev, res); 3254 if (IS_ERR(ud->mmrs[i])) 3255 return PTR_ERR(ud->mmrs[i]); 3256 } 3257 3258 return 0; 3259 } 3260 3261 static int udma_setup_resources(struct udma_dev *ud) 3262 { 3263 struct device *dev = ud->dev; 3264 int ch_count, ret, i, j; 3265 u32 cap2, cap3; 3266 struct ti_sci_resource_desc *rm_desc; 3267 struct ti_sci_resource *rm_res, irq_res; 3268 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 3269 static const char * const range_names[] = { "ti,sci-rm-range-tchan", 3270 "ti,sci-rm-range-rchan", 3271 "ti,sci-rm-range-rflow" }; 3272 3273 cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28); 3274 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 3275 3276 ud->rflow_cnt = cap3 & 0x3fff; 3277 ud->tchan_cnt = cap2 & 0x1ff; 3278 ud->echan_cnt = (cap2 >> 9) & 0x1ff; 3279 ud->rchan_cnt = (cap2 >> 18) & 0x1ff; 3280 ch_count = ud->tchan_cnt + ud->rchan_cnt; 3281 3282 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 3283 sizeof(unsigned long), GFP_KERNEL); 3284 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 3285 GFP_KERNEL); 3286 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 3287 sizeof(unsigned long), GFP_KERNEL); 3288 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 3289 GFP_KERNEL); 3290 ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt), 3291 sizeof(unsigned long), 3292 GFP_KERNEL); 3293 ud->rflow_gp_map_allocated = devm_kcalloc(dev, 3294 BITS_TO_LONGS(ud->rflow_cnt), 3295 sizeof(unsigned long), 3296 GFP_KERNEL); 3297 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), 3298 sizeof(unsigned long), 3299 GFP_KERNEL); 3300 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), 3301 GFP_KERNEL); 3302 3303 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map || 3304 !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans || 3305 !ud->rflows || !ud->rflow_in_use) 3306 return -ENOMEM; 3307 3308 /* 3309 * RX flows with the same Ids as RX channels are reserved to be used 3310 * as default flows if remote HW can't generate flow_ids. Those 3311 * RX flows can be requested only explicitly by id. 3312 */ 3313 bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt); 3314 3315 /* by default no GP rflows are assigned to Linux */ 3316 bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt); 3317 3318 /* Get resource ranges from tisci */ 3319 for (i = 0; i < RM_RANGE_LAST; i++) 3320 tisci_rm->rm_ranges[i] = 3321 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 3322 tisci_rm->tisci_dev_id, 3323 (char *)range_names[i]); 3324 3325 /* tchan ranges */ 3326 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 3327 if (IS_ERR(rm_res)) { 3328 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 3329 } else { 3330 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 3331 for (i = 0; i < rm_res->sets; i++) { 3332 rm_desc = &rm_res->desc[i]; 3333 bitmap_clear(ud->tchan_map, rm_desc->start, 3334 rm_desc->num); 3335 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n", 3336 rm_desc->start, rm_desc->num); 3337 } 3338 } 3339 irq_res.sets = rm_res->sets; 3340 3341 /* rchan and matching default flow ranges */ 3342 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 3343 if (IS_ERR(rm_res)) { 3344 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 3345 } else { 3346 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 3347 for (i = 0; i < rm_res->sets; i++) { 3348 rm_desc = &rm_res->desc[i]; 3349 bitmap_clear(ud->rchan_map, rm_desc->start, 3350 rm_desc->num); 3351 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n", 3352 rm_desc->start, rm_desc->num); 3353 } 3354 } 3355 3356 irq_res.sets += rm_res->sets; 3357 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 3358 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 3359 for (i = 0; i < rm_res->sets; i++) { 3360 irq_res.desc[i].start = rm_res->desc[i].start; 3361 irq_res.desc[i].num = rm_res->desc[i].num; 3362 } 3363 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 3364 for (j = 0; j < rm_res->sets; j++, i++) { 3365 irq_res.desc[i].start = rm_res->desc[j].start + 3366 ud->match_data->rchan_oes_offset; 3367 irq_res.desc[i].num = rm_res->desc[j].num; 3368 } 3369 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 3370 kfree(irq_res.desc); 3371 if (ret) { 3372 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 3373 return ret; 3374 } 3375 3376 /* GP rflow ranges */ 3377 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 3378 if (IS_ERR(rm_res)) { 3379 /* all gp flows are assigned exclusively to Linux */ 3380 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt, 3381 ud->rflow_cnt - ud->rchan_cnt); 3382 } else { 3383 for (i = 0; i < rm_res->sets; i++) { 3384 rm_desc = &rm_res->desc[i]; 3385 bitmap_clear(ud->rflow_gp_map, rm_desc->start, 3386 rm_desc->num); 3387 dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n", 3388 rm_desc->start, rm_desc->num); 3389 } 3390 } 3391 3392 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt); 3393 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt); 3394 if (!ch_count) 3395 return -ENODEV; 3396 3397 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels), 3398 GFP_KERNEL); 3399 if (!ud->channels) 3400 return -ENOMEM; 3401 3402 dev_info(dev, "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", 3403 ch_count, 3404 ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), 3405 ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt), 3406 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map, 3407 ud->rflow_cnt)); 3408 3409 return ch_count; 3410 } 3411 3412 static int udma_setup_rx_flush(struct udma_dev *ud) 3413 { 3414 struct udma_rx_flush *rx_flush = &ud->rx_flush; 3415 struct cppi5_desc_hdr_t *tr_desc; 3416 struct cppi5_tr_type1_t *tr_req; 3417 struct cppi5_host_desc_t *desc; 3418 struct device *dev = ud->dev; 3419 struct udma_hwdesc *hwdesc; 3420 size_t tr_size; 3421 3422 /* Allocate 1K buffer for discarded data on RX channel teardown */ 3423 rx_flush->buffer_size = SZ_1K; 3424 rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size, 3425 GFP_KERNEL); 3426 if (!rx_flush->buffer_vaddr) 3427 return -ENOMEM; 3428 3429 rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr, 3430 rx_flush->buffer_size, 3431 DMA_TO_DEVICE); 3432 if (dma_mapping_error(dev, rx_flush->buffer_paddr)) 3433 return -ENOMEM; 3434 3435 /* Set up descriptor to be used for TR mode */ 3436 hwdesc = &rx_flush->hwdescs[0]; 3437 tr_size = sizeof(struct cppi5_tr_type1_t); 3438 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1); 3439 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 3440 ud->desc_align); 3441 3442 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 3443 GFP_KERNEL); 3444 if (!hwdesc->cppi5_desc_vaddr) 3445 return -ENOMEM; 3446 3447 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 3448 hwdesc->cppi5_desc_size, 3449 DMA_TO_DEVICE); 3450 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 3451 return -ENOMEM; 3452 3453 /* Start of the TR req records */ 3454 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 3455 /* Start address of the TR response array */ 3456 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size; 3457 3458 tr_desc = hwdesc->cppi5_desc_vaddr; 3459 cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0); 3460 cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3461 cppi5_desc_set_retpolicy(tr_desc, 0, 0); 3462 3463 tr_req = hwdesc->tr_req_base; 3464 cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false, 3465 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3466 cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT); 3467 3468 tr_req->addr = rx_flush->buffer_paddr; 3469 tr_req->icnt0 = rx_flush->buffer_size; 3470 tr_req->icnt1 = 1; 3471 3472 /* Set up descriptor to be used for packet mode */ 3473 hwdesc = &rx_flush->hwdescs[1]; 3474 hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 3475 CPPI5_INFO0_HDESC_EPIB_SIZE + 3476 CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE, 3477 ud->desc_align); 3478 3479 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 3480 GFP_KERNEL); 3481 if (!hwdesc->cppi5_desc_vaddr) 3482 return -ENOMEM; 3483 3484 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 3485 hwdesc->cppi5_desc_size, 3486 DMA_TO_DEVICE); 3487 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 3488 return -ENOMEM; 3489 3490 desc = hwdesc->cppi5_desc_vaddr; 3491 cppi5_hdesc_init(desc, 0, 0); 3492 cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3493 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0); 3494 3495 cppi5_hdesc_attach_buf(desc, 3496 rx_flush->buffer_paddr, rx_flush->buffer_size, 3497 rx_flush->buffer_paddr, rx_flush->buffer_size); 3498 3499 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, 3500 hwdesc->cppi5_desc_size, DMA_TO_DEVICE); 3501 return 0; 3502 } 3503 3504 #ifdef CONFIG_DEBUG_FS 3505 static void udma_dbg_summary_show_chan(struct seq_file *s, 3506 struct dma_chan *chan) 3507 { 3508 struct udma_chan *uc = to_udma_chan(chan); 3509 struct udma_chan_config *ucc = &uc->config; 3510 3511 seq_printf(s, " %-13s| %s", dma_chan_name(chan), 3512 chan->dbg_client_name ?: "in-use"); 3513 seq_printf(s, " (%s, ", dmaengine_get_direction_text(uc->config.dir)); 3514 3515 switch (uc->config.dir) { 3516 case DMA_MEM_TO_MEM: 3517 seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id, 3518 ucc->src_thread, ucc->dst_thread); 3519 break; 3520 case DMA_DEV_TO_MEM: 3521 seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id, 3522 ucc->src_thread, ucc->dst_thread); 3523 break; 3524 case DMA_MEM_TO_DEV: 3525 seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id, 3526 ucc->src_thread, ucc->dst_thread); 3527 break; 3528 default: 3529 seq_printf(s, ")\n"); 3530 return; 3531 } 3532 3533 if (ucc->ep_type == PSIL_EP_NATIVE) { 3534 seq_printf(s, "PSI-L Native"); 3535 if (ucc->metadata_size) { 3536 seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : ""); 3537 if (ucc->psd_size) 3538 seq_printf(s, " PSDsize:%u", ucc->psd_size); 3539 seq_printf(s, " ]"); 3540 } 3541 } else { 3542 seq_printf(s, "PDMA"); 3543 if (ucc->enable_acc32 || ucc->enable_burst) 3544 seq_printf(s, "[%s%s ]", 3545 ucc->enable_acc32 ? " ACC32" : "", 3546 ucc->enable_burst ? " BURST" : ""); 3547 } 3548 3549 seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode"); 3550 } 3551 3552 static void udma_dbg_summary_show(struct seq_file *s, 3553 struct dma_device *dma_dev) 3554 { 3555 struct dma_chan *chan; 3556 3557 list_for_each_entry(chan, &dma_dev->channels, device_node) { 3558 if (chan->client_count) 3559 udma_dbg_summary_show_chan(s, chan); 3560 } 3561 } 3562 #endif /* CONFIG_DEBUG_FS */ 3563 3564 #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 3565 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 3566 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ 3567 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 3568 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 3569 3570 static int udma_probe(struct platform_device *pdev) 3571 { 3572 struct device_node *navss_node = pdev->dev.parent->of_node; 3573 struct device *dev = &pdev->dev; 3574 struct udma_dev *ud; 3575 const struct of_device_id *match; 3576 int i, ret; 3577 int ch_count; 3578 3579 ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48)); 3580 if (ret) 3581 dev_err(dev, "failed to set dma mask stuff\n"); 3582 3583 ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL); 3584 if (!ud) 3585 return -ENOMEM; 3586 3587 ret = udma_get_mmrs(pdev, ud); 3588 if (ret) 3589 return ret; 3590 3591 ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci"); 3592 if (IS_ERR(ud->tisci_rm.tisci)) 3593 return PTR_ERR(ud->tisci_rm.tisci); 3594 3595 ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id", 3596 &ud->tisci_rm.tisci_dev_id); 3597 if (ret) { 3598 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret); 3599 return ret; 3600 } 3601 pdev->id = ud->tisci_rm.tisci_dev_id; 3602 3603 ret = of_property_read_u32(navss_node, "ti,sci-dev-id", 3604 &ud->tisci_rm.tisci_navss_dev_id); 3605 if (ret) { 3606 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret); 3607 return ret; 3608 } 3609 3610 ret = of_property_read_u32(navss_node, "ti,udma-atype", &ud->atype); 3611 if (!ret && ud->atype > 2) { 3612 dev_err(dev, "Invalid atype: %u\n", ud->atype); 3613 return -EINVAL; 3614 } 3615 3616 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; 3617 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; 3618 3619 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc"); 3620 if (IS_ERR(ud->ringacc)) 3621 return PTR_ERR(ud->ringacc); 3622 3623 dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 3624 DOMAIN_BUS_TI_SCI_INTA_MSI); 3625 if (!dev->msi_domain) { 3626 dev_err(dev, "Failed to get MSI domain\n"); 3627 return -EPROBE_DEFER; 3628 } 3629 3630 match = of_match_node(udma_of_match, dev->of_node); 3631 if (!match) { 3632 dev_err(dev, "No compatible match found\n"); 3633 return -ENODEV; 3634 } 3635 ud->match_data = match->data; 3636 3637 dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask); 3638 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); 3639 3640 ud->ddev.device_alloc_chan_resources = udma_alloc_chan_resources; 3641 ud->ddev.device_config = udma_slave_config; 3642 ud->ddev.device_prep_slave_sg = udma_prep_slave_sg; 3643 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; 3644 ud->ddev.device_issue_pending = udma_issue_pending; 3645 ud->ddev.device_tx_status = udma_tx_status; 3646 ud->ddev.device_pause = udma_pause; 3647 ud->ddev.device_resume = udma_resume; 3648 ud->ddev.device_terminate_all = udma_terminate_all; 3649 ud->ddev.device_synchronize = udma_synchronize; 3650 #ifdef CONFIG_DEBUG_FS 3651 ud->ddev.dbg_summary_show = udma_dbg_summary_show; 3652 #endif 3653 3654 ud->ddev.device_free_chan_resources = udma_free_chan_resources; 3655 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; 3656 ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; 3657 ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 3658 ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 3659 ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES; 3660 ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | 3661 DESC_METADATA_ENGINE; 3662 if (ud->match_data->enable_memcpy_support) { 3663 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask); 3664 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy; 3665 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM); 3666 } 3667 3668 ud->ddev.dev = dev; 3669 ud->dev = dev; 3670 ud->psil_base = ud->match_data->psil_base; 3671 3672 INIT_LIST_HEAD(&ud->ddev.channels); 3673 INIT_LIST_HEAD(&ud->desc_to_purge); 3674 3675 ch_count = udma_setup_resources(ud); 3676 if (ch_count <= 0) 3677 return ch_count; 3678 3679 spin_lock_init(&ud->lock); 3680 INIT_WORK(&ud->purge_work, udma_purge_desc_work); 3681 3682 ud->desc_align = 64; 3683 if (ud->desc_align < dma_get_cache_alignment()) 3684 ud->desc_align = dma_get_cache_alignment(); 3685 3686 ret = udma_setup_rx_flush(ud); 3687 if (ret) 3688 return ret; 3689 3690 for (i = 0; i < ud->tchan_cnt; i++) { 3691 struct udma_tchan *tchan = &ud->tchans[i]; 3692 3693 tchan->id = i; 3694 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000; 3695 } 3696 3697 for (i = 0; i < ud->rchan_cnt; i++) { 3698 struct udma_rchan *rchan = &ud->rchans[i]; 3699 3700 rchan->id = i; 3701 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000; 3702 } 3703 3704 for (i = 0; i < ud->rflow_cnt; i++) { 3705 struct udma_rflow *rflow = &ud->rflows[i]; 3706 3707 rflow->id = i; 3708 } 3709 3710 for (i = 0; i < ch_count; i++) { 3711 struct udma_chan *uc = &ud->channels[i]; 3712 3713 uc->ud = ud; 3714 uc->vc.desc_free = udma_desc_free; 3715 uc->id = i; 3716 uc->tchan = NULL; 3717 uc->rchan = NULL; 3718 uc->config.remote_thread_id = -1; 3719 uc->config.dir = DMA_MEM_TO_MEM; 3720 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", 3721 dev_name(dev), i); 3722 3723 vchan_init(&uc->vc, &ud->ddev); 3724 /* Use custom vchan completion handling */ 3725 tasklet_init(&uc->vc.task, udma_vchan_complete, 3726 (unsigned long)&uc->vc); 3727 init_completion(&uc->teardown_completed); 3728 } 3729 3730 ret = dma_async_device_register(&ud->ddev); 3731 if (ret) { 3732 dev_err(dev, "failed to register slave DMA engine: %d\n", ret); 3733 return ret; 3734 } 3735 3736 platform_set_drvdata(pdev, ud); 3737 3738 ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud); 3739 if (ret) { 3740 dev_err(dev, "failed to register of_dma controller\n"); 3741 dma_async_device_unregister(&ud->ddev); 3742 } 3743 3744 return ret; 3745 } 3746 3747 static struct platform_driver udma_driver = { 3748 .driver = { 3749 .name = "ti-udma", 3750 .of_match_table = udma_of_match, 3751 .suppress_bind_attrs = true, 3752 }, 3753 .probe = udma_probe, 3754 }; 3755 builtin_platform_driver(udma_driver); 3756 3757 /* Private interfaces to UDMA */ 3758 #include "k3-udma-private.c" 3759