1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/delay.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <linux/err.h> 13 #include <linux/init.h> 14 #include <linux/interrupt.h> 15 #include <linux/list.h> 16 #include <linux/platform_device.h> 17 #include <linux/slab.h> 18 #include <linux/spinlock.h> 19 #include <linux/of.h> 20 #include <linux/of_dma.h> 21 #include <linux/of_device.h> 22 #include <linux/of_irq.h> 23 #include <linux/workqueue.h> 24 #include <linux/completion.h> 25 #include <linux/soc/ti/k3-ringacc.h> 26 #include <linux/soc/ti/ti_sci_protocol.h> 27 #include <linux/soc/ti/ti_sci_inta_msi.h> 28 #include <linux/dma/ti-cppi5.h> 29 30 #include "../virt-dma.h" 31 #include "k3-udma.h" 32 #include "k3-psil-priv.h" 33 34 struct udma_static_tr { 35 u8 elsize; /* RPSTR0 */ 36 u16 elcnt; /* RPSTR0 */ 37 u16 bstcnt; /* RPSTR1 */ 38 }; 39 40 #define K3_UDMA_MAX_RFLOWS 1024 41 #define K3_UDMA_DEFAULT_RING_SIZE 16 42 43 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */ 44 #define UDMA_RFLOW_SRCTAG_NONE 0 45 #define UDMA_RFLOW_SRCTAG_CFG_TAG 1 46 #define UDMA_RFLOW_SRCTAG_FLOW_ID 2 47 #define UDMA_RFLOW_SRCTAG_SRC_TAG 4 48 49 #define UDMA_RFLOW_DSTTAG_NONE 0 50 #define UDMA_RFLOW_DSTTAG_CFG_TAG 1 51 #define UDMA_RFLOW_DSTTAG_FLOW_ID 2 52 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO 4 53 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI 5 54 55 struct udma_chan; 56 57 enum udma_mmr { 58 MMR_GCFG = 0, 59 MMR_RCHANRT, 60 MMR_TCHANRT, 61 MMR_LAST, 62 }; 63 64 static const char * const mmr_names[] = { "gcfg", "rchanrt", "tchanrt" }; 65 66 struct udma_tchan { 67 void __iomem *reg_rt; 68 69 int id; 70 struct k3_ring *t_ring; /* Transmit ring */ 71 struct k3_ring *tc_ring; /* Transmit Completion ring */ 72 }; 73 74 struct udma_rflow { 75 int id; 76 struct k3_ring *fd_ring; /* Free Descriptor ring */ 77 struct k3_ring *r_ring; /* Receive ring */ 78 }; 79 80 struct udma_rchan { 81 void __iomem *reg_rt; 82 83 int id; 84 }; 85 86 #define UDMA_FLAG_PDMA_ACC32 BIT(0) 87 #define UDMA_FLAG_PDMA_BURST BIT(1) 88 89 struct udma_match_data { 90 u32 psil_base; 91 bool enable_memcpy_support; 92 u32 flags; 93 u32 statictr_z_mask; 94 u32 rchan_oes_offset; 95 96 u8 tpl_levels; 97 u32 level_start_idx[]; 98 }; 99 100 struct udma_hwdesc { 101 size_t cppi5_desc_size; 102 void *cppi5_desc_vaddr; 103 dma_addr_t cppi5_desc_paddr; 104 105 /* TR descriptor internal pointers */ 106 void *tr_req_base; 107 struct cppi5_tr_resp_t *tr_resp_base; 108 }; 109 110 struct udma_rx_flush { 111 struct udma_hwdesc hwdescs[2]; 112 113 size_t buffer_size; 114 void *buffer_vaddr; 115 dma_addr_t buffer_paddr; 116 }; 117 118 struct udma_dev { 119 struct dma_device ddev; 120 struct device *dev; 121 void __iomem *mmrs[MMR_LAST]; 122 const struct udma_match_data *match_data; 123 124 size_t desc_align; /* alignment to use for descriptors */ 125 126 struct udma_tisci_rm tisci_rm; 127 128 struct k3_ringacc *ringacc; 129 130 struct work_struct purge_work; 131 struct list_head desc_to_purge; 132 spinlock_t lock; 133 134 struct udma_rx_flush rx_flush; 135 136 int tchan_cnt; 137 int echan_cnt; 138 int rchan_cnt; 139 int rflow_cnt; 140 unsigned long *tchan_map; 141 unsigned long *rchan_map; 142 unsigned long *rflow_gp_map; 143 unsigned long *rflow_gp_map_allocated; 144 unsigned long *rflow_in_use; 145 146 struct udma_tchan *tchans; 147 struct udma_rchan *rchans; 148 struct udma_rflow *rflows; 149 150 struct udma_chan *channels; 151 u32 psil_base; 152 u32 atype; 153 }; 154 155 struct udma_desc { 156 struct virt_dma_desc vd; 157 158 bool terminated; 159 160 enum dma_transfer_direction dir; 161 162 struct udma_static_tr static_tr; 163 u32 residue; 164 165 unsigned int sglen; 166 unsigned int desc_idx; /* Only used for cyclic in packet mode */ 167 unsigned int tr_idx; 168 169 u32 metadata_size; 170 void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */ 171 172 unsigned int hwdesc_count; 173 struct udma_hwdesc hwdesc[0]; 174 }; 175 176 enum udma_chan_state { 177 UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */ 178 UDMA_CHAN_IS_ACTIVE, /* Normal operation */ 179 UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */ 180 }; 181 182 struct udma_tx_drain { 183 struct delayed_work work; 184 ktime_t tstamp; 185 u32 residue; 186 }; 187 188 struct udma_chan_config { 189 bool pkt_mode; /* TR or packet */ 190 bool needs_epib; /* EPIB is needed for the communication or not */ 191 u32 psd_size; /* size of Protocol Specific Data */ 192 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */ 193 u32 hdesc_size; /* Size of a packet descriptor in packet mode */ 194 bool notdpkt; /* Suppress sending TDC packet */ 195 int remote_thread_id; 196 u32 atype; 197 u32 src_thread; 198 u32 dst_thread; 199 enum psil_endpoint_type ep_type; 200 bool enable_acc32; 201 bool enable_burst; 202 enum udma_tp_level channel_tpl; /* Channel Throughput Level */ 203 204 enum dma_transfer_direction dir; 205 }; 206 207 struct udma_chan { 208 struct virt_dma_chan vc; 209 struct dma_slave_config cfg; 210 struct udma_dev *ud; 211 struct udma_desc *desc; 212 struct udma_desc *terminated_desc; 213 struct udma_static_tr static_tr; 214 char *name; 215 216 struct udma_tchan *tchan; 217 struct udma_rchan *rchan; 218 struct udma_rflow *rflow; 219 220 bool psil_paired; 221 222 int irq_num_ring; 223 int irq_num_udma; 224 225 bool cyclic; 226 bool paused; 227 228 enum udma_chan_state state; 229 struct completion teardown_completed; 230 231 struct udma_tx_drain tx_drain; 232 233 u32 bcnt; /* number of bytes completed since the start of the channel */ 234 u32 in_ring_cnt; /* number of descriptors in flight */ 235 236 /* Channel configuration parameters */ 237 struct udma_chan_config config; 238 239 /* dmapool for packet mode descriptors */ 240 bool use_dma_pool; 241 struct dma_pool *hdesc_pool; 242 243 u32 id; 244 }; 245 246 static inline struct udma_dev *to_udma_dev(struct dma_device *d) 247 { 248 return container_of(d, struct udma_dev, ddev); 249 } 250 251 static inline struct udma_chan *to_udma_chan(struct dma_chan *c) 252 { 253 return container_of(c, struct udma_chan, vc.chan); 254 } 255 256 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t) 257 { 258 return container_of(t, struct udma_desc, vd.tx); 259 } 260 261 /* Generic register access functions */ 262 static inline u32 udma_read(void __iomem *base, int reg) 263 { 264 return readl(base + reg); 265 } 266 267 static inline void udma_write(void __iomem *base, int reg, u32 val) 268 { 269 writel(val, base + reg); 270 } 271 272 static inline void udma_update_bits(void __iomem *base, int reg, 273 u32 mask, u32 val) 274 { 275 u32 tmp, orig; 276 277 orig = readl(base + reg); 278 tmp = orig & ~mask; 279 tmp |= (val & mask); 280 281 if (tmp != orig) 282 writel(tmp, base + reg); 283 } 284 285 /* TCHANRT */ 286 static inline u32 udma_tchanrt_read(struct udma_tchan *tchan, int reg) 287 { 288 if (!tchan) 289 return 0; 290 return udma_read(tchan->reg_rt, reg); 291 } 292 293 static inline void udma_tchanrt_write(struct udma_tchan *tchan, int reg, 294 u32 val) 295 { 296 if (!tchan) 297 return; 298 udma_write(tchan->reg_rt, reg, val); 299 } 300 301 static inline void udma_tchanrt_update_bits(struct udma_tchan *tchan, int reg, 302 u32 mask, u32 val) 303 { 304 if (!tchan) 305 return; 306 udma_update_bits(tchan->reg_rt, reg, mask, val); 307 } 308 309 /* RCHANRT */ 310 static inline u32 udma_rchanrt_read(struct udma_rchan *rchan, int reg) 311 { 312 if (!rchan) 313 return 0; 314 return udma_read(rchan->reg_rt, reg); 315 } 316 317 static inline void udma_rchanrt_write(struct udma_rchan *rchan, int reg, 318 u32 val) 319 { 320 if (!rchan) 321 return; 322 udma_write(rchan->reg_rt, reg, val); 323 } 324 325 static inline void udma_rchanrt_update_bits(struct udma_rchan *rchan, int reg, 326 u32 mask, u32 val) 327 { 328 if (!rchan) 329 return; 330 udma_update_bits(rchan->reg_rt, reg, mask, val); 331 } 332 333 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) 334 { 335 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 336 337 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 338 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci, 339 tisci_rm->tisci_navss_dev_id, 340 src_thread, dst_thread); 341 } 342 343 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread, 344 u32 dst_thread) 345 { 346 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 347 348 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 349 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci, 350 tisci_rm->tisci_navss_dev_id, 351 src_thread, dst_thread); 352 } 353 354 static void udma_reset_uchan(struct udma_chan *uc) 355 { 356 memset(&uc->config, 0, sizeof(uc->config)); 357 uc->config.remote_thread_id = -1; 358 uc->state = UDMA_CHAN_IS_IDLE; 359 } 360 361 static void udma_dump_chan_stdata(struct udma_chan *uc) 362 { 363 struct device *dev = uc->ud->dev; 364 u32 offset; 365 int i; 366 367 if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) { 368 dev_dbg(dev, "TCHAN State data:\n"); 369 for (i = 0; i < 32; i++) { 370 offset = UDMA_TCHAN_RT_STDATA_REG + i * 4; 371 dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i, 372 udma_tchanrt_read(uc->tchan, offset)); 373 } 374 } 375 376 if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) { 377 dev_dbg(dev, "RCHAN State data:\n"); 378 for (i = 0; i < 32; i++) { 379 offset = UDMA_RCHAN_RT_STDATA_REG + i * 4; 380 dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i, 381 udma_rchanrt_read(uc->rchan, offset)); 382 } 383 } 384 } 385 386 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d, 387 int idx) 388 { 389 return d->hwdesc[idx].cppi5_desc_paddr; 390 } 391 392 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx) 393 { 394 return d->hwdesc[idx].cppi5_desc_vaddr; 395 } 396 397 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc, 398 dma_addr_t paddr) 399 { 400 struct udma_desc *d = uc->terminated_desc; 401 402 if (d) { 403 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 404 d->desc_idx); 405 406 if (desc_paddr != paddr) 407 d = NULL; 408 } 409 410 if (!d) { 411 d = uc->desc; 412 if (d) { 413 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 414 d->desc_idx); 415 416 if (desc_paddr != paddr) 417 d = NULL; 418 } 419 } 420 421 return d; 422 } 423 424 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d) 425 { 426 if (uc->use_dma_pool) { 427 int i; 428 429 for (i = 0; i < d->hwdesc_count; i++) { 430 if (!d->hwdesc[i].cppi5_desc_vaddr) 431 continue; 432 433 dma_pool_free(uc->hdesc_pool, 434 d->hwdesc[i].cppi5_desc_vaddr, 435 d->hwdesc[i].cppi5_desc_paddr); 436 437 d->hwdesc[i].cppi5_desc_vaddr = NULL; 438 } 439 } else if (d->hwdesc[0].cppi5_desc_vaddr) { 440 struct udma_dev *ud = uc->ud; 441 442 dma_free_coherent(ud->dev, d->hwdesc[0].cppi5_desc_size, 443 d->hwdesc[0].cppi5_desc_vaddr, 444 d->hwdesc[0].cppi5_desc_paddr); 445 446 d->hwdesc[0].cppi5_desc_vaddr = NULL; 447 } 448 } 449 450 static void udma_purge_desc_work(struct work_struct *work) 451 { 452 struct udma_dev *ud = container_of(work, typeof(*ud), purge_work); 453 struct virt_dma_desc *vd, *_vd; 454 unsigned long flags; 455 LIST_HEAD(head); 456 457 spin_lock_irqsave(&ud->lock, flags); 458 list_splice_tail_init(&ud->desc_to_purge, &head); 459 spin_unlock_irqrestore(&ud->lock, flags); 460 461 list_for_each_entry_safe(vd, _vd, &head, node) { 462 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 463 struct udma_desc *d = to_udma_desc(&vd->tx); 464 465 udma_free_hwdesc(uc, d); 466 list_del(&vd->node); 467 kfree(d); 468 } 469 470 /* If more to purge, schedule the work again */ 471 if (!list_empty(&ud->desc_to_purge)) 472 schedule_work(&ud->purge_work); 473 } 474 475 static void udma_desc_free(struct virt_dma_desc *vd) 476 { 477 struct udma_dev *ud = to_udma_dev(vd->tx.chan->device); 478 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 479 struct udma_desc *d = to_udma_desc(&vd->tx); 480 unsigned long flags; 481 482 if (uc->terminated_desc == d) 483 uc->terminated_desc = NULL; 484 485 if (uc->use_dma_pool) { 486 udma_free_hwdesc(uc, d); 487 kfree(d); 488 return; 489 } 490 491 spin_lock_irqsave(&ud->lock, flags); 492 list_add_tail(&vd->node, &ud->desc_to_purge); 493 spin_unlock_irqrestore(&ud->lock, flags); 494 495 schedule_work(&ud->purge_work); 496 } 497 498 static bool udma_is_chan_running(struct udma_chan *uc) 499 { 500 u32 trt_ctl = 0; 501 u32 rrt_ctl = 0; 502 503 if (uc->tchan) 504 trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG); 505 if (uc->rchan) 506 rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG); 507 508 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN) 509 return true; 510 511 return false; 512 } 513 514 static bool udma_is_chan_paused(struct udma_chan *uc) 515 { 516 u32 val, pause_mask; 517 518 switch (uc->config.dir) { 519 case DMA_DEV_TO_MEM: 520 val = udma_rchanrt_read(uc->rchan, 521 UDMA_RCHAN_RT_PEER_RT_EN_REG); 522 pause_mask = UDMA_PEER_RT_EN_PAUSE; 523 break; 524 case DMA_MEM_TO_DEV: 525 val = udma_tchanrt_read(uc->tchan, 526 UDMA_TCHAN_RT_PEER_RT_EN_REG); 527 pause_mask = UDMA_PEER_RT_EN_PAUSE; 528 break; 529 case DMA_MEM_TO_MEM: 530 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG); 531 pause_mask = UDMA_CHAN_RT_CTL_PAUSE; 532 break; 533 default: 534 return false; 535 } 536 537 if (val & pause_mask) 538 return true; 539 540 return false; 541 } 542 543 static void udma_sync_for_device(struct udma_chan *uc, int idx) 544 { 545 struct udma_desc *d = uc->desc; 546 547 if (uc->cyclic && uc->config.pkt_mode) { 548 dma_sync_single_for_device(uc->ud->dev, 549 d->hwdesc[idx].cppi5_desc_paddr, 550 d->hwdesc[idx].cppi5_desc_size, 551 DMA_TO_DEVICE); 552 } else { 553 int i; 554 555 for (i = 0; i < d->hwdesc_count; i++) { 556 if (!d->hwdesc[i].cppi5_desc_vaddr) 557 continue; 558 559 dma_sync_single_for_device(uc->ud->dev, 560 d->hwdesc[i].cppi5_desc_paddr, 561 d->hwdesc[i].cppi5_desc_size, 562 DMA_TO_DEVICE); 563 } 564 } 565 } 566 567 static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc) 568 { 569 return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr; 570 } 571 572 static int udma_push_to_ring(struct udma_chan *uc, int idx) 573 { 574 struct udma_desc *d = uc->desc; 575 struct k3_ring *ring = NULL; 576 dma_addr_t paddr; 577 int ret; 578 579 switch (uc->config.dir) { 580 case DMA_DEV_TO_MEM: 581 ring = uc->rflow->fd_ring; 582 break; 583 case DMA_MEM_TO_DEV: 584 case DMA_MEM_TO_MEM: 585 ring = uc->tchan->t_ring; 586 break; 587 default: 588 return -EINVAL; 589 } 590 591 /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */ 592 if (idx == -1) { 593 paddr = udma_get_rx_flush_hwdesc_paddr(uc); 594 } else { 595 paddr = udma_curr_cppi5_desc_paddr(d, idx); 596 597 wmb(); /* Ensure that writes are not moved over this point */ 598 udma_sync_for_device(uc, idx); 599 } 600 601 ret = k3_ringacc_ring_push(ring, &paddr); 602 if (!ret) 603 uc->in_ring_cnt++; 604 605 return ret; 606 } 607 608 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr) 609 { 610 if (uc->config.dir != DMA_DEV_TO_MEM) 611 return false; 612 613 if (addr == udma_get_rx_flush_hwdesc_paddr(uc)) 614 return true; 615 616 return false; 617 } 618 619 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) 620 { 621 struct k3_ring *ring = NULL; 622 int ret = -ENOENT; 623 624 switch (uc->config.dir) { 625 case DMA_DEV_TO_MEM: 626 ring = uc->rflow->r_ring; 627 break; 628 case DMA_MEM_TO_DEV: 629 case DMA_MEM_TO_MEM: 630 ring = uc->tchan->tc_ring; 631 break; 632 default: 633 break; 634 } 635 636 if (ring && k3_ringacc_ring_get_occ(ring)) { 637 struct udma_desc *d = NULL; 638 639 ret = k3_ringacc_ring_pop(ring, addr); 640 if (ret) 641 return ret; 642 643 /* Teardown completion */ 644 if (cppi5_desc_is_tdcm(*addr)) 645 return ret; 646 647 /* Check for flush descriptor */ 648 if (udma_desc_is_rx_flush(uc, *addr)) 649 return -ENOENT; 650 651 d = udma_udma_desc_from_paddr(uc, *addr); 652 653 if (d) 654 dma_sync_single_for_cpu(uc->ud->dev, *addr, 655 d->hwdesc[0].cppi5_desc_size, 656 DMA_FROM_DEVICE); 657 rmb(); /* Ensure that reads are not moved before this point */ 658 659 if (!ret) 660 uc->in_ring_cnt--; 661 } 662 663 return ret; 664 } 665 666 static void udma_reset_rings(struct udma_chan *uc) 667 { 668 struct k3_ring *ring1 = NULL; 669 struct k3_ring *ring2 = NULL; 670 671 switch (uc->config.dir) { 672 case DMA_DEV_TO_MEM: 673 if (uc->rchan) { 674 ring1 = uc->rflow->fd_ring; 675 ring2 = uc->rflow->r_ring; 676 } 677 break; 678 case DMA_MEM_TO_DEV: 679 case DMA_MEM_TO_MEM: 680 if (uc->tchan) { 681 ring1 = uc->tchan->t_ring; 682 ring2 = uc->tchan->tc_ring; 683 } 684 break; 685 default: 686 break; 687 } 688 689 if (ring1) 690 k3_ringacc_ring_reset_dma(ring1, 691 k3_ringacc_ring_get_occ(ring1)); 692 if (ring2) 693 k3_ringacc_ring_reset(ring2); 694 695 /* make sure we are not leaking memory by stalled descriptor */ 696 if (uc->terminated_desc) { 697 udma_desc_free(&uc->terminated_desc->vd); 698 uc->terminated_desc = NULL; 699 } 700 701 uc->in_ring_cnt = 0; 702 } 703 704 static void udma_reset_counters(struct udma_chan *uc) 705 { 706 u32 val; 707 708 if (uc->tchan) { 709 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); 710 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_BCNT_REG, val); 711 712 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG); 713 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG, val); 714 715 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PCNT_REG); 716 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PCNT_REG, val); 717 718 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); 719 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG, val); 720 } 721 722 if (uc->rchan) { 723 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_BCNT_REG); 724 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_BCNT_REG, val); 725 726 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG); 727 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG, val); 728 729 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PCNT_REG); 730 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PCNT_REG, val); 731 732 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG); 733 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG, val); 734 } 735 736 uc->bcnt = 0; 737 } 738 739 static int udma_reset_chan(struct udma_chan *uc, bool hard) 740 { 741 switch (uc->config.dir) { 742 case DMA_DEV_TO_MEM: 743 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 0); 744 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0); 745 break; 746 case DMA_MEM_TO_DEV: 747 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0); 748 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 0); 749 break; 750 case DMA_MEM_TO_MEM: 751 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0); 752 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0); 753 break; 754 default: 755 return -EINVAL; 756 } 757 758 /* Reset all counters */ 759 udma_reset_counters(uc); 760 761 /* Hard reset: re-initialize the channel to reset */ 762 if (hard) { 763 struct udma_chan_config ucc_backup; 764 int ret; 765 766 memcpy(&ucc_backup, &uc->config, sizeof(uc->config)); 767 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan); 768 769 /* restore the channel configuration */ 770 memcpy(&uc->config, &ucc_backup, sizeof(uc->config)); 771 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan); 772 if (ret) 773 return ret; 774 775 /* 776 * Setting forced teardown after forced reset helps recovering 777 * the rchan. 778 */ 779 if (uc->config.dir == DMA_DEV_TO_MEM) 780 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 781 UDMA_CHAN_RT_CTL_EN | 782 UDMA_CHAN_RT_CTL_TDOWN | 783 UDMA_CHAN_RT_CTL_FTDOWN); 784 } 785 uc->state = UDMA_CHAN_IS_IDLE; 786 787 return 0; 788 } 789 790 static void udma_start_desc(struct udma_chan *uc) 791 { 792 struct udma_chan_config *ucc = &uc->config; 793 794 if (ucc->pkt_mode && (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) { 795 int i; 796 797 /* Push all descriptors to ring for packet mode cyclic or RX */ 798 for (i = 0; i < uc->desc->sglen; i++) 799 udma_push_to_ring(uc, i); 800 } else { 801 udma_push_to_ring(uc, 0); 802 } 803 } 804 805 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc) 806 { 807 /* Only PDMAs have staticTR */ 808 if (uc->config.ep_type == PSIL_EP_NATIVE) 809 return false; 810 811 /* Check if the staticTR configuration has changed for TX */ 812 if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr))) 813 return true; 814 815 return false; 816 } 817 818 static int udma_start(struct udma_chan *uc) 819 { 820 struct virt_dma_desc *vd = vchan_next_desc(&uc->vc); 821 822 if (!vd) { 823 uc->desc = NULL; 824 return -ENOENT; 825 } 826 827 list_del(&vd->node); 828 829 uc->desc = to_udma_desc(&vd->tx); 830 831 /* Channel is already running and does not need reconfiguration */ 832 if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) { 833 udma_start_desc(uc); 834 goto out; 835 } 836 837 /* Make sure that we clear the teardown bit, if it is set */ 838 udma_reset_chan(uc, false); 839 840 /* Push descriptors before we start the channel */ 841 udma_start_desc(uc); 842 843 switch (uc->desc->dir) { 844 case DMA_DEV_TO_MEM: 845 /* Config remote TR */ 846 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 847 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 848 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 849 const struct udma_match_data *match_data = 850 uc->ud->match_data; 851 852 if (uc->config.enable_acc32) 853 val |= PDMA_STATIC_TR_XY_ACC32; 854 if (uc->config.enable_burst) 855 val |= PDMA_STATIC_TR_XY_BURST; 856 857 udma_rchanrt_write(uc->rchan, 858 UDMA_RCHAN_RT_PEER_STATIC_TR_XY_REG, val); 859 860 udma_rchanrt_write(uc->rchan, 861 UDMA_RCHAN_RT_PEER_STATIC_TR_Z_REG, 862 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt, 863 match_data->statictr_z_mask)); 864 865 /* save the current staticTR configuration */ 866 memcpy(&uc->static_tr, &uc->desc->static_tr, 867 sizeof(uc->static_tr)); 868 } 869 870 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 871 UDMA_CHAN_RT_CTL_EN); 872 873 /* Enable remote */ 874 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 875 UDMA_PEER_RT_EN_ENABLE); 876 877 break; 878 case DMA_MEM_TO_DEV: 879 /* Config remote TR */ 880 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 881 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 882 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 883 884 if (uc->config.enable_acc32) 885 val |= PDMA_STATIC_TR_XY_ACC32; 886 if (uc->config.enable_burst) 887 val |= PDMA_STATIC_TR_XY_BURST; 888 889 udma_tchanrt_write(uc->tchan, 890 UDMA_TCHAN_RT_PEER_STATIC_TR_XY_REG, val); 891 892 /* save the current staticTR configuration */ 893 memcpy(&uc->static_tr, &uc->desc->static_tr, 894 sizeof(uc->static_tr)); 895 } 896 897 /* Enable remote */ 898 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 899 UDMA_PEER_RT_EN_ENABLE); 900 901 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 902 UDMA_CHAN_RT_CTL_EN); 903 904 break; 905 case DMA_MEM_TO_MEM: 906 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 907 UDMA_CHAN_RT_CTL_EN); 908 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 909 UDMA_CHAN_RT_CTL_EN); 910 911 break; 912 default: 913 return -EINVAL; 914 } 915 916 uc->state = UDMA_CHAN_IS_ACTIVE; 917 out: 918 919 return 0; 920 } 921 922 static int udma_stop(struct udma_chan *uc) 923 { 924 enum udma_chan_state old_state = uc->state; 925 926 uc->state = UDMA_CHAN_IS_TERMINATING; 927 reinit_completion(&uc->teardown_completed); 928 929 switch (uc->config.dir) { 930 case DMA_DEV_TO_MEM: 931 if (!uc->cyclic && !uc->desc) 932 udma_push_to_ring(uc, -1); 933 934 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 935 UDMA_PEER_RT_EN_ENABLE | 936 UDMA_PEER_RT_EN_TEARDOWN); 937 break; 938 case DMA_MEM_TO_DEV: 939 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 940 UDMA_PEER_RT_EN_ENABLE | 941 UDMA_PEER_RT_EN_FLUSH); 942 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 943 UDMA_CHAN_RT_CTL_EN | 944 UDMA_CHAN_RT_CTL_TDOWN); 945 break; 946 case DMA_MEM_TO_MEM: 947 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 948 UDMA_CHAN_RT_CTL_EN | 949 UDMA_CHAN_RT_CTL_TDOWN); 950 break; 951 default: 952 uc->state = old_state; 953 complete_all(&uc->teardown_completed); 954 return -EINVAL; 955 } 956 957 return 0; 958 } 959 960 static void udma_cyclic_packet_elapsed(struct udma_chan *uc) 961 { 962 struct udma_desc *d = uc->desc; 963 struct cppi5_host_desc_t *h_desc; 964 965 h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr; 966 cppi5_hdesc_reset_to_original(h_desc); 967 udma_push_to_ring(uc, d->desc_idx); 968 d->desc_idx = (d->desc_idx + 1) % d->sglen; 969 } 970 971 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d) 972 { 973 struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr; 974 975 memcpy(d->metadata, h_desc->epib, d->metadata_size); 976 } 977 978 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) 979 { 980 u32 peer_bcnt, bcnt; 981 982 /* Only TX towards PDMA is affected */ 983 if (uc->config.ep_type == PSIL_EP_NATIVE || 984 uc->config.dir != DMA_MEM_TO_DEV) 985 return true; 986 987 peer_bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); 988 bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); 989 990 /* Transfer is incomplete, store current residue and time stamp */ 991 if (peer_bcnt < bcnt) { 992 uc->tx_drain.residue = bcnt - peer_bcnt; 993 uc->tx_drain.tstamp = ktime_get(); 994 return false; 995 } 996 997 return true; 998 } 999 1000 static void udma_check_tx_completion(struct work_struct *work) 1001 { 1002 struct udma_chan *uc = container_of(work, typeof(*uc), 1003 tx_drain.work.work); 1004 bool desc_done = true; 1005 u32 residue_diff; 1006 ktime_t time_diff; 1007 unsigned long delay; 1008 1009 while (1) { 1010 if (uc->desc) { 1011 /* Get previous residue and time stamp */ 1012 residue_diff = uc->tx_drain.residue; 1013 time_diff = uc->tx_drain.tstamp; 1014 /* 1015 * Get current residue and time stamp or see if 1016 * transfer is complete 1017 */ 1018 desc_done = udma_is_desc_really_done(uc, uc->desc); 1019 } 1020 1021 if (!desc_done) { 1022 /* 1023 * Find the time delta and residue delta w.r.t 1024 * previous poll 1025 */ 1026 time_diff = ktime_sub(uc->tx_drain.tstamp, 1027 time_diff) + 1; 1028 residue_diff -= uc->tx_drain.residue; 1029 if (residue_diff) { 1030 /* 1031 * Try to guess when we should check 1032 * next time by calculating rate at 1033 * which data is being drained at the 1034 * peer device 1035 */ 1036 delay = (time_diff / residue_diff) * 1037 uc->tx_drain.residue; 1038 } else { 1039 /* No progress, check again in 1 second */ 1040 schedule_delayed_work(&uc->tx_drain.work, HZ); 1041 break; 1042 } 1043 1044 usleep_range(ktime_to_us(delay), 1045 ktime_to_us(delay) + 10); 1046 continue; 1047 } 1048 1049 if (uc->desc) { 1050 struct udma_desc *d = uc->desc; 1051 1052 uc->bcnt += d->residue; 1053 udma_start(uc); 1054 vchan_cookie_complete(&d->vd); 1055 break; 1056 } 1057 1058 break; 1059 } 1060 } 1061 1062 static irqreturn_t udma_ring_irq_handler(int irq, void *data) 1063 { 1064 struct udma_chan *uc = data; 1065 struct udma_desc *d; 1066 unsigned long flags; 1067 dma_addr_t paddr = 0; 1068 1069 if (udma_pop_from_ring(uc, &paddr) || !paddr) 1070 return IRQ_HANDLED; 1071 1072 spin_lock_irqsave(&uc->vc.lock, flags); 1073 1074 /* Teardown completion message */ 1075 if (cppi5_desc_is_tdcm(paddr)) { 1076 /* Compensate our internal pop/push counter */ 1077 uc->in_ring_cnt++; 1078 1079 complete_all(&uc->teardown_completed); 1080 1081 if (uc->terminated_desc) { 1082 udma_desc_free(&uc->terminated_desc->vd); 1083 uc->terminated_desc = NULL; 1084 } 1085 1086 if (!uc->desc) 1087 udma_start(uc); 1088 1089 goto out; 1090 } 1091 1092 d = udma_udma_desc_from_paddr(uc, paddr); 1093 1094 if (d) { 1095 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 1096 d->desc_idx); 1097 if (desc_paddr != paddr) { 1098 dev_err(uc->ud->dev, "not matching descriptors!\n"); 1099 goto out; 1100 } 1101 1102 if (d == uc->desc) { 1103 /* active descriptor */ 1104 if (uc->cyclic) { 1105 udma_cyclic_packet_elapsed(uc); 1106 vchan_cyclic_callback(&d->vd); 1107 } else { 1108 if (udma_is_desc_really_done(uc, d)) { 1109 uc->bcnt += d->residue; 1110 udma_start(uc); 1111 vchan_cookie_complete(&d->vd); 1112 } else { 1113 schedule_delayed_work(&uc->tx_drain.work, 1114 0); 1115 } 1116 } 1117 } else { 1118 /* 1119 * terminated descriptor, mark the descriptor as 1120 * completed to update the channel's cookie marker 1121 */ 1122 dma_cookie_complete(&d->vd.tx); 1123 } 1124 } 1125 out: 1126 spin_unlock_irqrestore(&uc->vc.lock, flags); 1127 1128 return IRQ_HANDLED; 1129 } 1130 1131 static irqreturn_t udma_udma_irq_handler(int irq, void *data) 1132 { 1133 struct udma_chan *uc = data; 1134 struct udma_desc *d; 1135 unsigned long flags; 1136 1137 spin_lock_irqsave(&uc->vc.lock, flags); 1138 d = uc->desc; 1139 if (d) { 1140 d->tr_idx = (d->tr_idx + 1) % d->sglen; 1141 1142 if (uc->cyclic) { 1143 vchan_cyclic_callback(&d->vd); 1144 } else { 1145 /* TODO: figure out the real amount of data */ 1146 uc->bcnt += d->residue; 1147 udma_start(uc); 1148 vchan_cookie_complete(&d->vd); 1149 } 1150 } 1151 1152 spin_unlock_irqrestore(&uc->vc.lock, flags); 1153 1154 return IRQ_HANDLED; 1155 } 1156 1157 /** 1158 * __udma_alloc_gp_rflow_range - alloc range of GP RX flows 1159 * @ud: UDMA device 1160 * @from: Start the search from this flow id number 1161 * @cnt: Number of consecutive flow ids to allocate 1162 * 1163 * Allocate range of RX flow ids for future use, those flows can be requested 1164 * only using explicit flow id number. if @from is set to -1 it will try to find 1165 * first free range. if @from is positive value it will force allocation only 1166 * of the specified range of flows. 1167 * 1168 * Returns -ENOMEM if can't find free range. 1169 * -EEXIST if requested range is busy. 1170 * -EINVAL if wrong input values passed. 1171 * Returns flow id on success. 1172 */ 1173 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1174 { 1175 int start, tmp_from; 1176 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS); 1177 1178 tmp_from = from; 1179 if (tmp_from < 0) 1180 tmp_from = ud->rchan_cnt; 1181 /* default flows can't be allocated and accessible only by id */ 1182 if (tmp_from < ud->rchan_cnt) 1183 return -EINVAL; 1184 1185 if (tmp_from + cnt > ud->rflow_cnt) 1186 return -EINVAL; 1187 1188 bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated, 1189 ud->rflow_cnt); 1190 1191 start = bitmap_find_next_zero_area(tmp, 1192 ud->rflow_cnt, 1193 tmp_from, cnt, 0); 1194 if (start >= ud->rflow_cnt) 1195 return -ENOMEM; 1196 1197 if (from >= 0 && start != from) 1198 return -EEXIST; 1199 1200 bitmap_set(ud->rflow_gp_map_allocated, start, cnt); 1201 return start; 1202 } 1203 1204 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1205 { 1206 if (from < ud->rchan_cnt) 1207 return -EINVAL; 1208 if (from + cnt > ud->rflow_cnt) 1209 return -EINVAL; 1210 1211 bitmap_clear(ud->rflow_gp_map_allocated, from, cnt); 1212 return 0; 1213 } 1214 1215 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id) 1216 { 1217 /* 1218 * Attempt to request rflow by ID can be made for any rflow 1219 * if not in use with assumption that caller knows what's doing. 1220 * TI-SCI FW will perform additional permission check ant way, it's 1221 * safe 1222 */ 1223 1224 if (id < 0 || id >= ud->rflow_cnt) 1225 return ERR_PTR(-ENOENT); 1226 1227 if (test_bit(id, ud->rflow_in_use)) 1228 return ERR_PTR(-ENOENT); 1229 1230 /* GP rflow has to be allocated first */ 1231 if (!test_bit(id, ud->rflow_gp_map) && 1232 !test_bit(id, ud->rflow_gp_map_allocated)) 1233 return ERR_PTR(-EINVAL); 1234 1235 dev_dbg(ud->dev, "get rflow%d\n", id); 1236 set_bit(id, ud->rflow_in_use); 1237 return &ud->rflows[id]; 1238 } 1239 1240 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow) 1241 { 1242 if (!test_bit(rflow->id, ud->rflow_in_use)) { 1243 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id); 1244 return; 1245 } 1246 1247 dev_dbg(ud->dev, "put rflow%d\n", rflow->id); 1248 clear_bit(rflow->id, ud->rflow_in_use); 1249 } 1250 1251 #define UDMA_RESERVE_RESOURCE(res) \ 1252 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ 1253 enum udma_tp_level tpl, \ 1254 int id) \ 1255 { \ 1256 if (id >= 0) { \ 1257 if (test_bit(id, ud->res##_map)) { \ 1258 dev_err(ud->dev, "res##%d is in use\n", id); \ 1259 return ERR_PTR(-ENOENT); \ 1260 } \ 1261 } else { \ 1262 int start; \ 1263 \ 1264 if (tpl >= ud->match_data->tpl_levels) \ 1265 tpl = ud->match_data->tpl_levels - 1; \ 1266 \ 1267 start = ud->match_data->level_start_idx[tpl]; \ 1268 \ 1269 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \ 1270 start); \ 1271 if (id == ud->res##_cnt) { \ 1272 return ERR_PTR(-ENOENT); \ 1273 } \ 1274 } \ 1275 \ 1276 set_bit(id, ud->res##_map); \ 1277 return &ud->res##s[id]; \ 1278 } 1279 1280 UDMA_RESERVE_RESOURCE(tchan); 1281 UDMA_RESERVE_RESOURCE(rchan); 1282 1283 static int udma_get_tchan(struct udma_chan *uc) 1284 { 1285 struct udma_dev *ud = uc->ud; 1286 1287 if (uc->tchan) { 1288 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n", 1289 uc->id, uc->tchan->id); 1290 return 0; 1291 } 1292 1293 uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, -1); 1294 if (IS_ERR(uc->tchan)) 1295 return PTR_ERR(uc->tchan); 1296 1297 return 0; 1298 } 1299 1300 static int udma_get_rchan(struct udma_chan *uc) 1301 { 1302 struct udma_dev *ud = uc->ud; 1303 1304 if (uc->rchan) { 1305 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n", 1306 uc->id, uc->rchan->id); 1307 return 0; 1308 } 1309 1310 uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, -1); 1311 if (IS_ERR(uc->rchan)) 1312 return PTR_ERR(uc->rchan); 1313 1314 return 0; 1315 } 1316 1317 static int udma_get_chan_pair(struct udma_chan *uc) 1318 { 1319 struct udma_dev *ud = uc->ud; 1320 const struct udma_match_data *match_data = ud->match_data; 1321 int chan_id, end; 1322 1323 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) { 1324 dev_info(ud->dev, "chan%d: already have %d pair allocated\n", 1325 uc->id, uc->tchan->id); 1326 return 0; 1327 } 1328 1329 if (uc->tchan) { 1330 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n", 1331 uc->id, uc->tchan->id); 1332 return -EBUSY; 1333 } else if (uc->rchan) { 1334 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n", 1335 uc->id, uc->rchan->id); 1336 return -EBUSY; 1337 } 1338 1339 /* Can be optimized, but let's have it like this for now */ 1340 end = min(ud->tchan_cnt, ud->rchan_cnt); 1341 /* Try to use the highest TPL channel pair for MEM_TO_MEM channels */ 1342 chan_id = match_data->level_start_idx[match_data->tpl_levels - 1]; 1343 for (; chan_id < end; chan_id++) { 1344 if (!test_bit(chan_id, ud->tchan_map) && 1345 !test_bit(chan_id, ud->rchan_map)) 1346 break; 1347 } 1348 1349 if (chan_id == end) 1350 return -ENOENT; 1351 1352 set_bit(chan_id, ud->tchan_map); 1353 set_bit(chan_id, ud->rchan_map); 1354 uc->tchan = &ud->tchans[chan_id]; 1355 uc->rchan = &ud->rchans[chan_id]; 1356 1357 return 0; 1358 } 1359 1360 static int udma_get_rflow(struct udma_chan *uc, int flow_id) 1361 { 1362 struct udma_dev *ud = uc->ud; 1363 1364 if (!uc->rchan) { 1365 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id); 1366 return -EINVAL; 1367 } 1368 1369 if (uc->rflow) { 1370 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n", 1371 uc->id, uc->rflow->id); 1372 return 0; 1373 } 1374 1375 uc->rflow = __udma_get_rflow(ud, flow_id); 1376 if (IS_ERR(uc->rflow)) 1377 return PTR_ERR(uc->rflow); 1378 1379 return 0; 1380 } 1381 1382 static void udma_put_rchan(struct udma_chan *uc) 1383 { 1384 struct udma_dev *ud = uc->ud; 1385 1386 if (uc->rchan) { 1387 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id, 1388 uc->rchan->id); 1389 clear_bit(uc->rchan->id, ud->rchan_map); 1390 uc->rchan = NULL; 1391 } 1392 } 1393 1394 static void udma_put_tchan(struct udma_chan *uc) 1395 { 1396 struct udma_dev *ud = uc->ud; 1397 1398 if (uc->tchan) { 1399 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id, 1400 uc->tchan->id); 1401 clear_bit(uc->tchan->id, ud->tchan_map); 1402 uc->tchan = NULL; 1403 } 1404 } 1405 1406 static void udma_put_rflow(struct udma_chan *uc) 1407 { 1408 struct udma_dev *ud = uc->ud; 1409 1410 if (uc->rflow) { 1411 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id, 1412 uc->rflow->id); 1413 __udma_put_rflow(ud, uc->rflow); 1414 uc->rflow = NULL; 1415 } 1416 } 1417 1418 static void udma_free_tx_resources(struct udma_chan *uc) 1419 { 1420 if (!uc->tchan) 1421 return; 1422 1423 k3_ringacc_ring_free(uc->tchan->t_ring); 1424 k3_ringacc_ring_free(uc->tchan->tc_ring); 1425 uc->tchan->t_ring = NULL; 1426 uc->tchan->tc_ring = NULL; 1427 1428 udma_put_tchan(uc); 1429 } 1430 1431 static int udma_alloc_tx_resources(struct udma_chan *uc) 1432 { 1433 struct k3_ring_cfg ring_cfg; 1434 struct udma_dev *ud = uc->ud; 1435 int ret; 1436 1437 ret = udma_get_tchan(uc); 1438 if (ret) 1439 return ret; 1440 1441 uc->tchan->t_ring = k3_ringacc_request_ring(ud->ringacc, 1442 uc->tchan->id, 0); 1443 if (!uc->tchan->t_ring) { 1444 ret = -EBUSY; 1445 goto err_tx_ring; 1446 } 1447 1448 uc->tchan->tc_ring = k3_ringacc_request_ring(ud->ringacc, -1, 0); 1449 if (!uc->tchan->tc_ring) { 1450 ret = -EBUSY; 1451 goto err_txc_ring; 1452 } 1453 1454 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1455 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1456 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1457 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1458 1459 ret = k3_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg); 1460 ret |= k3_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg); 1461 1462 if (ret) 1463 goto err_ringcfg; 1464 1465 return 0; 1466 1467 err_ringcfg: 1468 k3_ringacc_ring_free(uc->tchan->tc_ring); 1469 uc->tchan->tc_ring = NULL; 1470 err_txc_ring: 1471 k3_ringacc_ring_free(uc->tchan->t_ring); 1472 uc->tchan->t_ring = NULL; 1473 err_tx_ring: 1474 udma_put_tchan(uc); 1475 1476 return ret; 1477 } 1478 1479 static void udma_free_rx_resources(struct udma_chan *uc) 1480 { 1481 if (!uc->rchan) 1482 return; 1483 1484 if (uc->rflow) { 1485 struct udma_rflow *rflow = uc->rflow; 1486 1487 k3_ringacc_ring_free(rflow->fd_ring); 1488 k3_ringacc_ring_free(rflow->r_ring); 1489 rflow->fd_ring = NULL; 1490 rflow->r_ring = NULL; 1491 1492 udma_put_rflow(uc); 1493 } 1494 1495 udma_put_rchan(uc); 1496 } 1497 1498 static int udma_alloc_rx_resources(struct udma_chan *uc) 1499 { 1500 struct udma_dev *ud = uc->ud; 1501 struct k3_ring_cfg ring_cfg; 1502 struct udma_rflow *rflow; 1503 int fd_ring_id; 1504 int ret; 1505 1506 ret = udma_get_rchan(uc); 1507 if (ret) 1508 return ret; 1509 1510 /* For MEM_TO_MEM we don't need rflow or rings */ 1511 if (uc->config.dir == DMA_MEM_TO_MEM) 1512 return 0; 1513 1514 ret = udma_get_rflow(uc, uc->rchan->id); 1515 if (ret) { 1516 ret = -EBUSY; 1517 goto err_rflow; 1518 } 1519 1520 rflow = uc->rflow; 1521 fd_ring_id = ud->tchan_cnt + ud->echan_cnt + uc->rchan->id; 1522 rflow->fd_ring = k3_ringacc_request_ring(ud->ringacc, fd_ring_id, 0); 1523 if (!rflow->fd_ring) { 1524 ret = -EBUSY; 1525 goto err_rx_ring; 1526 } 1527 1528 rflow->r_ring = k3_ringacc_request_ring(ud->ringacc, -1, 0); 1529 if (!rflow->r_ring) { 1530 ret = -EBUSY; 1531 goto err_rxc_ring; 1532 } 1533 1534 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1535 1536 if (uc->config.pkt_mode) 1537 ring_cfg.size = SG_MAX_SEGMENTS; 1538 else 1539 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1540 1541 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1542 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1543 1544 ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg); 1545 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1546 ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg); 1547 1548 if (ret) 1549 goto err_ringcfg; 1550 1551 return 0; 1552 1553 err_ringcfg: 1554 k3_ringacc_ring_free(rflow->r_ring); 1555 rflow->r_ring = NULL; 1556 err_rxc_ring: 1557 k3_ringacc_ring_free(rflow->fd_ring); 1558 rflow->fd_ring = NULL; 1559 err_rx_ring: 1560 udma_put_rflow(uc); 1561 err_rflow: 1562 udma_put_rchan(uc); 1563 1564 return ret; 1565 } 1566 1567 #define TISCI_TCHAN_VALID_PARAMS ( \ 1568 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1569 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \ 1570 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \ 1571 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1572 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \ 1573 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1574 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1575 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1576 1577 #define TISCI_RCHAN_VALID_PARAMS ( \ 1578 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1579 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1580 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1581 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1582 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \ 1583 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \ 1584 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \ 1585 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \ 1586 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1587 1588 static int udma_tisci_m2m_channel_config(struct udma_chan *uc) 1589 { 1590 struct udma_dev *ud = uc->ud; 1591 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1592 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1593 struct udma_tchan *tchan = uc->tchan; 1594 struct udma_rchan *rchan = uc->rchan; 1595 int ret = 0; 1596 1597 /* Non synchronized - mem to mem type of transfer */ 1598 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1599 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1600 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1601 1602 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; 1603 req_tx.nav_id = tisci_rm->tisci_dev_id; 1604 req_tx.index = tchan->id; 1605 req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1606 req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1607 req_tx.txcq_qnum = tc_ring; 1608 req_tx.tx_atype = ud->atype; 1609 1610 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1611 if (ret) { 1612 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1613 return ret; 1614 } 1615 1616 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; 1617 req_rx.nav_id = tisci_rm->tisci_dev_id; 1618 req_rx.index = rchan->id; 1619 req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1620 req_rx.rxcq_qnum = tc_ring; 1621 req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1622 req_rx.rx_atype = ud->atype; 1623 1624 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1625 if (ret) 1626 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret); 1627 1628 return ret; 1629 } 1630 1631 static int udma_tisci_tx_channel_config(struct udma_chan *uc) 1632 { 1633 struct udma_dev *ud = uc->ud; 1634 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1635 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1636 struct udma_tchan *tchan = uc->tchan; 1637 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1638 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1639 u32 mode, fetch_size; 1640 int ret = 0; 1641 1642 if (uc->config.pkt_mode) { 1643 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1644 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1645 uc->config.psd_size, 0); 1646 } else { 1647 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1648 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1649 } 1650 1651 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; 1652 req_tx.nav_id = tisci_rm->tisci_dev_id; 1653 req_tx.index = tchan->id; 1654 req_tx.tx_chan_type = mode; 1655 req_tx.tx_supr_tdpkt = uc->config.notdpkt; 1656 req_tx.tx_fetch_size = fetch_size >> 2; 1657 req_tx.txcq_qnum = tc_ring; 1658 req_tx.tx_atype = uc->config.atype; 1659 1660 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1661 if (ret) 1662 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1663 1664 return ret; 1665 } 1666 1667 static int udma_tisci_rx_channel_config(struct udma_chan *uc) 1668 { 1669 struct udma_dev *ud = uc->ud; 1670 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1671 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1672 struct udma_rchan *rchan = uc->rchan; 1673 int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring); 1674 int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring); 1675 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1676 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; 1677 u32 mode, fetch_size; 1678 int ret = 0; 1679 1680 if (uc->config.pkt_mode) { 1681 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1682 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1683 uc->config.psd_size, 0); 1684 } else { 1685 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1686 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1687 } 1688 1689 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; 1690 req_rx.nav_id = tisci_rm->tisci_dev_id; 1691 req_rx.index = rchan->id; 1692 req_rx.rx_fetch_size = fetch_size >> 2; 1693 req_rx.rxcq_qnum = rx_ring; 1694 req_rx.rx_chan_type = mode; 1695 req_rx.rx_atype = uc->config.atype; 1696 1697 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1698 if (ret) { 1699 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); 1700 return ret; 1701 } 1702 1703 flow_req.valid_params = 1704 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 1705 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 1706 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 1707 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 1708 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1709 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 1710 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 1711 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 1712 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 1713 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1714 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1715 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1716 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1717 1718 flow_req.nav_id = tisci_rm->tisci_dev_id; 1719 flow_req.flow_index = rchan->id; 1720 1721 if (uc->config.needs_epib) 1722 flow_req.rx_einfo_present = 1; 1723 else 1724 flow_req.rx_einfo_present = 0; 1725 if (uc->config.psd_size) 1726 flow_req.rx_psinfo_present = 1; 1727 else 1728 flow_req.rx_psinfo_present = 0; 1729 flow_req.rx_error_handling = 1; 1730 flow_req.rx_dest_qnum = rx_ring; 1731 flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE; 1732 flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG; 1733 flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI; 1734 flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO; 1735 flow_req.rx_fdq0_sz0_qnum = fd_ring; 1736 flow_req.rx_fdq1_qnum = fd_ring; 1737 flow_req.rx_fdq2_qnum = fd_ring; 1738 flow_req.rx_fdq3_qnum = fd_ring; 1739 1740 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req); 1741 1742 if (ret) 1743 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret); 1744 1745 return 0; 1746 } 1747 1748 static int udma_alloc_chan_resources(struct dma_chan *chan) 1749 { 1750 struct udma_chan *uc = to_udma_chan(chan); 1751 struct udma_dev *ud = to_udma_dev(chan->device); 1752 const struct udma_match_data *match_data = ud->match_data; 1753 struct k3_ring *irq_ring; 1754 u32 irq_udma_idx; 1755 int ret; 1756 1757 if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) { 1758 uc->use_dma_pool = true; 1759 /* in case of MEM_TO_MEM we have maximum of two TRs */ 1760 if (uc->config.dir == DMA_MEM_TO_MEM) { 1761 uc->config.hdesc_size = cppi5_trdesc_calc_size( 1762 sizeof(struct cppi5_tr_type15_t), 2); 1763 uc->config.pkt_mode = false; 1764 } 1765 } 1766 1767 if (uc->use_dma_pool) { 1768 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, 1769 uc->config.hdesc_size, 1770 ud->desc_align, 1771 0); 1772 if (!uc->hdesc_pool) { 1773 dev_err(ud->ddev.dev, 1774 "Descriptor pool allocation failed\n"); 1775 uc->use_dma_pool = false; 1776 return -ENOMEM; 1777 } 1778 } 1779 1780 /* 1781 * Make sure that the completion is in a known state: 1782 * No teardown, the channel is idle 1783 */ 1784 reinit_completion(&uc->teardown_completed); 1785 complete_all(&uc->teardown_completed); 1786 uc->state = UDMA_CHAN_IS_IDLE; 1787 1788 switch (uc->config.dir) { 1789 case DMA_MEM_TO_MEM: 1790 /* Non synchronized - mem to mem type of transfer */ 1791 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, 1792 uc->id); 1793 1794 ret = udma_get_chan_pair(uc); 1795 if (ret) 1796 return ret; 1797 1798 ret = udma_alloc_tx_resources(uc); 1799 if (ret) 1800 return ret; 1801 1802 ret = udma_alloc_rx_resources(uc); 1803 if (ret) { 1804 udma_free_tx_resources(uc); 1805 return ret; 1806 } 1807 1808 uc->config.src_thread = ud->psil_base + uc->tchan->id; 1809 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 1810 K3_PSIL_DST_THREAD_ID_OFFSET; 1811 1812 irq_ring = uc->tchan->tc_ring; 1813 irq_udma_idx = uc->tchan->id; 1814 1815 ret = udma_tisci_m2m_channel_config(uc); 1816 break; 1817 case DMA_MEM_TO_DEV: 1818 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 1819 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 1820 uc->id); 1821 1822 ret = udma_alloc_tx_resources(uc); 1823 if (ret) { 1824 uc->config.remote_thread_id = -1; 1825 return ret; 1826 } 1827 1828 uc->config.src_thread = ud->psil_base + uc->tchan->id; 1829 uc->config.dst_thread = uc->config.remote_thread_id; 1830 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 1831 1832 irq_ring = uc->tchan->tc_ring; 1833 irq_udma_idx = uc->tchan->id; 1834 1835 ret = udma_tisci_tx_channel_config(uc); 1836 break; 1837 case DMA_DEV_TO_MEM: 1838 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 1839 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 1840 uc->id); 1841 1842 ret = udma_alloc_rx_resources(uc); 1843 if (ret) { 1844 uc->config.remote_thread_id = -1; 1845 return ret; 1846 } 1847 1848 uc->config.src_thread = uc->config.remote_thread_id; 1849 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 1850 K3_PSIL_DST_THREAD_ID_OFFSET; 1851 1852 irq_ring = uc->rflow->r_ring; 1853 irq_udma_idx = match_data->rchan_oes_offset + uc->rchan->id; 1854 1855 ret = udma_tisci_rx_channel_config(uc); 1856 break; 1857 default: 1858 /* Can not happen */ 1859 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 1860 __func__, uc->id, uc->config.dir); 1861 return -EINVAL; 1862 } 1863 1864 /* check if the channel configuration was successful */ 1865 if (ret) 1866 goto err_res_free; 1867 1868 if (udma_is_chan_running(uc)) { 1869 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 1870 udma_stop(uc); 1871 if (udma_is_chan_running(uc)) { 1872 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 1873 goto err_res_free; 1874 } 1875 } 1876 1877 /* PSI-L pairing */ 1878 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); 1879 if (ret) { 1880 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 1881 uc->config.src_thread, uc->config.dst_thread); 1882 goto err_res_free; 1883 } 1884 1885 uc->psil_paired = true; 1886 1887 uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring); 1888 if (uc->irq_num_ring <= 0) { 1889 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 1890 k3_ringacc_get_ring_id(irq_ring)); 1891 ret = -EINVAL; 1892 goto err_psi_free; 1893 } 1894 1895 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 1896 IRQF_TRIGGER_HIGH, uc->name, uc); 1897 if (ret) { 1898 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 1899 goto err_irq_free; 1900 } 1901 1902 /* Event from UDMA (TR events) only needed for slave TR mode channels */ 1903 if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) { 1904 uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, 1905 irq_udma_idx); 1906 if (uc->irq_num_udma <= 0) { 1907 dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", 1908 irq_udma_idx); 1909 free_irq(uc->irq_num_ring, uc); 1910 ret = -EINVAL; 1911 goto err_irq_free; 1912 } 1913 1914 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0, 1915 uc->name, uc); 1916 if (ret) { 1917 dev_err(ud->dev, "chan%d: UDMA irq request failed\n", 1918 uc->id); 1919 free_irq(uc->irq_num_ring, uc); 1920 goto err_irq_free; 1921 } 1922 } else { 1923 uc->irq_num_udma = 0; 1924 } 1925 1926 udma_reset_rings(uc); 1927 1928 INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work, 1929 udma_check_tx_completion); 1930 return 0; 1931 1932 err_irq_free: 1933 uc->irq_num_ring = 0; 1934 uc->irq_num_udma = 0; 1935 err_psi_free: 1936 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); 1937 uc->psil_paired = false; 1938 err_res_free: 1939 udma_free_tx_resources(uc); 1940 udma_free_rx_resources(uc); 1941 1942 udma_reset_uchan(uc); 1943 1944 if (uc->use_dma_pool) { 1945 dma_pool_destroy(uc->hdesc_pool); 1946 uc->use_dma_pool = false; 1947 } 1948 1949 return ret; 1950 } 1951 1952 static int udma_slave_config(struct dma_chan *chan, 1953 struct dma_slave_config *cfg) 1954 { 1955 struct udma_chan *uc = to_udma_chan(chan); 1956 1957 memcpy(&uc->cfg, cfg, sizeof(uc->cfg)); 1958 1959 return 0; 1960 } 1961 1962 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc, 1963 size_t tr_size, int tr_count, 1964 enum dma_transfer_direction dir) 1965 { 1966 struct udma_hwdesc *hwdesc; 1967 struct cppi5_desc_hdr_t *tr_desc; 1968 struct udma_desc *d; 1969 u32 reload_count = 0; 1970 u32 ring_id; 1971 1972 switch (tr_size) { 1973 case 16: 1974 case 32: 1975 case 64: 1976 case 128: 1977 break; 1978 default: 1979 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size); 1980 return NULL; 1981 } 1982 1983 /* We have only one descriptor containing multiple TRs */ 1984 d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT); 1985 if (!d) 1986 return NULL; 1987 1988 d->sglen = tr_count; 1989 1990 d->hwdesc_count = 1; 1991 hwdesc = &d->hwdesc[0]; 1992 1993 /* Allocate memory for DMA ring descriptor */ 1994 if (uc->use_dma_pool) { 1995 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 1996 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 1997 GFP_NOWAIT, 1998 &hwdesc->cppi5_desc_paddr); 1999 } else { 2000 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 2001 tr_count); 2002 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 2003 uc->ud->desc_align); 2004 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev, 2005 hwdesc->cppi5_desc_size, 2006 &hwdesc->cppi5_desc_paddr, 2007 GFP_NOWAIT); 2008 } 2009 2010 if (!hwdesc->cppi5_desc_vaddr) { 2011 kfree(d); 2012 return NULL; 2013 } 2014 2015 /* Start of the TR req records */ 2016 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 2017 /* Start address of the TR response array */ 2018 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count; 2019 2020 tr_desc = hwdesc->cppi5_desc_vaddr; 2021 2022 if (uc->cyclic) 2023 reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE; 2024 2025 if (dir == DMA_DEV_TO_MEM) 2026 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2027 else 2028 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2029 2030 cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count); 2031 cppi5_desc_set_pktids(tr_desc, uc->id, 2032 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2033 cppi5_desc_set_retpolicy(tr_desc, 0, ring_id); 2034 2035 return d; 2036 } 2037 2038 /** 2039 * udma_get_tr_counters - calculate TR counters for a given length 2040 * @len: Length of the trasnfer 2041 * @align_to: Preferred alignment 2042 * @tr0_cnt0: First TR icnt0 2043 * @tr0_cnt1: First TR icnt1 2044 * @tr1_cnt0: Second (if used) TR icnt0 2045 * 2046 * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated 2047 * For len >= SZ_64K two TRs are used in a simple way: 2048 * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1) 2049 * Second TR: the remaining length (tr1_cnt0) 2050 * 2051 * Returns the number of TRs the length needs (1 or 2) 2052 * -EINVAL if the length can not be supported 2053 */ 2054 static int udma_get_tr_counters(size_t len, unsigned long align_to, 2055 u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0) 2056 { 2057 if (len < SZ_64K) { 2058 *tr0_cnt0 = len; 2059 *tr0_cnt1 = 1; 2060 2061 return 1; 2062 } 2063 2064 if (align_to > 3) 2065 align_to = 3; 2066 2067 realign: 2068 *tr0_cnt0 = SZ_64K - BIT(align_to); 2069 if (len / *tr0_cnt0 >= SZ_64K) { 2070 if (align_to) { 2071 align_to--; 2072 goto realign; 2073 } 2074 return -EINVAL; 2075 } 2076 2077 *tr0_cnt1 = len / *tr0_cnt0; 2078 *tr1_cnt0 = len % *tr0_cnt0; 2079 2080 return 2; 2081 } 2082 2083 static struct udma_desc * 2084 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, 2085 unsigned int sglen, enum dma_transfer_direction dir, 2086 unsigned long tx_flags, void *context) 2087 { 2088 struct scatterlist *sgent; 2089 struct udma_desc *d; 2090 struct cppi5_tr_type1_t *tr_req = NULL; 2091 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2092 unsigned int i; 2093 size_t tr_size; 2094 int num_tr = 0; 2095 int tr_idx = 0; 2096 2097 if (!is_slave_direction(dir)) { 2098 dev_err(uc->ud->dev, "Only slave cyclic is supported\n"); 2099 return NULL; 2100 } 2101 2102 /* estimate the number of TRs we will need */ 2103 for_each_sg(sgl, sgent, sglen, i) { 2104 if (sg_dma_len(sgent) < SZ_64K) 2105 num_tr++; 2106 else 2107 num_tr += 2; 2108 } 2109 2110 /* Now allocate and setup the descriptor. */ 2111 tr_size = sizeof(struct cppi5_tr_type1_t); 2112 d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir); 2113 if (!d) 2114 return NULL; 2115 2116 d->sglen = sglen; 2117 2118 tr_req = d->hwdesc[0].tr_req_base; 2119 for_each_sg(sgl, sgent, sglen, i) { 2120 dma_addr_t sg_addr = sg_dma_address(sgent); 2121 2122 num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr), 2123 &tr0_cnt0, &tr0_cnt1, &tr1_cnt0); 2124 if (num_tr < 0) { 2125 dev_err(uc->ud->dev, "size %u is not supported\n", 2126 sg_dma_len(sgent)); 2127 udma_free_hwdesc(uc, d); 2128 kfree(d); 2129 return NULL; 2130 } 2131 2132 cppi5_tr_init(&tr_req[i].flags, CPPI5_TR_TYPE1, false, false, 2133 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2134 cppi5_tr_csf_set(&tr_req[i].flags, CPPI5_TR_CSF_SUPR_EVT); 2135 2136 tr_req[tr_idx].addr = sg_addr; 2137 tr_req[tr_idx].icnt0 = tr0_cnt0; 2138 tr_req[tr_idx].icnt1 = tr0_cnt1; 2139 tr_req[tr_idx].dim1 = tr0_cnt0; 2140 tr_idx++; 2141 2142 if (num_tr == 2) { 2143 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 2144 false, false, 2145 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2146 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2147 CPPI5_TR_CSF_SUPR_EVT); 2148 2149 tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0; 2150 tr_req[tr_idx].icnt0 = tr1_cnt0; 2151 tr_req[tr_idx].icnt1 = 1; 2152 tr_req[tr_idx].dim1 = tr1_cnt0; 2153 tr_idx++; 2154 } 2155 2156 d->residue += sg_dma_len(sgent); 2157 } 2158 2159 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, 2160 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); 2161 2162 return d; 2163 } 2164 2165 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d, 2166 enum dma_slave_buswidth dev_width, 2167 u16 elcnt) 2168 { 2169 if (uc->config.ep_type != PSIL_EP_PDMA_XY) 2170 return 0; 2171 2172 /* Bus width translates to the element size (ES) */ 2173 switch (dev_width) { 2174 case DMA_SLAVE_BUSWIDTH_1_BYTE: 2175 d->static_tr.elsize = 0; 2176 break; 2177 case DMA_SLAVE_BUSWIDTH_2_BYTES: 2178 d->static_tr.elsize = 1; 2179 break; 2180 case DMA_SLAVE_BUSWIDTH_3_BYTES: 2181 d->static_tr.elsize = 2; 2182 break; 2183 case DMA_SLAVE_BUSWIDTH_4_BYTES: 2184 d->static_tr.elsize = 3; 2185 break; 2186 case DMA_SLAVE_BUSWIDTH_8_BYTES: 2187 d->static_tr.elsize = 4; 2188 break; 2189 default: /* not reached */ 2190 return -EINVAL; 2191 } 2192 2193 d->static_tr.elcnt = elcnt; 2194 2195 /* 2196 * PDMA must to close the packet when the channel is in packet mode. 2197 * For TR mode when the channel is not cyclic we also need PDMA to close 2198 * the packet otherwise the transfer will stall because PDMA holds on 2199 * the data it has received from the peripheral. 2200 */ 2201 if (uc->config.pkt_mode || !uc->cyclic) { 2202 unsigned int div = dev_width * elcnt; 2203 2204 if (uc->cyclic) 2205 d->static_tr.bstcnt = d->residue / d->sglen / div; 2206 else 2207 d->static_tr.bstcnt = d->residue / div; 2208 2209 if (uc->config.dir == DMA_DEV_TO_MEM && 2210 d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) 2211 return -EINVAL; 2212 } else { 2213 d->static_tr.bstcnt = 0; 2214 } 2215 2216 return 0; 2217 } 2218 2219 static struct udma_desc * 2220 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, 2221 unsigned int sglen, enum dma_transfer_direction dir, 2222 unsigned long tx_flags, void *context) 2223 { 2224 struct scatterlist *sgent; 2225 struct cppi5_host_desc_t *h_desc = NULL; 2226 struct udma_desc *d; 2227 u32 ring_id; 2228 unsigned int i; 2229 2230 d = kzalloc(sizeof(*d) + sglen * sizeof(d->hwdesc[0]), GFP_NOWAIT); 2231 if (!d) 2232 return NULL; 2233 2234 d->sglen = sglen; 2235 d->hwdesc_count = sglen; 2236 2237 if (dir == DMA_DEV_TO_MEM) 2238 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2239 else 2240 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2241 2242 for_each_sg(sgl, sgent, sglen, i) { 2243 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 2244 dma_addr_t sg_addr = sg_dma_address(sgent); 2245 struct cppi5_host_desc_t *desc; 2246 size_t sg_len = sg_dma_len(sgent); 2247 2248 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2249 GFP_NOWAIT, 2250 &hwdesc->cppi5_desc_paddr); 2251 if (!hwdesc->cppi5_desc_vaddr) { 2252 dev_err(uc->ud->dev, 2253 "descriptor%d allocation failed\n", i); 2254 2255 udma_free_hwdesc(uc, d); 2256 kfree(d); 2257 return NULL; 2258 } 2259 2260 d->residue += sg_len; 2261 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2262 desc = hwdesc->cppi5_desc_vaddr; 2263 2264 if (i == 0) { 2265 cppi5_hdesc_init(desc, 0, 0); 2266 /* Flow and Packed ID */ 2267 cppi5_desc_set_pktids(&desc->hdr, uc->id, 2268 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2269 cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id); 2270 } else { 2271 cppi5_hdesc_reset_hbdesc(desc); 2272 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff); 2273 } 2274 2275 /* attach the sg buffer to the descriptor */ 2276 cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len); 2277 2278 /* Attach link as host buffer descriptor */ 2279 if (h_desc) 2280 cppi5_hdesc_link_hbdesc(h_desc, 2281 hwdesc->cppi5_desc_paddr); 2282 2283 if (dir == DMA_MEM_TO_DEV) 2284 h_desc = desc; 2285 } 2286 2287 if (d->residue >= SZ_4M) { 2288 dev_err(uc->ud->dev, 2289 "%s: Transfer size %u is over the supported 4M range\n", 2290 __func__, d->residue); 2291 udma_free_hwdesc(uc, d); 2292 kfree(d); 2293 return NULL; 2294 } 2295 2296 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2297 cppi5_hdesc_set_pktlen(h_desc, d->residue); 2298 2299 return d; 2300 } 2301 2302 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc, 2303 void *data, size_t len) 2304 { 2305 struct udma_desc *d = to_udma_desc(desc); 2306 struct udma_chan *uc = to_udma_chan(desc->chan); 2307 struct cppi5_host_desc_t *h_desc; 2308 u32 psd_size = len; 2309 u32 flags = 0; 2310 2311 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2312 return -ENOTSUPP; 2313 2314 if (!data || len > uc->config.metadata_size) 2315 return -EINVAL; 2316 2317 if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE) 2318 return -EINVAL; 2319 2320 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2321 if (d->dir == DMA_MEM_TO_DEV) 2322 memcpy(h_desc->epib, data, len); 2323 2324 if (uc->config.needs_epib) 2325 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 2326 2327 d->metadata = data; 2328 d->metadata_size = len; 2329 if (uc->config.needs_epib) 2330 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 2331 2332 cppi5_hdesc_update_flags(h_desc, flags); 2333 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 2334 2335 return 0; 2336 } 2337 2338 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc, 2339 size_t *payload_len, size_t *max_len) 2340 { 2341 struct udma_desc *d = to_udma_desc(desc); 2342 struct udma_chan *uc = to_udma_chan(desc->chan); 2343 struct cppi5_host_desc_t *h_desc; 2344 2345 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2346 return ERR_PTR(-ENOTSUPP); 2347 2348 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2349 2350 *max_len = uc->config.metadata_size; 2351 2352 *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ? 2353 CPPI5_INFO0_HDESC_EPIB_SIZE : 0; 2354 *payload_len += cppi5_hdesc_get_psdata_size(h_desc); 2355 2356 return h_desc->epib; 2357 } 2358 2359 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc, 2360 size_t payload_len) 2361 { 2362 struct udma_desc *d = to_udma_desc(desc); 2363 struct udma_chan *uc = to_udma_chan(desc->chan); 2364 struct cppi5_host_desc_t *h_desc; 2365 u32 psd_size = payload_len; 2366 u32 flags = 0; 2367 2368 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2369 return -ENOTSUPP; 2370 2371 if (payload_len > uc->config.metadata_size) 2372 return -EINVAL; 2373 2374 if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE) 2375 return -EINVAL; 2376 2377 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2378 2379 if (uc->config.needs_epib) { 2380 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 2381 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 2382 } 2383 2384 cppi5_hdesc_update_flags(h_desc, flags); 2385 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 2386 2387 return 0; 2388 } 2389 2390 static struct dma_descriptor_metadata_ops metadata_ops = { 2391 .attach = udma_attach_metadata, 2392 .get_ptr = udma_get_metadata_ptr, 2393 .set_len = udma_set_metadata_len, 2394 }; 2395 2396 static struct dma_async_tx_descriptor * 2397 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 2398 unsigned int sglen, enum dma_transfer_direction dir, 2399 unsigned long tx_flags, void *context) 2400 { 2401 struct udma_chan *uc = to_udma_chan(chan); 2402 enum dma_slave_buswidth dev_width; 2403 struct udma_desc *d; 2404 u32 burst; 2405 2406 if (dir != uc->config.dir) { 2407 dev_err(chan->device->dev, 2408 "%s: chan%d is for %s, not supporting %s\n", 2409 __func__, uc->id, 2410 dmaengine_get_direction_text(uc->config.dir), 2411 dmaengine_get_direction_text(dir)); 2412 return NULL; 2413 } 2414 2415 if (dir == DMA_DEV_TO_MEM) { 2416 dev_width = uc->cfg.src_addr_width; 2417 burst = uc->cfg.src_maxburst; 2418 } else if (dir == DMA_MEM_TO_DEV) { 2419 dev_width = uc->cfg.dst_addr_width; 2420 burst = uc->cfg.dst_maxburst; 2421 } else { 2422 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 2423 return NULL; 2424 } 2425 2426 if (!burst) 2427 burst = 1; 2428 2429 if (uc->config.pkt_mode) 2430 d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags, 2431 context); 2432 else 2433 d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags, 2434 context); 2435 2436 if (!d) 2437 return NULL; 2438 2439 d->dir = dir; 2440 d->desc_idx = 0; 2441 d->tr_idx = 0; 2442 2443 /* static TR for remote PDMA */ 2444 if (udma_configure_statictr(uc, d, dev_width, burst)) { 2445 dev_err(uc->ud->dev, 2446 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 2447 __func__, d->static_tr.bstcnt); 2448 2449 udma_free_hwdesc(uc, d); 2450 kfree(d); 2451 return NULL; 2452 } 2453 2454 if (uc->config.metadata_size) 2455 d->vd.tx.metadata_ops = &metadata_ops; 2456 2457 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 2458 } 2459 2460 static struct udma_desc * 2461 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, 2462 size_t buf_len, size_t period_len, 2463 enum dma_transfer_direction dir, unsigned long flags) 2464 { 2465 struct udma_desc *d; 2466 size_t tr_size, period_addr; 2467 struct cppi5_tr_type1_t *tr_req; 2468 unsigned int periods = buf_len / period_len; 2469 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2470 unsigned int i; 2471 int num_tr; 2472 2473 if (!is_slave_direction(dir)) { 2474 dev_err(uc->ud->dev, "Only slave cyclic is supported\n"); 2475 return NULL; 2476 } 2477 2478 num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0, 2479 &tr0_cnt1, &tr1_cnt0); 2480 if (num_tr < 0) { 2481 dev_err(uc->ud->dev, "size %zu is not supported\n", 2482 period_len); 2483 return NULL; 2484 } 2485 2486 /* Now allocate and setup the descriptor. */ 2487 tr_size = sizeof(struct cppi5_tr_type1_t); 2488 d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir); 2489 if (!d) 2490 return NULL; 2491 2492 tr_req = d->hwdesc[0].tr_req_base; 2493 period_addr = buf_addr; 2494 for (i = 0; i < periods; i++) { 2495 int tr_idx = i * num_tr; 2496 2497 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false, 2498 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2499 2500 tr_req[tr_idx].addr = period_addr; 2501 tr_req[tr_idx].icnt0 = tr0_cnt0; 2502 tr_req[tr_idx].icnt1 = tr0_cnt1; 2503 tr_req[tr_idx].dim1 = tr0_cnt0; 2504 2505 if (num_tr == 2) { 2506 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2507 CPPI5_TR_CSF_SUPR_EVT); 2508 tr_idx++; 2509 2510 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 2511 false, false, 2512 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2513 2514 tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0; 2515 tr_req[tr_idx].icnt0 = tr1_cnt0; 2516 tr_req[tr_idx].icnt1 = 1; 2517 tr_req[tr_idx].dim1 = tr1_cnt0; 2518 } 2519 2520 if (!(flags & DMA_PREP_INTERRUPT)) 2521 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2522 CPPI5_TR_CSF_SUPR_EVT); 2523 2524 period_addr += period_len; 2525 } 2526 2527 return d; 2528 } 2529 2530 static struct udma_desc * 2531 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr, 2532 size_t buf_len, size_t period_len, 2533 enum dma_transfer_direction dir, unsigned long flags) 2534 { 2535 struct udma_desc *d; 2536 u32 ring_id; 2537 int i; 2538 int periods = buf_len / period_len; 2539 2540 if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1)) 2541 return NULL; 2542 2543 if (period_len >= SZ_4M) 2544 return NULL; 2545 2546 d = kzalloc(sizeof(*d) + periods * sizeof(d->hwdesc[0]), GFP_NOWAIT); 2547 if (!d) 2548 return NULL; 2549 2550 d->hwdesc_count = periods; 2551 2552 /* TODO: re-check this... */ 2553 if (dir == DMA_DEV_TO_MEM) 2554 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2555 else 2556 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2557 2558 for (i = 0; i < periods; i++) { 2559 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 2560 dma_addr_t period_addr = buf_addr + (period_len * i); 2561 struct cppi5_host_desc_t *h_desc; 2562 2563 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2564 GFP_NOWAIT, 2565 &hwdesc->cppi5_desc_paddr); 2566 if (!hwdesc->cppi5_desc_vaddr) { 2567 dev_err(uc->ud->dev, 2568 "descriptor%d allocation failed\n", i); 2569 2570 udma_free_hwdesc(uc, d); 2571 kfree(d); 2572 return NULL; 2573 } 2574 2575 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2576 h_desc = hwdesc->cppi5_desc_vaddr; 2577 2578 cppi5_hdesc_init(h_desc, 0, 0); 2579 cppi5_hdesc_set_pktlen(h_desc, period_len); 2580 2581 /* Flow and Packed ID */ 2582 cppi5_desc_set_pktids(&h_desc->hdr, uc->id, 2583 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2584 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id); 2585 2586 /* attach each period to a new descriptor */ 2587 cppi5_hdesc_attach_buf(h_desc, 2588 period_addr, period_len, 2589 period_addr, period_len); 2590 } 2591 2592 return d; 2593 } 2594 2595 static struct dma_async_tx_descriptor * 2596 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 2597 size_t period_len, enum dma_transfer_direction dir, 2598 unsigned long flags) 2599 { 2600 struct udma_chan *uc = to_udma_chan(chan); 2601 enum dma_slave_buswidth dev_width; 2602 struct udma_desc *d; 2603 u32 burst; 2604 2605 if (dir != uc->config.dir) { 2606 dev_err(chan->device->dev, 2607 "%s: chan%d is for %s, not supporting %s\n", 2608 __func__, uc->id, 2609 dmaengine_get_direction_text(uc->config.dir), 2610 dmaengine_get_direction_text(dir)); 2611 return NULL; 2612 } 2613 2614 uc->cyclic = true; 2615 2616 if (dir == DMA_DEV_TO_MEM) { 2617 dev_width = uc->cfg.src_addr_width; 2618 burst = uc->cfg.src_maxburst; 2619 } else if (dir == DMA_MEM_TO_DEV) { 2620 dev_width = uc->cfg.dst_addr_width; 2621 burst = uc->cfg.dst_maxburst; 2622 } else { 2623 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 2624 return NULL; 2625 } 2626 2627 if (!burst) 2628 burst = 1; 2629 2630 if (uc->config.pkt_mode) 2631 d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len, 2632 dir, flags); 2633 else 2634 d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len, 2635 dir, flags); 2636 2637 if (!d) 2638 return NULL; 2639 2640 d->sglen = buf_len / period_len; 2641 2642 d->dir = dir; 2643 d->residue = buf_len; 2644 2645 /* static TR for remote PDMA */ 2646 if (udma_configure_statictr(uc, d, dev_width, burst)) { 2647 dev_err(uc->ud->dev, 2648 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 2649 __func__, d->static_tr.bstcnt); 2650 2651 udma_free_hwdesc(uc, d); 2652 kfree(d); 2653 return NULL; 2654 } 2655 2656 if (uc->config.metadata_size) 2657 d->vd.tx.metadata_ops = &metadata_ops; 2658 2659 return vchan_tx_prep(&uc->vc, &d->vd, flags); 2660 } 2661 2662 static struct dma_async_tx_descriptor * 2663 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 2664 size_t len, unsigned long tx_flags) 2665 { 2666 struct udma_chan *uc = to_udma_chan(chan); 2667 struct udma_desc *d; 2668 struct cppi5_tr_type15_t *tr_req; 2669 int num_tr; 2670 size_t tr_size = sizeof(struct cppi5_tr_type15_t); 2671 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2672 2673 if (uc->config.dir != DMA_MEM_TO_MEM) { 2674 dev_err(chan->device->dev, 2675 "%s: chan%d is for %s, not supporting %s\n", 2676 __func__, uc->id, 2677 dmaengine_get_direction_text(uc->config.dir), 2678 dmaengine_get_direction_text(DMA_MEM_TO_MEM)); 2679 return NULL; 2680 } 2681 2682 num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0, 2683 &tr0_cnt1, &tr1_cnt0); 2684 if (num_tr < 0) { 2685 dev_err(uc->ud->dev, "size %zu is not supported\n", 2686 len); 2687 return NULL; 2688 } 2689 2690 d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM); 2691 if (!d) 2692 return NULL; 2693 2694 d->dir = DMA_MEM_TO_MEM; 2695 d->desc_idx = 0; 2696 d->tr_idx = 0; 2697 d->residue = len; 2698 2699 tr_req = d->hwdesc[0].tr_req_base; 2700 2701 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true, 2702 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2703 cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT); 2704 2705 tr_req[0].addr = src; 2706 tr_req[0].icnt0 = tr0_cnt0; 2707 tr_req[0].icnt1 = tr0_cnt1; 2708 tr_req[0].icnt2 = 1; 2709 tr_req[0].icnt3 = 1; 2710 tr_req[0].dim1 = tr0_cnt0; 2711 2712 tr_req[0].daddr = dest; 2713 tr_req[0].dicnt0 = tr0_cnt0; 2714 tr_req[0].dicnt1 = tr0_cnt1; 2715 tr_req[0].dicnt2 = 1; 2716 tr_req[0].dicnt3 = 1; 2717 tr_req[0].ddim1 = tr0_cnt0; 2718 2719 if (num_tr == 2) { 2720 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true, 2721 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2722 cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT); 2723 2724 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0; 2725 tr_req[1].icnt0 = tr1_cnt0; 2726 tr_req[1].icnt1 = 1; 2727 tr_req[1].icnt2 = 1; 2728 tr_req[1].icnt3 = 1; 2729 2730 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0; 2731 tr_req[1].dicnt0 = tr1_cnt0; 2732 tr_req[1].dicnt1 = 1; 2733 tr_req[1].dicnt2 = 1; 2734 tr_req[1].dicnt3 = 1; 2735 } 2736 2737 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, 2738 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); 2739 2740 if (uc->config.metadata_size) 2741 d->vd.tx.metadata_ops = &metadata_ops; 2742 2743 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 2744 } 2745 2746 static void udma_issue_pending(struct dma_chan *chan) 2747 { 2748 struct udma_chan *uc = to_udma_chan(chan); 2749 unsigned long flags; 2750 2751 spin_lock_irqsave(&uc->vc.lock, flags); 2752 2753 /* If we have something pending and no active descriptor, then */ 2754 if (vchan_issue_pending(&uc->vc) && !uc->desc) { 2755 /* 2756 * start a descriptor if the channel is NOT [marked as 2757 * terminating _and_ it is still running (teardown has not 2758 * completed yet)]. 2759 */ 2760 if (!(uc->state == UDMA_CHAN_IS_TERMINATING && 2761 udma_is_chan_running(uc))) 2762 udma_start(uc); 2763 } 2764 2765 spin_unlock_irqrestore(&uc->vc.lock, flags); 2766 } 2767 2768 static enum dma_status udma_tx_status(struct dma_chan *chan, 2769 dma_cookie_t cookie, 2770 struct dma_tx_state *txstate) 2771 { 2772 struct udma_chan *uc = to_udma_chan(chan); 2773 enum dma_status ret; 2774 unsigned long flags; 2775 2776 spin_lock_irqsave(&uc->vc.lock, flags); 2777 2778 ret = dma_cookie_status(chan, cookie, txstate); 2779 2780 if (!udma_is_chan_running(uc)) 2781 ret = DMA_COMPLETE; 2782 2783 if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc)) 2784 ret = DMA_PAUSED; 2785 2786 if (ret == DMA_COMPLETE || !txstate) 2787 goto out; 2788 2789 if (uc->desc && uc->desc->vd.tx.cookie == cookie) { 2790 u32 peer_bcnt = 0; 2791 u32 bcnt = 0; 2792 u32 residue = uc->desc->residue; 2793 u32 delay = 0; 2794 2795 if (uc->desc->dir == DMA_MEM_TO_DEV) { 2796 bcnt = udma_tchanrt_read(uc->tchan, 2797 UDMA_TCHAN_RT_SBCNT_REG); 2798 2799 if (uc->config.ep_type != PSIL_EP_NATIVE) { 2800 peer_bcnt = udma_tchanrt_read(uc->tchan, 2801 UDMA_TCHAN_RT_PEER_BCNT_REG); 2802 2803 if (bcnt > peer_bcnt) 2804 delay = bcnt - peer_bcnt; 2805 } 2806 } else if (uc->desc->dir == DMA_DEV_TO_MEM) { 2807 bcnt = udma_rchanrt_read(uc->rchan, 2808 UDMA_RCHAN_RT_BCNT_REG); 2809 2810 if (uc->config.ep_type != PSIL_EP_NATIVE) { 2811 peer_bcnt = udma_rchanrt_read(uc->rchan, 2812 UDMA_RCHAN_RT_PEER_BCNT_REG); 2813 2814 if (peer_bcnt > bcnt) 2815 delay = peer_bcnt - bcnt; 2816 } 2817 } else { 2818 bcnt = udma_tchanrt_read(uc->tchan, 2819 UDMA_TCHAN_RT_BCNT_REG); 2820 } 2821 2822 bcnt -= uc->bcnt; 2823 if (bcnt && !(bcnt % uc->desc->residue)) 2824 residue = 0; 2825 else 2826 residue -= bcnt % uc->desc->residue; 2827 2828 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) { 2829 ret = DMA_COMPLETE; 2830 delay = 0; 2831 } 2832 2833 dma_set_residue(txstate, residue); 2834 dma_set_in_flight_bytes(txstate, delay); 2835 2836 } else { 2837 ret = DMA_COMPLETE; 2838 } 2839 2840 out: 2841 spin_unlock_irqrestore(&uc->vc.lock, flags); 2842 return ret; 2843 } 2844 2845 static int udma_pause(struct dma_chan *chan) 2846 { 2847 struct udma_chan *uc = to_udma_chan(chan); 2848 2849 /* pause the channel */ 2850 switch (uc->config.dir) { 2851 case DMA_DEV_TO_MEM: 2852 udma_rchanrt_update_bits(uc->rchan, 2853 UDMA_RCHAN_RT_PEER_RT_EN_REG, 2854 UDMA_PEER_RT_EN_PAUSE, 2855 UDMA_PEER_RT_EN_PAUSE); 2856 break; 2857 case DMA_MEM_TO_DEV: 2858 udma_tchanrt_update_bits(uc->tchan, 2859 UDMA_TCHAN_RT_PEER_RT_EN_REG, 2860 UDMA_PEER_RT_EN_PAUSE, 2861 UDMA_PEER_RT_EN_PAUSE); 2862 break; 2863 case DMA_MEM_TO_MEM: 2864 udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 2865 UDMA_CHAN_RT_CTL_PAUSE, 2866 UDMA_CHAN_RT_CTL_PAUSE); 2867 break; 2868 default: 2869 return -EINVAL; 2870 } 2871 2872 return 0; 2873 } 2874 2875 static int udma_resume(struct dma_chan *chan) 2876 { 2877 struct udma_chan *uc = to_udma_chan(chan); 2878 2879 /* resume the channel */ 2880 switch (uc->config.dir) { 2881 case DMA_DEV_TO_MEM: 2882 udma_rchanrt_update_bits(uc->rchan, 2883 UDMA_RCHAN_RT_PEER_RT_EN_REG, 2884 UDMA_PEER_RT_EN_PAUSE, 0); 2885 2886 break; 2887 case DMA_MEM_TO_DEV: 2888 udma_tchanrt_update_bits(uc->tchan, 2889 UDMA_TCHAN_RT_PEER_RT_EN_REG, 2890 UDMA_PEER_RT_EN_PAUSE, 0); 2891 break; 2892 case DMA_MEM_TO_MEM: 2893 udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 2894 UDMA_CHAN_RT_CTL_PAUSE, 0); 2895 break; 2896 default: 2897 return -EINVAL; 2898 } 2899 2900 return 0; 2901 } 2902 2903 static int udma_terminate_all(struct dma_chan *chan) 2904 { 2905 struct udma_chan *uc = to_udma_chan(chan); 2906 unsigned long flags; 2907 LIST_HEAD(head); 2908 2909 spin_lock_irqsave(&uc->vc.lock, flags); 2910 2911 if (udma_is_chan_running(uc)) 2912 udma_stop(uc); 2913 2914 if (uc->desc) { 2915 uc->terminated_desc = uc->desc; 2916 uc->desc = NULL; 2917 uc->terminated_desc->terminated = true; 2918 cancel_delayed_work(&uc->tx_drain.work); 2919 } 2920 2921 uc->paused = false; 2922 2923 vchan_get_all_descriptors(&uc->vc, &head); 2924 spin_unlock_irqrestore(&uc->vc.lock, flags); 2925 vchan_dma_desc_free_list(&uc->vc, &head); 2926 2927 return 0; 2928 } 2929 2930 static void udma_synchronize(struct dma_chan *chan) 2931 { 2932 struct udma_chan *uc = to_udma_chan(chan); 2933 unsigned long timeout = msecs_to_jiffies(1000); 2934 2935 vchan_synchronize(&uc->vc); 2936 2937 if (uc->state == UDMA_CHAN_IS_TERMINATING) { 2938 timeout = wait_for_completion_timeout(&uc->teardown_completed, 2939 timeout); 2940 if (!timeout) { 2941 dev_warn(uc->ud->dev, "chan%d teardown timeout!\n", 2942 uc->id); 2943 udma_dump_chan_stdata(uc); 2944 udma_reset_chan(uc, true); 2945 } 2946 } 2947 2948 udma_reset_chan(uc, false); 2949 if (udma_is_chan_running(uc)) 2950 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id); 2951 2952 cancel_delayed_work_sync(&uc->tx_drain.work); 2953 udma_reset_rings(uc); 2954 } 2955 2956 static void udma_desc_pre_callback(struct virt_dma_chan *vc, 2957 struct virt_dma_desc *vd, 2958 struct dmaengine_result *result) 2959 { 2960 struct udma_chan *uc = to_udma_chan(&vc->chan); 2961 struct udma_desc *d; 2962 2963 if (!vd) 2964 return; 2965 2966 d = to_udma_desc(&vd->tx); 2967 2968 if (d->metadata_size) 2969 udma_fetch_epib(uc, d); 2970 2971 /* Provide residue information for the client */ 2972 if (result) { 2973 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx); 2974 2975 if (cppi5_desc_get_type(desc_vaddr) == 2976 CPPI5_INFO0_DESC_TYPE_VAL_HOST) { 2977 result->residue = d->residue - 2978 cppi5_hdesc_get_pktlen(desc_vaddr); 2979 if (result->residue) 2980 result->result = DMA_TRANS_ABORTED; 2981 else 2982 result->result = DMA_TRANS_NOERROR; 2983 } else { 2984 result->residue = 0; 2985 result->result = DMA_TRANS_NOERROR; 2986 } 2987 } 2988 } 2989 2990 /* 2991 * This tasklet handles the completion of a DMA descriptor by 2992 * calling its callback and freeing it. 2993 */ 2994 static void udma_vchan_complete(unsigned long arg) 2995 { 2996 struct virt_dma_chan *vc = (struct virt_dma_chan *)arg; 2997 struct virt_dma_desc *vd, *_vd; 2998 struct dmaengine_desc_callback cb; 2999 LIST_HEAD(head); 3000 3001 spin_lock_irq(&vc->lock); 3002 list_splice_tail_init(&vc->desc_completed, &head); 3003 vd = vc->cyclic; 3004 if (vd) { 3005 vc->cyclic = NULL; 3006 dmaengine_desc_get_callback(&vd->tx, &cb); 3007 } else { 3008 memset(&cb, 0, sizeof(cb)); 3009 } 3010 spin_unlock_irq(&vc->lock); 3011 3012 udma_desc_pre_callback(vc, vd, NULL); 3013 dmaengine_desc_callback_invoke(&cb, NULL); 3014 3015 list_for_each_entry_safe(vd, _vd, &head, node) { 3016 struct dmaengine_result result; 3017 3018 dmaengine_desc_get_callback(&vd->tx, &cb); 3019 3020 list_del(&vd->node); 3021 3022 udma_desc_pre_callback(vc, vd, &result); 3023 dmaengine_desc_callback_invoke(&cb, &result); 3024 3025 vchan_vdesc_fini(vd); 3026 } 3027 } 3028 3029 static void udma_free_chan_resources(struct dma_chan *chan) 3030 { 3031 struct udma_chan *uc = to_udma_chan(chan); 3032 struct udma_dev *ud = to_udma_dev(chan->device); 3033 3034 udma_terminate_all(chan); 3035 if (uc->terminated_desc) { 3036 udma_reset_chan(uc, false); 3037 udma_reset_rings(uc); 3038 } 3039 3040 cancel_delayed_work_sync(&uc->tx_drain.work); 3041 destroy_delayed_work_on_stack(&uc->tx_drain.work); 3042 3043 if (uc->irq_num_ring > 0) { 3044 free_irq(uc->irq_num_ring, uc); 3045 3046 uc->irq_num_ring = 0; 3047 } 3048 if (uc->irq_num_udma > 0) { 3049 free_irq(uc->irq_num_udma, uc); 3050 3051 uc->irq_num_udma = 0; 3052 } 3053 3054 /* Release PSI-L pairing */ 3055 if (uc->psil_paired) { 3056 navss_psil_unpair(ud, uc->config.src_thread, 3057 uc->config.dst_thread); 3058 uc->psil_paired = false; 3059 } 3060 3061 vchan_free_chan_resources(&uc->vc); 3062 tasklet_kill(&uc->vc.task); 3063 3064 udma_free_tx_resources(uc); 3065 udma_free_rx_resources(uc); 3066 udma_reset_uchan(uc); 3067 3068 if (uc->use_dma_pool) { 3069 dma_pool_destroy(uc->hdesc_pool); 3070 uc->use_dma_pool = false; 3071 } 3072 } 3073 3074 static struct platform_driver udma_driver; 3075 3076 struct udma_filter_param { 3077 int remote_thread_id; 3078 u32 atype; 3079 }; 3080 3081 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) 3082 { 3083 struct udma_chan_config *ucc; 3084 struct psil_endpoint_config *ep_config; 3085 struct udma_filter_param *filter_param; 3086 struct udma_chan *uc; 3087 struct udma_dev *ud; 3088 3089 if (chan->device->dev->driver != &udma_driver.driver) 3090 return false; 3091 3092 uc = to_udma_chan(chan); 3093 ucc = &uc->config; 3094 ud = uc->ud; 3095 filter_param = param; 3096 3097 if (filter_param->atype > 2) { 3098 dev_err(ud->dev, "Invalid channel atype: %u\n", 3099 filter_param->atype); 3100 return false; 3101 } 3102 3103 ucc->remote_thread_id = filter_param->remote_thread_id; 3104 ucc->atype = filter_param->atype; 3105 3106 if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) 3107 ucc->dir = DMA_MEM_TO_DEV; 3108 else 3109 ucc->dir = DMA_DEV_TO_MEM; 3110 3111 ep_config = psil_get_ep_config(ucc->remote_thread_id); 3112 if (IS_ERR(ep_config)) { 3113 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n", 3114 ucc->remote_thread_id); 3115 ucc->dir = DMA_MEM_TO_MEM; 3116 ucc->remote_thread_id = -1; 3117 ucc->atype = 0; 3118 return false; 3119 } 3120 3121 ucc->pkt_mode = ep_config->pkt_mode; 3122 ucc->channel_tpl = ep_config->channel_tpl; 3123 ucc->notdpkt = ep_config->notdpkt; 3124 ucc->ep_type = ep_config->ep_type; 3125 3126 if (ucc->ep_type != PSIL_EP_NATIVE) { 3127 const struct udma_match_data *match_data = ud->match_data; 3128 3129 if (match_data->flags & UDMA_FLAG_PDMA_ACC32) 3130 ucc->enable_acc32 = ep_config->pdma_acc32; 3131 if (match_data->flags & UDMA_FLAG_PDMA_BURST) 3132 ucc->enable_burst = ep_config->pdma_burst; 3133 } 3134 3135 ucc->needs_epib = ep_config->needs_epib; 3136 ucc->psd_size = ep_config->psd_size; 3137 ucc->metadata_size = 3138 (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) + 3139 ucc->psd_size; 3140 3141 if (ucc->pkt_mode) 3142 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 3143 ucc->metadata_size, ud->desc_align); 3144 3145 dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id, 3146 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir)); 3147 3148 return true; 3149 } 3150 3151 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, 3152 struct of_dma *ofdma) 3153 { 3154 struct udma_dev *ud = ofdma->of_dma_data; 3155 dma_cap_mask_t mask = ud->ddev.cap_mask; 3156 struct udma_filter_param filter_param; 3157 struct dma_chan *chan; 3158 3159 if (dma_spec->args_count != 1 && dma_spec->args_count != 2) 3160 return NULL; 3161 3162 filter_param.remote_thread_id = dma_spec->args[0]; 3163 if (dma_spec->args_count == 2) 3164 filter_param.atype = dma_spec->args[1]; 3165 else 3166 filter_param.atype = 0; 3167 3168 chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, 3169 ofdma->of_node); 3170 if (!chan) { 3171 dev_err(ud->dev, "get channel fail in %s.\n", __func__); 3172 return ERR_PTR(-EINVAL); 3173 } 3174 3175 return chan; 3176 } 3177 3178 static struct udma_match_data am654_main_data = { 3179 .psil_base = 0x1000, 3180 .enable_memcpy_support = true, 3181 .statictr_z_mask = GENMASK(11, 0), 3182 .rchan_oes_offset = 0x2000, 3183 .tpl_levels = 2, 3184 .level_start_idx = { 3185 [0] = 8, /* Normal channels */ 3186 [1] = 0, /* High Throughput channels */ 3187 }, 3188 }; 3189 3190 static struct udma_match_data am654_mcu_data = { 3191 .psil_base = 0x6000, 3192 .enable_memcpy_support = true, /* TEST: DMA domains */ 3193 .statictr_z_mask = GENMASK(11, 0), 3194 .rchan_oes_offset = 0x2000, 3195 .tpl_levels = 2, 3196 .level_start_idx = { 3197 [0] = 2, /* Normal channels */ 3198 [1] = 0, /* High Throughput channels */ 3199 }, 3200 }; 3201 3202 static struct udma_match_data j721e_main_data = { 3203 .psil_base = 0x1000, 3204 .enable_memcpy_support = true, 3205 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, 3206 .statictr_z_mask = GENMASK(23, 0), 3207 .rchan_oes_offset = 0x400, 3208 .tpl_levels = 3, 3209 .level_start_idx = { 3210 [0] = 16, /* Normal channels */ 3211 [1] = 4, /* High Throughput channels */ 3212 [2] = 0, /* Ultra High Throughput channels */ 3213 }, 3214 }; 3215 3216 static struct udma_match_data j721e_mcu_data = { 3217 .psil_base = 0x6000, 3218 .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ 3219 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, 3220 .statictr_z_mask = GENMASK(23, 0), 3221 .rchan_oes_offset = 0x400, 3222 .tpl_levels = 2, 3223 .level_start_idx = { 3224 [0] = 2, /* Normal channels */ 3225 [1] = 0, /* High Throughput channels */ 3226 }, 3227 }; 3228 3229 static const struct of_device_id udma_of_match[] = { 3230 { 3231 .compatible = "ti,am654-navss-main-udmap", 3232 .data = &am654_main_data, 3233 }, 3234 { 3235 .compatible = "ti,am654-navss-mcu-udmap", 3236 .data = &am654_mcu_data, 3237 }, { 3238 .compatible = "ti,j721e-navss-main-udmap", 3239 .data = &j721e_main_data, 3240 }, { 3241 .compatible = "ti,j721e-navss-mcu-udmap", 3242 .data = &j721e_mcu_data, 3243 }, 3244 { /* Sentinel */ }, 3245 }; 3246 3247 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) 3248 { 3249 struct resource *res; 3250 int i; 3251 3252 for (i = 0; i < MMR_LAST; i++) { 3253 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 3254 mmr_names[i]); 3255 ud->mmrs[i] = devm_ioremap_resource(&pdev->dev, res); 3256 if (IS_ERR(ud->mmrs[i])) 3257 return PTR_ERR(ud->mmrs[i]); 3258 } 3259 3260 return 0; 3261 } 3262 3263 static int udma_setup_resources(struct udma_dev *ud) 3264 { 3265 struct device *dev = ud->dev; 3266 int ch_count, ret, i, j; 3267 u32 cap2, cap3; 3268 struct ti_sci_resource_desc *rm_desc; 3269 struct ti_sci_resource *rm_res, irq_res; 3270 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 3271 static const char * const range_names[] = { "ti,sci-rm-range-tchan", 3272 "ti,sci-rm-range-rchan", 3273 "ti,sci-rm-range-rflow" }; 3274 3275 cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28); 3276 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 3277 3278 ud->rflow_cnt = cap3 & 0x3fff; 3279 ud->tchan_cnt = cap2 & 0x1ff; 3280 ud->echan_cnt = (cap2 >> 9) & 0x1ff; 3281 ud->rchan_cnt = (cap2 >> 18) & 0x1ff; 3282 ch_count = ud->tchan_cnt + ud->rchan_cnt; 3283 3284 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 3285 sizeof(unsigned long), GFP_KERNEL); 3286 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 3287 GFP_KERNEL); 3288 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 3289 sizeof(unsigned long), GFP_KERNEL); 3290 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 3291 GFP_KERNEL); 3292 ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt), 3293 sizeof(unsigned long), 3294 GFP_KERNEL); 3295 ud->rflow_gp_map_allocated = devm_kcalloc(dev, 3296 BITS_TO_LONGS(ud->rflow_cnt), 3297 sizeof(unsigned long), 3298 GFP_KERNEL); 3299 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), 3300 sizeof(unsigned long), 3301 GFP_KERNEL); 3302 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), 3303 GFP_KERNEL); 3304 3305 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map || 3306 !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans || 3307 !ud->rflows || !ud->rflow_in_use) 3308 return -ENOMEM; 3309 3310 /* 3311 * RX flows with the same Ids as RX channels are reserved to be used 3312 * as default flows if remote HW can't generate flow_ids. Those 3313 * RX flows can be requested only explicitly by id. 3314 */ 3315 bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt); 3316 3317 /* by default no GP rflows are assigned to Linux */ 3318 bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt); 3319 3320 /* Get resource ranges from tisci */ 3321 for (i = 0; i < RM_RANGE_LAST; i++) 3322 tisci_rm->rm_ranges[i] = 3323 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 3324 tisci_rm->tisci_dev_id, 3325 (char *)range_names[i]); 3326 3327 /* tchan ranges */ 3328 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 3329 if (IS_ERR(rm_res)) { 3330 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 3331 } else { 3332 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 3333 for (i = 0; i < rm_res->sets; i++) { 3334 rm_desc = &rm_res->desc[i]; 3335 bitmap_clear(ud->tchan_map, rm_desc->start, 3336 rm_desc->num); 3337 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n", 3338 rm_desc->start, rm_desc->num); 3339 } 3340 } 3341 irq_res.sets = rm_res->sets; 3342 3343 /* rchan and matching default flow ranges */ 3344 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 3345 if (IS_ERR(rm_res)) { 3346 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 3347 } else { 3348 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 3349 for (i = 0; i < rm_res->sets; i++) { 3350 rm_desc = &rm_res->desc[i]; 3351 bitmap_clear(ud->rchan_map, rm_desc->start, 3352 rm_desc->num); 3353 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n", 3354 rm_desc->start, rm_desc->num); 3355 } 3356 } 3357 3358 irq_res.sets += rm_res->sets; 3359 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 3360 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 3361 for (i = 0; i < rm_res->sets; i++) { 3362 irq_res.desc[i].start = rm_res->desc[i].start; 3363 irq_res.desc[i].num = rm_res->desc[i].num; 3364 } 3365 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 3366 for (j = 0; j < rm_res->sets; j++, i++) { 3367 irq_res.desc[i].start = rm_res->desc[j].start + 3368 ud->match_data->rchan_oes_offset; 3369 irq_res.desc[i].num = rm_res->desc[j].num; 3370 } 3371 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 3372 kfree(irq_res.desc); 3373 if (ret) { 3374 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 3375 return ret; 3376 } 3377 3378 /* GP rflow ranges */ 3379 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 3380 if (IS_ERR(rm_res)) { 3381 /* all gp flows are assigned exclusively to Linux */ 3382 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt, 3383 ud->rflow_cnt - ud->rchan_cnt); 3384 } else { 3385 for (i = 0; i < rm_res->sets; i++) { 3386 rm_desc = &rm_res->desc[i]; 3387 bitmap_clear(ud->rflow_gp_map, rm_desc->start, 3388 rm_desc->num); 3389 dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n", 3390 rm_desc->start, rm_desc->num); 3391 } 3392 } 3393 3394 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt); 3395 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt); 3396 if (!ch_count) 3397 return -ENODEV; 3398 3399 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels), 3400 GFP_KERNEL); 3401 if (!ud->channels) 3402 return -ENOMEM; 3403 3404 dev_info(dev, "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", 3405 ch_count, 3406 ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), 3407 ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt), 3408 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map, 3409 ud->rflow_cnt)); 3410 3411 return ch_count; 3412 } 3413 3414 static int udma_setup_rx_flush(struct udma_dev *ud) 3415 { 3416 struct udma_rx_flush *rx_flush = &ud->rx_flush; 3417 struct cppi5_desc_hdr_t *tr_desc; 3418 struct cppi5_tr_type1_t *tr_req; 3419 struct cppi5_host_desc_t *desc; 3420 struct device *dev = ud->dev; 3421 struct udma_hwdesc *hwdesc; 3422 size_t tr_size; 3423 3424 /* Allocate 1K buffer for discarded data on RX channel teardown */ 3425 rx_flush->buffer_size = SZ_1K; 3426 rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size, 3427 GFP_KERNEL); 3428 if (!rx_flush->buffer_vaddr) 3429 return -ENOMEM; 3430 3431 rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr, 3432 rx_flush->buffer_size, 3433 DMA_TO_DEVICE); 3434 if (dma_mapping_error(dev, rx_flush->buffer_paddr)) 3435 return -ENOMEM; 3436 3437 /* Set up descriptor to be used for TR mode */ 3438 hwdesc = &rx_flush->hwdescs[0]; 3439 tr_size = sizeof(struct cppi5_tr_type1_t); 3440 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1); 3441 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 3442 ud->desc_align); 3443 3444 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 3445 GFP_KERNEL); 3446 if (!hwdesc->cppi5_desc_vaddr) 3447 return -ENOMEM; 3448 3449 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 3450 hwdesc->cppi5_desc_size, 3451 DMA_TO_DEVICE); 3452 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 3453 return -ENOMEM; 3454 3455 /* Start of the TR req records */ 3456 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 3457 /* Start address of the TR response array */ 3458 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size; 3459 3460 tr_desc = hwdesc->cppi5_desc_vaddr; 3461 cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0); 3462 cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3463 cppi5_desc_set_retpolicy(tr_desc, 0, 0); 3464 3465 tr_req = hwdesc->tr_req_base; 3466 cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false, 3467 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3468 cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT); 3469 3470 tr_req->addr = rx_flush->buffer_paddr; 3471 tr_req->icnt0 = rx_flush->buffer_size; 3472 tr_req->icnt1 = 1; 3473 3474 /* Set up descriptor to be used for packet mode */ 3475 hwdesc = &rx_flush->hwdescs[1]; 3476 hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 3477 CPPI5_INFO0_HDESC_EPIB_SIZE + 3478 CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE, 3479 ud->desc_align); 3480 3481 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 3482 GFP_KERNEL); 3483 if (!hwdesc->cppi5_desc_vaddr) 3484 return -ENOMEM; 3485 3486 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 3487 hwdesc->cppi5_desc_size, 3488 DMA_TO_DEVICE); 3489 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 3490 return -ENOMEM; 3491 3492 desc = hwdesc->cppi5_desc_vaddr; 3493 cppi5_hdesc_init(desc, 0, 0); 3494 cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3495 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0); 3496 3497 cppi5_hdesc_attach_buf(desc, 3498 rx_flush->buffer_paddr, rx_flush->buffer_size, 3499 rx_flush->buffer_paddr, rx_flush->buffer_size); 3500 3501 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, 3502 hwdesc->cppi5_desc_size, DMA_TO_DEVICE); 3503 return 0; 3504 } 3505 3506 #ifdef CONFIG_DEBUG_FS 3507 static void udma_dbg_summary_show_chan(struct seq_file *s, 3508 struct dma_chan *chan) 3509 { 3510 struct udma_chan *uc = to_udma_chan(chan); 3511 struct udma_chan_config *ucc = &uc->config; 3512 3513 seq_printf(s, " %-13s| %s", dma_chan_name(chan), 3514 chan->dbg_client_name ?: "in-use"); 3515 seq_printf(s, " (%s, ", dmaengine_get_direction_text(uc->config.dir)); 3516 3517 switch (uc->config.dir) { 3518 case DMA_MEM_TO_MEM: 3519 seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id, 3520 ucc->src_thread, ucc->dst_thread); 3521 break; 3522 case DMA_DEV_TO_MEM: 3523 seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id, 3524 ucc->src_thread, ucc->dst_thread); 3525 break; 3526 case DMA_MEM_TO_DEV: 3527 seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id, 3528 ucc->src_thread, ucc->dst_thread); 3529 break; 3530 default: 3531 seq_printf(s, ")\n"); 3532 return; 3533 } 3534 3535 if (ucc->ep_type == PSIL_EP_NATIVE) { 3536 seq_printf(s, "PSI-L Native"); 3537 if (ucc->metadata_size) { 3538 seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : ""); 3539 if (ucc->psd_size) 3540 seq_printf(s, " PSDsize:%u", ucc->psd_size); 3541 seq_printf(s, " ]"); 3542 } 3543 } else { 3544 seq_printf(s, "PDMA"); 3545 if (ucc->enable_acc32 || ucc->enable_burst) 3546 seq_printf(s, "[%s%s ]", 3547 ucc->enable_acc32 ? " ACC32" : "", 3548 ucc->enable_burst ? " BURST" : ""); 3549 } 3550 3551 seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode"); 3552 } 3553 3554 static void udma_dbg_summary_show(struct seq_file *s, 3555 struct dma_device *dma_dev) 3556 { 3557 struct dma_chan *chan; 3558 3559 list_for_each_entry(chan, &dma_dev->channels, device_node) { 3560 if (chan->client_count) 3561 udma_dbg_summary_show_chan(s, chan); 3562 } 3563 } 3564 #endif /* CONFIG_DEBUG_FS */ 3565 3566 #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 3567 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 3568 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ 3569 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 3570 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 3571 3572 static int udma_probe(struct platform_device *pdev) 3573 { 3574 struct device_node *navss_node = pdev->dev.parent->of_node; 3575 struct device *dev = &pdev->dev; 3576 struct udma_dev *ud; 3577 const struct of_device_id *match; 3578 int i, ret; 3579 int ch_count; 3580 3581 ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48)); 3582 if (ret) 3583 dev_err(dev, "failed to set dma mask stuff\n"); 3584 3585 ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL); 3586 if (!ud) 3587 return -ENOMEM; 3588 3589 ret = udma_get_mmrs(pdev, ud); 3590 if (ret) 3591 return ret; 3592 3593 ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci"); 3594 if (IS_ERR(ud->tisci_rm.tisci)) 3595 return PTR_ERR(ud->tisci_rm.tisci); 3596 3597 ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id", 3598 &ud->tisci_rm.tisci_dev_id); 3599 if (ret) { 3600 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret); 3601 return ret; 3602 } 3603 pdev->id = ud->tisci_rm.tisci_dev_id; 3604 3605 ret = of_property_read_u32(navss_node, "ti,sci-dev-id", 3606 &ud->tisci_rm.tisci_navss_dev_id); 3607 if (ret) { 3608 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret); 3609 return ret; 3610 } 3611 3612 ret = of_property_read_u32(navss_node, "ti,udma-atype", &ud->atype); 3613 if (!ret && ud->atype > 2) { 3614 dev_err(dev, "Invalid atype: %u\n", ud->atype); 3615 return -EINVAL; 3616 } 3617 3618 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; 3619 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; 3620 3621 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc"); 3622 if (IS_ERR(ud->ringacc)) 3623 return PTR_ERR(ud->ringacc); 3624 3625 dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 3626 DOMAIN_BUS_TI_SCI_INTA_MSI); 3627 if (!dev->msi_domain) { 3628 dev_err(dev, "Failed to get MSI domain\n"); 3629 return -EPROBE_DEFER; 3630 } 3631 3632 match = of_match_node(udma_of_match, dev->of_node); 3633 if (!match) { 3634 dev_err(dev, "No compatible match found\n"); 3635 return -ENODEV; 3636 } 3637 ud->match_data = match->data; 3638 3639 dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask); 3640 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); 3641 3642 ud->ddev.device_alloc_chan_resources = udma_alloc_chan_resources; 3643 ud->ddev.device_config = udma_slave_config; 3644 ud->ddev.device_prep_slave_sg = udma_prep_slave_sg; 3645 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; 3646 ud->ddev.device_issue_pending = udma_issue_pending; 3647 ud->ddev.device_tx_status = udma_tx_status; 3648 ud->ddev.device_pause = udma_pause; 3649 ud->ddev.device_resume = udma_resume; 3650 ud->ddev.device_terminate_all = udma_terminate_all; 3651 ud->ddev.device_synchronize = udma_synchronize; 3652 #ifdef CONFIG_DEBUG_FS 3653 ud->ddev.dbg_summary_show = udma_dbg_summary_show; 3654 #endif 3655 3656 ud->ddev.device_free_chan_resources = udma_free_chan_resources; 3657 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; 3658 ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; 3659 ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 3660 ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 3661 ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES; 3662 ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | 3663 DESC_METADATA_ENGINE; 3664 if (ud->match_data->enable_memcpy_support) { 3665 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask); 3666 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy; 3667 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM); 3668 } 3669 3670 ud->ddev.dev = dev; 3671 ud->dev = dev; 3672 ud->psil_base = ud->match_data->psil_base; 3673 3674 INIT_LIST_HEAD(&ud->ddev.channels); 3675 INIT_LIST_HEAD(&ud->desc_to_purge); 3676 3677 ch_count = udma_setup_resources(ud); 3678 if (ch_count <= 0) 3679 return ch_count; 3680 3681 spin_lock_init(&ud->lock); 3682 INIT_WORK(&ud->purge_work, udma_purge_desc_work); 3683 3684 ud->desc_align = 64; 3685 if (ud->desc_align < dma_get_cache_alignment()) 3686 ud->desc_align = dma_get_cache_alignment(); 3687 3688 ret = udma_setup_rx_flush(ud); 3689 if (ret) 3690 return ret; 3691 3692 for (i = 0; i < ud->tchan_cnt; i++) { 3693 struct udma_tchan *tchan = &ud->tchans[i]; 3694 3695 tchan->id = i; 3696 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000; 3697 } 3698 3699 for (i = 0; i < ud->rchan_cnt; i++) { 3700 struct udma_rchan *rchan = &ud->rchans[i]; 3701 3702 rchan->id = i; 3703 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000; 3704 } 3705 3706 for (i = 0; i < ud->rflow_cnt; i++) { 3707 struct udma_rflow *rflow = &ud->rflows[i]; 3708 3709 rflow->id = i; 3710 } 3711 3712 for (i = 0; i < ch_count; i++) { 3713 struct udma_chan *uc = &ud->channels[i]; 3714 3715 uc->ud = ud; 3716 uc->vc.desc_free = udma_desc_free; 3717 uc->id = i; 3718 uc->tchan = NULL; 3719 uc->rchan = NULL; 3720 uc->config.remote_thread_id = -1; 3721 uc->config.dir = DMA_MEM_TO_MEM; 3722 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", 3723 dev_name(dev), i); 3724 3725 vchan_init(&uc->vc, &ud->ddev); 3726 /* Use custom vchan completion handling */ 3727 tasklet_init(&uc->vc.task, udma_vchan_complete, 3728 (unsigned long)&uc->vc); 3729 init_completion(&uc->teardown_completed); 3730 } 3731 3732 ret = dma_async_device_register(&ud->ddev); 3733 if (ret) { 3734 dev_err(dev, "failed to register slave DMA engine: %d\n", ret); 3735 return ret; 3736 } 3737 3738 platform_set_drvdata(pdev, ud); 3739 3740 ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud); 3741 if (ret) { 3742 dev_err(dev, "failed to register of_dma controller\n"); 3743 dma_async_device_unregister(&ud->ddev); 3744 } 3745 3746 return ret; 3747 } 3748 3749 static struct platform_driver udma_driver = { 3750 .driver = { 3751 .name = "ti-udma", 3752 .of_match_table = udma_of_match, 3753 .suppress_bind_attrs = true, 3754 }, 3755 .probe = udma_probe, 3756 }; 3757 builtin_platform_driver(udma_driver); 3758 3759 /* Private interfaces to UDMA */ 3760 #include "k3-udma-private.c" 3761