1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/delay.h> 10 #include <linux/dmaengine.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/dmapool.h> 13 #include <linux/err.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/list.h> 17 #include <linux/platform_device.h> 18 #include <linux/slab.h> 19 #include <linux/spinlock.h> 20 #include <linux/sys_soc.h> 21 #include <linux/of.h> 22 #include <linux/of_dma.h> 23 #include <linux/of_irq.h> 24 #include <linux/workqueue.h> 25 #include <linux/completion.h> 26 #include <linux/soc/ti/k3-ringacc.h> 27 #include <linux/soc/ti/ti_sci_protocol.h> 28 #include <linux/soc/ti/ti_sci_inta_msi.h> 29 #include <linux/dma/k3-event-router.h> 30 #include <linux/dma/ti-cppi5.h> 31 32 #include "../virt-dma.h" 33 #include "k3-udma.h" 34 #include "k3-psil-priv.h" 35 36 struct udma_static_tr { 37 u8 elsize; /* RPSTR0 */ 38 u16 elcnt; /* RPSTR0 */ 39 u16 bstcnt; /* RPSTR1 */ 40 }; 41 42 #define K3_UDMA_MAX_RFLOWS 1024 43 #define K3_UDMA_DEFAULT_RING_SIZE 16 44 45 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */ 46 #define UDMA_RFLOW_SRCTAG_NONE 0 47 #define UDMA_RFLOW_SRCTAG_CFG_TAG 1 48 #define UDMA_RFLOW_SRCTAG_FLOW_ID 2 49 #define UDMA_RFLOW_SRCTAG_SRC_TAG 4 50 51 #define UDMA_RFLOW_DSTTAG_NONE 0 52 #define UDMA_RFLOW_DSTTAG_CFG_TAG 1 53 #define UDMA_RFLOW_DSTTAG_FLOW_ID 2 54 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO 4 55 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI 5 56 57 struct udma_chan; 58 59 enum k3_dma_type { 60 DMA_TYPE_UDMA = 0, 61 DMA_TYPE_BCDMA, 62 DMA_TYPE_PKTDMA, 63 }; 64 65 enum udma_mmr { 66 MMR_GCFG = 0, 67 MMR_BCHANRT, 68 MMR_RCHANRT, 69 MMR_TCHANRT, 70 MMR_LAST, 71 }; 72 73 static const char * const mmr_names[] = { 74 [MMR_GCFG] = "gcfg", 75 [MMR_BCHANRT] = "bchanrt", 76 [MMR_RCHANRT] = "rchanrt", 77 [MMR_TCHANRT] = "tchanrt", 78 }; 79 80 struct udma_tchan { 81 void __iomem *reg_rt; 82 83 int id; 84 struct k3_ring *t_ring; /* Transmit ring */ 85 struct k3_ring *tc_ring; /* Transmit Completion ring */ 86 int tflow_id; /* applicable only for PKTDMA */ 87 88 }; 89 90 #define udma_bchan udma_tchan 91 92 struct udma_rflow { 93 int id; 94 struct k3_ring *fd_ring; /* Free Descriptor ring */ 95 struct k3_ring *r_ring; /* Receive ring */ 96 }; 97 98 struct udma_rchan { 99 void __iomem *reg_rt; 100 101 int id; 102 }; 103 104 struct udma_oes_offsets { 105 /* K3 UDMA Output Event Offset */ 106 u32 udma_rchan; 107 108 /* BCDMA Output Event Offsets */ 109 u32 bcdma_bchan_data; 110 u32 bcdma_bchan_ring; 111 u32 bcdma_tchan_data; 112 u32 bcdma_tchan_ring; 113 u32 bcdma_rchan_data; 114 u32 bcdma_rchan_ring; 115 116 /* PKTDMA Output Event Offsets */ 117 u32 pktdma_tchan_flow; 118 u32 pktdma_rchan_flow; 119 }; 120 121 #define UDMA_FLAG_PDMA_ACC32 BIT(0) 122 #define UDMA_FLAG_PDMA_BURST BIT(1) 123 #define UDMA_FLAG_TDTYPE BIT(2) 124 #define UDMA_FLAG_BURST_SIZE BIT(3) 125 #define UDMA_FLAGS_J7_CLASS (UDMA_FLAG_PDMA_ACC32 | \ 126 UDMA_FLAG_PDMA_BURST | \ 127 UDMA_FLAG_TDTYPE | \ 128 UDMA_FLAG_BURST_SIZE) 129 130 struct udma_match_data { 131 enum k3_dma_type type; 132 u32 psil_base; 133 bool enable_memcpy_support; 134 u32 flags; 135 u32 statictr_z_mask; 136 u8 burst_size[3]; 137 struct udma_soc_data *soc_data; 138 }; 139 140 struct udma_soc_data { 141 struct udma_oes_offsets oes; 142 u32 bcdma_trigger_event_offset; 143 }; 144 145 struct udma_hwdesc { 146 size_t cppi5_desc_size; 147 void *cppi5_desc_vaddr; 148 dma_addr_t cppi5_desc_paddr; 149 150 /* TR descriptor internal pointers */ 151 void *tr_req_base; 152 struct cppi5_tr_resp_t *tr_resp_base; 153 }; 154 155 struct udma_rx_flush { 156 struct udma_hwdesc hwdescs[2]; 157 158 size_t buffer_size; 159 void *buffer_vaddr; 160 dma_addr_t buffer_paddr; 161 }; 162 163 struct udma_tpl { 164 u8 levels; 165 u32 start_idx[3]; 166 }; 167 168 struct udma_dev { 169 struct dma_device ddev; 170 struct device *dev; 171 void __iomem *mmrs[MMR_LAST]; 172 const struct udma_match_data *match_data; 173 const struct udma_soc_data *soc_data; 174 175 struct udma_tpl bchan_tpl; 176 struct udma_tpl tchan_tpl; 177 struct udma_tpl rchan_tpl; 178 179 size_t desc_align; /* alignment to use for descriptors */ 180 181 struct udma_tisci_rm tisci_rm; 182 183 struct k3_ringacc *ringacc; 184 185 struct work_struct purge_work; 186 struct list_head desc_to_purge; 187 spinlock_t lock; 188 189 struct udma_rx_flush rx_flush; 190 191 int bchan_cnt; 192 int tchan_cnt; 193 int echan_cnt; 194 int rchan_cnt; 195 int rflow_cnt; 196 int tflow_cnt; 197 unsigned long *bchan_map; 198 unsigned long *tchan_map; 199 unsigned long *rchan_map; 200 unsigned long *rflow_gp_map; 201 unsigned long *rflow_gp_map_allocated; 202 unsigned long *rflow_in_use; 203 unsigned long *tflow_map; 204 205 struct udma_bchan *bchans; 206 struct udma_tchan *tchans; 207 struct udma_rchan *rchans; 208 struct udma_rflow *rflows; 209 210 struct udma_chan *channels; 211 u32 psil_base; 212 u32 atype; 213 u32 asel; 214 }; 215 216 struct udma_desc { 217 struct virt_dma_desc vd; 218 219 bool terminated; 220 221 enum dma_transfer_direction dir; 222 223 struct udma_static_tr static_tr; 224 u32 residue; 225 226 unsigned int sglen; 227 unsigned int desc_idx; /* Only used for cyclic in packet mode */ 228 unsigned int tr_idx; 229 230 u32 metadata_size; 231 void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */ 232 233 unsigned int hwdesc_count; 234 struct udma_hwdesc hwdesc[]; 235 }; 236 237 enum udma_chan_state { 238 UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */ 239 UDMA_CHAN_IS_ACTIVE, /* Normal operation */ 240 UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */ 241 }; 242 243 struct udma_tx_drain { 244 struct delayed_work work; 245 ktime_t tstamp; 246 u32 residue; 247 }; 248 249 struct udma_chan_config { 250 bool pkt_mode; /* TR or packet */ 251 bool needs_epib; /* EPIB is needed for the communication or not */ 252 u32 psd_size; /* size of Protocol Specific Data */ 253 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */ 254 u32 hdesc_size; /* Size of a packet descriptor in packet mode */ 255 bool notdpkt; /* Suppress sending TDC packet */ 256 int remote_thread_id; 257 u32 atype; 258 u32 asel; 259 u32 src_thread; 260 u32 dst_thread; 261 enum psil_endpoint_type ep_type; 262 bool enable_acc32; 263 bool enable_burst; 264 enum udma_tp_level channel_tpl; /* Channel Throughput Level */ 265 266 u32 tr_trigger_type; 267 unsigned long tx_flags; 268 269 /* PKDMA mapped channel */ 270 int mapped_channel_id; 271 /* PKTDMA default tflow or rflow for mapped channel */ 272 int default_flow_id; 273 274 enum dma_transfer_direction dir; 275 }; 276 277 struct udma_chan { 278 struct virt_dma_chan vc; 279 struct dma_slave_config cfg; 280 struct udma_dev *ud; 281 struct device *dma_dev; 282 struct udma_desc *desc; 283 struct udma_desc *terminated_desc; 284 struct udma_static_tr static_tr; 285 char *name; 286 287 struct udma_bchan *bchan; 288 struct udma_tchan *tchan; 289 struct udma_rchan *rchan; 290 struct udma_rflow *rflow; 291 292 bool psil_paired; 293 294 int irq_num_ring; 295 int irq_num_udma; 296 297 bool cyclic; 298 bool paused; 299 300 enum udma_chan_state state; 301 struct completion teardown_completed; 302 303 struct udma_tx_drain tx_drain; 304 305 /* Channel configuration parameters */ 306 struct udma_chan_config config; 307 /* Channel configuration parameters (backup) */ 308 struct udma_chan_config backup_config; 309 310 /* dmapool for packet mode descriptors */ 311 bool use_dma_pool; 312 struct dma_pool *hdesc_pool; 313 314 u32 id; 315 }; 316 317 static inline struct udma_dev *to_udma_dev(struct dma_device *d) 318 { 319 return container_of(d, struct udma_dev, ddev); 320 } 321 322 static inline struct udma_chan *to_udma_chan(struct dma_chan *c) 323 { 324 return container_of(c, struct udma_chan, vc.chan); 325 } 326 327 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t) 328 { 329 return container_of(t, struct udma_desc, vd.tx); 330 } 331 332 /* Generic register access functions */ 333 static inline u32 udma_read(void __iomem *base, int reg) 334 { 335 return readl(base + reg); 336 } 337 338 static inline void udma_write(void __iomem *base, int reg, u32 val) 339 { 340 writel(val, base + reg); 341 } 342 343 static inline void udma_update_bits(void __iomem *base, int reg, 344 u32 mask, u32 val) 345 { 346 u32 tmp, orig; 347 348 orig = readl(base + reg); 349 tmp = orig & ~mask; 350 tmp |= (val & mask); 351 352 if (tmp != orig) 353 writel(tmp, base + reg); 354 } 355 356 /* TCHANRT */ 357 static inline u32 udma_tchanrt_read(struct udma_chan *uc, int reg) 358 { 359 if (!uc->tchan) 360 return 0; 361 return udma_read(uc->tchan->reg_rt, reg); 362 } 363 364 static inline void udma_tchanrt_write(struct udma_chan *uc, int reg, u32 val) 365 { 366 if (!uc->tchan) 367 return; 368 udma_write(uc->tchan->reg_rt, reg, val); 369 } 370 371 static inline void udma_tchanrt_update_bits(struct udma_chan *uc, int reg, 372 u32 mask, u32 val) 373 { 374 if (!uc->tchan) 375 return; 376 udma_update_bits(uc->tchan->reg_rt, reg, mask, val); 377 } 378 379 /* RCHANRT */ 380 static inline u32 udma_rchanrt_read(struct udma_chan *uc, int reg) 381 { 382 if (!uc->rchan) 383 return 0; 384 return udma_read(uc->rchan->reg_rt, reg); 385 } 386 387 static inline void udma_rchanrt_write(struct udma_chan *uc, int reg, u32 val) 388 { 389 if (!uc->rchan) 390 return; 391 udma_write(uc->rchan->reg_rt, reg, val); 392 } 393 394 static inline void udma_rchanrt_update_bits(struct udma_chan *uc, int reg, 395 u32 mask, u32 val) 396 { 397 if (!uc->rchan) 398 return; 399 udma_update_bits(uc->rchan->reg_rt, reg, mask, val); 400 } 401 402 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) 403 { 404 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 405 406 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 407 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci, 408 tisci_rm->tisci_navss_dev_id, 409 src_thread, dst_thread); 410 } 411 412 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread, 413 u32 dst_thread) 414 { 415 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 416 417 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 418 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci, 419 tisci_rm->tisci_navss_dev_id, 420 src_thread, dst_thread); 421 } 422 423 static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel) 424 { 425 struct device *chan_dev = &chan->dev->device; 426 427 if (asel == 0) { 428 /* No special handling for the channel */ 429 chan->dev->chan_dma_dev = false; 430 431 chan_dev->dma_coherent = false; 432 chan_dev->dma_parms = NULL; 433 } else if (asel == 14 || asel == 15) { 434 chan->dev->chan_dma_dev = true; 435 436 chan_dev->dma_coherent = true; 437 dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48)); 438 chan_dev->dma_parms = chan_dev->parent->dma_parms; 439 } else { 440 dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel); 441 442 chan_dev->dma_coherent = false; 443 chan_dev->dma_parms = NULL; 444 } 445 } 446 447 static u8 udma_get_chan_tpl_index(struct udma_tpl *tpl_map, int chan_id) 448 { 449 int i; 450 451 for (i = 0; i < tpl_map->levels; i++) { 452 if (chan_id >= tpl_map->start_idx[i]) 453 return i; 454 } 455 456 return 0; 457 } 458 459 static void udma_reset_uchan(struct udma_chan *uc) 460 { 461 memset(&uc->config, 0, sizeof(uc->config)); 462 uc->config.remote_thread_id = -1; 463 uc->config.mapped_channel_id = -1; 464 uc->config.default_flow_id = -1; 465 uc->state = UDMA_CHAN_IS_IDLE; 466 } 467 468 static void udma_dump_chan_stdata(struct udma_chan *uc) 469 { 470 struct device *dev = uc->ud->dev; 471 u32 offset; 472 int i; 473 474 if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) { 475 dev_dbg(dev, "TCHAN State data:\n"); 476 for (i = 0; i < 32; i++) { 477 offset = UDMA_CHAN_RT_STDATA_REG + i * 4; 478 dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i, 479 udma_tchanrt_read(uc, offset)); 480 } 481 } 482 483 if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) { 484 dev_dbg(dev, "RCHAN State data:\n"); 485 for (i = 0; i < 32; i++) { 486 offset = UDMA_CHAN_RT_STDATA_REG + i * 4; 487 dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i, 488 udma_rchanrt_read(uc, offset)); 489 } 490 } 491 } 492 493 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d, 494 int idx) 495 { 496 return d->hwdesc[idx].cppi5_desc_paddr; 497 } 498 499 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx) 500 { 501 return d->hwdesc[idx].cppi5_desc_vaddr; 502 } 503 504 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc, 505 dma_addr_t paddr) 506 { 507 struct udma_desc *d = uc->terminated_desc; 508 509 if (d) { 510 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 511 d->desc_idx); 512 513 if (desc_paddr != paddr) 514 d = NULL; 515 } 516 517 if (!d) { 518 d = uc->desc; 519 if (d) { 520 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 521 d->desc_idx); 522 523 if (desc_paddr != paddr) 524 d = NULL; 525 } 526 } 527 528 return d; 529 } 530 531 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d) 532 { 533 if (uc->use_dma_pool) { 534 int i; 535 536 for (i = 0; i < d->hwdesc_count; i++) { 537 if (!d->hwdesc[i].cppi5_desc_vaddr) 538 continue; 539 540 dma_pool_free(uc->hdesc_pool, 541 d->hwdesc[i].cppi5_desc_vaddr, 542 d->hwdesc[i].cppi5_desc_paddr); 543 544 d->hwdesc[i].cppi5_desc_vaddr = NULL; 545 } 546 } else if (d->hwdesc[0].cppi5_desc_vaddr) { 547 dma_free_coherent(uc->dma_dev, d->hwdesc[0].cppi5_desc_size, 548 d->hwdesc[0].cppi5_desc_vaddr, 549 d->hwdesc[0].cppi5_desc_paddr); 550 551 d->hwdesc[0].cppi5_desc_vaddr = NULL; 552 } 553 } 554 555 static void udma_purge_desc_work(struct work_struct *work) 556 { 557 struct udma_dev *ud = container_of(work, typeof(*ud), purge_work); 558 struct virt_dma_desc *vd, *_vd; 559 unsigned long flags; 560 LIST_HEAD(head); 561 562 spin_lock_irqsave(&ud->lock, flags); 563 list_splice_tail_init(&ud->desc_to_purge, &head); 564 spin_unlock_irqrestore(&ud->lock, flags); 565 566 list_for_each_entry_safe(vd, _vd, &head, node) { 567 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 568 struct udma_desc *d = to_udma_desc(&vd->tx); 569 570 udma_free_hwdesc(uc, d); 571 list_del(&vd->node); 572 kfree(d); 573 } 574 575 /* If more to purge, schedule the work again */ 576 if (!list_empty(&ud->desc_to_purge)) 577 schedule_work(&ud->purge_work); 578 } 579 580 static void udma_desc_free(struct virt_dma_desc *vd) 581 { 582 struct udma_dev *ud = to_udma_dev(vd->tx.chan->device); 583 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 584 struct udma_desc *d = to_udma_desc(&vd->tx); 585 unsigned long flags; 586 587 if (uc->terminated_desc == d) 588 uc->terminated_desc = NULL; 589 590 if (uc->use_dma_pool) { 591 udma_free_hwdesc(uc, d); 592 kfree(d); 593 return; 594 } 595 596 spin_lock_irqsave(&ud->lock, flags); 597 list_add_tail(&vd->node, &ud->desc_to_purge); 598 spin_unlock_irqrestore(&ud->lock, flags); 599 600 schedule_work(&ud->purge_work); 601 } 602 603 static bool udma_is_chan_running(struct udma_chan *uc) 604 { 605 u32 trt_ctl = 0; 606 u32 rrt_ctl = 0; 607 608 if (uc->tchan) 609 trt_ctl = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); 610 if (uc->rchan) 611 rrt_ctl = udma_rchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); 612 613 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN) 614 return true; 615 616 return false; 617 } 618 619 static bool udma_is_chan_paused(struct udma_chan *uc) 620 { 621 u32 val, pause_mask; 622 623 switch (uc->config.dir) { 624 case DMA_DEV_TO_MEM: 625 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG); 626 pause_mask = UDMA_PEER_RT_EN_PAUSE; 627 break; 628 case DMA_MEM_TO_DEV: 629 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG); 630 pause_mask = UDMA_PEER_RT_EN_PAUSE; 631 break; 632 case DMA_MEM_TO_MEM: 633 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); 634 pause_mask = UDMA_CHAN_RT_CTL_PAUSE; 635 break; 636 default: 637 return false; 638 } 639 640 if (val & pause_mask) 641 return true; 642 643 return false; 644 } 645 646 static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc) 647 { 648 return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr; 649 } 650 651 static int udma_push_to_ring(struct udma_chan *uc, int idx) 652 { 653 struct udma_desc *d = uc->desc; 654 struct k3_ring *ring = NULL; 655 dma_addr_t paddr; 656 657 switch (uc->config.dir) { 658 case DMA_DEV_TO_MEM: 659 ring = uc->rflow->fd_ring; 660 break; 661 case DMA_MEM_TO_DEV: 662 case DMA_MEM_TO_MEM: 663 ring = uc->tchan->t_ring; 664 break; 665 default: 666 return -EINVAL; 667 } 668 669 /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */ 670 if (idx == -1) { 671 paddr = udma_get_rx_flush_hwdesc_paddr(uc); 672 } else { 673 paddr = udma_curr_cppi5_desc_paddr(d, idx); 674 675 wmb(); /* Ensure that writes are not moved over this point */ 676 } 677 678 return k3_ringacc_ring_push(ring, &paddr); 679 } 680 681 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr) 682 { 683 if (uc->config.dir != DMA_DEV_TO_MEM) 684 return false; 685 686 if (addr == udma_get_rx_flush_hwdesc_paddr(uc)) 687 return true; 688 689 return false; 690 } 691 692 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) 693 { 694 struct k3_ring *ring = NULL; 695 int ret; 696 697 switch (uc->config.dir) { 698 case DMA_DEV_TO_MEM: 699 ring = uc->rflow->r_ring; 700 break; 701 case DMA_MEM_TO_DEV: 702 case DMA_MEM_TO_MEM: 703 ring = uc->tchan->tc_ring; 704 break; 705 default: 706 return -ENOENT; 707 } 708 709 ret = k3_ringacc_ring_pop(ring, addr); 710 if (ret) 711 return ret; 712 713 rmb(); /* Ensure that reads are not moved before this point */ 714 715 /* Teardown completion */ 716 if (cppi5_desc_is_tdcm(*addr)) 717 return 0; 718 719 /* Check for flush descriptor */ 720 if (udma_desc_is_rx_flush(uc, *addr)) 721 return -ENOENT; 722 723 return 0; 724 } 725 726 static void udma_reset_rings(struct udma_chan *uc) 727 { 728 struct k3_ring *ring1 = NULL; 729 struct k3_ring *ring2 = NULL; 730 731 switch (uc->config.dir) { 732 case DMA_DEV_TO_MEM: 733 if (uc->rchan) { 734 ring1 = uc->rflow->fd_ring; 735 ring2 = uc->rflow->r_ring; 736 } 737 break; 738 case DMA_MEM_TO_DEV: 739 case DMA_MEM_TO_MEM: 740 if (uc->tchan) { 741 ring1 = uc->tchan->t_ring; 742 ring2 = uc->tchan->tc_ring; 743 } 744 break; 745 default: 746 break; 747 } 748 749 if (ring1) 750 k3_ringacc_ring_reset_dma(ring1, 751 k3_ringacc_ring_get_occ(ring1)); 752 if (ring2) 753 k3_ringacc_ring_reset(ring2); 754 755 /* make sure we are not leaking memory by stalled descriptor */ 756 if (uc->terminated_desc) { 757 udma_desc_free(&uc->terminated_desc->vd); 758 uc->terminated_desc = NULL; 759 } 760 } 761 762 static void udma_decrement_byte_counters(struct udma_chan *uc, u32 val) 763 { 764 if (uc->desc->dir == DMA_DEV_TO_MEM) { 765 udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); 766 udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); 767 if (uc->config.ep_type != PSIL_EP_NATIVE) 768 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); 769 } else { 770 udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); 771 udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); 772 if (!uc->bchan && uc->config.ep_type != PSIL_EP_NATIVE) 773 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); 774 } 775 } 776 777 static void udma_reset_counters(struct udma_chan *uc) 778 { 779 u32 val; 780 781 if (uc->tchan) { 782 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 783 udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); 784 785 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); 786 udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); 787 788 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG); 789 udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val); 790 791 if (!uc->bchan) { 792 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); 793 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); 794 } 795 } 796 797 if (uc->rchan) { 798 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 799 udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); 800 801 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); 802 udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); 803 804 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG); 805 udma_rchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val); 806 807 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); 808 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); 809 } 810 } 811 812 static int udma_reset_chan(struct udma_chan *uc, bool hard) 813 { 814 switch (uc->config.dir) { 815 case DMA_DEV_TO_MEM: 816 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0); 817 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 818 break; 819 case DMA_MEM_TO_DEV: 820 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 821 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0); 822 break; 823 case DMA_MEM_TO_MEM: 824 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 825 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); 826 break; 827 default: 828 return -EINVAL; 829 } 830 831 /* Reset all counters */ 832 udma_reset_counters(uc); 833 834 /* Hard reset: re-initialize the channel to reset */ 835 if (hard) { 836 struct udma_chan_config ucc_backup; 837 int ret; 838 839 memcpy(&ucc_backup, &uc->config, sizeof(uc->config)); 840 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan); 841 842 /* restore the channel configuration */ 843 memcpy(&uc->config, &ucc_backup, sizeof(uc->config)); 844 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan); 845 if (ret) 846 return ret; 847 848 /* 849 * Setting forced teardown after forced reset helps recovering 850 * the rchan. 851 */ 852 if (uc->config.dir == DMA_DEV_TO_MEM) 853 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 854 UDMA_CHAN_RT_CTL_EN | 855 UDMA_CHAN_RT_CTL_TDOWN | 856 UDMA_CHAN_RT_CTL_FTDOWN); 857 } 858 uc->state = UDMA_CHAN_IS_IDLE; 859 860 return 0; 861 } 862 863 static void udma_start_desc(struct udma_chan *uc) 864 { 865 struct udma_chan_config *ucc = &uc->config; 866 867 if (uc->ud->match_data->type == DMA_TYPE_UDMA && ucc->pkt_mode && 868 (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) { 869 int i; 870 871 /* 872 * UDMA only: Push all descriptors to ring for packet mode 873 * cyclic or RX 874 * PKTDMA supports pre-linked descriptor and cyclic is not 875 * supported 876 */ 877 for (i = 0; i < uc->desc->sglen; i++) 878 udma_push_to_ring(uc, i); 879 } else { 880 udma_push_to_ring(uc, 0); 881 } 882 } 883 884 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc) 885 { 886 /* Only PDMAs have staticTR */ 887 if (uc->config.ep_type == PSIL_EP_NATIVE) 888 return false; 889 890 /* Check if the staticTR configuration has changed for TX */ 891 if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr))) 892 return true; 893 894 return false; 895 } 896 897 static int udma_start(struct udma_chan *uc) 898 { 899 struct virt_dma_desc *vd = vchan_next_desc(&uc->vc); 900 901 if (!vd) { 902 uc->desc = NULL; 903 return -ENOENT; 904 } 905 906 list_del(&vd->node); 907 908 uc->desc = to_udma_desc(&vd->tx); 909 910 /* Channel is already running and does not need reconfiguration */ 911 if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) { 912 udma_start_desc(uc); 913 goto out; 914 } 915 916 /* Make sure that we clear the teardown bit, if it is set */ 917 udma_reset_chan(uc, false); 918 919 /* Push descriptors before we start the channel */ 920 udma_start_desc(uc); 921 922 switch (uc->desc->dir) { 923 case DMA_DEV_TO_MEM: 924 /* Config remote TR */ 925 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 926 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 927 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 928 const struct udma_match_data *match_data = 929 uc->ud->match_data; 930 931 if (uc->config.enable_acc32) 932 val |= PDMA_STATIC_TR_XY_ACC32; 933 if (uc->config.enable_burst) 934 val |= PDMA_STATIC_TR_XY_BURST; 935 936 udma_rchanrt_write(uc, 937 UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG, 938 val); 939 940 udma_rchanrt_write(uc, 941 UDMA_CHAN_RT_PEER_STATIC_TR_Z_REG, 942 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt, 943 match_data->statictr_z_mask)); 944 945 /* save the current staticTR configuration */ 946 memcpy(&uc->static_tr, &uc->desc->static_tr, 947 sizeof(uc->static_tr)); 948 } 949 950 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 951 UDMA_CHAN_RT_CTL_EN); 952 953 /* Enable remote */ 954 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 955 UDMA_PEER_RT_EN_ENABLE); 956 957 break; 958 case DMA_MEM_TO_DEV: 959 /* Config remote TR */ 960 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 961 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 962 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 963 964 if (uc->config.enable_acc32) 965 val |= PDMA_STATIC_TR_XY_ACC32; 966 if (uc->config.enable_burst) 967 val |= PDMA_STATIC_TR_XY_BURST; 968 969 udma_tchanrt_write(uc, 970 UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG, 971 val); 972 973 /* save the current staticTR configuration */ 974 memcpy(&uc->static_tr, &uc->desc->static_tr, 975 sizeof(uc->static_tr)); 976 } 977 978 /* Enable remote */ 979 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 980 UDMA_PEER_RT_EN_ENABLE); 981 982 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 983 UDMA_CHAN_RT_CTL_EN); 984 985 break; 986 case DMA_MEM_TO_MEM: 987 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 988 UDMA_CHAN_RT_CTL_EN); 989 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 990 UDMA_CHAN_RT_CTL_EN); 991 992 break; 993 default: 994 return -EINVAL; 995 } 996 997 uc->state = UDMA_CHAN_IS_ACTIVE; 998 out: 999 1000 return 0; 1001 } 1002 1003 static int udma_stop(struct udma_chan *uc) 1004 { 1005 enum udma_chan_state old_state = uc->state; 1006 1007 uc->state = UDMA_CHAN_IS_TERMINATING; 1008 reinit_completion(&uc->teardown_completed); 1009 1010 switch (uc->config.dir) { 1011 case DMA_DEV_TO_MEM: 1012 if (!uc->cyclic && !uc->desc) 1013 udma_push_to_ring(uc, -1); 1014 1015 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 1016 UDMA_PEER_RT_EN_ENABLE | 1017 UDMA_PEER_RT_EN_TEARDOWN); 1018 break; 1019 case DMA_MEM_TO_DEV: 1020 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 1021 UDMA_PEER_RT_EN_ENABLE | 1022 UDMA_PEER_RT_EN_FLUSH); 1023 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 1024 UDMA_CHAN_RT_CTL_EN | 1025 UDMA_CHAN_RT_CTL_TDOWN); 1026 break; 1027 case DMA_MEM_TO_MEM: 1028 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 1029 UDMA_CHAN_RT_CTL_EN | 1030 UDMA_CHAN_RT_CTL_TDOWN); 1031 break; 1032 default: 1033 uc->state = old_state; 1034 complete_all(&uc->teardown_completed); 1035 return -EINVAL; 1036 } 1037 1038 return 0; 1039 } 1040 1041 static void udma_cyclic_packet_elapsed(struct udma_chan *uc) 1042 { 1043 struct udma_desc *d = uc->desc; 1044 struct cppi5_host_desc_t *h_desc; 1045 1046 h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr; 1047 cppi5_hdesc_reset_to_original(h_desc); 1048 udma_push_to_ring(uc, d->desc_idx); 1049 d->desc_idx = (d->desc_idx + 1) % d->sglen; 1050 } 1051 1052 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d) 1053 { 1054 struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr; 1055 1056 memcpy(d->metadata, h_desc->epib, d->metadata_size); 1057 } 1058 1059 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) 1060 { 1061 u32 peer_bcnt, bcnt; 1062 1063 /* 1064 * Only TX towards PDMA is affected. 1065 * If DMA_PREP_INTERRUPT is not set by consumer then skip the transfer 1066 * completion calculation, consumer must ensure that there is no stale 1067 * data in DMA fabric in this case. 1068 */ 1069 if (uc->config.ep_type == PSIL_EP_NATIVE || 1070 uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT)) 1071 return true; 1072 1073 peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); 1074 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 1075 1076 /* Transfer is incomplete, store current residue and time stamp */ 1077 if (peer_bcnt < bcnt) { 1078 uc->tx_drain.residue = bcnt - peer_bcnt; 1079 uc->tx_drain.tstamp = ktime_get(); 1080 return false; 1081 } 1082 1083 return true; 1084 } 1085 1086 static void udma_check_tx_completion(struct work_struct *work) 1087 { 1088 struct udma_chan *uc = container_of(work, typeof(*uc), 1089 tx_drain.work.work); 1090 bool desc_done = true; 1091 u32 residue_diff; 1092 ktime_t time_diff; 1093 unsigned long delay; 1094 1095 while (1) { 1096 if (uc->desc) { 1097 /* Get previous residue and time stamp */ 1098 residue_diff = uc->tx_drain.residue; 1099 time_diff = uc->tx_drain.tstamp; 1100 /* 1101 * Get current residue and time stamp or see if 1102 * transfer is complete 1103 */ 1104 desc_done = udma_is_desc_really_done(uc, uc->desc); 1105 } 1106 1107 if (!desc_done) { 1108 /* 1109 * Find the time delta and residue delta w.r.t 1110 * previous poll 1111 */ 1112 time_diff = ktime_sub(uc->tx_drain.tstamp, 1113 time_diff) + 1; 1114 residue_diff -= uc->tx_drain.residue; 1115 if (residue_diff) { 1116 /* 1117 * Try to guess when we should check 1118 * next time by calculating rate at 1119 * which data is being drained at the 1120 * peer device 1121 */ 1122 delay = (time_diff / residue_diff) * 1123 uc->tx_drain.residue; 1124 } else { 1125 /* No progress, check again in 1 second */ 1126 schedule_delayed_work(&uc->tx_drain.work, HZ); 1127 break; 1128 } 1129 1130 usleep_range(ktime_to_us(delay), 1131 ktime_to_us(delay) + 10); 1132 continue; 1133 } 1134 1135 if (uc->desc) { 1136 struct udma_desc *d = uc->desc; 1137 1138 udma_decrement_byte_counters(uc, d->residue); 1139 udma_start(uc); 1140 vchan_cookie_complete(&d->vd); 1141 break; 1142 } 1143 1144 break; 1145 } 1146 } 1147 1148 static irqreturn_t udma_ring_irq_handler(int irq, void *data) 1149 { 1150 struct udma_chan *uc = data; 1151 struct udma_desc *d; 1152 dma_addr_t paddr = 0; 1153 1154 if (udma_pop_from_ring(uc, &paddr) || !paddr) 1155 return IRQ_HANDLED; 1156 1157 spin_lock(&uc->vc.lock); 1158 1159 /* Teardown completion message */ 1160 if (cppi5_desc_is_tdcm(paddr)) { 1161 complete_all(&uc->teardown_completed); 1162 1163 if (uc->terminated_desc) { 1164 udma_desc_free(&uc->terminated_desc->vd); 1165 uc->terminated_desc = NULL; 1166 } 1167 1168 if (!uc->desc) 1169 udma_start(uc); 1170 1171 goto out; 1172 } 1173 1174 d = udma_udma_desc_from_paddr(uc, paddr); 1175 1176 if (d) { 1177 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 1178 d->desc_idx); 1179 if (desc_paddr != paddr) { 1180 dev_err(uc->ud->dev, "not matching descriptors!\n"); 1181 goto out; 1182 } 1183 1184 if (d == uc->desc) { 1185 /* active descriptor */ 1186 if (uc->cyclic) { 1187 udma_cyclic_packet_elapsed(uc); 1188 vchan_cyclic_callback(&d->vd); 1189 } else { 1190 if (udma_is_desc_really_done(uc, d)) { 1191 udma_decrement_byte_counters(uc, d->residue); 1192 udma_start(uc); 1193 vchan_cookie_complete(&d->vd); 1194 } else { 1195 schedule_delayed_work(&uc->tx_drain.work, 1196 0); 1197 } 1198 } 1199 } else { 1200 /* 1201 * terminated descriptor, mark the descriptor as 1202 * completed to update the channel's cookie marker 1203 */ 1204 dma_cookie_complete(&d->vd.tx); 1205 } 1206 } 1207 out: 1208 spin_unlock(&uc->vc.lock); 1209 1210 return IRQ_HANDLED; 1211 } 1212 1213 static irqreturn_t udma_udma_irq_handler(int irq, void *data) 1214 { 1215 struct udma_chan *uc = data; 1216 struct udma_desc *d; 1217 1218 spin_lock(&uc->vc.lock); 1219 d = uc->desc; 1220 if (d) { 1221 d->tr_idx = (d->tr_idx + 1) % d->sglen; 1222 1223 if (uc->cyclic) { 1224 vchan_cyclic_callback(&d->vd); 1225 } else { 1226 /* TODO: figure out the real amount of data */ 1227 udma_decrement_byte_counters(uc, d->residue); 1228 udma_start(uc); 1229 vchan_cookie_complete(&d->vd); 1230 } 1231 } 1232 1233 spin_unlock(&uc->vc.lock); 1234 1235 return IRQ_HANDLED; 1236 } 1237 1238 /** 1239 * __udma_alloc_gp_rflow_range - alloc range of GP RX flows 1240 * @ud: UDMA device 1241 * @from: Start the search from this flow id number 1242 * @cnt: Number of consecutive flow ids to allocate 1243 * 1244 * Allocate range of RX flow ids for future use, those flows can be requested 1245 * only using explicit flow id number. if @from is set to -1 it will try to find 1246 * first free range. if @from is positive value it will force allocation only 1247 * of the specified range of flows. 1248 * 1249 * Returns -ENOMEM if can't find free range. 1250 * -EEXIST if requested range is busy. 1251 * -EINVAL if wrong input values passed. 1252 * Returns flow id on success. 1253 */ 1254 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1255 { 1256 int start, tmp_from; 1257 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS); 1258 1259 tmp_from = from; 1260 if (tmp_from < 0) 1261 tmp_from = ud->rchan_cnt; 1262 /* default flows can't be allocated and accessible only by id */ 1263 if (tmp_from < ud->rchan_cnt) 1264 return -EINVAL; 1265 1266 if (tmp_from + cnt > ud->rflow_cnt) 1267 return -EINVAL; 1268 1269 bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated, 1270 ud->rflow_cnt); 1271 1272 start = bitmap_find_next_zero_area(tmp, 1273 ud->rflow_cnt, 1274 tmp_from, cnt, 0); 1275 if (start >= ud->rflow_cnt) 1276 return -ENOMEM; 1277 1278 if (from >= 0 && start != from) 1279 return -EEXIST; 1280 1281 bitmap_set(ud->rflow_gp_map_allocated, start, cnt); 1282 return start; 1283 } 1284 1285 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1286 { 1287 if (from < ud->rchan_cnt) 1288 return -EINVAL; 1289 if (from + cnt > ud->rflow_cnt) 1290 return -EINVAL; 1291 1292 bitmap_clear(ud->rflow_gp_map_allocated, from, cnt); 1293 return 0; 1294 } 1295 1296 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id) 1297 { 1298 /* 1299 * Attempt to request rflow by ID can be made for any rflow 1300 * if not in use with assumption that caller knows what's doing. 1301 * TI-SCI FW will perform additional permission check ant way, it's 1302 * safe 1303 */ 1304 1305 if (id < 0 || id >= ud->rflow_cnt) 1306 return ERR_PTR(-ENOENT); 1307 1308 if (test_bit(id, ud->rflow_in_use)) 1309 return ERR_PTR(-ENOENT); 1310 1311 if (ud->rflow_gp_map) { 1312 /* GP rflow has to be allocated first */ 1313 if (!test_bit(id, ud->rflow_gp_map) && 1314 !test_bit(id, ud->rflow_gp_map_allocated)) 1315 return ERR_PTR(-EINVAL); 1316 } 1317 1318 dev_dbg(ud->dev, "get rflow%d\n", id); 1319 set_bit(id, ud->rflow_in_use); 1320 return &ud->rflows[id]; 1321 } 1322 1323 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow) 1324 { 1325 if (!test_bit(rflow->id, ud->rflow_in_use)) { 1326 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id); 1327 return; 1328 } 1329 1330 dev_dbg(ud->dev, "put rflow%d\n", rflow->id); 1331 clear_bit(rflow->id, ud->rflow_in_use); 1332 } 1333 1334 #define UDMA_RESERVE_RESOURCE(res) \ 1335 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ 1336 enum udma_tp_level tpl, \ 1337 int id) \ 1338 { \ 1339 if (id >= 0) { \ 1340 if (test_bit(id, ud->res##_map)) { \ 1341 dev_err(ud->dev, "res##%d is in use\n", id); \ 1342 return ERR_PTR(-ENOENT); \ 1343 } \ 1344 } else { \ 1345 int start; \ 1346 \ 1347 if (tpl >= ud->res##_tpl.levels) \ 1348 tpl = ud->res##_tpl.levels - 1; \ 1349 \ 1350 start = ud->res##_tpl.start_idx[tpl]; \ 1351 \ 1352 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \ 1353 start); \ 1354 if (id == ud->res##_cnt) { \ 1355 return ERR_PTR(-ENOENT); \ 1356 } \ 1357 } \ 1358 \ 1359 set_bit(id, ud->res##_map); \ 1360 return &ud->res##s[id]; \ 1361 } 1362 1363 UDMA_RESERVE_RESOURCE(bchan); 1364 UDMA_RESERVE_RESOURCE(tchan); 1365 UDMA_RESERVE_RESOURCE(rchan); 1366 1367 static int bcdma_get_bchan(struct udma_chan *uc) 1368 { 1369 struct udma_dev *ud = uc->ud; 1370 enum udma_tp_level tpl; 1371 int ret; 1372 1373 if (uc->bchan) { 1374 dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n", 1375 uc->id, uc->bchan->id); 1376 return 0; 1377 } 1378 1379 /* 1380 * Use normal channels for peripherals, and highest TPL channel for 1381 * mem2mem 1382 */ 1383 if (uc->config.tr_trigger_type) 1384 tpl = 0; 1385 else 1386 tpl = ud->bchan_tpl.levels - 1; 1387 1388 uc->bchan = __udma_reserve_bchan(ud, tpl, -1); 1389 if (IS_ERR(uc->bchan)) { 1390 ret = PTR_ERR(uc->bchan); 1391 uc->bchan = NULL; 1392 return ret; 1393 } 1394 1395 uc->tchan = uc->bchan; 1396 1397 return 0; 1398 } 1399 1400 static int udma_get_tchan(struct udma_chan *uc) 1401 { 1402 struct udma_dev *ud = uc->ud; 1403 int ret; 1404 1405 if (uc->tchan) { 1406 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n", 1407 uc->id, uc->tchan->id); 1408 return 0; 1409 } 1410 1411 /* 1412 * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels. 1413 * For PKTDMA mapped channels it is configured to a channel which must 1414 * be used to service the peripheral. 1415 */ 1416 uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, 1417 uc->config.mapped_channel_id); 1418 if (IS_ERR(uc->tchan)) { 1419 ret = PTR_ERR(uc->tchan); 1420 uc->tchan = NULL; 1421 return ret; 1422 } 1423 1424 if (ud->tflow_cnt) { 1425 int tflow_id; 1426 1427 /* Only PKTDMA have support for tx flows */ 1428 if (uc->config.default_flow_id >= 0) 1429 tflow_id = uc->config.default_flow_id; 1430 else 1431 tflow_id = uc->tchan->id; 1432 1433 if (test_bit(tflow_id, ud->tflow_map)) { 1434 dev_err(ud->dev, "tflow%d is in use\n", tflow_id); 1435 clear_bit(uc->tchan->id, ud->tchan_map); 1436 uc->tchan = NULL; 1437 return -ENOENT; 1438 } 1439 1440 uc->tchan->tflow_id = tflow_id; 1441 set_bit(tflow_id, ud->tflow_map); 1442 } else { 1443 uc->tchan->tflow_id = -1; 1444 } 1445 1446 return 0; 1447 } 1448 1449 static int udma_get_rchan(struct udma_chan *uc) 1450 { 1451 struct udma_dev *ud = uc->ud; 1452 int ret; 1453 1454 if (uc->rchan) { 1455 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n", 1456 uc->id, uc->rchan->id); 1457 return 0; 1458 } 1459 1460 /* 1461 * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels. 1462 * For PKTDMA mapped channels it is configured to a channel which must 1463 * be used to service the peripheral. 1464 */ 1465 uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, 1466 uc->config.mapped_channel_id); 1467 if (IS_ERR(uc->rchan)) { 1468 ret = PTR_ERR(uc->rchan); 1469 uc->rchan = NULL; 1470 return ret; 1471 } 1472 1473 return 0; 1474 } 1475 1476 static int udma_get_chan_pair(struct udma_chan *uc) 1477 { 1478 struct udma_dev *ud = uc->ud; 1479 int chan_id, end; 1480 1481 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) { 1482 dev_info(ud->dev, "chan%d: already have %d pair allocated\n", 1483 uc->id, uc->tchan->id); 1484 return 0; 1485 } 1486 1487 if (uc->tchan) { 1488 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n", 1489 uc->id, uc->tchan->id); 1490 return -EBUSY; 1491 } else if (uc->rchan) { 1492 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n", 1493 uc->id, uc->rchan->id); 1494 return -EBUSY; 1495 } 1496 1497 /* Can be optimized, but let's have it like this for now */ 1498 end = min(ud->tchan_cnt, ud->rchan_cnt); 1499 /* 1500 * Try to use the highest TPL channel pair for MEM_TO_MEM channels 1501 * Note: in UDMAP the channel TPL is symmetric between tchan and rchan 1502 */ 1503 chan_id = ud->tchan_tpl.start_idx[ud->tchan_tpl.levels - 1]; 1504 for (; chan_id < end; chan_id++) { 1505 if (!test_bit(chan_id, ud->tchan_map) && 1506 !test_bit(chan_id, ud->rchan_map)) 1507 break; 1508 } 1509 1510 if (chan_id == end) 1511 return -ENOENT; 1512 1513 set_bit(chan_id, ud->tchan_map); 1514 set_bit(chan_id, ud->rchan_map); 1515 uc->tchan = &ud->tchans[chan_id]; 1516 uc->rchan = &ud->rchans[chan_id]; 1517 1518 /* UDMA does not use tx flows */ 1519 uc->tchan->tflow_id = -1; 1520 1521 return 0; 1522 } 1523 1524 static int udma_get_rflow(struct udma_chan *uc, int flow_id) 1525 { 1526 struct udma_dev *ud = uc->ud; 1527 int ret; 1528 1529 if (!uc->rchan) { 1530 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id); 1531 return -EINVAL; 1532 } 1533 1534 if (uc->rflow) { 1535 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n", 1536 uc->id, uc->rflow->id); 1537 return 0; 1538 } 1539 1540 uc->rflow = __udma_get_rflow(ud, flow_id); 1541 if (IS_ERR(uc->rflow)) { 1542 ret = PTR_ERR(uc->rflow); 1543 uc->rflow = NULL; 1544 return ret; 1545 } 1546 1547 return 0; 1548 } 1549 1550 static void bcdma_put_bchan(struct udma_chan *uc) 1551 { 1552 struct udma_dev *ud = uc->ud; 1553 1554 if (uc->bchan) { 1555 dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id, 1556 uc->bchan->id); 1557 clear_bit(uc->bchan->id, ud->bchan_map); 1558 uc->bchan = NULL; 1559 uc->tchan = NULL; 1560 } 1561 } 1562 1563 static void udma_put_rchan(struct udma_chan *uc) 1564 { 1565 struct udma_dev *ud = uc->ud; 1566 1567 if (uc->rchan) { 1568 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id, 1569 uc->rchan->id); 1570 clear_bit(uc->rchan->id, ud->rchan_map); 1571 uc->rchan = NULL; 1572 } 1573 } 1574 1575 static void udma_put_tchan(struct udma_chan *uc) 1576 { 1577 struct udma_dev *ud = uc->ud; 1578 1579 if (uc->tchan) { 1580 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id, 1581 uc->tchan->id); 1582 clear_bit(uc->tchan->id, ud->tchan_map); 1583 1584 if (uc->tchan->tflow_id >= 0) 1585 clear_bit(uc->tchan->tflow_id, ud->tflow_map); 1586 1587 uc->tchan = NULL; 1588 } 1589 } 1590 1591 static void udma_put_rflow(struct udma_chan *uc) 1592 { 1593 struct udma_dev *ud = uc->ud; 1594 1595 if (uc->rflow) { 1596 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id, 1597 uc->rflow->id); 1598 __udma_put_rflow(ud, uc->rflow); 1599 uc->rflow = NULL; 1600 } 1601 } 1602 1603 static void bcdma_free_bchan_resources(struct udma_chan *uc) 1604 { 1605 if (!uc->bchan) 1606 return; 1607 1608 k3_ringacc_ring_free(uc->bchan->tc_ring); 1609 k3_ringacc_ring_free(uc->bchan->t_ring); 1610 uc->bchan->tc_ring = NULL; 1611 uc->bchan->t_ring = NULL; 1612 k3_configure_chan_coherency(&uc->vc.chan, 0); 1613 1614 bcdma_put_bchan(uc); 1615 } 1616 1617 static int bcdma_alloc_bchan_resources(struct udma_chan *uc) 1618 { 1619 struct k3_ring_cfg ring_cfg; 1620 struct udma_dev *ud = uc->ud; 1621 int ret; 1622 1623 ret = bcdma_get_bchan(uc); 1624 if (ret) 1625 return ret; 1626 1627 ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1, 1628 &uc->bchan->t_ring, 1629 &uc->bchan->tc_ring); 1630 if (ret) { 1631 ret = -EBUSY; 1632 goto err_ring; 1633 } 1634 1635 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1636 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1637 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1638 ring_cfg.mode = K3_RINGACC_RING_MODE_RING; 1639 1640 k3_configure_chan_coherency(&uc->vc.chan, ud->asel); 1641 ring_cfg.asel = ud->asel; 1642 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan); 1643 1644 ret = k3_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg); 1645 if (ret) 1646 goto err_ringcfg; 1647 1648 return 0; 1649 1650 err_ringcfg: 1651 k3_ringacc_ring_free(uc->bchan->tc_ring); 1652 uc->bchan->tc_ring = NULL; 1653 k3_ringacc_ring_free(uc->bchan->t_ring); 1654 uc->bchan->t_ring = NULL; 1655 k3_configure_chan_coherency(&uc->vc.chan, 0); 1656 err_ring: 1657 bcdma_put_bchan(uc); 1658 1659 return ret; 1660 } 1661 1662 static void udma_free_tx_resources(struct udma_chan *uc) 1663 { 1664 if (!uc->tchan) 1665 return; 1666 1667 k3_ringacc_ring_free(uc->tchan->t_ring); 1668 k3_ringacc_ring_free(uc->tchan->tc_ring); 1669 uc->tchan->t_ring = NULL; 1670 uc->tchan->tc_ring = NULL; 1671 1672 udma_put_tchan(uc); 1673 } 1674 1675 static int udma_alloc_tx_resources(struct udma_chan *uc) 1676 { 1677 struct k3_ring_cfg ring_cfg; 1678 struct udma_dev *ud = uc->ud; 1679 struct udma_tchan *tchan; 1680 int ring_idx, ret; 1681 1682 ret = udma_get_tchan(uc); 1683 if (ret) 1684 return ret; 1685 1686 tchan = uc->tchan; 1687 if (tchan->tflow_id >= 0) 1688 ring_idx = tchan->tflow_id; 1689 else 1690 ring_idx = ud->bchan_cnt + tchan->id; 1691 1692 ret = k3_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1, 1693 &tchan->t_ring, 1694 &tchan->tc_ring); 1695 if (ret) { 1696 ret = -EBUSY; 1697 goto err_ring; 1698 } 1699 1700 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1701 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1702 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1703 if (ud->match_data->type == DMA_TYPE_UDMA) { 1704 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1705 } else { 1706 ring_cfg.mode = K3_RINGACC_RING_MODE_RING; 1707 1708 k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel); 1709 ring_cfg.asel = uc->config.asel; 1710 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan); 1711 } 1712 1713 ret = k3_ringacc_ring_cfg(tchan->t_ring, &ring_cfg); 1714 ret |= k3_ringacc_ring_cfg(tchan->tc_ring, &ring_cfg); 1715 1716 if (ret) 1717 goto err_ringcfg; 1718 1719 return 0; 1720 1721 err_ringcfg: 1722 k3_ringacc_ring_free(uc->tchan->tc_ring); 1723 uc->tchan->tc_ring = NULL; 1724 k3_ringacc_ring_free(uc->tchan->t_ring); 1725 uc->tchan->t_ring = NULL; 1726 err_ring: 1727 udma_put_tchan(uc); 1728 1729 return ret; 1730 } 1731 1732 static void udma_free_rx_resources(struct udma_chan *uc) 1733 { 1734 if (!uc->rchan) 1735 return; 1736 1737 if (uc->rflow) { 1738 struct udma_rflow *rflow = uc->rflow; 1739 1740 k3_ringacc_ring_free(rflow->fd_ring); 1741 k3_ringacc_ring_free(rflow->r_ring); 1742 rflow->fd_ring = NULL; 1743 rflow->r_ring = NULL; 1744 1745 udma_put_rflow(uc); 1746 } 1747 1748 udma_put_rchan(uc); 1749 } 1750 1751 static int udma_alloc_rx_resources(struct udma_chan *uc) 1752 { 1753 struct udma_dev *ud = uc->ud; 1754 struct k3_ring_cfg ring_cfg; 1755 struct udma_rflow *rflow; 1756 int fd_ring_id; 1757 int ret; 1758 1759 ret = udma_get_rchan(uc); 1760 if (ret) 1761 return ret; 1762 1763 /* For MEM_TO_MEM we don't need rflow or rings */ 1764 if (uc->config.dir == DMA_MEM_TO_MEM) 1765 return 0; 1766 1767 if (uc->config.default_flow_id >= 0) 1768 ret = udma_get_rflow(uc, uc->config.default_flow_id); 1769 else 1770 ret = udma_get_rflow(uc, uc->rchan->id); 1771 1772 if (ret) { 1773 ret = -EBUSY; 1774 goto err_rflow; 1775 } 1776 1777 rflow = uc->rflow; 1778 if (ud->tflow_cnt) 1779 fd_ring_id = ud->tflow_cnt + rflow->id; 1780 else 1781 fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt + 1782 uc->rchan->id; 1783 1784 ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1, 1785 &rflow->fd_ring, &rflow->r_ring); 1786 if (ret) { 1787 ret = -EBUSY; 1788 goto err_ring; 1789 } 1790 1791 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1792 1793 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1794 if (ud->match_data->type == DMA_TYPE_UDMA) { 1795 if (uc->config.pkt_mode) 1796 ring_cfg.size = SG_MAX_SEGMENTS; 1797 else 1798 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1799 1800 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1801 } else { 1802 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1803 ring_cfg.mode = K3_RINGACC_RING_MODE_RING; 1804 1805 k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel); 1806 ring_cfg.asel = uc->config.asel; 1807 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan); 1808 } 1809 1810 ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg); 1811 1812 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1813 ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg); 1814 1815 if (ret) 1816 goto err_ringcfg; 1817 1818 return 0; 1819 1820 err_ringcfg: 1821 k3_ringacc_ring_free(rflow->r_ring); 1822 rflow->r_ring = NULL; 1823 k3_ringacc_ring_free(rflow->fd_ring); 1824 rflow->fd_ring = NULL; 1825 err_ring: 1826 udma_put_rflow(uc); 1827 err_rflow: 1828 udma_put_rchan(uc); 1829 1830 return ret; 1831 } 1832 1833 #define TISCI_BCDMA_BCHAN_VALID_PARAMS ( \ 1834 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1835 TI_SCI_MSG_VALUE_RM_UDMAP_CH_EXTENDED_CH_TYPE_VALID) 1836 1837 #define TISCI_BCDMA_TCHAN_VALID_PARAMS ( \ 1838 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1839 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID) 1840 1841 #define TISCI_BCDMA_RCHAN_VALID_PARAMS ( \ 1842 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID) 1843 1844 #define TISCI_UDMA_TCHAN_VALID_PARAMS ( \ 1845 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1846 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \ 1847 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \ 1848 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1849 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \ 1850 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1851 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1852 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1853 1854 #define TISCI_UDMA_RCHAN_VALID_PARAMS ( \ 1855 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1856 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1857 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1858 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1859 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \ 1860 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \ 1861 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \ 1862 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \ 1863 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1864 1865 static int udma_tisci_m2m_channel_config(struct udma_chan *uc) 1866 { 1867 struct udma_dev *ud = uc->ud; 1868 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1869 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1870 struct udma_tchan *tchan = uc->tchan; 1871 struct udma_rchan *rchan = uc->rchan; 1872 u8 burst_size = 0; 1873 int ret; 1874 u8 tpl; 1875 1876 /* Non synchronized - mem to mem type of transfer */ 1877 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1878 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1879 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1880 1881 if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) { 1882 tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, tchan->id); 1883 1884 burst_size = ud->match_data->burst_size[tpl]; 1885 } 1886 1887 req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS; 1888 req_tx.nav_id = tisci_rm->tisci_dev_id; 1889 req_tx.index = tchan->id; 1890 req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1891 req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1892 req_tx.txcq_qnum = tc_ring; 1893 req_tx.tx_atype = ud->atype; 1894 if (burst_size) { 1895 req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; 1896 req_tx.tx_burst_size = burst_size; 1897 } 1898 1899 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1900 if (ret) { 1901 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1902 return ret; 1903 } 1904 1905 req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS; 1906 req_rx.nav_id = tisci_rm->tisci_dev_id; 1907 req_rx.index = rchan->id; 1908 req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1909 req_rx.rxcq_qnum = tc_ring; 1910 req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1911 req_rx.rx_atype = ud->atype; 1912 if (burst_size) { 1913 req_rx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; 1914 req_rx.rx_burst_size = burst_size; 1915 } 1916 1917 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1918 if (ret) 1919 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret); 1920 1921 return ret; 1922 } 1923 1924 static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc) 1925 { 1926 struct udma_dev *ud = uc->ud; 1927 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1928 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1929 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1930 struct udma_bchan *bchan = uc->bchan; 1931 u8 burst_size = 0; 1932 int ret; 1933 u8 tpl; 1934 1935 if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) { 1936 tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, bchan->id); 1937 1938 burst_size = ud->match_data->burst_size[tpl]; 1939 } 1940 1941 req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS; 1942 req_tx.nav_id = tisci_rm->tisci_dev_id; 1943 req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN; 1944 req_tx.index = bchan->id; 1945 if (burst_size) { 1946 req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; 1947 req_tx.tx_burst_size = burst_size; 1948 } 1949 1950 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1951 if (ret) 1952 dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret); 1953 1954 return ret; 1955 } 1956 1957 static int udma_tisci_tx_channel_config(struct udma_chan *uc) 1958 { 1959 struct udma_dev *ud = uc->ud; 1960 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1961 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1962 struct udma_tchan *tchan = uc->tchan; 1963 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1964 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1965 u32 mode, fetch_size; 1966 int ret; 1967 1968 if (uc->config.pkt_mode) { 1969 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1970 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1971 uc->config.psd_size, 0); 1972 } else { 1973 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1974 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1975 } 1976 1977 req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS; 1978 req_tx.nav_id = tisci_rm->tisci_dev_id; 1979 req_tx.index = tchan->id; 1980 req_tx.tx_chan_type = mode; 1981 req_tx.tx_supr_tdpkt = uc->config.notdpkt; 1982 req_tx.tx_fetch_size = fetch_size >> 2; 1983 req_tx.txcq_qnum = tc_ring; 1984 req_tx.tx_atype = uc->config.atype; 1985 if (uc->config.ep_type == PSIL_EP_PDMA_XY && 1986 ud->match_data->flags & UDMA_FLAG_TDTYPE) { 1987 /* wait for peer to complete the teardown for PDMAs */ 1988 req_tx.valid_params |= 1989 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID; 1990 req_tx.tx_tdtype = 1; 1991 } 1992 1993 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1994 if (ret) 1995 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1996 1997 return ret; 1998 } 1999 2000 static int bcdma_tisci_tx_channel_config(struct udma_chan *uc) 2001 { 2002 struct udma_dev *ud = uc->ud; 2003 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 2004 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 2005 struct udma_tchan *tchan = uc->tchan; 2006 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 2007 int ret; 2008 2009 req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS; 2010 req_tx.nav_id = tisci_rm->tisci_dev_id; 2011 req_tx.index = tchan->id; 2012 req_tx.tx_supr_tdpkt = uc->config.notdpkt; 2013 if (ud->match_data->flags & UDMA_FLAG_TDTYPE) { 2014 /* wait for peer to complete the teardown for PDMAs */ 2015 req_tx.valid_params |= 2016 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID; 2017 req_tx.tx_tdtype = 1; 2018 } 2019 2020 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 2021 if (ret) 2022 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 2023 2024 return ret; 2025 } 2026 2027 #define pktdma_tisci_tx_channel_config bcdma_tisci_tx_channel_config 2028 2029 static int udma_tisci_rx_channel_config(struct udma_chan *uc) 2030 { 2031 struct udma_dev *ud = uc->ud; 2032 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 2033 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 2034 struct udma_rchan *rchan = uc->rchan; 2035 int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring); 2036 int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2037 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 2038 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; 2039 u32 mode, fetch_size; 2040 int ret; 2041 2042 if (uc->config.pkt_mode) { 2043 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 2044 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 2045 uc->config.psd_size, 0); 2046 } else { 2047 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 2048 fetch_size = sizeof(struct cppi5_desc_hdr_t); 2049 } 2050 2051 req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS; 2052 req_rx.nav_id = tisci_rm->tisci_dev_id; 2053 req_rx.index = rchan->id; 2054 req_rx.rx_fetch_size = fetch_size >> 2; 2055 req_rx.rxcq_qnum = rx_ring; 2056 req_rx.rx_chan_type = mode; 2057 req_rx.rx_atype = uc->config.atype; 2058 2059 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 2060 if (ret) { 2061 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); 2062 return ret; 2063 } 2064 2065 flow_req.valid_params = 2066 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 2067 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 2068 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 2069 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 2070 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 2071 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 2072 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 2073 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 2074 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 2075 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 2076 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 2077 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 2078 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 2079 2080 flow_req.nav_id = tisci_rm->tisci_dev_id; 2081 flow_req.flow_index = rchan->id; 2082 2083 if (uc->config.needs_epib) 2084 flow_req.rx_einfo_present = 1; 2085 else 2086 flow_req.rx_einfo_present = 0; 2087 if (uc->config.psd_size) 2088 flow_req.rx_psinfo_present = 1; 2089 else 2090 flow_req.rx_psinfo_present = 0; 2091 flow_req.rx_error_handling = 1; 2092 flow_req.rx_dest_qnum = rx_ring; 2093 flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE; 2094 flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG; 2095 flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI; 2096 flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO; 2097 flow_req.rx_fdq0_sz0_qnum = fd_ring; 2098 flow_req.rx_fdq1_qnum = fd_ring; 2099 flow_req.rx_fdq2_qnum = fd_ring; 2100 flow_req.rx_fdq3_qnum = fd_ring; 2101 2102 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req); 2103 2104 if (ret) 2105 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret); 2106 2107 return 0; 2108 } 2109 2110 static int bcdma_tisci_rx_channel_config(struct udma_chan *uc) 2111 { 2112 struct udma_dev *ud = uc->ud; 2113 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 2114 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 2115 struct udma_rchan *rchan = uc->rchan; 2116 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 2117 int ret; 2118 2119 req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; 2120 req_rx.nav_id = tisci_rm->tisci_dev_id; 2121 req_rx.index = rchan->id; 2122 2123 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 2124 if (ret) 2125 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); 2126 2127 return ret; 2128 } 2129 2130 static int pktdma_tisci_rx_channel_config(struct udma_chan *uc) 2131 { 2132 struct udma_dev *ud = uc->ud; 2133 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 2134 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 2135 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 2136 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; 2137 int ret; 2138 2139 req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; 2140 req_rx.nav_id = tisci_rm->tisci_dev_id; 2141 req_rx.index = uc->rchan->id; 2142 2143 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 2144 if (ret) { 2145 dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret); 2146 return ret; 2147 } 2148 2149 flow_req.valid_params = 2150 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 2151 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 2152 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID; 2153 2154 flow_req.nav_id = tisci_rm->tisci_dev_id; 2155 flow_req.flow_index = uc->rflow->id; 2156 2157 if (uc->config.needs_epib) 2158 flow_req.rx_einfo_present = 1; 2159 else 2160 flow_req.rx_einfo_present = 0; 2161 if (uc->config.psd_size) 2162 flow_req.rx_psinfo_present = 1; 2163 else 2164 flow_req.rx_psinfo_present = 0; 2165 flow_req.rx_error_handling = 1; 2166 2167 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req); 2168 2169 if (ret) 2170 dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id, 2171 ret); 2172 2173 return ret; 2174 } 2175 2176 static int udma_alloc_chan_resources(struct dma_chan *chan) 2177 { 2178 struct udma_chan *uc = to_udma_chan(chan); 2179 struct udma_dev *ud = to_udma_dev(chan->device); 2180 const struct udma_soc_data *soc_data = ud->soc_data; 2181 struct k3_ring *irq_ring; 2182 u32 irq_udma_idx; 2183 int ret; 2184 2185 uc->dma_dev = ud->dev; 2186 2187 if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) { 2188 uc->use_dma_pool = true; 2189 /* in case of MEM_TO_MEM we have maximum of two TRs */ 2190 if (uc->config.dir == DMA_MEM_TO_MEM) { 2191 uc->config.hdesc_size = cppi5_trdesc_calc_size( 2192 sizeof(struct cppi5_tr_type15_t), 2); 2193 uc->config.pkt_mode = false; 2194 } 2195 } 2196 2197 if (uc->use_dma_pool) { 2198 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, 2199 uc->config.hdesc_size, 2200 ud->desc_align, 2201 0); 2202 if (!uc->hdesc_pool) { 2203 dev_err(ud->ddev.dev, 2204 "Descriptor pool allocation failed\n"); 2205 uc->use_dma_pool = false; 2206 ret = -ENOMEM; 2207 goto err_cleanup; 2208 } 2209 } 2210 2211 /* 2212 * Make sure that the completion is in a known state: 2213 * No teardown, the channel is idle 2214 */ 2215 reinit_completion(&uc->teardown_completed); 2216 complete_all(&uc->teardown_completed); 2217 uc->state = UDMA_CHAN_IS_IDLE; 2218 2219 switch (uc->config.dir) { 2220 case DMA_MEM_TO_MEM: 2221 /* Non synchronized - mem to mem type of transfer */ 2222 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, 2223 uc->id); 2224 2225 ret = udma_get_chan_pair(uc); 2226 if (ret) 2227 goto err_cleanup; 2228 2229 ret = udma_alloc_tx_resources(uc); 2230 if (ret) { 2231 udma_put_rchan(uc); 2232 goto err_cleanup; 2233 } 2234 2235 ret = udma_alloc_rx_resources(uc); 2236 if (ret) { 2237 udma_free_tx_resources(uc); 2238 goto err_cleanup; 2239 } 2240 2241 uc->config.src_thread = ud->psil_base + uc->tchan->id; 2242 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 2243 K3_PSIL_DST_THREAD_ID_OFFSET; 2244 2245 irq_ring = uc->tchan->tc_ring; 2246 irq_udma_idx = uc->tchan->id; 2247 2248 ret = udma_tisci_m2m_channel_config(uc); 2249 break; 2250 case DMA_MEM_TO_DEV: 2251 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 2252 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 2253 uc->id); 2254 2255 ret = udma_alloc_tx_resources(uc); 2256 if (ret) 2257 goto err_cleanup; 2258 2259 uc->config.src_thread = ud->psil_base + uc->tchan->id; 2260 uc->config.dst_thread = uc->config.remote_thread_id; 2261 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 2262 2263 irq_ring = uc->tchan->tc_ring; 2264 irq_udma_idx = uc->tchan->id; 2265 2266 ret = udma_tisci_tx_channel_config(uc); 2267 break; 2268 case DMA_DEV_TO_MEM: 2269 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 2270 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 2271 uc->id); 2272 2273 ret = udma_alloc_rx_resources(uc); 2274 if (ret) 2275 goto err_cleanup; 2276 2277 uc->config.src_thread = uc->config.remote_thread_id; 2278 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 2279 K3_PSIL_DST_THREAD_ID_OFFSET; 2280 2281 irq_ring = uc->rflow->r_ring; 2282 irq_udma_idx = soc_data->oes.udma_rchan + uc->rchan->id; 2283 2284 ret = udma_tisci_rx_channel_config(uc); 2285 break; 2286 default: 2287 /* Can not happen */ 2288 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 2289 __func__, uc->id, uc->config.dir); 2290 ret = -EINVAL; 2291 goto err_cleanup; 2292 2293 } 2294 2295 /* check if the channel configuration was successful */ 2296 if (ret) 2297 goto err_res_free; 2298 2299 if (udma_is_chan_running(uc)) { 2300 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 2301 udma_reset_chan(uc, false); 2302 if (udma_is_chan_running(uc)) { 2303 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 2304 ret = -EBUSY; 2305 goto err_res_free; 2306 } 2307 } 2308 2309 /* PSI-L pairing */ 2310 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); 2311 if (ret) { 2312 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 2313 uc->config.src_thread, uc->config.dst_thread); 2314 goto err_res_free; 2315 } 2316 2317 uc->psil_paired = true; 2318 2319 uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring); 2320 if (uc->irq_num_ring <= 0) { 2321 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 2322 k3_ringacc_get_ring_id(irq_ring)); 2323 ret = -EINVAL; 2324 goto err_psi_free; 2325 } 2326 2327 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 2328 IRQF_TRIGGER_HIGH, uc->name, uc); 2329 if (ret) { 2330 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 2331 goto err_irq_free; 2332 } 2333 2334 /* Event from UDMA (TR events) only needed for slave TR mode channels */ 2335 if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) { 2336 uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx); 2337 if (uc->irq_num_udma <= 0) { 2338 dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", 2339 irq_udma_idx); 2340 free_irq(uc->irq_num_ring, uc); 2341 ret = -EINVAL; 2342 goto err_irq_free; 2343 } 2344 2345 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0, 2346 uc->name, uc); 2347 if (ret) { 2348 dev_err(ud->dev, "chan%d: UDMA irq request failed\n", 2349 uc->id); 2350 free_irq(uc->irq_num_ring, uc); 2351 goto err_irq_free; 2352 } 2353 } else { 2354 uc->irq_num_udma = 0; 2355 } 2356 2357 udma_reset_rings(uc); 2358 2359 return 0; 2360 2361 err_irq_free: 2362 uc->irq_num_ring = 0; 2363 uc->irq_num_udma = 0; 2364 err_psi_free: 2365 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); 2366 uc->psil_paired = false; 2367 err_res_free: 2368 udma_free_tx_resources(uc); 2369 udma_free_rx_resources(uc); 2370 err_cleanup: 2371 udma_reset_uchan(uc); 2372 2373 if (uc->use_dma_pool) { 2374 dma_pool_destroy(uc->hdesc_pool); 2375 uc->use_dma_pool = false; 2376 } 2377 2378 return ret; 2379 } 2380 2381 static int bcdma_alloc_chan_resources(struct dma_chan *chan) 2382 { 2383 struct udma_chan *uc = to_udma_chan(chan); 2384 struct udma_dev *ud = to_udma_dev(chan->device); 2385 const struct udma_oes_offsets *oes = &ud->soc_data->oes; 2386 u32 irq_udma_idx, irq_ring_idx; 2387 int ret; 2388 2389 /* Only TR mode is supported */ 2390 uc->config.pkt_mode = false; 2391 2392 /* 2393 * Make sure that the completion is in a known state: 2394 * No teardown, the channel is idle 2395 */ 2396 reinit_completion(&uc->teardown_completed); 2397 complete_all(&uc->teardown_completed); 2398 uc->state = UDMA_CHAN_IS_IDLE; 2399 2400 switch (uc->config.dir) { 2401 case DMA_MEM_TO_MEM: 2402 /* Non synchronized - mem to mem type of transfer */ 2403 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, 2404 uc->id); 2405 2406 ret = bcdma_alloc_bchan_resources(uc); 2407 if (ret) 2408 return ret; 2409 2410 irq_ring_idx = uc->bchan->id + oes->bcdma_bchan_ring; 2411 irq_udma_idx = uc->bchan->id + oes->bcdma_bchan_data; 2412 2413 ret = bcdma_tisci_m2m_channel_config(uc); 2414 break; 2415 case DMA_MEM_TO_DEV: 2416 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 2417 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 2418 uc->id); 2419 2420 ret = udma_alloc_tx_resources(uc); 2421 if (ret) { 2422 uc->config.remote_thread_id = -1; 2423 return ret; 2424 } 2425 2426 uc->config.src_thread = ud->psil_base + uc->tchan->id; 2427 uc->config.dst_thread = uc->config.remote_thread_id; 2428 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 2429 2430 irq_ring_idx = uc->tchan->id + oes->bcdma_tchan_ring; 2431 irq_udma_idx = uc->tchan->id + oes->bcdma_tchan_data; 2432 2433 ret = bcdma_tisci_tx_channel_config(uc); 2434 break; 2435 case DMA_DEV_TO_MEM: 2436 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 2437 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 2438 uc->id); 2439 2440 ret = udma_alloc_rx_resources(uc); 2441 if (ret) { 2442 uc->config.remote_thread_id = -1; 2443 return ret; 2444 } 2445 2446 uc->config.src_thread = uc->config.remote_thread_id; 2447 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 2448 K3_PSIL_DST_THREAD_ID_OFFSET; 2449 2450 irq_ring_idx = uc->rchan->id + oes->bcdma_rchan_ring; 2451 irq_udma_idx = uc->rchan->id + oes->bcdma_rchan_data; 2452 2453 ret = bcdma_tisci_rx_channel_config(uc); 2454 break; 2455 default: 2456 /* Can not happen */ 2457 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 2458 __func__, uc->id, uc->config.dir); 2459 return -EINVAL; 2460 } 2461 2462 /* check if the channel configuration was successful */ 2463 if (ret) 2464 goto err_res_free; 2465 2466 if (udma_is_chan_running(uc)) { 2467 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 2468 udma_reset_chan(uc, false); 2469 if (udma_is_chan_running(uc)) { 2470 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 2471 ret = -EBUSY; 2472 goto err_res_free; 2473 } 2474 } 2475 2476 uc->dma_dev = dmaengine_get_dma_device(chan); 2477 if (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type) { 2478 uc->config.hdesc_size = cppi5_trdesc_calc_size( 2479 sizeof(struct cppi5_tr_type15_t), 2); 2480 2481 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, 2482 uc->config.hdesc_size, 2483 ud->desc_align, 2484 0); 2485 if (!uc->hdesc_pool) { 2486 dev_err(ud->ddev.dev, 2487 "Descriptor pool allocation failed\n"); 2488 uc->use_dma_pool = false; 2489 ret = -ENOMEM; 2490 goto err_res_free; 2491 } 2492 2493 uc->use_dma_pool = true; 2494 } else if (uc->config.dir != DMA_MEM_TO_MEM) { 2495 /* PSI-L pairing */ 2496 ret = navss_psil_pair(ud, uc->config.src_thread, 2497 uc->config.dst_thread); 2498 if (ret) { 2499 dev_err(ud->dev, 2500 "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 2501 uc->config.src_thread, uc->config.dst_thread); 2502 goto err_res_free; 2503 } 2504 2505 uc->psil_paired = true; 2506 } 2507 2508 uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx); 2509 if (uc->irq_num_ring <= 0) { 2510 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 2511 irq_ring_idx); 2512 ret = -EINVAL; 2513 goto err_psi_free; 2514 } 2515 2516 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 2517 IRQF_TRIGGER_HIGH, uc->name, uc); 2518 if (ret) { 2519 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 2520 goto err_irq_free; 2521 } 2522 2523 /* Event from BCDMA (TR events) only needed for slave channels */ 2524 if (is_slave_direction(uc->config.dir)) { 2525 uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx); 2526 if (uc->irq_num_udma <= 0) { 2527 dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n", 2528 irq_udma_idx); 2529 free_irq(uc->irq_num_ring, uc); 2530 ret = -EINVAL; 2531 goto err_irq_free; 2532 } 2533 2534 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0, 2535 uc->name, uc); 2536 if (ret) { 2537 dev_err(ud->dev, "chan%d: BCDMA irq request failed\n", 2538 uc->id); 2539 free_irq(uc->irq_num_ring, uc); 2540 goto err_irq_free; 2541 } 2542 } else { 2543 uc->irq_num_udma = 0; 2544 } 2545 2546 udma_reset_rings(uc); 2547 2548 INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work, 2549 udma_check_tx_completion); 2550 return 0; 2551 2552 err_irq_free: 2553 uc->irq_num_ring = 0; 2554 uc->irq_num_udma = 0; 2555 err_psi_free: 2556 if (uc->psil_paired) 2557 navss_psil_unpair(ud, uc->config.src_thread, 2558 uc->config.dst_thread); 2559 uc->psil_paired = false; 2560 err_res_free: 2561 bcdma_free_bchan_resources(uc); 2562 udma_free_tx_resources(uc); 2563 udma_free_rx_resources(uc); 2564 2565 udma_reset_uchan(uc); 2566 2567 if (uc->use_dma_pool) { 2568 dma_pool_destroy(uc->hdesc_pool); 2569 uc->use_dma_pool = false; 2570 } 2571 2572 return ret; 2573 } 2574 2575 static int bcdma_router_config(struct dma_chan *chan) 2576 { 2577 struct k3_event_route_data *router_data = chan->route_data; 2578 struct udma_chan *uc = to_udma_chan(chan); 2579 u32 trigger_event; 2580 2581 if (!uc->bchan) 2582 return -EINVAL; 2583 2584 if (uc->config.tr_trigger_type != 1 && uc->config.tr_trigger_type != 2) 2585 return -EINVAL; 2586 2587 trigger_event = uc->ud->soc_data->bcdma_trigger_event_offset; 2588 trigger_event += (uc->bchan->id * 2) + uc->config.tr_trigger_type - 1; 2589 2590 return router_data->set_event(router_data->priv, trigger_event); 2591 } 2592 2593 static int pktdma_alloc_chan_resources(struct dma_chan *chan) 2594 { 2595 struct udma_chan *uc = to_udma_chan(chan); 2596 struct udma_dev *ud = to_udma_dev(chan->device); 2597 const struct udma_oes_offsets *oes = &ud->soc_data->oes; 2598 u32 irq_ring_idx; 2599 int ret; 2600 2601 /* 2602 * Make sure that the completion is in a known state: 2603 * No teardown, the channel is idle 2604 */ 2605 reinit_completion(&uc->teardown_completed); 2606 complete_all(&uc->teardown_completed); 2607 uc->state = UDMA_CHAN_IS_IDLE; 2608 2609 switch (uc->config.dir) { 2610 case DMA_MEM_TO_DEV: 2611 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 2612 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 2613 uc->id); 2614 2615 ret = udma_alloc_tx_resources(uc); 2616 if (ret) { 2617 uc->config.remote_thread_id = -1; 2618 return ret; 2619 } 2620 2621 uc->config.src_thread = ud->psil_base + uc->tchan->id; 2622 uc->config.dst_thread = uc->config.remote_thread_id; 2623 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 2624 2625 irq_ring_idx = uc->tchan->tflow_id + oes->pktdma_tchan_flow; 2626 2627 ret = pktdma_tisci_tx_channel_config(uc); 2628 break; 2629 case DMA_DEV_TO_MEM: 2630 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 2631 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 2632 uc->id); 2633 2634 ret = udma_alloc_rx_resources(uc); 2635 if (ret) { 2636 uc->config.remote_thread_id = -1; 2637 return ret; 2638 } 2639 2640 uc->config.src_thread = uc->config.remote_thread_id; 2641 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 2642 K3_PSIL_DST_THREAD_ID_OFFSET; 2643 2644 irq_ring_idx = uc->rflow->id + oes->pktdma_rchan_flow; 2645 2646 ret = pktdma_tisci_rx_channel_config(uc); 2647 break; 2648 default: 2649 /* Can not happen */ 2650 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 2651 __func__, uc->id, uc->config.dir); 2652 return -EINVAL; 2653 } 2654 2655 /* check if the channel configuration was successful */ 2656 if (ret) 2657 goto err_res_free; 2658 2659 if (udma_is_chan_running(uc)) { 2660 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 2661 udma_reset_chan(uc, false); 2662 if (udma_is_chan_running(uc)) { 2663 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 2664 ret = -EBUSY; 2665 goto err_res_free; 2666 } 2667 } 2668 2669 uc->dma_dev = dmaengine_get_dma_device(chan); 2670 uc->hdesc_pool = dma_pool_create(uc->name, uc->dma_dev, 2671 uc->config.hdesc_size, ud->desc_align, 2672 0); 2673 if (!uc->hdesc_pool) { 2674 dev_err(ud->ddev.dev, 2675 "Descriptor pool allocation failed\n"); 2676 uc->use_dma_pool = false; 2677 ret = -ENOMEM; 2678 goto err_res_free; 2679 } 2680 2681 uc->use_dma_pool = true; 2682 2683 /* PSI-L pairing */ 2684 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); 2685 if (ret) { 2686 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 2687 uc->config.src_thread, uc->config.dst_thread); 2688 goto err_res_free; 2689 } 2690 2691 uc->psil_paired = true; 2692 2693 uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx); 2694 if (uc->irq_num_ring <= 0) { 2695 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 2696 irq_ring_idx); 2697 ret = -EINVAL; 2698 goto err_psi_free; 2699 } 2700 2701 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 2702 IRQF_TRIGGER_HIGH, uc->name, uc); 2703 if (ret) { 2704 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 2705 goto err_irq_free; 2706 } 2707 2708 uc->irq_num_udma = 0; 2709 2710 udma_reset_rings(uc); 2711 2712 INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work, 2713 udma_check_tx_completion); 2714 2715 if (uc->tchan) 2716 dev_dbg(ud->dev, 2717 "chan%d: tchan%d, tflow%d, Remote thread: 0x%04x\n", 2718 uc->id, uc->tchan->id, uc->tchan->tflow_id, 2719 uc->config.remote_thread_id); 2720 else if (uc->rchan) 2721 dev_dbg(ud->dev, 2722 "chan%d: rchan%d, rflow%d, Remote thread: 0x%04x\n", 2723 uc->id, uc->rchan->id, uc->rflow->id, 2724 uc->config.remote_thread_id); 2725 return 0; 2726 2727 err_irq_free: 2728 uc->irq_num_ring = 0; 2729 err_psi_free: 2730 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); 2731 uc->psil_paired = false; 2732 err_res_free: 2733 udma_free_tx_resources(uc); 2734 udma_free_rx_resources(uc); 2735 2736 udma_reset_uchan(uc); 2737 2738 dma_pool_destroy(uc->hdesc_pool); 2739 uc->use_dma_pool = false; 2740 2741 return ret; 2742 } 2743 2744 static int udma_slave_config(struct dma_chan *chan, 2745 struct dma_slave_config *cfg) 2746 { 2747 struct udma_chan *uc = to_udma_chan(chan); 2748 2749 memcpy(&uc->cfg, cfg, sizeof(uc->cfg)); 2750 2751 return 0; 2752 } 2753 2754 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc, 2755 size_t tr_size, int tr_count, 2756 enum dma_transfer_direction dir) 2757 { 2758 struct udma_hwdesc *hwdesc; 2759 struct cppi5_desc_hdr_t *tr_desc; 2760 struct udma_desc *d; 2761 u32 reload_count = 0; 2762 u32 ring_id; 2763 2764 switch (tr_size) { 2765 case 16: 2766 case 32: 2767 case 64: 2768 case 128: 2769 break; 2770 default: 2771 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size); 2772 return NULL; 2773 } 2774 2775 /* We have only one descriptor containing multiple TRs */ 2776 d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT); 2777 if (!d) 2778 return NULL; 2779 2780 d->sglen = tr_count; 2781 2782 d->hwdesc_count = 1; 2783 hwdesc = &d->hwdesc[0]; 2784 2785 /* Allocate memory for DMA ring descriptor */ 2786 if (uc->use_dma_pool) { 2787 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2788 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2789 GFP_NOWAIT, 2790 &hwdesc->cppi5_desc_paddr); 2791 } else { 2792 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 2793 tr_count); 2794 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 2795 uc->ud->desc_align); 2796 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev, 2797 hwdesc->cppi5_desc_size, 2798 &hwdesc->cppi5_desc_paddr, 2799 GFP_NOWAIT); 2800 } 2801 2802 if (!hwdesc->cppi5_desc_vaddr) { 2803 kfree(d); 2804 return NULL; 2805 } 2806 2807 /* Start of the TR req records */ 2808 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 2809 /* Start address of the TR response array */ 2810 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count; 2811 2812 tr_desc = hwdesc->cppi5_desc_vaddr; 2813 2814 if (uc->cyclic) 2815 reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE; 2816 2817 if (dir == DMA_DEV_TO_MEM) 2818 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2819 else 2820 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2821 2822 cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count); 2823 cppi5_desc_set_pktids(tr_desc, uc->id, 2824 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2825 cppi5_desc_set_retpolicy(tr_desc, 0, ring_id); 2826 2827 return d; 2828 } 2829 2830 /** 2831 * udma_get_tr_counters - calculate TR counters for a given length 2832 * @len: Length of the trasnfer 2833 * @align_to: Preferred alignment 2834 * @tr0_cnt0: First TR icnt0 2835 * @tr0_cnt1: First TR icnt1 2836 * @tr1_cnt0: Second (if used) TR icnt0 2837 * 2838 * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated 2839 * For len >= SZ_64K two TRs are used in a simple way: 2840 * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1) 2841 * Second TR: the remaining length (tr1_cnt0) 2842 * 2843 * Returns the number of TRs the length needs (1 or 2) 2844 * -EINVAL if the length can not be supported 2845 */ 2846 static int udma_get_tr_counters(size_t len, unsigned long align_to, 2847 u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0) 2848 { 2849 if (len < SZ_64K) { 2850 *tr0_cnt0 = len; 2851 *tr0_cnt1 = 1; 2852 2853 return 1; 2854 } 2855 2856 if (align_to > 3) 2857 align_to = 3; 2858 2859 realign: 2860 *tr0_cnt0 = SZ_64K - BIT(align_to); 2861 if (len / *tr0_cnt0 >= SZ_64K) { 2862 if (align_to) { 2863 align_to--; 2864 goto realign; 2865 } 2866 return -EINVAL; 2867 } 2868 2869 *tr0_cnt1 = len / *tr0_cnt0; 2870 *tr1_cnt0 = len % *tr0_cnt0; 2871 2872 return 2; 2873 } 2874 2875 static struct udma_desc * 2876 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, 2877 unsigned int sglen, enum dma_transfer_direction dir, 2878 unsigned long tx_flags, void *context) 2879 { 2880 struct scatterlist *sgent; 2881 struct udma_desc *d; 2882 struct cppi5_tr_type1_t *tr_req = NULL; 2883 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2884 unsigned int i; 2885 size_t tr_size; 2886 int num_tr = 0; 2887 int tr_idx = 0; 2888 u64 asel; 2889 2890 /* estimate the number of TRs we will need */ 2891 for_each_sg(sgl, sgent, sglen, i) { 2892 if (sg_dma_len(sgent) < SZ_64K) 2893 num_tr++; 2894 else 2895 num_tr += 2; 2896 } 2897 2898 /* Now allocate and setup the descriptor. */ 2899 tr_size = sizeof(struct cppi5_tr_type1_t); 2900 d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir); 2901 if (!d) 2902 return NULL; 2903 2904 d->sglen = sglen; 2905 2906 if (uc->ud->match_data->type == DMA_TYPE_UDMA) 2907 asel = 0; 2908 else 2909 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; 2910 2911 tr_req = d->hwdesc[0].tr_req_base; 2912 for_each_sg(sgl, sgent, sglen, i) { 2913 dma_addr_t sg_addr = sg_dma_address(sgent); 2914 2915 num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr), 2916 &tr0_cnt0, &tr0_cnt1, &tr1_cnt0); 2917 if (num_tr < 0) { 2918 dev_err(uc->ud->dev, "size %u is not supported\n", 2919 sg_dma_len(sgent)); 2920 udma_free_hwdesc(uc, d); 2921 kfree(d); 2922 return NULL; 2923 } 2924 2925 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false, 2926 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2927 cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT); 2928 2929 sg_addr |= asel; 2930 tr_req[tr_idx].addr = sg_addr; 2931 tr_req[tr_idx].icnt0 = tr0_cnt0; 2932 tr_req[tr_idx].icnt1 = tr0_cnt1; 2933 tr_req[tr_idx].dim1 = tr0_cnt0; 2934 tr_idx++; 2935 2936 if (num_tr == 2) { 2937 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 2938 false, false, 2939 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2940 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 2941 CPPI5_TR_CSF_SUPR_EVT); 2942 2943 tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0; 2944 tr_req[tr_idx].icnt0 = tr1_cnt0; 2945 tr_req[tr_idx].icnt1 = 1; 2946 tr_req[tr_idx].dim1 = tr1_cnt0; 2947 tr_idx++; 2948 } 2949 2950 d->residue += sg_dma_len(sgent); 2951 } 2952 2953 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, 2954 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); 2955 2956 return d; 2957 } 2958 2959 static struct udma_desc * 2960 udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, 2961 unsigned int sglen, 2962 enum dma_transfer_direction dir, 2963 unsigned long tx_flags, void *context) 2964 { 2965 struct scatterlist *sgent; 2966 struct cppi5_tr_type15_t *tr_req = NULL; 2967 enum dma_slave_buswidth dev_width; 2968 u32 csf = CPPI5_TR_CSF_SUPR_EVT; 2969 u16 tr_cnt0, tr_cnt1; 2970 dma_addr_t dev_addr; 2971 struct udma_desc *d; 2972 unsigned int i; 2973 size_t tr_size, sg_len; 2974 int num_tr = 0; 2975 int tr_idx = 0; 2976 u32 burst, trigger_size, port_window; 2977 u64 asel; 2978 2979 if (dir == DMA_DEV_TO_MEM) { 2980 dev_addr = uc->cfg.src_addr; 2981 dev_width = uc->cfg.src_addr_width; 2982 burst = uc->cfg.src_maxburst; 2983 port_window = uc->cfg.src_port_window_size; 2984 } else if (dir == DMA_MEM_TO_DEV) { 2985 dev_addr = uc->cfg.dst_addr; 2986 dev_width = uc->cfg.dst_addr_width; 2987 burst = uc->cfg.dst_maxburst; 2988 port_window = uc->cfg.dst_port_window_size; 2989 } else { 2990 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 2991 return NULL; 2992 } 2993 2994 if (!burst) 2995 burst = 1; 2996 2997 if (port_window) { 2998 if (port_window != burst) { 2999 dev_err(uc->ud->dev, 3000 "The burst must be equal to port_window\n"); 3001 return NULL; 3002 } 3003 3004 tr_cnt0 = dev_width * port_window; 3005 tr_cnt1 = 1; 3006 } else { 3007 tr_cnt0 = dev_width; 3008 tr_cnt1 = burst; 3009 } 3010 trigger_size = tr_cnt0 * tr_cnt1; 3011 3012 /* estimate the number of TRs we will need */ 3013 for_each_sg(sgl, sgent, sglen, i) { 3014 sg_len = sg_dma_len(sgent); 3015 3016 if (sg_len % trigger_size) { 3017 dev_err(uc->ud->dev, 3018 "Not aligned SG entry (%zu for %u)\n", sg_len, 3019 trigger_size); 3020 return NULL; 3021 } 3022 3023 if (sg_len / trigger_size < SZ_64K) 3024 num_tr++; 3025 else 3026 num_tr += 2; 3027 } 3028 3029 /* Now allocate and setup the descriptor. */ 3030 tr_size = sizeof(struct cppi5_tr_type15_t); 3031 d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir); 3032 if (!d) 3033 return NULL; 3034 3035 d->sglen = sglen; 3036 3037 if (uc->ud->match_data->type == DMA_TYPE_UDMA) { 3038 asel = 0; 3039 csf |= CPPI5_TR_CSF_EOL_ICNT0; 3040 } else { 3041 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; 3042 dev_addr |= asel; 3043 } 3044 3045 tr_req = d->hwdesc[0].tr_req_base; 3046 for_each_sg(sgl, sgent, sglen, i) { 3047 u16 tr0_cnt2, tr0_cnt3, tr1_cnt2; 3048 dma_addr_t sg_addr = sg_dma_address(sgent); 3049 3050 sg_len = sg_dma_len(sgent); 3051 num_tr = udma_get_tr_counters(sg_len / trigger_size, 0, 3052 &tr0_cnt2, &tr0_cnt3, &tr1_cnt2); 3053 if (num_tr < 0) { 3054 dev_err(uc->ud->dev, "size %zu is not supported\n", 3055 sg_len); 3056 udma_free_hwdesc(uc, d); 3057 kfree(d); 3058 return NULL; 3059 } 3060 3061 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false, 3062 true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3063 cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf); 3064 cppi5_tr_set_trigger(&tr_req[tr_idx].flags, 3065 uc->config.tr_trigger_type, 3066 CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0); 3067 3068 sg_addr |= asel; 3069 if (dir == DMA_DEV_TO_MEM) { 3070 tr_req[tr_idx].addr = dev_addr; 3071 tr_req[tr_idx].icnt0 = tr_cnt0; 3072 tr_req[tr_idx].icnt1 = tr_cnt1; 3073 tr_req[tr_idx].icnt2 = tr0_cnt2; 3074 tr_req[tr_idx].icnt3 = tr0_cnt3; 3075 tr_req[tr_idx].dim1 = (-1) * tr_cnt0; 3076 3077 tr_req[tr_idx].daddr = sg_addr; 3078 tr_req[tr_idx].dicnt0 = tr_cnt0; 3079 tr_req[tr_idx].dicnt1 = tr_cnt1; 3080 tr_req[tr_idx].dicnt2 = tr0_cnt2; 3081 tr_req[tr_idx].dicnt3 = tr0_cnt3; 3082 tr_req[tr_idx].ddim1 = tr_cnt0; 3083 tr_req[tr_idx].ddim2 = trigger_size; 3084 tr_req[tr_idx].ddim3 = trigger_size * tr0_cnt2; 3085 } else { 3086 tr_req[tr_idx].addr = sg_addr; 3087 tr_req[tr_idx].icnt0 = tr_cnt0; 3088 tr_req[tr_idx].icnt1 = tr_cnt1; 3089 tr_req[tr_idx].icnt2 = tr0_cnt2; 3090 tr_req[tr_idx].icnt3 = tr0_cnt3; 3091 tr_req[tr_idx].dim1 = tr_cnt0; 3092 tr_req[tr_idx].dim2 = trigger_size; 3093 tr_req[tr_idx].dim3 = trigger_size * tr0_cnt2; 3094 3095 tr_req[tr_idx].daddr = dev_addr; 3096 tr_req[tr_idx].dicnt0 = tr_cnt0; 3097 tr_req[tr_idx].dicnt1 = tr_cnt1; 3098 tr_req[tr_idx].dicnt2 = tr0_cnt2; 3099 tr_req[tr_idx].dicnt3 = tr0_cnt3; 3100 tr_req[tr_idx].ddim1 = (-1) * tr_cnt0; 3101 } 3102 3103 tr_idx++; 3104 3105 if (num_tr == 2) { 3106 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, 3107 false, true, 3108 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3109 cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf); 3110 cppi5_tr_set_trigger(&tr_req[tr_idx].flags, 3111 uc->config.tr_trigger_type, 3112 CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 3113 0, 0); 3114 3115 sg_addr += trigger_size * tr0_cnt2 * tr0_cnt3; 3116 if (dir == DMA_DEV_TO_MEM) { 3117 tr_req[tr_idx].addr = dev_addr; 3118 tr_req[tr_idx].icnt0 = tr_cnt0; 3119 tr_req[tr_idx].icnt1 = tr_cnt1; 3120 tr_req[tr_idx].icnt2 = tr1_cnt2; 3121 tr_req[tr_idx].icnt3 = 1; 3122 tr_req[tr_idx].dim1 = (-1) * tr_cnt0; 3123 3124 tr_req[tr_idx].daddr = sg_addr; 3125 tr_req[tr_idx].dicnt0 = tr_cnt0; 3126 tr_req[tr_idx].dicnt1 = tr_cnt1; 3127 tr_req[tr_idx].dicnt2 = tr1_cnt2; 3128 tr_req[tr_idx].dicnt3 = 1; 3129 tr_req[tr_idx].ddim1 = tr_cnt0; 3130 tr_req[tr_idx].ddim2 = trigger_size; 3131 } else { 3132 tr_req[tr_idx].addr = sg_addr; 3133 tr_req[tr_idx].icnt0 = tr_cnt0; 3134 tr_req[tr_idx].icnt1 = tr_cnt1; 3135 tr_req[tr_idx].icnt2 = tr1_cnt2; 3136 tr_req[tr_idx].icnt3 = 1; 3137 tr_req[tr_idx].dim1 = tr_cnt0; 3138 tr_req[tr_idx].dim2 = trigger_size; 3139 3140 tr_req[tr_idx].daddr = dev_addr; 3141 tr_req[tr_idx].dicnt0 = tr_cnt0; 3142 tr_req[tr_idx].dicnt1 = tr_cnt1; 3143 tr_req[tr_idx].dicnt2 = tr1_cnt2; 3144 tr_req[tr_idx].dicnt3 = 1; 3145 tr_req[tr_idx].ddim1 = (-1) * tr_cnt0; 3146 } 3147 tr_idx++; 3148 } 3149 3150 d->residue += sg_len; 3151 } 3152 3153 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, csf | CPPI5_TR_CSF_EOP); 3154 3155 return d; 3156 } 3157 3158 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d, 3159 enum dma_slave_buswidth dev_width, 3160 u16 elcnt) 3161 { 3162 if (uc->config.ep_type != PSIL_EP_PDMA_XY) 3163 return 0; 3164 3165 /* Bus width translates to the element size (ES) */ 3166 switch (dev_width) { 3167 case DMA_SLAVE_BUSWIDTH_1_BYTE: 3168 d->static_tr.elsize = 0; 3169 break; 3170 case DMA_SLAVE_BUSWIDTH_2_BYTES: 3171 d->static_tr.elsize = 1; 3172 break; 3173 case DMA_SLAVE_BUSWIDTH_3_BYTES: 3174 d->static_tr.elsize = 2; 3175 break; 3176 case DMA_SLAVE_BUSWIDTH_4_BYTES: 3177 d->static_tr.elsize = 3; 3178 break; 3179 case DMA_SLAVE_BUSWIDTH_8_BYTES: 3180 d->static_tr.elsize = 4; 3181 break; 3182 default: /* not reached */ 3183 return -EINVAL; 3184 } 3185 3186 d->static_tr.elcnt = elcnt; 3187 3188 /* 3189 * PDMA must to close the packet when the channel is in packet mode. 3190 * For TR mode when the channel is not cyclic we also need PDMA to close 3191 * the packet otherwise the transfer will stall because PDMA holds on 3192 * the data it has received from the peripheral. 3193 */ 3194 if (uc->config.pkt_mode || !uc->cyclic) { 3195 unsigned int div = dev_width * elcnt; 3196 3197 if (uc->cyclic) 3198 d->static_tr.bstcnt = d->residue / d->sglen / div; 3199 else 3200 d->static_tr.bstcnt = d->residue / div; 3201 3202 if (uc->config.dir == DMA_DEV_TO_MEM && 3203 d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) 3204 return -EINVAL; 3205 } else { 3206 d->static_tr.bstcnt = 0; 3207 } 3208 3209 return 0; 3210 } 3211 3212 static struct udma_desc * 3213 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, 3214 unsigned int sglen, enum dma_transfer_direction dir, 3215 unsigned long tx_flags, void *context) 3216 { 3217 struct scatterlist *sgent; 3218 struct cppi5_host_desc_t *h_desc = NULL; 3219 struct udma_desc *d; 3220 u32 ring_id; 3221 unsigned int i; 3222 u64 asel; 3223 3224 d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT); 3225 if (!d) 3226 return NULL; 3227 3228 d->sglen = sglen; 3229 d->hwdesc_count = sglen; 3230 3231 if (dir == DMA_DEV_TO_MEM) 3232 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 3233 else 3234 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 3235 3236 if (uc->ud->match_data->type == DMA_TYPE_UDMA) 3237 asel = 0; 3238 else 3239 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; 3240 3241 for_each_sg(sgl, sgent, sglen, i) { 3242 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 3243 dma_addr_t sg_addr = sg_dma_address(sgent); 3244 struct cppi5_host_desc_t *desc; 3245 size_t sg_len = sg_dma_len(sgent); 3246 3247 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 3248 GFP_NOWAIT, 3249 &hwdesc->cppi5_desc_paddr); 3250 if (!hwdesc->cppi5_desc_vaddr) { 3251 dev_err(uc->ud->dev, 3252 "descriptor%d allocation failed\n", i); 3253 3254 udma_free_hwdesc(uc, d); 3255 kfree(d); 3256 return NULL; 3257 } 3258 3259 d->residue += sg_len; 3260 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 3261 desc = hwdesc->cppi5_desc_vaddr; 3262 3263 if (i == 0) { 3264 cppi5_hdesc_init(desc, 0, 0); 3265 /* Flow and Packed ID */ 3266 cppi5_desc_set_pktids(&desc->hdr, uc->id, 3267 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3268 cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id); 3269 } else { 3270 cppi5_hdesc_reset_hbdesc(desc); 3271 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff); 3272 } 3273 3274 /* attach the sg buffer to the descriptor */ 3275 sg_addr |= asel; 3276 cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len); 3277 3278 /* Attach link as host buffer descriptor */ 3279 if (h_desc) 3280 cppi5_hdesc_link_hbdesc(h_desc, 3281 hwdesc->cppi5_desc_paddr | asel); 3282 3283 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA || 3284 dir == DMA_MEM_TO_DEV) 3285 h_desc = desc; 3286 } 3287 3288 if (d->residue >= SZ_4M) { 3289 dev_err(uc->ud->dev, 3290 "%s: Transfer size %u is over the supported 4M range\n", 3291 __func__, d->residue); 3292 udma_free_hwdesc(uc, d); 3293 kfree(d); 3294 return NULL; 3295 } 3296 3297 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 3298 cppi5_hdesc_set_pktlen(h_desc, d->residue); 3299 3300 return d; 3301 } 3302 3303 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc, 3304 void *data, size_t len) 3305 { 3306 struct udma_desc *d = to_udma_desc(desc); 3307 struct udma_chan *uc = to_udma_chan(desc->chan); 3308 struct cppi5_host_desc_t *h_desc; 3309 u32 psd_size = len; 3310 u32 flags = 0; 3311 3312 if (!uc->config.pkt_mode || !uc->config.metadata_size) 3313 return -ENOTSUPP; 3314 3315 if (!data || len > uc->config.metadata_size) 3316 return -EINVAL; 3317 3318 if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE) 3319 return -EINVAL; 3320 3321 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 3322 if (d->dir == DMA_MEM_TO_DEV) 3323 memcpy(h_desc->epib, data, len); 3324 3325 if (uc->config.needs_epib) 3326 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 3327 3328 d->metadata = data; 3329 d->metadata_size = len; 3330 if (uc->config.needs_epib) 3331 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 3332 3333 cppi5_hdesc_update_flags(h_desc, flags); 3334 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 3335 3336 return 0; 3337 } 3338 3339 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc, 3340 size_t *payload_len, size_t *max_len) 3341 { 3342 struct udma_desc *d = to_udma_desc(desc); 3343 struct udma_chan *uc = to_udma_chan(desc->chan); 3344 struct cppi5_host_desc_t *h_desc; 3345 3346 if (!uc->config.pkt_mode || !uc->config.metadata_size) 3347 return ERR_PTR(-ENOTSUPP); 3348 3349 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 3350 3351 *max_len = uc->config.metadata_size; 3352 3353 *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ? 3354 CPPI5_INFO0_HDESC_EPIB_SIZE : 0; 3355 *payload_len += cppi5_hdesc_get_psdata_size(h_desc); 3356 3357 return h_desc->epib; 3358 } 3359 3360 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc, 3361 size_t payload_len) 3362 { 3363 struct udma_desc *d = to_udma_desc(desc); 3364 struct udma_chan *uc = to_udma_chan(desc->chan); 3365 struct cppi5_host_desc_t *h_desc; 3366 u32 psd_size = payload_len; 3367 u32 flags = 0; 3368 3369 if (!uc->config.pkt_mode || !uc->config.metadata_size) 3370 return -ENOTSUPP; 3371 3372 if (payload_len > uc->config.metadata_size) 3373 return -EINVAL; 3374 3375 if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE) 3376 return -EINVAL; 3377 3378 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 3379 3380 if (uc->config.needs_epib) { 3381 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 3382 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 3383 } 3384 3385 cppi5_hdesc_update_flags(h_desc, flags); 3386 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 3387 3388 return 0; 3389 } 3390 3391 static struct dma_descriptor_metadata_ops metadata_ops = { 3392 .attach = udma_attach_metadata, 3393 .get_ptr = udma_get_metadata_ptr, 3394 .set_len = udma_set_metadata_len, 3395 }; 3396 3397 static struct dma_async_tx_descriptor * 3398 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 3399 unsigned int sglen, enum dma_transfer_direction dir, 3400 unsigned long tx_flags, void *context) 3401 { 3402 struct udma_chan *uc = to_udma_chan(chan); 3403 enum dma_slave_buswidth dev_width; 3404 struct udma_desc *d; 3405 u32 burst; 3406 3407 if (dir != uc->config.dir && 3408 (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type)) { 3409 dev_err(chan->device->dev, 3410 "%s: chan%d is for %s, not supporting %s\n", 3411 __func__, uc->id, 3412 dmaengine_get_direction_text(uc->config.dir), 3413 dmaengine_get_direction_text(dir)); 3414 return NULL; 3415 } 3416 3417 if (dir == DMA_DEV_TO_MEM) { 3418 dev_width = uc->cfg.src_addr_width; 3419 burst = uc->cfg.src_maxburst; 3420 } else if (dir == DMA_MEM_TO_DEV) { 3421 dev_width = uc->cfg.dst_addr_width; 3422 burst = uc->cfg.dst_maxburst; 3423 } else { 3424 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 3425 return NULL; 3426 } 3427 3428 if (!burst) 3429 burst = 1; 3430 3431 uc->config.tx_flags = tx_flags; 3432 3433 if (uc->config.pkt_mode) 3434 d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags, 3435 context); 3436 else if (is_slave_direction(uc->config.dir)) 3437 d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags, 3438 context); 3439 else 3440 d = udma_prep_slave_sg_triggered_tr(uc, sgl, sglen, dir, 3441 tx_flags, context); 3442 3443 if (!d) 3444 return NULL; 3445 3446 d->dir = dir; 3447 d->desc_idx = 0; 3448 d->tr_idx = 0; 3449 3450 /* static TR for remote PDMA */ 3451 if (udma_configure_statictr(uc, d, dev_width, burst)) { 3452 dev_err(uc->ud->dev, 3453 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 3454 __func__, d->static_tr.bstcnt); 3455 3456 udma_free_hwdesc(uc, d); 3457 kfree(d); 3458 return NULL; 3459 } 3460 3461 if (uc->config.metadata_size) 3462 d->vd.tx.metadata_ops = &metadata_ops; 3463 3464 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 3465 } 3466 3467 static struct udma_desc * 3468 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, 3469 size_t buf_len, size_t period_len, 3470 enum dma_transfer_direction dir, unsigned long flags) 3471 { 3472 struct udma_desc *d; 3473 size_t tr_size, period_addr; 3474 struct cppi5_tr_type1_t *tr_req; 3475 unsigned int periods = buf_len / period_len; 3476 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 3477 unsigned int i; 3478 int num_tr; 3479 3480 num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0, 3481 &tr0_cnt1, &tr1_cnt0); 3482 if (num_tr < 0) { 3483 dev_err(uc->ud->dev, "size %zu is not supported\n", 3484 period_len); 3485 return NULL; 3486 } 3487 3488 /* Now allocate and setup the descriptor. */ 3489 tr_size = sizeof(struct cppi5_tr_type1_t); 3490 d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir); 3491 if (!d) 3492 return NULL; 3493 3494 tr_req = d->hwdesc[0].tr_req_base; 3495 if (uc->ud->match_data->type == DMA_TYPE_UDMA) 3496 period_addr = buf_addr; 3497 else 3498 period_addr = buf_addr | 3499 ((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT); 3500 3501 for (i = 0; i < periods; i++) { 3502 int tr_idx = i * num_tr; 3503 3504 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false, 3505 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3506 3507 tr_req[tr_idx].addr = period_addr; 3508 tr_req[tr_idx].icnt0 = tr0_cnt0; 3509 tr_req[tr_idx].icnt1 = tr0_cnt1; 3510 tr_req[tr_idx].dim1 = tr0_cnt0; 3511 3512 if (num_tr == 2) { 3513 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 3514 CPPI5_TR_CSF_SUPR_EVT); 3515 tr_idx++; 3516 3517 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, 3518 false, false, 3519 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3520 3521 tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0; 3522 tr_req[tr_idx].icnt0 = tr1_cnt0; 3523 tr_req[tr_idx].icnt1 = 1; 3524 tr_req[tr_idx].dim1 = tr1_cnt0; 3525 } 3526 3527 if (!(flags & DMA_PREP_INTERRUPT)) 3528 cppi5_tr_csf_set(&tr_req[tr_idx].flags, 3529 CPPI5_TR_CSF_SUPR_EVT); 3530 3531 period_addr += period_len; 3532 } 3533 3534 return d; 3535 } 3536 3537 static struct udma_desc * 3538 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr, 3539 size_t buf_len, size_t period_len, 3540 enum dma_transfer_direction dir, unsigned long flags) 3541 { 3542 struct udma_desc *d; 3543 u32 ring_id; 3544 int i; 3545 int periods = buf_len / period_len; 3546 3547 if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1)) 3548 return NULL; 3549 3550 if (period_len >= SZ_4M) 3551 return NULL; 3552 3553 d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT); 3554 if (!d) 3555 return NULL; 3556 3557 d->hwdesc_count = periods; 3558 3559 /* TODO: re-check this... */ 3560 if (dir == DMA_DEV_TO_MEM) 3561 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 3562 else 3563 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 3564 3565 if (uc->ud->match_data->type != DMA_TYPE_UDMA) 3566 buf_addr |= (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; 3567 3568 for (i = 0; i < periods; i++) { 3569 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 3570 dma_addr_t period_addr = buf_addr + (period_len * i); 3571 struct cppi5_host_desc_t *h_desc; 3572 3573 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 3574 GFP_NOWAIT, 3575 &hwdesc->cppi5_desc_paddr); 3576 if (!hwdesc->cppi5_desc_vaddr) { 3577 dev_err(uc->ud->dev, 3578 "descriptor%d allocation failed\n", i); 3579 3580 udma_free_hwdesc(uc, d); 3581 kfree(d); 3582 return NULL; 3583 } 3584 3585 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 3586 h_desc = hwdesc->cppi5_desc_vaddr; 3587 3588 cppi5_hdesc_init(h_desc, 0, 0); 3589 cppi5_hdesc_set_pktlen(h_desc, period_len); 3590 3591 /* Flow and Packed ID */ 3592 cppi5_desc_set_pktids(&h_desc->hdr, uc->id, 3593 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 3594 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id); 3595 3596 /* attach each period to a new descriptor */ 3597 cppi5_hdesc_attach_buf(h_desc, 3598 period_addr, period_len, 3599 period_addr, period_len); 3600 } 3601 3602 return d; 3603 } 3604 3605 static struct dma_async_tx_descriptor * 3606 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 3607 size_t period_len, enum dma_transfer_direction dir, 3608 unsigned long flags) 3609 { 3610 struct udma_chan *uc = to_udma_chan(chan); 3611 enum dma_slave_buswidth dev_width; 3612 struct udma_desc *d; 3613 u32 burst; 3614 3615 if (dir != uc->config.dir) { 3616 dev_err(chan->device->dev, 3617 "%s: chan%d is for %s, not supporting %s\n", 3618 __func__, uc->id, 3619 dmaengine_get_direction_text(uc->config.dir), 3620 dmaengine_get_direction_text(dir)); 3621 return NULL; 3622 } 3623 3624 uc->cyclic = true; 3625 3626 if (dir == DMA_DEV_TO_MEM) { 3627 dev_width = uc->cfg.src_addr_width; 3628 burst = uc->cfg.src_maxburst; 3629 } else if (dir == DMA_MEM_TO_DEV) { 3630 dev_width = uc->cfg.dst_addr_width; 3631 burst = uc->cfg.dst_maxburst; 3632 } else { 3633 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 3634 return NULL; 3635 } 3636 3637 if (!burst) 3638 burst = 1; 3639 3640 if (uc->config.pkt_mode) 3641 d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len, 3642 dir, flags); 3643 else 3644 d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len, 3645 dir, flags); 3646 3647 if (!d) 3648 return NULL; 3649 3650 d->sglen = buf_len / period_len; 3651 3652 d->dir = dir; 3653 d->residue = buf_len; 3654 3655 /* static TR for remote PDMA */ 3656 if (udma_configure_statictr(uc, d, dev_width, burst)) { 3657 dev_err(uc->ud->dev, 3658 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 3659 __func__, d->static_tr.bstcnt); 3660 3661 udma_free_hwdesc(uc, d); 3662 kfree(d); 3663 return NULL; 3664 } 3665 3666 if (uc->config.metadata_size) 3667 d->vd.tx.metadata_ops = &metadata_ops; 3668 3669 return vchan_tx_prep(&uc->vc, &d->vd, flags); 3670 } 3671 3672 static struct dma_async_tx_descriptor * 3673 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 3674 size_t len, unsigned long tx_flags) 3675 { 3676 struct udma_chan *uc = to_udma_chan(chan); 3677 struct udma_desc *d; 3678 struct cppi5_tr_type15_t *tr_req; 3679 int num_tr; 3680 size_t tr_size = sizeof(struct cppi5_tr_type15_t); 3681 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 3682 u32 csf = CPPI5_TR_CSF_SUPR_EVT; 3683 3684 if (uc->config.dir != DMA_MEM_TO_MEM) { 3685 dev_err(chan->device->dev, 3686 "%s: chan%d is for %s, not supporting %s\n", 3687 __func__, uc->id, 3688 dmaengine_get_direction_text(uc->config.dir), 3689 dmaengine_get_direction_text(DMA_MEM_TO_MEM)); 3690 return NULL; 3691 } 3692 3693 num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0, 3694 &tr0_cnt1, &tr1_cnt0); 3695 if (num_tr < 0) { 3696 dev_err(uc->ud->dev, "size %zu is not supported\n", 3697 len); 3698 return NULL; 3699 } 3700 3701 d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM); 3702 if (!d) 3703 return NULL; 3704 3705 d->dir = DMA_MEM_TO_MEM; 3706 d->desc_idx = 0; 3707 d->tr_idx = 0; 3708 d->residue = len; 3709 3710 if (uc->ud->match_data->type != DMA_TYPE_UDMA) { 3711 src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT; 3712 dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT; 3713 } else { 3714 csf |= CPPI5_TR_CSF_EOL_ICNT0; 3715 } 3716 3717 tr_req = d->hwdesc[0].tr_req_base; 3718 3719 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true, 3720 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3721 cppi5_tr_csf_set(&tr_req[0].flags, csf); 3722 3723 tr_req[0].addr = src; 3724 tr_req[0].icnt0 = tr0_cnt0; 3725 tr_req[0].icnt1 = tr0_cnt1; 3726 tr_req[0].icnt2 = 1; 3727 tr_req[0].icnt3 = 1; 3728 tr_req[0].dim1 = tr0_cnt0; 3729 3730 tr_req[0].daddr = dest; 3731 tr_req[0].dicnt0 = tr0_cnt0; 3732 tr_req[0].dicnt1 = tr0_cnt1; 3733 tr_req[0].dicnt2 = 1; 3734 tr_req[0].dicnt3 = 1; 3735 tr_req[0].ddim1 = tr0_cnt0; 3736 3737 if (num_tr == 2) { 3738 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true, 3739 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 3740 cppi5_tr_csf_set(&tr_req[1].flags, csf); 3741 3742 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0; 3743 tr_req[1].icnt0 = tr1_cnt0; 3744 tr_req[1].icnt1 = 1; 3745 tr_req[1].icnt2 = 1; 3746 tr_req[1].icnt3 = 1; 3747 3748 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0; 3749 tr_req[1].dicnt0 = tr1_cnt0; 3750 tr_req[1].dicnt1 = 1; 3751 tr_req[1].dicnt2 = 1; 3752 tr_req[1].dicnt3 = 1; 3753 } 3754 3755 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, csf | CPPI5_TR_CSF_EOP); 3756 3757 if (uc->config.metadata_size) 3758 d->vd.tx.metadata_ops = &metadata_ops; 3759 3760 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 3761 } 3762 3763 static void udma_issue_pending(struct dma_chan *chan) 3764 { 3765 struct udma_chan *uc = to_udma_chan(chan); 3766 unsigned long flags; 3767 3768 spin_lock_irqsave(&uc->vc.lock, flags); 3769 3770 /* If we have something pending and no active descriptor, then */ 3771 if (vchan_issue_pending(&uc->vc) && !uc->desc) { 3772 /* 3773 * start a descriptor if the channel is NOT [marked as 3774 * terminating _and_ it is still running (teardown has not 3775 * completed yet)]. 3776 */ 3777 if (!(uc->state == UDMA_CHAN_IS_TERMINATING && 3778 udma_is_chan_running(uc))) 3779 udma_start(uc); 3780 } 3781 3782 spin_unlock_irqrestore(&uc->vc.lock, flags); 3783 } 3784 3785 static enum dma_status udma_tx_status(struct dma_chan *chan, 3786 dma_cookie_t cookie, 3787 struct dma_tx_state *txstate) 3788 { 3789 struct udma_chan *uc = to_udma_chan(chan); 3790 enum dma_status ret; 3791 unsigned long flags; 3792 3793 spin_lock_irqsave(&uc->vc.lock, flags); 3794 3795 ret = dma_cookie_status(chan, cookie, txstate); 3796 3797 if (!udma_is_chan_running(uc)) 3798 ret = DMA_COMPLETE; 3799 3800 if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc)) 3801 ret = DMA_PAUSED; 3802 3803 if (ret == DMA_COMPLETE || !txstate) 3804 goto out; 3805 3806 if (uc->desc && uc->desc->vd.tx.cookie == cookie) { 3807 u32 peer_bcnt = 0; 3808 u32 bcnt = 0; 3809 u32 residue = uc->desc->residue; 3810 u32 delay = 0; 3811 3812 if (uc->desc->dir == DMA_MEM_TO_DEV) { 3813 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); 3814 3815 if (uc->config.ep_type != PSIL_EP_NATIVE) { 3816 peer_bcnt = udma_tchanrt_read(uc, 3817 UDMA_CHAN_RT_PEER_BCNT_REG); 3818 3819 if (bcnt > peer_bcnt) 3820 delay = bcnt - peer_bcnt; 3821 } 3822 } else if (uc->desc->dir == DMA_DEV_TO_MEM) { 3823 bcnt = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 3824 3825 if (uc->config.ep_type != PSIL_EP_NATIVE) { 3826 peer_bcnt = udma_rchanrt_read(uc, 3827 UDMA_CHAN_RT_PEER_BCNT_REG); 3828 3829 if (peer_bcnt > bcnt) 3830 delay = peer_bcnt - bcnt; 3831 } 3832 } else { 3833 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); 3834 } 3835 3836 if (bcnt && !(bcnt % uc->desc->residue)) 3837 residue = 0; 3838 else 3839 residue -= bcnt % uc->desc->residue; 3840 3841 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) { 3842 ret = DMA_COMPLETE; 3843 delay = 0; 3844 } 3845 3846 dma_set_residue(txstate, residue); 3847 dma_set_in_flight_bytes(txstate, delay); 3848 3849 } else { 3850 ret = DMA_COMPLETE; 3851 } 3852 3853 out: 3854 spin_unlock_irqrestore(&uc->vc.lock, flags); 3855 return ret; 3856 } 3857 3858 static int udma_pause(struct dma_chan *chan) 3859 { 3860 struct udma_chan *uc = to_udma_chan(chan); 3861 3862 /* pause the channel */ 3863 switch (uc->config.dir) { 3864 case DMA_DEV_TO_MEM: 3865 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 3866 UDMA_PEER_RT_EN_PAUSE, 3867 UDMA_PEER_RT_EN_PAUSE); 3868 break; 3869 case DMA_MEM_TO_DEV: 3870 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 3871 UDMA_PEER_RT_EN_PAUSE, 3872 UDMA_PEER_RT_EN_PAUSE); 3873 break; 3874 case DMA_MEM_TO_MEM: 3875 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG, 3876 UDMA_CHAN_RT_CTL_PAUSE, 3877 UDMA_CHAN_RT_CTL_PAUSE); 3878 break; 3879 default: 3880 return -EINVAL; 3881 } 3882 3883 return 0; 3884 } 3885 3886 static int udma_resume(struct dma_chan *chan) 3887 { 3888 struct udma_chan *uc = to_udma_chan(chan); 3889 3890 /* resume the channel */ 3891 switch (uc->config.dir) { 3892 case DMA_DEV_TO_MEM: 3893 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 3894 UDMA_PEER_RT_EN_PAUSE, 0); 3895 3896 break; 3897 case DMA_MEM_TO_DEV: 3898 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 3899 UDMA_PEER_RT_EN_PAUSE, 0); 3900 break; 3901 case DMA_MEM_TO_MEM: 3902 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG, 3903 UDMA_CHAN_RT_CTL_PAUSE, 0); 3904 break; 3905 default: 3906 return -EINVAL; 3907 } 3908 3909 return 0; 3910 } 3911 3912 static int udma_terminate_all(struct dma_chan *chan) 3913 { 3914 struct udma_chan *uc = to_udma_chan(chan); 3915 unsigned long flags; 3916 LIST_HEAD(head); 3917 3918 spin_lock_irqsave(&uc->vc.lock, flags); 3919 3920 if (udma_is_chan_running(uc)) 3921 udma_stop(uc); 3922 3923 if (uc->desc) { 3924 uc->terminated_desc = uc->desc; 3925 uc->desc = NULL; 3926 uc->terminated_desc->terminated = true; 3927 cancel_delayed_work(&uc->tx_drain.work); 3928 } 3929 3930 uc->paused = false; 3931 3932 vchan_get_all_descriptors(&uc->vc, &head); 3933 spin_unlock_irqrestore(&uc->vc.lock, flags); 3934 vchan_dma_desc_free_list(&uc->vc, &head); 3935 3936 return 0; 3937 } 3938 3939 static void udma_synchronize(struct dma_chan *chan) 3940 { 3941 struct udma_chan *uc = to_udma_chan(chan); 3942 unsigned long timeout = msecs_to_jiffies(1000); 3943 3944 vchan_synchronize(&uc->vc); 3945 3946 if (uc->state == UDMA_CHAN_IS_TERMINATING) { 3947 timeout = wait_for_completion_timeout(&uc->teardown_completed, 3948 timeout); 3949 if (!timeout) { 3950 dev_warn(uc->ud->dev, "chan%d teardown timeout!\n", 3951 uc->id); 3952 udma_dump_chan_stdata(uc); 3953 udma_reset_chan(uc, true); 3954 } 3955 } 3956 3957 udma_reset_chan(uc, false); 3958 if (udma_is_chan_running(uc)) 3959 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id); 3960 3961 cancel_delayed_work_sync(&uc->tx_drain.work); 3962 udma_reset_rings(uc); 3963 } 3964 3965 static void udma_desc_pre_callback(struct virt_dma_chan *vc, 3966 struct virt_dma_desc *vd, 3967 struct dmaengine_result *result) 3968 { 3969 struct udma_chan *uc = to_udma_chan(&vc->chan); 3970 struct udma_desc *d; 3971 3972 if (!vd) 3973 return; 3974 3975 d = to_udma_desc(&vd->tx); 3976 3977 if (d->metadata_size) 3978 udma_fetch_epib(uc, d); 3979 3980 /* Provide residue information for the client */ 3981 if (result) { 3982 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx); 3983 3984 if (cppi5_desc_get_type(desc_vaddr) == 3985 CPPI5_INFO0_DESC_TYPE_VAL_HOST) { 3986 result->residue = d->residue - 3987 cppi5_hdesc_get_pktlen(desc_vaddr); 3988 if (result->residue) 3989 result->result = DMA_TRANS_ABORTED; 3990 else 3991 result->result = DMA_TRANS_NOERROR; 3992 } else { 3993 result->residue = 0; 3994 result->result = DMA_TRANS_NOERROR; 3995 } 3996 } 3997 } 3998 3999 /* 4000 * This tasklet handles the completion of a DMA descriptor by 4001 * calling its callback and freeing it. 4002 */ 4003 static void udma_vchan_complete(struct tasklet_struct *t) 4004 { 4005 struct virt_dma_chan *vc = from_tasklet(vc, t, task); 4006 struct virt_dma_desc *vd, *_vd; 4007 struct dmaengine_desc_callback cb; 4008 LIST_HEAD(head); 4009 4010 spin_lock_irq(&vc->lock); 4011 list_splice_tail_init(&vc->desc_completed, &head); 4012 vd = vc->cyclic; 4013 if (vd) { 4014 vc->cyclic = NULL; 4015 dmaengine_desc_get_callback(&vd->tx, &cb); 4016 } else { 4017 memset(&cb, 0, sizeof(cb)); 4018 } 4019 spin_unlock_irq(&vc->lock); 4020 4021 udma_desc_pre_callback(vc, vd, NULL); 4022 dmaengine_desc_callback_invoke(&cb, NULL); 4023 4024 list_for_each_entry_safe(vd, _vd, &head, node) { 4025 struct dmaengine_result result; 4026 4027 dmaengine_desc_get_callback(&vd->tx, &cb); 4028 4029 list_del(&vd->node); 4030 4031 udma_desc_pre_callback(vc, vd, &result); 4032 dmaengine_desc_callback_invoke(&cb, &result); 4033 4034 vchan_vdesc_fini(vd); 4035 } 4036 } 4037 4038 static void udma_free_chan_resources(struct dma_chan *chan) 4039 { 4040 struct udma_chan *uc = to_udma_chan(chan); 4041 struct udma_dev *ud = to_udma_dev(chan->device); 4042 4043 udma_terminate_all(chan); 4044 if (uc->terminated_desc) { 4045 udma_reset_chan(uc, false); 4046 udma_reset_rings(uc); 4047 } 4048 4049 cancel_delayed_work_sync(&uc->tx_drain.work); 4050 4051 if (uc->irq_num_ring > 0) { 4052 free_irq(uc->irq_num_ring, uc); 4053 4054 uc->irq_num_ring = 0; 4055 } 4056 if (uc->irq_num_udma > 0) { 4057 free_irq(uc->irq_num_udma, uc); 4058 4059 uc->irq_num_udma = 0; 4060 } 4061 4062 /* Release PSI-L pairing */ 4063 if (uc->psil_paired) { 4064 navss_psil_unpair(ud, uc->config.src_thread, 4065 uc->config.dst_thread); 4066 uc->psil_paired = false; 4067 } 4068 4069 vchan_free_chan_resources(&uc->vc); 4070 tasklet_kill(&uc->vc.task); 4071 4072 bcdma_free_bchan_resources(uc); 4073 udma_free_tx_resources(uc); 4074 udma_free_rx_resources(uc); 4075 udma_reset_uchan(uc); 4076 4077 if (uc->use_dma_pool) { 4078 dma_pool_destroy(uc->hdesc_pool); 4079 uc->use_dma_pool = false; 4080 } 4081 } 4082 4083 static struct platform_driver udma_driver; 4084 static struct platform_driver bcdma_driver; 4085 static struct platform_driver pktdma_driver; 4086 4087 struct udma_filter_param { 4088 int remote_thread_id; 4089 u32 atype; 4090 u32 asel; 4091 u32 tr_trigger_type; 4092 }; 4093 4094 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) 4095 { 4096 struct udma_chan_config *ucc; 4097 struct psil_endpoint_config *ep_config; 4098 struct udma_filter_param *filter_param; 4099 struct udma_chan *uc; 4100 struct udma_dev *ud; 4101 4102 if (chan->device->dev->driver != &udma_driver.driver && 4103 chan->device->dev->driver != &bcdma_driver.driver && 4104 chan->device->dev->driver != &pktdma_driver.driver) 4105 return false; 4106 4107 uc = to_udma_chan(chan); 4108 ucc = &uc->config; 4109 ud = uc->ud; 4110 filter_param = param; 4111 4112 if (filter_param->atype > 2) { 4113 dev_err(ud->dev, "Invalid channel atype: %u\n", 4114 filter_param->atype); 4115 return false; 4116 } 4117 4118 if (filter_param->asel > 15) { 4119 dev_err(ud->dev, "Invalid channel asel: %u\n", 4120 filter_param->asel); 4121 return false; 4122 } 4123 4124 ucc->remote_thread_id = filter_param->remote_thread_id; 4125 ucc->atype = filter_param->atype; 4126 ucc->asel = filter_param->asel; 4127 ucc->tr_trigger_type = filter_param->tr_trigger_type; 4128 4129 if (ucc->tr_trigger_type) { 4130 ucc->dir = DMA_MEM_TO_MEM; 4131 goto triggered_bchan; 4132 } else if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) { 4133 ucc->dir = DMA_MEM_TO_DEV; 4134 } else { 4135 ucc->dir = DMA_DEV_TO_MEM; 4136 } 4137 4138 ep_config = psil_get_ep_config(ucc->remote_thread_id); 4139 if (IS_ERR(ep_config)) { 4140 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n", 4141 ucc->remote_thread_id); 4142 ucc->dir = DMA_MEM_TO_MEM; 4143 ucc->remote_thread_id = -1; 4144 ucc->atype = 0; 4145 ucc->asel = 0; 4146 return false; 4147 } 4148 4149 if (ud->match_data->type == DMA_TYPE_BCDMA && 4150 ep_config->pkt_mode) { 4151 dev_err(ud->dev, 4152 "Only TR mode is supported (psi-l thread 0x%04x)\n", 4153 ucc->remote_thread_id); 4154 ucc->dir = DMA_MEM_TO_MEM; 4155 ucc->remote_thread_id = -1; 4156 ucc->atype = 0; 4157 ucc->asel = 0; 4158 return false; 4159 } 4160 4161 ucc->pkt_mode = ep_config->pkt_mode; 4162 ucc->channel_tpl = ep_config->channel_tpl; 4163 ucc->notdpkt = ep_config->notdpkt; 4164 ucc->ep_type = ep_config->ep_type; 4165 4166 if (ud->match_data->type == DMA_TYPE_PKTDMA && 4167 ep_config->mapped_channel_id >= 0) { 4168 ucc->mapped_channel_id = ep_config->mapped_channel_id; 4169 ucc->default_flow_id = ep_config->default_flow_id; 4170 } else { 4171 ucc->mapped_channel_id = -1; 4172 ucc->default_flow_id = -1; 4173 } 4174 4175 if (ucc->ep_type != PSIL_EP_NATIVE) { 4176 const struct udma_match_data *match_data = ud->match_data; 4177 4178 if (match_data->flags & UDMA_FLAG_PDMA_ACC32) 4179 ucc->enable_acc32 = ep_config->pdma_acc32; 4180 if (match_data->flags & UDMA_FLAG_PDMA_BURST) 4181 ucc->enable_burst = ep_config->pdma_burst; 4182 } 4183 4184 ucc->needs_epib = ep_config->needs_epib; 4185 ucc->psd_size = ep_config->psd_size; 4186 ucc->metadata_size = 4187 (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) + 4188 ucc->psd_size; 4189 4190 if (ucc->pkt_mode) 4191 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 4192 ucc->metadata_size, ud->desc_align); 4193 4194 dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id, 4195 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir)); 4196 4197 return true; 4198 4199 triggered_bchan: 4200 dev_dbg(ud->dev, "chan%d: triggered channel (type: %u)\n", uc->id, 4201 ucc->tr_trigger_type); 4202 4203 return true; 4204 4205 } 4206 4207 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, 4208 struct of_dma *ofdma) 4209 { 4210 struct udma_dev *ud = ofdma->of_dma_data; 4211 dma_cap_mask_t mask = ud->ddev.cap_mask; 4212 struct udma_filter_param filter_param; 4213 struct dma_chan *chan; 4214 4215 if (ud->match_data->type == DMA_TYPE_BCDMA) { 4216 if (dma_spec->args_count != 3) 4217 return NULL; 4218 4219 filter_param.tr_trigger_type = dma_spec->args[0]; 4220 filter_param.remote_thread_id = dma_spec->args[1]; 4221 filter_param.asel = dma_spec->args[2]; 4222 filter_param.atype = 0; 4223 } else { 4224 if (dma_spec->args_count != 1 && dma_spec->args_count != 2) 4225 return NULL; 4226 4227 filter_param.remote_thread_id = dma_spec->args[0]; 4228 filter_param.tr_trigger_type = 0; 4229 if (dma_spec->args_count == 2) { 4230 if (ud->match_data->type == DMA_TYPE_UDMA) { 4231 filter_param.atype = dma_spec->args[1]; 4232 filter_param.asel = 0; 4233 } else { 4234 filter_param.atype = 0; 4235 filter_param.asel = dma_spec->args[1]; 4236 } 4237 } else { 4238 filter_param.atype = 0; 4239 filter_param.asel = 0; 4240 } 4241 } 4242 4243 chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, 4244 ofdma->of_node); 4245 if (!chan) { 4246 dev_err(ud->dev, "get channel fail in %s.\n", __func__); 4247 return ERR_PTR(-EINVAL); 4248 } 4249 4250 return chan; 4251 } 4252 4253 static struct udma_match_data am654_main_data = { 4254 .type = DMA_TYPE_UDMA, 4255 .psil_base = 0x1000, 4256 .enable_memcpy_support = true, 4257 .statictr_z_mask = GENMASK(11, 0), 4258 .burst_size = { 4259 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4260 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */ 4261 0, /* No UH Channels */ 4262 }, 4263 }; 4264 4265 static struct udma_match_data am654_mcu_data = { 4266 .type = DMA_TYPE_UDMA, 4267 .psil_base = 0x6000, 4268 .enable_memcpy_support = false, 4269 .statictr_z_mask = GENMASK(11, 0), 4270 .burst_size = { 4271 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4272 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */ 4273 0, /* No UH Channels */ 4274 }, 4275 }; 4276 4277 static struct udma_match_data j721e_main_data = { 4278 .type = DMA_TYPE_UDMA, 4279 .psil_base = 0x1000, 4280 .enable_memcpy_support = true, 4281 .flags = UDMA_FLAGS_J7_CLASS, 4282 .statictr_z_mask = GENMASK(23, 0), 4283 .burst_size = { 4284 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4285 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* H Channels */ 4286 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* UH Channels */ 4287 }, 4288 }; 4289 4290 static struct udma_match_data j721e_mcu_data = { 4291 .type = DMA_TYPE_UDMA, 4292 .psil_base = 0x6000, 4293 .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ 4294 .flags = UDMA_FLAGS_J7_CLASS, 4295 .statictr_z_mask = GENMASK(23, 0), 4296 .burst_size = { 4297 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4298 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES, /* H Channels */ 4299 0, /* No UH Channels */ 4300 }, 4301 }; 4302 4303 static struct udma_soc_data am62a_dmss_csi_soc_data = { 4304 .oes = { 4305 .bcdma_rchan_data = 0xe00, 4306 .bcdma_rchan_ring = 0x1000, 4307 }, 4308 }; 4309 4310 static struct udma_soc_data j721s2_bcdma_csi_soc_data = { 4311 .oes = { 4312 .bcdma_tchan_data = 0x800, 4313 .bcdma_tchan_ring = 0xa00, 4314 .bcdma_rchan_data = 0xe00, 4315 .bcdma_rchan_ring = 0x1000, 4316 }, 4317 }; 4318 4319 static struct udma_match_data am62a_bcdma_csirx_data = { 4320 .type = DMA_TYPE_BCDMA, 4321 .psil_base = 0x3100, 4322 .enable_memcpy_support = false, 4323 .burst_size = { 4324 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4325 0, /* No H Channels */ 4326 0, /* No UH Channels */ 4327 }, 4328 .soc_data = &am62a_dmss_csi_soc_data, 4329 }; 4330 4331 static struct udma_match_data am64_bcdma_data = { 4332 .type = DMA_TYPE_BCDMA, 4333 .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */ 4334 .enable_memcpy_support = true, /* Supported via bchan */ 4335 .flags = UDMA_FLAGS_J7_CLASS, 4336 .statictr_z_mask = GENMASK(23, 0), 4337 .burst_size = { 4338 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4339 0, /* No H Channels */ 4340 0, /* No UH Channels */ 4341 }, 4342 }; 4343 4344 static struct udma_match_data am64_pktdma_data = { 4345 .type = DMA_TYPE_PKTDMA, 4346 .psil_base = 0x1000, 4347 .enable_memcpy_support = false, /* PKTDMA does not support MEM_TO_MEM */ 4348 .flags = UDMA_FLAGS_J7_CLASS, 4349 .statictr_z_mask = GENMASK(23, 0), 4350 .burst_size = { 4351 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4352 0, /* No H Channels */ 4353 0, /* No UH Channels */ 4354 }, 4355 }; 4356 4357 static struct udma_match_data j721s2_bcdma_csi_data = { 4358 .type = DMA_TYPE_BCDMA, 4359 .psil_base = 0x2000, 4360 .enable_memcpy_support = false, 4361 .burst_size = { 4362 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ 4363 0, /* No H Channels */ 4364 0, /* No UH Channels */ 4365 }, 4366 .soc_data = &j721s2_bcdma_csi_soc_data, 4367 }; 4368 4369 static const struct of_device_id udma_of_match[] = { 4370 { 4371 .compatible = "ti,am654-navss-main-udmap", 4372 .data = &am654_main_data, 4373 }, 4374 { 4375 .compatible = "ti,am654-navss-mcu-udmap", 4376 .data = &am654_mcu_data, 4377 }, { 4378 .compatible = "ti,j721e-navss-main-udmap", 4379 .data = &j721e_main_data, 4380 }, { 4381 .compatible = "ti,j721e-navss-mcu-udmap", 4382 .data = &j721e_mcu_data, 4383 }, 4384 { 4385 .compatible = "ti,am64-dmss-bcdma", 4386 .data = &am64_bcdma_data, 4387 }, 4388 { 4389 .compatible = "ti,am64-dmss-pktdma", 4390 .data = &am64_pktdma_data, 4391 }, 4392 { 4393 .compatible = "ti,am62a-dmss-bcdma-csirx", 4394 .data = &am62a_bcdma_csirx_data, 4395 }, 4396 { 4397 .compatible = "ti,j721s2-dmss-bcdma-csi", 4398 .data = &j721s2_bcdma_csi_data, 4399 }, 4400 { /* Sentinel */ }, 4401 }; 4402 4403 static struct udma_soc_data am654_soc_data = { 4404 .oes = { 4405 .udma_rchan = 0x200, 4406 }, 4407 }; 4408 4409 static struct udma_soc_data j721e_soc_data = { 4410 .oes = { 4411 .udma_rchan = 0x400, 4412 }, 4413 }; 4414 4415 static struct udma_soc_data j7200_soc_data = { 4416 .oes = { 4417 .udma_rchan = 0x80, 4418 }, 4419 }; 4420 4421 static struct udma_soc_data am64_soc_data = { 4422 .oes = { 4423 .bcdma_bchan_data = 0x2200, 4424 .bcdma_bchan_ring = 0x2400, 4425 .bcdma_tchan_data = 0x2800, 4426 .bcdma_tchan_ring = 0x2a00, 4427 .bcdma_rchan_data = 0x2e00, 4428 .bcdma_rchan_ring = 0x3000, 4429 .pktdma_tchan_flow = 0x1200, 4430 .pktdma_rchan_flow = 0x1600, 4431 }, 4432 .bcdma_trigger_event_offset = 0xc400, 4433 }; 4434 4435 static const struct soc_device_attribute k3_soc_devices[] = { 4436 { .family = "AM65X", .data = &am654_soc_data }, 4437 { .family = "J721E", .data = &j721e_soc_data }, 4438 { .family = "J7200", .data = &j7200_soc_data }, 4439 { .family = "AM64X", .data = &am64_soc_data }, 4440 { .family = "J721S2", .data = &j721e_soc_data}, 4441 { .family = "AM62X", .data = &am64_soc_data }, 4442 { .family = "AM62AX", .data = &am64_soc_data }, 4443 { .family = "J784S4", .data = &j721e_soc_data }, 4444 { /* sentinel */ } 4445 }; 4446 4447 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) 4448 { 4449 u32 cap2, cap3, cap4; 4450 int i; 4451 4452 ud->mmrs[MMR_GCFG] = devm_platform_ioremap_resource_byname(pdev, mmr_names[MMR_GCFG]); 4453 if (IS_ERR(ud->mmrs[MMR_GCFG])) 4454 return PTR_ERR(ud->mmrs[MMR_GCFG]); 4455 4456 cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28); 4457 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 4458 4459 switch (ud->match_data->type) { 4460 case DMA_TYPE_UDMA: 4461 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3); 4462 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2); 4463 ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2); 4464 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); 4465 break; 4466 case DMA_TYPE_BCDMA: 4467 ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2); 4468 ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2); 4469 ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2); 4470 ud->rflow_cnt = ud->rchan_cnt; 4471 break; 4472 case DMA_TYPE_PKTDMA: 4473 cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30); 4474 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2); 4475 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); 4476 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3); 4477 ud->tflow_cnt = PKTDMA_CAP4_TFLOW_CNT(cap4); 4478 break; 4479 default: 4480 return -EINVAL; 4481 } 4482 4483 for (i = 1; i < MMR_LAST; i++) { 4484 if (i == MMR_BCHANRT && ud->bchan_cnt == 0) 4485 continue; 4486 if (i == MMR_TCHANRT && ud->tchan_cnt == 0) 4487 continue; 4488 if (i == MMR_RCHANRT && ud->rchan_cnt == 0) 4489 continue; 4490 4491 ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]); 4492 if (IS_ERR(ud->mmrs[i])) 4493 return PTR_ERR(ud->mmrs[i]); 4494 } 4495 4496 return 0; 4497 } 4498 4499 static void udma_mark_resource_ranges(struct udma_dev *ud, unsigned long *map, 4500 struct ti_sci_resource_desc *rm_desc, 4501 char *name) 4502 { 4503 bitmap_clear(map, rm_desc->start, rm_desc->num); 4504 bitmap_clear(map, rm_desc->start_sec, rm_desc->num_sec); 4505 dev_dbg(ud->dev, "ti_sci resource range for %s: %d:%d | %d:%d\n", name, 4506 rm_desc->start, rm_desc->num, rm_desc->start_sec, 4507 rm_desc->num_sec); 4508 } 4509 4510 static const char * const range_names[] = { 4511 [RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan", 4512 [RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan", 4513 [RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan", 4514 [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow", 4515 [RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow", 4516 }; 4517 4518 static int udma_setup_resources(struct udma_dev *ud) 4519 { 4520 int ret, i, j; 4521 struct device *dev = ud->dev; 4522 struct ti_sci_resource *rm_res, irq_res; 4523 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 4524 u32 cap3; 4525 4526 /* Set up the throughput level start indexes */ 4527 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 4528 if (of_device_is_compatible(dev->of_node, 4529 "ti,am654-navss-main-udmap")) { 4530 ud->tchan_tpl.levels = 2; 4531 ud->tchan_tpl.start_idx[0] = 8; 4532 } else if (of_device_is_compatible(dev->of_node, 4533 "ti,am654-navss-mcu-udmap")) { 4534 ud->tchan_tpl.levels = 2; 4535 ud->tchan_tpl.start_idx[0] = 2; 4536 } else if (UDMA_CAP3_UCHAN_CNT(cap3)) { 4537 ud->tchan_tpl.levels = 3; 4538 ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); 4539 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); 4540 } else if (UDMA_CAP3_HCHAN_CNT(cap3)) { 4541 ud->tchan_tpl.levels = 2; 4542 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); 4543 } else { 4544 ud->tchan_tpl.levels = 1; 4545 } 4546 4547 ud->rchan_tpl.levels = ud->tchan_tpl.levels; 4548 ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0]; 4549 ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1]; 4550 4551 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 4552 sizeof(unsigned long), GFP_KERNEL); 4553 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 4554 GFP_KERNEL); 4555 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 4556 sizeof(unsigned long), GFP_KERNEL); 4557 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 4558 GFP_KERNEL); 4559 ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt), 4560 sizeof(unsigned long), 4561 GFP_KERNEL); 4562 ud->rflow_gp_map_allocated = devm_kcalloc(dev, 4563 BITS_TO_LONGS(ud->rflow_cnt), 4564 sizeof(unsigned long), 4565 GFP_KERNEL); 4566 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), 4567 sizeof(unsigned long), 4568 GFP_KERNEL); 4569 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), 4570 GFP_KERNEL); 4571 4572 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map || 4573 !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans || 4574 !ud->rflows || !ud->rflow_in_use) 4575 return -ENOMEM; 4576 4577 /* 4578 * RX flows with the same Ids as RX channels are reserved to be used 4579 * as default flows if remote HW can't generate flow_ids. Those 4580 * RX flows can be requested only explicitly by id. 4581 */ 4582 bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt); 4583 4584 /* by default no GP rflows are assigned to Linux */ 4585 bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt); 4586 4587 /* Get resource ranges from tisci */ 4588 for (i = 0; i < RM_RANGE_LAST; i++) { 4589 if (i == RM_RANGE_BCHAN || i == RM_RANGE_TFLOW) 4590 continue; 4591 4592 tisci_rm->rm_ranges[i] = 4593 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 4594 tisci_rm->tisci_dev_id, 4595 (char *)range_names[i]); 4596 } 4597 4598 /* tchan ranges */ 4599 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 4600 if (IS_ERR(rm_res)) { 4601 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 4602 irq_res.sets = 1; 4603 } else { 4604 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 4605 for (i = 0; i < rm_res->sets; i++) 4606 udma_mark_resource_ranges(ud, ud->tchan_map, 4607 &rm_res->desc[i], "tchan"); 4608 irq_res.sets = rm_res->sets; 4609 } 4610 4611 /* rchan and matching default flow ranges */ 4612 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 4613 if (IS_ERR(rm_res)) { 4614 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 4615 irq_res.sets++; 4616 } else { 4617 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 4618 for (i = 0; i < rm_res->sets; i++) 4619 udma_mark_resource_ranges(ud, ud->rchan_map, 4620 &rm_res->desc[i], "rchan"); 4621 irq_res.sets += rm_res->sets; 4622 } 4623 4624 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 4625 if (!irq_res.desc) 4626 return -ENOMEM; 4627 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 4628 if (IS_ERR(rm_res)) { 4629 irq_res.desc[0].start = 0; 4630 irq_res.desc[0].num = ud->tchan_cnt; 4631 i = 1; 4632 } else { 4633 for (i = 0; i < rm_res->sets; i++) { 4634 irq_res.desc[i].start = rm_res->desc[i].start; 4635 irq_res.desc[i].num = rm_res->desc[i].num; 4636 irq_res.desc[i].start_sec = rm_res->desc[i].start_sec; 4637 irq_res.desc[i].num_sec = rm_res->desc[i].num_sec; 4638 } 4639 } 4640 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 4641 if (IS_ERR(rm_res)) { 4642 irq_res.desc[i].start = 0; 4643 irq_res.desc[i].num = ud->rchan_cnt; 4644 } else { 4645 for (j = 0; j < rm_res->sets; j++, i++) { 4646 if (rm_res->desc[j].num) { 4647 irq_res.desc[i].start = rm_res->desc[j].start + 4648 ud->soc_data->oes.udma_rchan; 4649 irq_res.desc[i].num = rm_res->desc[j].num; 4650 } 4651 if (rm_res->desc[j].num_sec) { 4652 irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + 4653 ud->soc_data->oes.udma_rchan; 4654 irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; 4655 } 4656 } 4657 } 4658 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 4659 kfree(irq_res.desc); 4660 if (ret) { 4661 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 4662 return ret; 4663 } 4664 4665 /* GP rflow ranges */ 4666 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 4667 if (IS_ERR(rm_res)) { 4668 /* all gp flows are assigned exclusively to Linux */ 4669 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt, 4670 ud->rflow_cnt - ud->rchan_cnt); 4671 } else { 4672 for (i = 0; i < rm_res->sets; i++) 4673 udma_mark_resource_ranges(ud, ud->rflow_gp_map, 4674 &rm_res->desc[i], "gp-rflow"); 4675 } 4676 4677 return 0; 4678 } 4679 4680 static int bcdma_setup_resources(struct udma_dev *ud) 4681 { 4682 int ret, i, j; 4683 struct device *dev = ud->dev; 4684 struct ti_sci_resource *rm_res, irq_res; 4685 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 4686 const struct udma_oes_offsets *oes = &ud->soc_data->oes; 4687 u32 cap; 4688 4689 /* Set up the throughput level start indexes */ 4690 cap = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 4691 if (BCDMA_CAP3_UBCHAN_CNT(cap)) { 4692 ud->bchan_tpl.levels = 3; 4693 ud->bchan_tpl.start_idx[1] = BCDMA_CAP3_UBCHAN_CNT(cap); 4694 ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap); 4695 } else if (BCDMA_CAP3_HBCHAN_CNT(cap)) { 4696 ud->bchan_tpl.levels = 2; 4697 ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap); 4698 } else { 4699 ud->bchan_tpl.levels = 1; 4700 } 4701 4702 cap = udma_read(ud->mmrs[MMR_GCFG], 0x30); 4703 if (BCDMA_CAP4_URCHAN_CNT(cap)) { 4704 ud->rchan_tpl.levels = 3; 4705 ud->rchan_tpl.start_idx[1] = BCDMA_CAP4_URCHAN_CNT(cap); 4706 ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap); 4707 } else if (BCDMA_CAP4_HRCHAN_CNT(cap)) { 4708 ud->rchan_tpl.levels = 2; 4709 ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap); 4710 } else { 4711 ud->rchan_tpl.levels = 1; 4712 } 4713 4714 if (BCDMA_CAP4_UTCHAN_CNT(cap)) { 4715 ud->tchan_tpl.levels = 3; 4716 ud->tchan_tpl.start_idx[1] = BCDMA_CAP4_UTCHAN_CNT(cap); 4717 ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap); 4718 } else if (BCDMA_CAP4_HTCHAN_CNT(cap)) { 4719 ud->tchan_tpl.levels = 2; 4720 ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap); 4721 } else { 4722 ud->tchan_tpl.levels = 1; 4723 } 4724 4725 ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt), 4726 sizeof(unsigned long), GFP_KERNEL); 4727 ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans), 4728 GFP_KERNEL); 4729 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 4730 sizeof(unsigned long), GFP_KERNEL); 4731 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 4732 GFP_KERNEL); 4733 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 4734 sizeof(unsigned long), GFP_KERNEL); 4735 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 4736 GFP_KERNEL); 4737 /* BCDMA do not really have flows, but the driver expect it */ 4738 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rchan_cnt), 4739 sizeof(unsigned long), 4740 GFP_KERNEL); 4741 ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows), 4742 GFP_KERNEL); 4743 4744 if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map || 4745 !ud->rflow_in_use || !ud->bchans || !ud->tchans || !ud->rchans || 4746 !ud->rflows) 4747 return -ENOMEM; 4748 4749 /* Get resource ranges from tisci */ 4750 for (i = 0; i < RM_RANGE_LAST; i++) { 4751 if (i == RM_RANGE_RFLOW || i == RM_RANGE_TFLOW) 4752 continue; 4753 if (i == RM_RANGE_BCHAN && ud->bchan_cnt == 0) 4754 continue; 4755 if (i == RM_RANGE_TCHAN && ud->tchan_cnt == 0) 4756 continue; 4757 if (i == RM_RANGE_RCHAN && ud->rchan_cnt == 0) 4758 continue; 4759 4760 tisci_rm->rm_ranges[i] = 4761 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 4762 tisci_rm->tisci_dev_id, 4763 (char *)range_names[i]); 4764 } 4765 4766 irq_res.sets = 0; 4767 4768 /* bchan ranges */ 4769 if (ud->bchan_cnt) { 4770 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN]; 4771 if (IS_ERR(rm_res)) { 4772 bitmap_zero(ud->bchan_map, ud->bchan_cnt); 4773 irq_res.sets++; 4774 } else { 4775 bitmap_fill(ud->bchan_map, ud->bchan_cnt); 4776 for (i = 0; i < rm_res->sets; i++) 4777 udma_mark_resource_ranges(ud, ud->bchan_map, 4778 &rm_res->desc[i], 4779 "bchan"); 4780 irq_res.sets += rm_res->sets; 4781 } 4782 } 4783 4784 /* tchan ranges */ 4785 if (ud->tchan_cnt) { 4786 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 4787 if (IS_ERR(rm_res)) { 4788 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 4789 irq_res.sets += 2; 4790 } else { 4791 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 4792 for (i = 0; i < rm_res->sets; i++) 4793 udma_mark_resource_ranges(ud, ud->tchan_map, 4794 &rm_res->desc[i], 4795 "tchan"); 4796 irq_res.sets += rm_res->sets * 2; 4797 } 4798 } 4799 4800 /* rchan ranges */ 4801 if (ud->rchan_cnt) { 4802 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 4803 if (IS_ERR(rm_res)) { 4804 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 4805 irq_res.sets += 2; 4806 } else { 4807 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 4808 for (i = 0; i < rm_res->sets; i++) 4809 udma_mark_resource_ranges(ud, ud->rchan_map, 4810 &rm_res->desc[i], 4811 "rchan"); 4812 irq_res.sets += rm_res->sets * 2; 4813 } 4814 } 4815 4816 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 4817 if (!irq_res.desc) 4818 return -ENOMEM; 4819 if (ud->bchan_cnt) { 4820 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN]; 4821 if (IS_ERR(rm_res)) { 4822 irq_res.desc[0].start = oes->bcdma_bchan_ring; 4823 irq_res.desc[0].num = ud->bchan_cnt; 4824 i = 1; 4825 } else { 4826 for (i = 0; i < rm_res->sets; i++) { 4827 irq_res.desc[i].start = rm_res->desc[i].start + 4828 oes->bcdma_bchan_ring; 4829 irq_res.desc[i].num = rm_res->desc[i].num; 4830 } 4831 } 4832 } else { 4833 i = 0; 4834 } 4835 4836 if (ud->tchan_cnt) { 4837 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 4838 if (IS_ERR(rm_res)) { 4839 irq_res.desc[i].start = oes->bcdma_tchan_data; 4840 irq_res.desc[i].num = ud->tchan_cnt; 4841 irq_res.desc[i + 1].start = oes->bcdma_tchan_ring; 4842 irq_res.desc[i + 1].num = ud->tchan_cnt; 4843 i += 2; 4844 } else { 4845 for (j = 0; j < rm_res->sets; j++, i += 2) { 4846 irq_res.desc[i].start = rm_res->desc[j].start + 4847 oes->bcdma_tchan_data; 4848 irq_res.desc[i].num = rm_res->desc[j].num; 4849 4850 irq_res.desc[i + 1].start = rm_res->desc[j].start + 4851 oes->bcdma_tchan_ring; 4852 irq_res.desc[i + 1].num = rm_res->desc[j].num; 4853 } 4854 } 4855 } 4856 if (ud->rchan_cnt) { 4857 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 4858 if (IS_ERR(rm_res)) { 4859 irq_res.desc[i].start = oes->bcdma_rchan_data; 4860 irq_res.desc[i].num = ud->rchan_cnt; 4861 irq_res.desc[i + 1].start = oes->bcdma_rchan_ring; 4862 irq_res.desc[i + 1].num = ud->rchan_cnt; 4863 i += 2; 4864 } else { 4865 for (j = 0; j < rm_res->sets; j++, i += 2) { 4866 irq_res.desc[i].start = rm_res->desc[j].start + 4867 oes->bcdma_rchan_data; 4868 irq_res.desc[i].num = rm_res->desc[j].num; 4869 4870 irq_res.desc[i + 1].start = rm_res->desc[j].start + 4871 oes->bcdma_rchan_ring; 4872 irq_res.desc[i + 1].num = rm_res->desc[j].num; 4873 } 4874 } 4875 } 4876 4877 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 4878 kfree(irq_res.desc); 4879 if (ret) { 4880 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 4881 return ret; 4882 } 4883 4884 return 0; 4885 } 4886 4887 static int pktdma_setup_resources(struct udma_dev *ud) 4888 { 4889 int ret, i, j; 4890 struct device *dev = ud->dev; 4891 struct ti_sci_resource *rm_res, irq_res; 4892 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 4893 const struct udma_oes_offsets *oes = &ud->soc_data->oes; 4894 u32 cap3; 4895 4896 /* Set up the throughput level start indexes */ 4897 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 4898 if (UDMA_CAP3_UCHAN_CNT(cap3)) { 4899 ud->tchan_tpl.levels = 3; 4900 ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); 4901 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); 4902 } else if (UDMA_CAP3_HCHAN_CNT(cap3)) { 4903 ud->tchan_tpl.levels = 2; 4904 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); 4905 } else { 4906 ud->tchan_tpl.levels = 1; 4907 } 4908 4909 ud->rchan_tpl.levels = ud->tchan_tpl.levels; 4910 ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0]; 4911 ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1]; 4912 4913 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 4914 sizeof(unsigned long), GFP_KERNEL); 4915 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 4916 GFP_KERNEL); 4917 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 4918 sizeof(unsigned long), GFP_KERNEL); 4919 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 4920 GFP_KERNEL); 4921 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), 4922 sizeof(unsigned long), 4923 GFP_KERNEL); 4924 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), 4925 GFP_KERNEL); 4926 ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt), 4927 sizeof(unsigned long), GFP_KERNEL); 4928 4929 if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans || 4930 !ud->rchans || !ud->rflows || !ud->rflow_in_use) 4931 return -ENOMEM; 4932 4933 /* Get resource ranges from tisci */ 4934 for (i = 0; i < RM_RANGE_LAST; i++) { 4935 if (i == RM_RANGE_BCHAN) 4936 continue; 4937 4938 tisci_rm->rm_ranges[i] = 4939 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 4940 tisci_rm->tisci_dev_id, 4941 (char *)range_names[i]); 4942 } 4943 4944 /* tchan ranges */ 4945 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 4946 if (IS_ERR(rm_res)) { 4947 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 4948 } else { 4949 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 4950 for (i = 0; i < rm_res->sets; i++) 4951 udma_mark_resource_ranges(ud, ud->tchan_map, 4952 &rm_res->desc[i], "tchan"); 4953 } 4954 4955 /* rchan ranges */ 4956 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 4957 if (IS_ERR(rm_res)) { 4958 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 4959 } else { 4960 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 4961 for (i = 0; i < rm_res->sets; i++) 4962 udma_mark_resource_ranges(ud, ud->rchan_map, 4963 &rm_res->desc[i], "rchan"); 4964 } 4965 4966 /* rflow ranges */ 4967 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 4968 if (IS_ERR(rm_res)) { 4969 /* all rflows are assigned exclusively to Linux */ 4970 bitmap_zero(ud->rflow_in_use, ud->rflow_cnt); 4971 irq_res.sets = 1; 4972 } else { 4973 bitmap_fill(ud->rflow_in_use, ud->rflow_cnt); 4974 for (i = 0; i < rm_res->sets; i++) 4975 udma_mark_resource_ranges(ud, ud->rflow_in_use, 4976 &rm_res->desc[i], "rflow"); 4977 irq_res.sets = rm_res->sets; 4978 } 4979 4980 /* tflow ranges */ 4981 rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW]; 4982 if (IS_ERR(rm_res)) { 4983 /* all tflows are assigned exclusively to Linux */ 4984 bitmap_zero(ud->tflow_map, ud->tflow_cnt); 4985 irq_res.sets++; 4986 } else { 4987 bitmap_fill(ud->tflow_map, ud->tflow_cnt); 4988 for (i = 0; i < rm_res->sets; i++) 4989 udma_mark_resource_ranges(ud, ud->tflow_map, 4990 &rm_res->desc[i], "tflow"); 4991 irq_res.sets += rm_res->sets; 4992 } 4993 4994 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 4995 if (!irq_res.desc) 4996 return -ENOMEM; 4997 rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW]; 4998 if (IS_ERR(rm_res)) { 4999 irq_res.desc[0].start = oes->pktdma_tchan_flow; 5000 irq_res.desc[0].num = ud->tflow_cnt; 5001 i = 1; 5002 } else { 5003 for (i = 0; i < rm_res->sets; i++) { 5004 irq_res.desc[i].start = rm_res->desc[i].start + 5005 oes->pktdma_tchan_flow; 5006 irq_res.desc[i].num = rm_res->desc[i].num; 5007 } 5008 } 5009 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 5010 if (IS_ERR(rm_res)) { 5011 irq_res.desc[i].start = oes->pktdma_rchan_flow; 5012 irq_res.desc[i].num = ud->rflow_cnt; 5013 } else { 5014 for (j = 0; j < rm_res->sets; j++, i++) { 5015 irq_res.desc[i].start = rm_res->desc[j].start + 5016 oes->pktdma_rchan_flow; 5017 irq_res.desc[i].num = rm_res->desc[j].num; 5018 } 5019 } 5020 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 5021 kfree(irq_res.desc); 5022 if (ret) { 5023 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 5024 return ret; 5025 } 5026 5027 return 0; 5028 } 5029 5030 static int setup_resources(struct udma_dev *ud) 5031 { 5032 struct device *dev = ud->dev; 5033 int ch_count, ret; 5034 5035 switch (ud->match_data->type) { 5036 case DMA_TYPE_UDMA: 5037 ret = udma_setup_resources(ud); 5038 break; 5039 case DMA_TYPE_BCDMA: 5040 ret = bcdma_setup_resources(ud); 5041 break; 5042 case DMA_TYPE_PKTDMA: 5043 ret = pktdma_setup_resources(ud); 5044 break; 5045 default: 5046 return -EINVAL; 5047 } 5048 5049 if (ret) 5050 return ret; 5051 5052 ch_count = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt; 5053 if (ud->bchan_cnt) 5054 ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt); 5055 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt); 5056 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt); 5057 if (!ch_count) 5058 return -ENODEV; 5059 5060 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels), 5061 GFP_KERNEL); 5062 if (!ud->channels) 5063 return -ENOMEM; 5064 5065 switch (ud->match_data->type) { 5066 case DMA_TYPE_UDMA: 5067 dev_info(dev, 5068 "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", 5069 ch_count, 5070 ud->tchan_cnt - bitmap_weight(ud->tchan_map, 5071 ud->tchan_cnt), 5072 ud->rchan_cnt - bitmap_weight(ud->rchan_map, 5073 ud->rchan_cnt), 5074 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map, 5075 ud->rflow_cnt)); 5076 break; 5077 case DMA_TYPE_BCDMA: 5078 dev_info(dev, 5079 "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n", 5080 ch_count, 5081 ud->bchan_cnt - bitmap_weight(ud->bchan_map, 5082 ud->bchan_cnt), 5083 ud->tchan_cnt - bitmap_weight(ud->tchan_map, 5084 ud->tchan_cnt), 5085 ud->rchan_cnt - bitmap_weight(ud->rchan_map, 5086 ud->rchan_cnt)); 5087 break; 5088 case DMA_TYPE_PKTDMA: 5089 dev_info(dev, 5090 "Channels: %d (tchan: %u, rchan: %u)\n", 5091 ch_count, 5092 ud->tchan_cnt - bitmap_weight(ud->tchan_map, 5093 ud->tchan_cnt), 5094 ud->rchan_cnt - bitmap_weight(ud->rchan_map, 5095 ud->rchan_cnt)); 5096 break; 5097 default: 5098 break; 5099 } 5100 5101 return ch_count; 5102 } 5103 5104 static int udma_setup_rx_flush(struct udma_dev *ud) 5105 { 5106 struct udma_rx_flush *rx_flush = &ud->rx_flush; 5107 struct cppi5_desc_hdr_t *tr_desc; 5108 struct cppi5_tr_type1_t *tr_req; 5109 struct cppi5_host_desc_t *desc; 5110 struct device *dev = ud->dev; 5111 struct udma_hwdesc *hwdesc; 5112 size_t tr_size; 5113 5114 /* Allocate 1K buffer for discarded data on RX channel teardown */ 5115 rx_flush->buffer_size = SZ_1K; 5116 rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size, 5117 GFP_KERNEL); 5118 if (!rx_flush->buffer_vaddr) 5119 return -ENOMEM; 5120 5121 rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr, 5122 rx_flush->buffer_size, 5123 DMA_TO_DEVICE); 5124 if (dma_mapping_error(dev, rx_flush->buffer_paddr)) 5125 return -ENOMEM; 5126 5127 /* Set up descriptor to be used for TR mode */ 5128 hwdesc = &rx_flush->hwdescs[0]; 5129 tr_size = sizeof(struct cppi5_tr_type1_t); 5130 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1); 5131 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 5132 ud->desc_align); 5133 5134 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 5135 GFP_KERNEL); 5136 if (!hwdesc->cppi5_desc_vaddr) 5137 return -ENOMEM; 5138 5139 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 5140 hwdesc->cppi5_desc_size, 5141 DMA_TO_DEVICE); 5142 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 5143 return -ENOMEM; 5144 5145 /* Start of the TR req records */ 5146 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 5147 /* Start address of the TR response array */ 5148 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size; 5149 5150 tr_desc = hwdesc->cppi5_desc_vaddr; 5151 cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0); 5152 cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 5153 cppi5_desc_set_retpolicy(tr_desc, 0, 0); 5154 5155 tr_req = hwdesc->tr_req_base; 5156 cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false, 5157 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 5158 cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT); 5159 5160 tr_req->addr = rx_flush->buffer_paddr; 5161 tr_req->icnt0 = rx_flush->buffer_size; 5162 tr_req->icnt1 = 1; 5163 5164 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, 5165 hwdesc->cppi5_desc_size, DMA_TO_DEVICE); 5166 5167 /* Set up descriptor to be used for packet mode */ 5168 hwdesc = &rx_flush->hwdescs[1]; 5169 hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 5170 CPPI5_INFO0_HDESC_EPIB_SIZE + 5171 CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE, 5172 ud->desc_align); 5173 5174 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, 5175 GFP_KERNEL); 5176 if (!hwdesc->cppi5_desc_vaddr) 5177 return -ENOMEM; 5178 5179 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, 5180 hwdesc->cppi5_desc_size, 5181 DMA_TO_DEVICE); 5182 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) 5183 return -ENOMEM; 5184 5185 desc = hwdesc->cppi5_desc_vaddr; 5186 cppi5_hdesc_init(desc, 0, 0); 5187 cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); 5188 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0); 5189 5190 cppi5_hdesc_attach_buf(desc, 5191 rx_flush->buffer_paddr, rx_flush->buffer_size, 5192 rx_flush->buffer_paddr, rx_flush->buffer_size); 5193 5194 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, 5195 hwdesc->cppi5_desc_size, DMA_TO_DEVICE); 5196 return 0; 5197 } 5198 5199 #ifdef CONFIG_DEBUG_FS 5200 static void udma_dbg_summary_show_chan(struct seq_file *s, 5201 struct dma_chan *chan) 5202 { 5203 struct udma_chan *uc = to_udma_chan(chan); 5204 struct udma_chan_config *ucc = &uc->config; 5205 5206 seq_printf(s, " %-13s| %s", dma_chan_name(chan), 5207 chan->dbg_client_name ?: "in-use"); 5208 if (ucc->tr_trigger_type) 5209 seq_puts(s, " (triggered, "); 5210 else 5211 seq_printf(s, " (%s, ", 5212 dmaengine_get_direction_text(uc->config.dir)); 5213 5214 switch (uc->config.dir) { 5215 case DMA_MEM_TO_MEM: 5216 if (uc->ud->match_data->type == DMA_TYPE_BCDMA) { 5217 seq_printf(s, "bchan%d)\n", uc->bchan->id); 5218 return; 5219 } 5220 5221 seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id, 5222 ucc->src_thread, ucc->dst_thread); 5223 break; 5224 case DMA_DEV_TO_MEM: 5225 seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id, 5226 ucc->src_thread, ucc->dst_thread); 5227 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA) 5228 seq_printf(s, "rflow%d, ", uc->rflow->id); 5229 break; 5230 case DMA_MEM_TO_DEV: 5231 seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id, 5232 ucc->src_thread, ucc->dst_thread); 5233 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA) 5234 seq_printf(s, "tflow%d, ", uc->tchan->tflow_id); 5235 break; 5236 default: 5237 seq_printf(s, ")\n"); 5238 return; 5239 } 5240 5241 if (ucc->ep_type == PSIL_EP_NATIVE) { 5242 seq_printf(s, "PSI-L Native"); 5243 if (ucc->metadata_size) { 5244 seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : ""); 5245 if (ucc->psd_size) 5246 seq_printf(s, " PSDsize:%u", ucc->psd_size); 5247 seq_printf(s, " ]"); 5248 } 5249 } else { 5250 seq_printf(s, "PDMA"); 5251 if (ucc->enable_acc32 || ucc->enable_burst) 5252 seq_printf(s, "[%s%s ]", 5253 ucc->enable_acc32 ? " ACC32" : "", 5254 ucc->enable_burst ? " BURST" : ""); 5255 } 5256 5257 seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode"); 5258 } 5259 5260 static void udma_dbg_summary_show(struct seq_file *s, 5261 struct dma_device *dma_dev) 5262 { 5263 struct dma_chan *chan; 5264 5265 list_for_each_entry(chan, &dma_dev->channels, device_node) { 5266 if (chan->client_count) 5267 udma_dbg_summary_show_chan(s, chan); 5268 } 5269 } 5270 #endif /* CONFIG_DEBUG_FS */ 5271 5272 static enum dmaengine_alignment udma_get_copy_align(struct udma_dev *ud) 5273 { 5274 const struct udma_match_data *match_data = ud->match_data; 5275 u8 tpl; 5276 5277 if (!match_data->enable_memcpy_support) 5278 return DMAENGINE_ALIGN_8_BYTES; 5279 5280 /* Get the highest TPL level the device supports for memcpy */ 5281 if (ud->bchan_cnt) 5282 tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, 0); 5283 else if (ud->tchan_cnt) 5284 tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, 0); 5285 else 5286 return DMAENGINE_ALIGN_8_BYTES; 5287 5288 switch (match_data->burst_size[tpl]) { 5289 case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES: 5290 return DMAENGINE_ALIGN_256_BYTES; 5291 case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES: 5292 return DMAENGINE_ALIGN_128_BYTES; 5293 case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES: 5294 fallthrough; 5295 default: 5296 return DMAENGINE_ALIGN_64_BYTES; 5297 } 5298 } 5299 5300 #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 5301 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 5302 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ 5303 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 5304 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 5305 5306 static int udma_probe(struct platform_device *pdev) 5307 { 5308 struct device_node *navss_node = pdev->dev.parent->of_node; 5309 const struct soc_device_attribute *soc; 5310 struct device *dev = &pdev->dev; 5311 struct udma_dev *ud; 5312 const struct of_device_id *match; 5313 int i, ret; 5314 int ch_count; 5315 5316 ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48)); 5317 if (ret) 5318 dev_err(dev, "failed to set dma mask stuff\n"); 5319 5320 ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL); 5321 if (!ud) 5322 return -ENOMEM; 5323 5324 match = of_match_node(udma_of_match, dev->of_node); 5325 if (!match) { 5326 dev_err(dev, "No compatible match found\n"); 5327 return -ENODEV; 5328 } 5329 ud->match_data = match->data; 5330 5331 ud->soc_data = ud->match_data->soc_data; 5332 if (!ud->soc_data) { 5333 soc = soc_device_match(k3_soc_devices); 5334 if (!soc) { 5335 dev_err(dev, "No compatible SoC found\n"); 5336 return -ENODEV; 5337 } 5338 ud->soc_data = soc->data; 5339 } 5340 5341 ret = udma_get_mmrs(pdev, ud); 5342 if (ret) 5343 return ret; 5344 5345 ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci"); 5346 if (IS_ERR(ud->tisci_rm.tisci)) 5347 return PTR_ERR(ud->tisci_rm.tisci); 5348 5349 ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id", 5350 &ud->tisci_rm.tisci_dev_id); 5351 if (ret) { 5352 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret); 5353 return ret; 5354 } 5355 pdev->id = ud->tisci_rm.tisci_dev_id; 5356 5357 ret = of_property_read_u32(navss_node, "ti,sci-dev-id", 5358 &ud->tisci_rm.tisci_navss_dev_id); 5359 if (ret) { 5360 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret); 5361 return ret; 5362 } 5363 5364 if (ud->match_data->type == DMA_TYPE_UDMA) { 5365 ret = of_property_read_u32(dev->of_node, "ti,udma-atype", 5366 &ud->atype); 5367 if (!ret && ud->atype > 2) { 5368 dev_err(dev, "Invalid atype: %u\n", ud->atype); 5369 return -EINVAL; 5370 } 5371 } else { 5372 ret = of_property_read_u32(dev->of_node, "ti,asel", 5373 &ud->asel); 5374 if (!ret && ud->asel > 15) { 5375 dev_err(dev, "Invalid asel: %u\n", ud->asel); 5376 return -EINVAL; 5377 } 5378 } 5379 5380 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; 5381 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; 5382 5383 if (ud->match_data->type == DMA_TYPE_UDMA) { 5384 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc"); 5385 } else { 5386 struct k3_ringacc_init_data ring_init_data; 5387 5388 ring_init_data.tisci = ud->tisci_rm.tisci; 5389 ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id; 5390 if (ud->match_data->type == DMA_TYPE_BCDMA) { 5391 ring_init_data.num_rings = ud->bchan_cnt + 5392 ud->tchan_cnt + 5393 ud->rchan_cnt; 5394 } else { 5395 ring_init_data.num_rings = ud->rflow_cnt + 5396 ud->tflow_cnt; 5397 } 5398 5399 ud->ringacc = k3_ringacc_dmarings_init(pdev, &ring_init_data); 5400 } 5401 5402 if (IS_ERR(ud->ringacc)) 5403 return PTR_ERR(ud->ringacc); 5404 5405 dev->msi.domain = of_msi_get_domain(dev, dev->of_node, 5406 DOMAIN_BUS_TI_SCI_INTA_MSI); 5407 if (!dev->msi.domain) { 5408 return -EPROBE_DEFER; 5409 } 5410 5411 dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask); 5412 /* cyclic operation is not supported via PKTDMA */ 5413 if (ud->match_data->type != DMA_TYPE_PKTDMA) { 5414 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); 5415 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; 5416 } 5417 5418 ud->ddev.device_config = udma_slave_config; 5419 ud->ddev.device_prep_slave_sg = udma_prep_slave_sg; 5420 ud->ddev.device_issue_pending = udma_issue_pending; 5421 ud->ddev.device_tx_status = udma_tx_status; 5422 ud->ddev.device_pause = udma_pause; 5423 ud->ddev.device_resume = udma_resume; 5424 ud->ddev.device_terminate_all = udma_terminate_all; 5425 ud->ddev.device_synchronize = udma_synchronize; 5426 #ifdef CONFIG_DEBUG_FS 5427 ud->ddev.dbg_summary_show = udma_dbg_summary_show; 5428 #endif 5429 5430 switch (ud->match_data->type) { 5431 case DMA_TYPE_UDMA: 5432 ud->ddev.device_alloc_chan_resources = 5433 udma_alloc_chan_resources; 5434 break; 5435 case DMA_TYPE_BCDMA: 5436 ud->ddev.device_alloc_chan_resources = 5437 bcdma_alloc_chan_resources; 5438 ud->ddev.device_router_config = bcdma_router_config; 5439 break; 5440 case DMA_TYPE_PKTDMA: 5441 ud->ddev.device_alloc_chan_resources = 5442 pktdma_alloc_chan_resources; 5443 break; 5444 default: 5445 return -EINVAL; 5446 } 5447 ud->ddev.device_free_chan_resources = udma_free_chan_resources; 5448 5449 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; 5450 ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; 5451 ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 5452 ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 5453 ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | 5454 DESC_METADATA_ENGINE; 5455 if (ud->match_data->enable_memcpy_support && 5456 !(ud->match_data->type == DMA_TYPE_BCDMA && ud->bchan_cnt == 0)) { 5457 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask); 5458 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy; 5459 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM); 5460 } 5461 5462 ud->ddev.dev = dev; 5463 ud->dev = dev; 5464 ud->psil_base = ud->match_data->psil_base; 5465 5466 INIT_LIST_HEAD(&ud->ddev.channels); 5467 INIT_LIST_HEAD(&ud->desc_to_purge); 5468 5469 ch_count = setup_resources(ud); 5470 if (ch_count <= 0) 5471 return ch_count; 5472 5473 spin_lock_init(&ud->lock); 5474 INIT_WORK(&ud->purge_work, udma_purge_desc_work); 5475 5476 ud->desc_align = 64; 5477 if (ud->desc_align < dma_get_cache_alignment()) 5478 ud->desc_align = dma_get_cache_alignment(); 5479 5480 ret = udma_setup_rx_flush(ud); 5481 if (ret) 5482 return ret; 5483 5484 for (i = 0; i < ud->bchan_cnt; i++) { 5485 struct udma_bchan *bchan = &ud->bchans[i]; 5486 5487 bchan->id = i; 5488 bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000; 5489 } 5490 5491 for (i = 0; i < ud->tchan_cnt; i++) { 5492 struct udma_tchan *tchan = &ud->tchans[i]; 5493 5494 tchan->id = i; 5495 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000; 5496 } 5497 5498 for (i = 0; i < ud->rchan_cnt; i++) { 5499 struct udma_rchan *rchan = &ud->rchans[i]; 5500 5501 rchan->id = i; 5502 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000; 5503 } 5504 5505 for (i = 0; i < ud->rflow_cnt; i++) { 5506 struct udma_rflow *rflow = &ud->rflows[i]; 5507 5508 rflow->id = i; 5509 } 5510 5511 for (i = 0; i < ch_count; i++) { 5512 struct udma_chan *uc = &ud->channels[i]; 5513 5514 uc->ud = ud; 5515 uc->vc.desc_free = udma_desc_free; 5516 uc->id = i; 5517 uc->bchan = NULL; 5518 uc->tchan = NULL; 5519 uc->rchan = NULL; 5520 uc->config.remote_thread_id = -1; 5521 uc->config.mapped_channel_id = -1; 5522 uc->config.default_flow_id = -1; 5523 uc->config.dir = DMA_MEM_TO_MEM; 5524 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", 5525 dev_name(dev), i); 5526 5527 vchan_init(&uc->vc, &ud->ddev); 5528 /* Use custom vchan completion handling */ 5529 tasklet_setup(&uc->vc.task, udma_vchan_complete); 5530 init_completion(&uc->teardown_completed); 5531 INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion); 5532 } 5533 5534 /* Configure the copy_align to the maximum burst size the device supports */ 5535 ud->ddev.copy_align = udma_get_copy_align(ud); 5536 5537 ret = dma_async_device_register(&ud->ddev); 5538 if (ret) { 5539 dev_err(dev, "failed to register slave DMA engine: %d\n", ret); 5540 return ret; 5541 } 5542 5543 platform_set_drvdata(pdev, ud); 5544 5545 ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud); 5546 if (ret) { 5547 dev_err(dev, "failed to register of_dma controller\n"); 5548 dma_async_device_unregister(&ud->ddev); 5549 } 5550 5551 return ret; 5552 } 5553 5554 static int __maybe_unused udma_pm_suspend(struct device *dev) 5555 { 5556 struct udma_dev *ud = dev_get_drvdata(dev); 5557 struct dma_device *dma_dev = &ud->ddev; 5558 struct dma_chan *chan; 5559 struct udma_chan *uc; 5560 5561 list_for_each_entry(chan, &dma_dev->channels, device_node) { 5562 if (chan->client_count) { 5563 uc = to_udma_chan(chan); 5564 /* backup the channel configuration */ 5565 memcpy(&uc->backup_config, &uc->config, 5566 sizeof(struct udma_chan_config)); 5567 dev_dbg(dev, "Suspending channel %s\n", 5568 dma_chan_name(chan)); 5569 ud->ddev.device_free_chan_resources(chan); 5570 } 5571 } 5572 5573 return 0; 5574 } 5575 5576 static int __maybe_unused udma_pm_resume(struct device *dev) 5577 { 5578 struct udma_dev *ud = dev_get_drvdata(dev); 5579 struct dma_device *dma_dev = &ud->ddev; 5580 struct dma_chan *chan; 5581 struct udma_chan *uc; 5582 int ret; 5583 5584 list_for_each_entry(chan, &dma_dev->channels, device_node) { 5585 if (chan->client_count) { 5586 uc = to_udma_chan(chan); 5587 /* restore the channel configuration */ 5588 memcpy(&uc->config, &uc->backup_config, 5589 sizeof(struct udma_chan_config)); 5590 dev_dbg(dev, "Resuming channel %s\n", 5591 dma_chan_name(chan)); 5592 ret = ud->ddev.device_alloc_chan_resources(chan); 5593 if (ret) 5594 return ret; 5595 } 5596 } 5597 5598 return 0; 5599 } 5600 5601 static const struct dev_pm_ops udma_pm_ops = { 5602 SET_LATE_SYSTEM_SLEEP_PM_OPS(udma_pm_suspend, udma_pm_resume) 5603 }; 5604 5605 static struct platform_driver udma_driver = { 5606 .driver = { 5607 .name = "ti-udma", 5608 .of_match_table = udma_of_match, 5609 .suppress_bind_attrs = true, 5610 .pm = &udma_pm_ops, 5611 }, 5612 .probe = udma_probe, 5613 }; 5614 5615 module_platform_driver(udma_driver); 5616 MODULE_LICENSE("GPL v2"); 5617 5618 /* Private interfaces to UDMA */ 5619 #include "k3-udma-private.c" 5620