1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/dmaengine.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/dmapool.h> 11 #include <linux/err.h> 12 #include <linux/init.h> 13 #include <linux/interrupt.h> 14 #include <linux/list.h> 15 #include <linux/platform_device.h> 16 #include <linux/slab.h> 17 #include <linux/spinlock.h> 18 #include <linux/of.h> 19 #include <linux/of_dma.h> 20 #include <linux/of_device.h> 21 #include <linux/of_irq.h> 22 #include <linux/workqueue.h> 23 #include <linux/completion.h> 24 #include <linux/soc/ti/k3-ringacc.h> 25 #include <linux/soc/ti/ti_sci_protocol.h> 26 #include <linux/soc/ti/ti_sci_inta_msi.h> 27 #include <linux/dma/ti-cppi5.h> 28 29 #include "../virt-dma.h" 30 #include "k3-udma.h" 31 #include "k3-psil-priv.h" 32 33 struct udma_static_tr { 34 u8 elsize; /* RPSTR0 */ 35 u16 elcnt; /* RPSTR0 */ 36 u16 bstcnt; /* RPSTR1 */ 37 }; 38 39 #define K3_UDMA_MAX_RFLOWS 1024 40 #define K3_UDMA_DEFAULT_RING_SIZE 16 41 42 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */ 43 #define UDMA_RFLOW_SRCTAG_NONE 0 44 #define UDMA_RFLOW_SRCTAG_CFG_TAG 1 45 #define UDMA_RFLOW_SRCTAG_FLOW_ID 2 46 #define UDMA_RFLOW_SRCTAG_SRC_TAG 4 47 48 #define UDMA_RFLOW_DSTTAG_NONE 0 49 #define UDMA_RFLOW_DSTTAG_CFG_TAG 1 50 #define UDMA_RFLOW_DSTTAG_FLOW_ID 2 51 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO 4 52 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI 5 53 54 struct udma_chan; 55 56 enum udma_mmr { 57 MMR_GCFG = 0, 58 MMR_RCHANRT, 59 MMR_TCHANRT, 60 MMR_LAST, 61 }; 62 63 static const char * const mmr_names[] = { "gcfg", "rchanrt", "tchanrt" }; 64 65 struct udma_tchan { 66 void __iomem *reg_rt; 67 68 int id; 69 struct k3_ring *t_ring; /* Transmit ring */ 70 struct k3_ring *tc_ring; /* Transmit Completion ring */ 71 }; 72 73 struct udma_rflow { 74 int id; 75 struct k3_ring *fd_ring; /* Free Descriptor ring */ 76 struct k3_ring *r_ring; /* Receive ring */ 77 }; 78 79 struct udma_rchan { 80 void __iomem *reg_rt; 81 82 int id; 83 }; 84 85 #define UDMA_FLAG_PDMA_ACC32 BIT(0) 86 #define UDMA_FLAG_PDMA_BURST BIT(1) 87 88 struct udma_match_data { 89 u32 psil_base; 90 bool enable_memcpy_support; 91 u32 flags; 92 u32 statictr_z_mask; 93 u32 rchan_oes_offset; 94 95 u8 tpl_levels; 96 u32 level_start_idx[]; 97 }; 98 99 struct udma_dev { 100 struct dma_device ddev; 101 struct device *dev; 102 void __iomem *mmrs[MMR_LAST]; 103 const struct udma_match_data *match_data; 104 105 size_t desc_align; /* alignment to use for descriptors */ 106 107 struct udma_tisci_rm tisci_rm; 108 109 struct k3_ringacc *ringacc; 110 111 struct work_struct purge_work; 112 struct list_head desc_to_purge; 113 spinlock_t lock; 114 115 int tchan_cnt; 116 int echan_cnt; 117 int rchan_cnt; 118 int rflow_cnt; 119 unsigned long *tchan_map; 120 unsigned long *rchan_map; 121 unsigned long *rflow_gp_map; 122 unsigned long *rflow_gp_map_allocated; 123 unsigned long *rflow_in_use; 124 125 struct udma_tchan *tchans; 126 struct udma_rchan *rchans; 127 struct udma_rflow *rflows; 128 129 struct udma_chan *channels; 130 u32 psil_base; 131 }; 132 133 struct udma_hwdesc { 134 size_t cppi5_desc_size; 135 void *cppi5_desc_vaddr; 136 dma_addr_t cppi5_desc_paddr; 137 138 /* TR descriptor internal pointers */ 139 void *tr_req_base; 140 struct cppi5_tr_resp_t *tr_resp_base; 141 }; 142 143 struct udma_desc { 144 struct virt_dma_desc vd; 145 146 bool terminated; 147 148 enum dma_transfer_direction dir; 149 150 struct udma_static_tr static_tr; 151 u32 residue; 152 153 unsigned int sglen; 154 unsigned int desc_idx; /* Only used for cyclic in packet mode */ 155 unsigned int tr_idx; 156 157 u32 metadata_size; 158 void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */ 159 160 unsigned int hwdesc_count; 161 struct udma_hwdesc hwdesc[0]; 162 }; 163 164 enum udma_chan_state { 165 UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */ 166 UDMA_CHAN_IS_ACTIVE, /* Normal operation */ 167 UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */ 168 }; 169 170 struct udma_tx_drain { 171 struct delayed_work work; 172 unsigned long jiffie; 173 u32 residue; 174 }; 175 176 struct udma_chan_config { 177 bool pkt_mode; /* TR or packet */ 178 bool needs_epib; /* EPIB is needed for the communication or not */ 179 u32 psd_size; /* size of Protocol Specific Data */ 180 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */ 181 u32 hdesc_size; /* Size of a packet descriptor in packet mode */ 182 bool notdpkt; /* Suppress sending TDC packet */ 183 int remote_thread_id; 184 u32 src_thread; 185 u32 dst_thread; 186 enum psil_endpoint_type ep_type; 187 bool enable_acc32; 188 bool enable_burst; 189 enum udma_tp_level channel_tpl; /* Channel Throughput Level */ 190 191 enum dma_transfer_direction dir; 192 }; 193 194 struct udma_chan { 195 struct virt_dma_chan vc; 196 struct dma_slave_config cfg; 197 struct udma_dev *ud; 198 struct udma_desc *desc; 199 struct udma_desc *terminated_desc; 200 struct udma_static_tr static_tr; 201 char *name; 202 203 struct udma_tchan *tchan; 204 struct udma_rchan *rchan; 205 struct udma_rflow *rflow; 206 207 bool psil_paired; 208 209 int irq_num_ring; 210 int irq_num_udma; 211 212 bool cyclic; 213 bool paused; 214 215 enum udma_chan_state state; 216 struct completion teardown_completed; 217 218 struct udma_tx_drain tx_drain; 219 220 u32 bcnt; /* number of bytes completed since the start of the channel */ 221 u32 in_ring_cnt; /* number of descriptors in flight */ 222 223 /* Channel configuration parameters */ 224 struct udma_chan_config config; 225 226 /* dmapool for packet mode descriptors */ 227 bool use_dma_pool; 228 struct dma_pool *hdesc_pool; 229 230 u32 id; 231 }; 232 233 static inline struct udma_dev *to_udma_dev(struct dma_device *d) 234 { 235 return container_of(d, struct udma_dev, ddev); 236 } 237 238 static inline struct udma_chan *to_udma_chan(struct dma_chan *c) 239 { 240 return container_of(c, struct udma_chan, vc.chan); 241 } 242 243 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t) 244 { 245 return container_of(t, struct udma_desc, vd.tx); 246 } 247 248 /* Generic register access functions */ 249 static inline u32 udma_read(void __iomem *base, int reg) 250 { 251 return readl(base + reg); 252 } 253 254 static inline void udma_write(void __iomem *base, int reg, u32 val) 255 { 256 writel(val, base + reg); 257 } 258 259 static inline void udma_update_bits(void __iomem *base, int reg, 260 u32 mask, u32 val) 261 { 262 u32 tmp, orig; 263 264 orig = readl(base + reg); 265 tmp = orig & ~mask; 266 tmp |= (val & mask); 267 268 if (tmp != orig) 269 writel(tmp, base + reg); 270 } 271 272 /* TCHANRT */ 273 static inline u32 udma_tchanrt_read(struct udma_tchan *tchan, int reg) 274 { 275 if (!tchan) 276 return 0; 277 return udma_read(tchan->reg_rt, reg); 278 } 279 280 static inline void udma_tchanrt_write(struct udma_tchan *tchan, int reg, 281 u32 val) 282 { 283 if (!tchan) 284 return; 285 udma_write(tchan->reg_rt, reg, val); 286 } 287 288 static inline void udma_tchanrt_update_bits(struct udma_tchan *tchan, int reg, 289 u32 mask, u32 val) 290 { 291 if (!tchan) 292 return; 293 udma_update_bits(tchan->reg_rt, reg, mask, val); 294 } 295 296 /* RCHANRT */ 297 static inline u32 udma_rchanrt_read(struct udma_rchan *rchan, int reg) 298 { 299 if (!rchan) 300 return 0; 301 return udma_read(rchan->reg_rt, reg); 302 } 303 304 static inline void udma_rchanrt_write(struct udma_rchan *rchan, int reg, 305 u32 val) 306 { 307 if (!rchan) 308 return; 309 udma_write(rchan->reg_rt, reg, val); 310 } 311 312 static inline void udma_rchanrt_update_bits(struct udma_rchan *rchan, int reg, 313 u32 mask, u32 val) 314 { 315 if (!rchan) 316 return; 317 udma_update_bits(rchan->reg_rt, reg, mask, val); 318 } 319 320 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) 321 { 322 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 323 324 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 325 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci, 326 tisci_rm->tisci_navss_dev_id, 327 src_thread, dst_thread); 328 } 329 330 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread, 331 u32 dst_thread) 332 { 333 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 334 335 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 336 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci, 337 tisci_rm->tisci_navss_dev_id, 338 src_thread, dst_thread); 339 } 340 341 static void udma_reset_uchan(struct udma_chan *uc) 342 { 343 memset(&uc->config, 0, sizeof(uc->config)); 344 uc->config.remote_thread_id = -1; 345 uc->state = UDMA_CHAN_IS_IDLE; 346 } 347 348 static void udma_dump_chan_stdata(struct udma_chan *uc) 349 { 350 struct device *dev = uc->ud->dev; 351 u32 offset; 352 int i; 353 354 if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) { 355 dev_dbg(dev, "TCHAN State data:\n"); 356 for (i = 0; i < 32; i++) { 357 offset = UDMA_TCHAN_RT_STDATA_REG + i * 4; 358 dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i, 359 udma_tchanrt_read(uc->tchan, offset)); 360 } 361 } 362 363 if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) { 364 dev_dbg(dev, "RCHAN State data:\n"); 365 for (i = 0; i < 32; i++) { 366 offset = UDMA_RCHAN_RT_STDATA_REG + i * 4; 367 dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i, 368 udma_rchanrt_read(uc->rchan, offset)); 369 } 370 } 371 } 372 373 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d, 374 int idx) 375 { 376 return d->hwdesc[idx].cppi5_desc_paddr; 377 } 378 379 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx) 380 { 381 return d->hwdesc[idx].cppi5_desc_vaddr; 382 } 383 384 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc, 385 dma_addr_t paddr) 386 { 387 struct udma_desc *d = uc->terminated_desc; 388 389 if (d) { 390 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 391 d->desc_idx); 392 393 if (desc_paddr != paddr) 394 d = NULL; 395 } 396 397 if (!d) { 398 d = uc->desc; 399 if (d) { 400 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 401 d->desc_idx); 402 403 if (desc_paddr != paddr) 404 d = NULL; 405 } 406 } 407 408 return d; 409 } 410 411 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d) 412 { 413 if (uc->use_dma_pool) { 414 int i; 415 416 for (i = 0; i < d->hwdesc_count; i++) { 417 if (!d->hwdesc[i].cppi5_desc_vaddr) 418 continue; 419 420 dma_pool_free(uc->hdesc_pool, 421 d->hwdesc[i].cppi5_desc_vaddr, 422 d->hwdesc[i].cppi5_desc_paddr); 423 424 d->hwdesc[i].cppi5_desc_vaddr = NULL; 425 } 426 } else if (d->hwdesc[0].cppi5_desc_vaddr) { 427 struct udma_dev *ud = uc->ud; 428 429 dma_free_coherent(ud->dev, d->hwdesc[0].cppi5_desc_size, 430 d->hwdesc[0].cppi5_desc_vaddr, 431 d->hwdesc[0].cppi5_desc_paddr); 432 433 d->hwdesc[0].cppi5_desc_vaddr = NULL; 434 } 435 } 436 437 static void udma_purge_desc_work(struct work_struct *work) 438 { 439 struct udma_dev *ud = container_of(work, typeof(*ud), purge_work); 440 struct virt_dma_desc *vd, *_vd; 441 unsigned long flags; 442 LIST_HEAD(head); 443 444 spin_lock_irqsave(&ud->lock, flags); 445 list_splice_tail_init(&ud->desc_to_purge, &head); 446 spin_unlock_irqrestore(&ud->lock, flags); 447 448 list_for_each_entry_safe(vd, _vd, &head, node) { 449 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 450 struct udma_desc *d = to_udma_desc(&vd->tx); 451 452 udma_free_hwdesc(uc, d); 453 list_del(&vd->node); 454 kfree(d); 455 } 456 457 /* If more to purge, schedule the work again */ 458 if (!list_empty(&ud->desc_to_purge)) 459 schedule_work(&ud->purge_work); 460 } 461 462 static void udma_desc_free(struct virt_dma_desc *vd) 463 { 464 struct udma_dev *ud = to_udma_dev(vd->tx.chan->device); 465 struct udma_chan *uc = to_udma_chan(vd->tx.chan); 466 struct udma_desc *d = to_udma_desc(&vd->tx); 467 unsigned long flags; 468 469 if (uc->terminated_desc == d) 470 uc->terminated_desc = NULL; 471 472 if (uc->use_dma_pool) { 473 udma_free_hwdesc(uc, d); 474 kfree(d); 475 return; 476 } 477 478 spin_lock_irqsave(&ud->lock, flags); 479 list_add_tail(&vd->node, &ud->desc_to_purge); 480 spin_unlock_irqrestore(&ud->lock, flags); 481 482 schedule_work(&ud->purge_work); 483 } 484 485 static bool udma_is_chan_running(struct udma_chan *uc) 486 { 487 u32 trt_ctl = 0; 488 u32 rrt_ctl = 0; 489 490 if (uc->tchan) 491 trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG); 492 if (uc->rchan) 493 rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG); 494 495 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN) 496 return true; 497 498 return false; 499 } 500 501 static bool udma_is_chan_paused(struct udma_chan *uc) 502 { 503 u32 val, pause_mask; 504 505 switch (uc->desc->dir) { 506 case DMA_DEV_TO_MEM: 507 val = udma_rchanrt_read(uc->rchan, 508 UDMA_RCHAN_RT_PEER_RT_EN_REG); 509 pause_mask = UDMA_PEER_RT_EN_PAUSE; 510 break; 511 case DMA_MEM_TO_DEV: 512 val = udma_tchanrt_read(uc->tchan, 513 UDMA_TCHAN_RT_PEER_RT_EN_REG); 514 pause_mask = UDMA_PEER_RT_EN_PAUSE; 515 break; 516 case DMA_MEM_TO_MEM: 517 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG); 518 pause_mask = UDMA_CHAN_RT_CTL_PAUSE; 519 break; 520 default: 521 return false; 522 } 523 524 if (val & pause_mask) 525 return true; 526 527 return false; 528 } 529 530 static void udma_sync_for_device(struct udma_chan *uc, int idx) 531 { 532 struct udma_desc *d = uc->desc; 533 534 if (uc->cyclic && uc->config.pkt_mode) { 535 dma_sync_single_for_device(uc->ud->dev, 536 d->hwdesc[idx].cppi5_desc_paddr, 537 d->hwdesc[idx].cppi5_desc_size, 538 DMA_TO_DEVICE); 539 } else { 540 int i; 541 542 for (i = 0; i < d->hwdesc_count; i++) { 543 if (!d->hwdesc[i].cppi5_desc_vaddr) 544 continue; 545 546 dma_sync_single_for_device(uc->ud->dev, 547 d->hwdesc[i].cppi5_desc_paddr, 548 d->hwdesc[i].cppi5_desc_size, 549 DMA_TO_DEVICE); 550 } 551 } 552 } 553 554 static int udma_push_to_ring(struct udma_chan *uc, int idx) 555 { 556 struct udma_desc *d = uc->desc; 557 558 struct k3_ring *ring = NULL; 559 int ret = -EINVAL; 560 561 switch (uc->config.dir) { 562 case DMA_DEV_TO_MEM: 563 ring = uc->rflow->fd_ring; 564 break; 565 case DMA_MEM_TO_DEV: 566 case DMA_MEM_TO_MEM: 567 ring = uc->tchan->t_ring; 568 break; 569 default: 570 break; 571 } 572 573 if (ring) { 574 dma_addr_t desc_addr = udma_curr_cppi5_desc_paddr(d, idx); 575 576 wmb(); /* Ensure that writes are not moved over this point */ 577 udma_sync_for_device(uc, idx); 578 ret = k3_ringacc_ring_push(ring, &desc_addr); 579 uc->in_ring_cnt++; 580 } 581 582 return ret; 583 } 584 585 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) 586 { 587 struct k3_ring *ring = NULL; 588 int ret = -ENOENT; 589 590 switch (uc->config.dir) { 591 case DMA_DEV_TO_MEM: 592 ring = uc->rflow->r_ring; 593 break; 594 case DMA_MEM_TO_DEV: 595 case DMA_MEM_TO_MEM: 596 ring = uc->tchan->tc_ring; 597 break; 598 default: 599 break; 600 } 601 602 if (ring && k3_ringacc_ring_get_occ(ring)) { 603 struct udma_desc *d = NULL; 604 605 ret = k3_ringacc_ring_pop(ring, addr); 606 if (ret) 607 return ret; 608 609 /* Teardown completion */ 610 if (cppi5_desc_is_tdcm(*addr)) 611 return ret; 612 613 d = udma_udma_desc_from_paddr(uc, *addr); 614 615 if (d) 616 dma_sync_single_for_cpu(uc->ud->dev, *addr, 617 d->hwdesc[0].cppi5_desc_size, 618 DMA_FROM_DEVICE); 619 rmb(); /* Ensure that reads are not moved before this point */ 620 621 if (!ret) 622 uc->in_ring_cnt--; 623 } 624 625 return ret; 626 } 627 628 static void udma_reset_rings(struct udma_chan *uc) 629 { 630 struct k3_ring *ring1 = NULL; 631 struct k3_ring *ring2 = NULL; 632 633 switch (uc->config.dir) { 634 case DMA_DEV_TO_MEM: 635 if (uc->rchan) { 636 ring1 = uc->rflow->fd_ring; 637 ring2 = uc->rflow->r_ring; 638 } 639 break; 640 case DMA_MEM_TO_DEV: 641 case DMA_MEM_TO_MEM: 642 if (uc->tchan) { 643 ring1 = uc->tchan->t_ring; 644 ring2 = uc->tchan->tc_ring; 645 } 646 break; 647 default: 648 break; 649 } 650 651 if (ring1) 652 k3_ringacc_ring_reset_dma(ring1, 653 k3_ringacc_ring_get_occ(ring1)); 654 if (ring2) 655 k3_ringacc_ring_reset(ring2); 656 657 /* make sure we are not leaking memory by stalled descriptor */ 658 if (uc->terminated_desc) { 659 udma_desc_free(&uc->terminated_desc->vd); 660 uc->terminated_desc = NULL; 661 } 662 663 uc->in_ring_cnt = 0; 664 } 665 666 static void udma_reset_counters(struct udma_chan *uc) 667 { 668 u32 val; 669 670 if (uc->tchan) { 671 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); 672 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_BCNT_REG, val); 673 674 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG); 675 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG, val); 676 677 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PCNT_REG); 678 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PCNT_REG, val); 679 680 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); 681 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG, val); 682 } 683 684 if (uc->rchan) { 685 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_BCNT_REG); 686 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_BCNT_REG, val); 687 688 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG); 689 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG, val); 690 691 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PCNT_REG); 692 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PCNT_REG, val); 693 694 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG); 695 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG, val); 696 } 697 698 uc->bcnt = 0; 699 } 700 701 static int udma_reset_chan(struct udma_chan *uc, bool hard) 702 { 703 switch (uc->config.dir) { 704 case DMA_DEV_TO_MEM: 705 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 0); 706 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0); 707 break; 708 case DMA_MEM_TO_DEV: 709 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0); 710 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 0); 711 break; 712 case DMA_MEM_TO_MEM: 713 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0); 714 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0); 715 break; 716 default: 717 return -EINVAL; 718 } 719 720 /* Reset all counters */ 721 udma_reset_counters(uc); 722 723 /* Hard reset: re-initialize the channel to reset */ 724 if (hard) { 725 struct udma_chan_config ucc_backup; 726 int ret; 727 728 memcpy(&ucc_backup, &uc->config, sizeof(uc->config)); 729 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan); 730 731 /* restore the channel configuration */ 732 memcpy(&uc->config, &ucc_backup, sizeof(uc->config)); 733 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan); 734 if (ret) 735 return ret; 736 737 /* 738 * Setting forced teardown after forced reset helps recovering 739 * the rchan. 740 */ 741 if (uc->config.dir == DMA_DEV_TO_MEM) 742 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 743 UDMA_CHAN_RT_CTL_EN | 744 UDMA_CHAN_RT_CTL_TDOWN | 745 UDMA_CHAN_RT_CTL_FTDOWN); 746 } 747 uc->state = UDMA_CHAN_IS_IDLE; 748 749 return 0; 750 } 751 752 static void udma_start_desc(struct udma_chan *uc) 753 { 754 struct udma_chan_config *ucc = &uc->config; 755 756 if (ucc->pkt_mode && (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) { 757 int i; 758 759 /* Push all descriptors to ring for packet mode cyclic or RX */ 760 for (i = 0; i < uc->desc->sglen; i++) 761 udma_push_to_ring(uc, i); 762 } else { 763 udma_push_to_ring(uc, 0); 764 } 765 } 766 767 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc) 768 { 769 /* Only PDMAs have staticTR */ 770 if (uc->config.ep_type == PSIL_EP_NATIVE) 771 return false; 772 773 /* Check if the staticTR configuration has changed for TX */ 774 if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr))) 775 return true; 776 777 return false; 778 } 779 780 static int udma_start(struct udma_chan *uc) 781 { 782 struct virt_dma_desc *vd = vchan_next_desc(&uc->vc); 783 784 if (!vd) { 785 uc->desc = NULL; 786 return -ENOENT; 787 } 788 789 list_del(&vd->node); 790 791 uc->desc = to_udma_desc(&vd->tx); 792 793 /* Channel is already running and does not need reconfiguration */ 794 if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) { 795 udma_start_desc(uc); 796 goto out; 797 } 798 799 /* Make sure that we clear the teardown bit, if it is set */ 800 udma_reset_chan(uc, false); 801 802 /* Push descriptors before we start the channel */ 803 udma_start_desc(uc); 804 805 switch (uc->desc->dir) { 806 case DMA_DEV_TO_MEM: 807 /* Config remote TR */ 808 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 809 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 810 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 811 const struct udma_match_data *match_data = 812 uc->ud->match_data; 813 814 if (uc->config.enable_acc32) 815 val |= PDMA_STATIC_TR_XY_ACC32; 816 if (uc->config.enable_burst) 817 val |= PDMA_STATIC_TR_XY_BURST; 818 819 udma_rchanrt_write(uc->rchan, 820 UDMA_RCHAN_RT_PEER_STATIC_TR_XY_REG, val); 821 822 udma_rchanrt_write(uc->rchan, 823 UDMA_RCHAN_RT_PEER_STATIC_TR_Z_REG, 824 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt, 825 match_data->statictr_z_mask)); 826 827 /* save the current staticTR configuration */ 828 memcpy(&uc->static_tr, &uc->desc->static_tr, 829 sizeof(uc->static_tr)); 830 } 831 832 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 833 UDMA_CHAN_RT_CTL_EN); 834 835 /* Enable remote */ 836 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 837 UDMA_PEER_RT_EN_ENABLE); 838 839 break; 840 case DMA_MEM_TO_DEV: 841 /* Config remote TR */ 842 if (uc->config.ep_type == PSIL_EP_PDMA_XY) { 843 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) | 844 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize); 845 846 if (uc->config.enable_acc32) 847 val |= PDMA_STATIC_TR_XY_ACC32; 848 if (uc->config.enable_burst) 849 val |= PDMA_STATIC_TR_XY_BURST; 850 851 udma_tchanrt_write(uc->tchan, 852 UDMA_TCHAN_RT_PEER_STATIC_TR_XY_REG, val); 853 854 /* save the current staticTR configuration */ 855 memcpy(&uc->static_tr, &uc->desc->static_tr, 856 sizeof(uc->static_tr)); 857 } 858 859 /* Enable remote */ 860 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 861 UDMA_PEER_RT_EN_ENABLE); 862 863 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 864 UDMA_CHAN_RT_CTL_EN); 865 866 break; 867 case DMA_MEM_TO_MEM: 868 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 869 UDMA_CHAN_RT_CTL_EN); 870 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 871 UDMA_CHAN_RT_CTL_EN); 872 873 break; 874 default: 875 return -EINVAL; 876 } 877 878 uc->state = UDMA_CHAN_IS_ACTIVE; 879 out: 880 881 return 0; 882 } 883 884 static int udma_stop(struct udma_chan *uc) 885 { 886 enum udma_chan_state old_state = uc->state; 887 888 uc->state = UDMA_CHAN_IS_TERMINATING; 889 reinit_completion(&uc->teardown_completed); 890 891 switch (uc->config.dir) { 892 case DMA_DEV_TO_MEM: 893 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 894 UDMA_PEER_RT_EN_ENABLE | 895 UDMA_PEER_RT_EN_TEARDOWN); 896 break; 897 case DMA_MEM_TO_DEV: 898 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 899 UDMA_PEER_RT_EN_ENABLE | 900 UDMA_PEER_RT_EN_FLUSH); 901 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 902 UDMA_CHAN_RT_CTL_EN | 903 UDMA_CHAN_RT_CTL_TDOWN); 904 break; 905 case DMA_MEM_TO_MEM: 906 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 907 UDMA_CHAN_RT_CTL_EN | 908 UDMA_CHAN_RT_CTL_TDOWN); 909 break; 910 default: 911 uc->state = old_state; 912 complete_all(&uc->teardown_completed); 913 return -EINVAL; 914 } 915 916 return 0; 917 } 918 919 static void udma_cyclic_packet_elapsed(struct udma_chan *uc) 920 { 921 struct udma_desc *d = uc->desc; 922 struct cppi5_host_desc_t *h_desc; 923 924 h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr; 925 cppi5_hdesc_reset_to_original(h_desc); 926 udma_push_to_ring(uc, d->desc_idx); 927 d->desc_idx = (d->desc_idx + 1) % d->sglen; 928 } 929 930 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d) 931 { 932 struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr; 933 934 memcpy(d->metadata, h_desc->epib, d->metadata_size); 935 } 936 937 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) 938 { 939 u32 peer_bcnt, bcnt; 940 941 /* Only TX towards PDMA is affected */ 942 if (uc->config.ep_type == PSIL_EP_NATIVE || 943 uc->config.dir != DMA_MEM_TO_DEV) 944 return true; 945 946 peer_bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); 947 bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); 948 949 if (peer_bcnt < bcnt) { 950 uc->tx_drain.residue = bcnt - peer_bcnt; 951 uc->tx_drain.jiffie = jiffies; 952 return false; 953 } 954 955 return true; 956 } 957 958 static void udma_check_tx_completion(struct work_struct *work) 959 { 960 struct udma_chan *uc = container_of(work, typeof(*uc), 961 tx_drain.work.work); 962 bool desc_done = true; 963 u32 residue_diff; 964 unsigned long jiffie_diff, delay; 965 966 if (uc->desc) { 967 residue_diff = uc->tx_drain.residue; 968 jiffie_diff = uc->tx_drain.jiffie; 969 desc_done = udma_is_desc_really_done(uc, uc->desc); 970 } 971 972 if (!desc_done) { 973 jiffie_diff = uc->tx_drain.jiffie - jiffie_diff; 974 residue_diff -= uc->tx_drain.residue; 975 if (residue_diff) { 976 /* Try to guess when we should check next time */ 977 residue_diff /= jiffie_diff; 978 delay = uc->tx_drain.residue / residue_diff / 3; 979 if (jiffies_to_msecs(delay) < 5) 980 delay = 0; 981 } else { 982 /* No progress, check again in 1 second */ 983 delay = HZ; 984 } 985 986 schedule_delayed_work(&uc->tx_drain.work, delay); 987 } else if (uc->desc) { 988 struct udma_desc *d = uc->desc; 989 990 uc->bcnt += d->residue; 991 udma_start(uc); 992 vchan_cookie_complete(&d->vd); 993 } 994 } 995 996 static irqreturn_t udma_ring_irq_handler(int irq, void *data) 997 { 998 struct udma_chan *uc = data; 999 struct udma_desc *d; 1000 unsigned long flags; 1001 dma_addr_t paddr = 0; 1002 1003 if (udma_pop_from_ring(uc, &paddr) || !paddr) 1004 return IRQ_HANDLED; 1005 1006 spin_lock_irqsave(&uc->vc.lock, flags); 1007 1008 /* Teardown completion message */ 1009 if (cppi5_desc_is_tdcm(paddr)) { 1010 /* Compensate our internal pop/push counter */ 1011 uc->in_ring_cnt++; 1012 1013 complete_all(&uc->teardown_completed); 1014 1015 if (uc->terminated_desc) { 1016 udma_desc_free(&uc->terminated_desc->vd); 1017 uc->terminated_desc = NULL; 1018 } 1019 1020 if (!uc->desc) 1021 udma_start(uc); 1022 1023 goto out; 1024 } 1025 1026 d = udma_udma_desc_from_paddr(uc, paddr); 1027 1028 if (d) { 1029 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d, 1030 d->desc_idx); 1031 if (desc_paddr != paddr) { 1032 dev_err(uc->ud->dev, "not matching descriptors!\n"); 1033 goto out; 1034 } 1035 1036 if (uc->cyclic) { 1037 /* push the descriptor back to the ring */ 1038 if (d == uc->desc) { 1039 udma_cyclic_packet_elapsed(uc); 1040 vchan_cyclic_callback(&d->vd); 1041 } 1042 } else { 1043 bool desc_done = false; 1044 1045 if (d == uc->desc) { 1046 desc_done = udma_is_desc_really_done(uc, d); 1047 1048 if (desc_done) { 1049 uc->bcnt += d->residue; 1050 udma_start(uc); 1051 } else { 1052 schedule_delayed_work(&uc->tx_drain.work, 1053 0); 1054 } 1055 } 1056 1057 if (desc_done) 1058 vchan_cookie_complete(&d->vd); 1059 } 1060 } 1061 out: 1062 spin_unlock_irqrestore(&uc->vc.lock, flags); 1063 1064 return IRQ_HANDLED; 1065 } 1066 1067 static irqreturn_t udma_udma_irq_handler(int irq, void *data) 1068 { 1069 struct udma_chan *uc = data; 1070 struct udma_desc *d; 1071 unsigned long flags; 1072 1073 spin_lock_irqsave(&uc->vc.lock, flags); 1074 d = uc->desc; 1075 if (d) { 1076 d->tr_idx = (d->tr_idx + 1) % d->sglen; 1077 1078 if (uc->cyclic) { 1079 vchan_cyclic_callback(&d->vd); 1080 } else { 1081 /* TODO: figure out the real amount of data */ 1082 uc->bcnt += d->residue; 1083 udma_start(uc); 1084 vchan_cookie_complete(&d->vd); 1085 } 1086 } 1087 1088 spin_unlock_irqrestore(&uc->vc.lock, flags); 1089 1090 return IRQ_HANDLED; 1091 } 1092 1093 /** 1094 * __udma_alloc_gp_rflow_range - alloc range of GP RX flows 1095 * @ud: UDMA device 1096 * @from: Start the search from this flow id number 1097 * @cnt: Number of consecutive flow ids to allocate 1098 * 1099 * Allocate range of RX flow ids for future use, those flows can be requested 1100 * only using explicit flow id number. if @from is set to -1 it will try to find 1101 * first free range. if @from is positive value it will force allocation only 1102 * of the specified range of flows. 1103 * 1104 * Returns -ENOMEM if can't find free range. 1105 * -EEXIST if requested range is busy. 1106 * -EINVAL if wrong input values passed. 1107 * Returns flow id on success. 1108 */ 1109 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1110 { 1111 int start, tmp_from; 1112 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS); 1113 1114 tmp_from = from; 1115 if (tmp_from < 0) 1116 tmp_from = ud->rchan_cnt; 1117 /* default flows can't be allocated and accessible only by id */ 1118 if (tmp_from < ud->rchan_cnt) 1119 return -EINVAL; 1120 1121 if (tmp_from + cnt > ud->rflow_cnt) 1122 return -EINVAL; 1123 1124 bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated, 1125 ud->rflow_cnt); 1126 1127 start = bitmap_find_next_zero_area(tmp, 1128 ud->rflow_cnt, 1129 tmp_from, cnt, 0); 1130 if (start >= ud->rflow_cnt) 1131 return -ENOMEM; 1132 1133 if (from >= 0 && start != from) 1134 return -EEXIST; 1135 1136 bitmap_set(ud->rflow_gp_map_allocated, start, cnt); 1137 return start; 1138 } 1139 1140 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt) 1141 { 1142 if (from < ud->rchan_cnt) 1143 return -EINVAL; 1144 if (from + cnt > ud->rflow_cnt) 1145 return -EINVAL; 1146 1147 bitmap_clear(ud->rflow_gp_map_allocated, from, cnt); 1148 return 0; 1149 } 1150 1151 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id) 1152 { 1153 /* 1154 * Attempt to request rflow by ID can be made for any rflow 1155 * if not in use with assumption that caller knows what's doing. 1156 * TI-SCI FW will perform additional permission check ant way, it's 1157 * safe 1158 */ 1159 1160 if (id < 0 || id >= ud->rflow_cnt) 1161 return ERR_PTR(-ENOENT); 1162 1163 if (test_bit(id, ud->rflow_in_use)) 1164 return ERR_PTR(-ENOENT); 1165 1166 /* GP rflow has to be allocated first */ 1167 if (!test_bit(id, ud->rflow_gp_map) && 1168 !test_bit(id, ud->rflow_gp_map_allocated)) 1169 return ERR_PTR(-EINVAL); 1170 1171 dev_dbg(ud->dev, "get rflow%d\n", id); 1172 set_bit(id, ud->rflow_in_use); 1173 return &ud->rflows[id]; 1174 } 1175 1176 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow) 1177 { 1178 if (!test_bit(rflow->id, ud->rflow_in_use)) { 1179 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id); 1180 return; 1181 } 1182 1183 dev_dbg(ud->dev, "put rflow%d\n", rflow->id); 1184 clear_bit(rflow->id, ud->rflow_in_use); 1185 } 1186 1187 #define UDMA_RESERVE_RESOURCE(res) \ 1188 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ 1189 enum udma_tp_level tpl, \ 1190 int id) \ 1191 { \ 1192 if (id >= 0) { \ 1193 if (test_bit(id, ud->res##_map)) { \ 1194 dev_err(ud->dev, "res##%d is in use\n", id); \ 1195 return ERR_PTR(-ENOENT); \ 1196 } \ 1197 } else { \ 1198 int start; \ 1199 \ 1200 if (tpl >= ud->match_data->tpl_levels) \ 1201 tpl = ud->match_data->tpl_levels - 1; \ 1202 \ 1203 start = ud->match_data->level_start_idx[tpl]; \ 1204 \ 1205 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \ 1206 start); \ 1207 if (id == ud->res##_cnt) { \ 1208 return ERR_PTR(-ENOENT); \ 1209 } \ 1210 } \ 1211 \ 1212 set_bit(id, ud->res##_map); \ 1213 return &ud->res##s[id]; \ 1214 } 1215 1216 UDMA_RESERVE_RESOURCE(tchan); 1217 UDMA_RESERVE_RESOURCE(rchan); 1218 1219 static int udma_get_tchan(struct udma_chan *uc) 1220 { 1221 struct udma_dev *ud = uc->ud; 1222 1223 if (uc->tchan) { 1224 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n", 1225 uc->id, uc->tchan->id); 1226 return 0; 1227 } 1228 1229 uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, -1); 1230 if (IS_ERR(uc->tchan)) 1231 return PTR_ERR(uc->tchan); 1232 1233 return 0; 1234 } 1235 1236 static int udma_get_rchan(struct udma_chan *uc) 1237 { 1238 struct udma_dev *ud = uc->ud; 1239 1240 if (uc->rchan) { 1241 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n", 1242 uc->id, uc->rchan->id); 1243 return 0; 1244 } 1245 1246 uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, -1); 1247 if (IS_ERR(uc->rchan)) 1248 return PTR_ERR(uc->rchan); 1249 1250 return 0; 1251 } 1252 1253 static int udma_get_chan_pair(struct udma_chan *uc) 1254 { 1255 struct udma_dev *ud = uc->ud; 1256 const struct udma_match_data *match_data = ud->match_data; 1257 int chan_id, end; 1258 1259 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) { 1260 dev_info(ud->dev, "chan%d: already have %d pair allocated\n", 1261 uc->id, uc->tchan->id); 1262 return 0; 1263 } 1264 1265 if (uc->tchan) { 1266 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n", 1267 uc->id, uc->tchan->id); 1268 return -EBUSY; 1269 } else if (uc->rchan) { 1270 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n", 1271 uc->id, uc->rchan->id); 1272 return -EBUSY; 1273 } 1274 1275 /* Can be optimized, but let's have it like this for now */ 1276 end = min(ud->tchan_cnt, ud->rchan_cnt); 1277 /* Try to use the highest TPL channel pair for MEM_TO_MEM channels */ 1278 chan_id = match_data->level_start_idx[match_data->tpl_levels - 1]; 1279 for (; chan_id < end; chan_id++) { 1280 if (!test_bit(chan_id, ud->tchan_map) && 1281 !test_bit(chan_id, ud->rchan_map)) 1282 break; 1283 } 1284 1285 if (chan_id == end) 1286 return -ENOENT; 1287 1288 set_bit(chan_id, ud->tchan_map); 1289 set_bit(chan_id, ud->rchan_map); 1290 uc->tchan = &ud->tchans[chan_id]; 1291 uc->rchan = &ud->rchans[chan_id]; 1292 1293 return 0; 1294 } 1295 1296 static int udma_get_rflow(struct udma_chan *uc, int flow_id) 1297 { 1298 struct udma_dev *ud = uc->ud; 1299 1300 if (!uc->rchan) { 1301 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id); 1302 return -EINVAL; 1303 } 1304 1305 if (uc->rflow) { 1306 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n", 1307 uc->id, uc->rflow->id); 1308 return 0; 1309 } 1310 1311 uc->rflow = __udma_get_rflow(ud, flow_id); 1312 if (IS_ERR(uc->rflow)) 1313 return PTR_ERR(uc->rflow); 1314 1315 return 0; 1316 } 1317 1318 static void udma_put_rchan(struct udma_chan *uc) 1319 { 1320 struct udma_dev *ud = uc->ud; 1321 1322 if (uc->rchan) { 1323 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id, 1324 uc->rchan->id); 1325 clear_bit(uc->rchan->id, ud->rchan_map); 1326 uc->rchan = NULL; 1327 } 1328 } 1329 1330 static void udma_put_tchan(struct udma_chan *uc) 1331 { 1332 struct udma_dev *ud = uc->ud; 1333 1334 if (uc->tchan) { 1335 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id, 1336 uc->tchan->id); 1337 clear_bit(uc->tchan->id, ud->tchan_map); 1338 uc->tchan = NULL; 1339 } 1340 } 1341 1342 static void udma_put_rflow(struct udma_chan *uc) 1343 { 1344 struct udma_dev *ud = uc->ud; 1345 1346 if (uc->rflow) { 1347 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id, 1348 uc->rflow->id); 1349 __udma_put_rflow(ud, uc->rflow); 1350 uc->rflow = NULL; 1351 } 1352 } 1353 1354 static void udma_free_tx_resources(struct udma_chan *uc) 1355 { 1356 if (!uc->tchan) 1357 return; 1358 1359 k3_ringacc_ring_free(uc->tchan->t_ring); 1360 k3_ringacc_ring_free(uc->tchan->tc_ring); 1361 uc->tchan->t_ring = NULL; 1362 uc->tchan->tc_ring = NULL; 1363 1364 udma_put_tchan(uc); 1365 } 1366 1367 static int udma_alloc_tx_resources(struct udma_chan *uc) 1368 { 1369 struct k3_ring_cfg ring_cfg; 1370 struct udma_dev *ud = uc->ud; 1371 int ret; 1372 1373 ret = udma_get_tchan(uc); 1374 if (ret) 1375 return ret; 1376 1377 uc->tchan->t_ring = k3_ringacc_request_ring(ud->ringacc, 1378 uc->tchan->id, 0); 1379 if (!uc->tchan->t_ring) { 1380 ret = -EBUSY; 1381 goto err_tx_ring; 1382 } 1383 1384 uc->tchan->tc_ring = k3_ringacc_request_ring(ud->ringacc, -1, 0); 1385 if (!uc->tchan->tc_ring) { 1386 ret = -EBUSY; 1387 goto err_txc_ring; 1388 } 1389 1390 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1391 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1392 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1393 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1394 1395 ret = k3_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg); 1396 ret |= k3_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg); 1397 1398 if (ret) 1399 goto err_ringcfg; 1400 1401 return 0; 1402 1403 err_ringcfg: 1404 k3_ringacc_ring_free(uc->tchan->tc_ring); 1405 uc->tchan->tc_ring = NULL; 1406 err_txc_ring: 1407 k3_ringacc_ring_free(uc->tchan->t_ring); 1408 uc->tchan->t_ring = NULL; 1409 err_tx_ring: 1410 udma_put_tchan(uc); 1411 1412 return ret; 1413 } 1414 1415 static void udma_free_rx_resources(struct udma_chan *uc) 1416 { 1417 if (!uc->rchan) 1418 return; 1419 1420 if (uc->rflow) { 1421 struct udma_rflow *rflow = uc->rflow; 1422 1423 k3_ringacc_ring_free(rflow->fd_ring); 1424 k3_ringacc_ring_free(rflow->r_ring); 1425 rflow->fd_ring = NULL; 1426 rflow->r_ring = NULL; 1427 1428 udma_put_rflow(uc); 1429 } 1430 1431 udma_put_rchan(uc); 1432 } 1433 1434 static int udma_alloc_rx_resources(struct udma_chan *uc) 1435 { 1436 struct udma_dev *ud = uc->ud; 1437 struct k3_ring_cfg ring_cfg; 1438 struct udma_rflow *rflow; 1439 int fd_ring_id; 1440 int ret; 1441 1442 ret = udma_get_rchan(uc); 1443 if (ret) 1444 return ret; 1445 1446 /* For MEM_TO_MEM we don't need rflow or rings */ 1447 if (uc->config.dir == DMA_MEM_TO_MEM) 1448 return 0; 1449 1450 ret = udma_get_rflow(uc, uc->rchan->id); 1451 if (ret) { 1452 ret = -EBUSY; 1453 goto err_rflow; 1454 } 1455 1456 rflow = uc->rflow; 1457 fd_ring_id = ud->tchan_cnt + ud->echan_cnt + uc->rchan->id; 1458 rflow->fd_ring = k3_ringacc_request_ring(ud->ringacc, fd_ring_id, 0); 1459 if (!rflow->fd_ring) { 1460 ret = -EBUSY; 1461 goto err_rx_ring; 1462 } 1463 1464 rflow->r_ring = k3_ringacc_request_ring(ud->ringacc, -1, 0); 1465 if (!rflow->r_ring) { 1466 ret = -EBUSY; 1467 goto err_rxc_ring; 1468 } 1469 1470 memset(&ring_cfg, 0, sizeof(ring_cfg)); 1471 1472 if (uc->config.pkt_mode) 1473 ring_cfg.size = SG_MAX_SEGMENTS; 1474 else 1475 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1476 1477 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; 1478 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; 1479 1480 ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg); 1481 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; 1482 ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg); 1483 1484 if (ret) 1485 goto err_ringcfg; 1486 1487 return 0; 1488 1489 err_ringcfg: 1490 k3_ringacc_ring_free(rflow->r_ring); 1491 rflow->r_ring = NULL; 1492 err_rxc_ring: 1493 k3_ringacc_ring_free(rflow->fd_ring); 1494 rflow->fd_ring = NULL; 1495 err_rx_ring: 1496 udma_put_rflow(uc); 1497 err_rflow: 1498 udma_put_rchan(uc); 1499 1500 return ret; 1501 } 1502 1503 #define TISCI_TCHAN_VALID_PARAMS ( \ 1504 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1505 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \ 1506 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \ 1507 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1508 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \ 1509 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1510 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID) 1511 1512 #define TISCI_RCHAN_VALID_PARAMS ( \ 1513 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ 1514 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1515 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1516 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1517 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \ 1518 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \ 1519 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \ 1520 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID) 1521 1522 static int udma_tisci_m2m_channel_config(struct udma_chan *uc) 1523 { 1524 struct udma_dev *ud = uc->ud; 1525 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1526 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1527 struct udma_tchan *tchan = uc->tchan; 1528 struct udma_rchan *rchan = uc->rchan; 1529 int ret = 0; 1530 1531 /* Non synchronized - mem to mem type of transfer */ 1532 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1533 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1534 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1535 1536 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; 1537 req_tx.nav_id = tisci_rm->tisci_dev_id; 1538 req_tx.index = tchan->id; 1539 req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1540 req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1541 req_tx.txcq_qnum = tc_ring; 1542 1543 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1544 if (ret) { 1545 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1546 return ret; 1547 } 1548 1549 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; 1550 req_rx.nav_id = tisci_rm->tisci_dev_id; 1551 req_rx.index = rchan->id; 1552 req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1553 req_rx.rxcq_qnum = tc_ring; 1554 req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1555 1556 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1557 if (ret) 1558 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret); 1559 1560 return ret; 1561 } 1562 1563 static int udma_tisci_tx_channel_config(struct udma_chan *uc) 1564 { 1565 struct udma_dev *ud = uc->ud; 1566 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1567 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1568 struct udma_tchan *tchan = uc->tchan; 1569 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); 1570 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; 1571 u32 mode, fetch_size; 1572 int ret = 0; 1573 1574 if (uc->config.pkt_mode) { 1575 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1576 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1577 uc->config.psd_size, 0); 1578 } else { 1579 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1580 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1581 } 1582 1583 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; 1584 req_tx.nav_id = tisci_rm->tisci_dev_id; 1585 req_tx.index = tchan->id; 1586 req_tx.tx_chan_type = mode; 1587 req_tx.tx_supr_tdpkt = uc->config.notdpkt; 1588 req_tx.tx_fetch_size = fetch_size >> 2; 1589 req_tx.txcq_qnum = tc_ring; 1590 1591 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1592 if (ret) 1593 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); 1594 1595 return ret; 1596 } 1597 1598 static int udma_tisci_rx_channel_config(struct udma_chan *uc) 1599 { 1600 struct udma_dev *ud = uc->ud; 1601 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 1602 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; 1603 struct udma_rchan *rchan = uc->rchan; 1604 int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring); 1605 int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring); 1606 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; 1607 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; 1608 u32 mode, fetch_size; 1609 int ret = 0; 1610 1611 if (uc->config.pkt_mode) { 1612 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 1613 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib, 1614 uc->config.psd_size, 0); 1615 } else { 1616 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR; 1617 fetch_size = sizeof(struct cppi5_desc_hdr_t); 1618 } 1619 1620 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; 1621 req_rx.nav_id = tisci_rm->tisci_dev_id; 1622 req_rx.index = rchan->id; 1623 req_rx.rx_fetch_size = fetch_size >> 2; 1624 req_rx.rxcq_qnum = rx_ring; 1625 req_rx.rx_chan_type = mode; 1626 1627 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1628 if (ret) { 1629 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); 1630 return ret; 1631 } 1632 1633 flow_req.valid_params = 1634 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 1635 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 1636 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 1637 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 1638 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1639 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 1640 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 1641 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 1642 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 1643 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1644 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1645 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1646 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1647 1648 flow_req.nav_id = tisci_rm->tisci_dev_id; 1649 flow_req.flow_index = rchan->id; 1650 1651 if (uc->config.needs_epib) 1652 flow_req.rx_einfo_present = 1; 1653 else 1654 flow_req.rx_einfo_present = 0; 1655 if (uc->config.psd_size) 1656 flow_req.rx_psinfo_present = 1; 1657 else 1658 flow_req.rx_psinfo_present = 0; 1659 flow_req.rx_error_handling = 1; 1660 flow_req.rx_dest_qnum = rx_ring; 1661 flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE; 1662 flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG; 1663 flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI; 1664 flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO; 1665 flow_req.rx_fdq0_sz0_qnum = fd_ring; 1666 flow_req.rx_fdq1_qnum = fd_ring; 1667 flow_req.rx_fdq2_qnum = fd_ring; 1668 flow_req.rx_fdq3_qnum = fd_ring; 1669 1670 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req); 1671 1672 if (ret) 1673 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret); 1674 1675 return 0; 1676 } 1677 1678 static int udma_alloc_chan_resources(struct dma_chan *chan) 1679 { 1680 struct udma_chan *uc = to_udma_chan(chan); 1681 struct udma_dev *ud = to_udma_dev(chan->device); 1682 const struct udma_match_data *match_data = ud->match_data; 1683 struct k3_ring *irq_ring; 1684 u32 irq_udma_idx; 1685 int ret; 1686 1687 if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) { 1688 uc->use_dma_pool = true; 1689 /* in case of MEM_TO_MEM we have maximum of two TRs */ 1690 if (uc->config.dir == DMA_MEM_TO_MEM) { 1691 uc->config.hdesc_size = cppi5_trdesc_calc_size( 1692 sizeof(struct cppi5_tr_type15_t), 2); 1693 uc->config.pkt_mode = false; 1694 } 1695 } 1696 1697 if (uc->use_dma_pool) { 1698 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, 1699 uc->config.hdesc_size, 1700 ud->desc_align, 1701 0); 1702 if (!uc->hdesc_pool) { 1703 dev_err(ud->ddev.dev, 1704 "Descriptor pool allocation failed\n"); 1705 uc->use_dma_pool = false; 1706 return -ENOMEM; 1707 } 1708 } 1709 1710 /* 1711 * Make sure that the completion is in a known state: 1712 * No teardown, the channel is idle 1713 */ 1714 reinit_completion(&uc->teardown_completed); 1715 complete_all(&uc->teardown_completed); 1716 uc->state = UDMA_CHAN_IS_IDLE; 1717 1718 switch (uc->config.dir) { 1719 case DMA_MEM_TO_MEM: 1720 /* Non synchronized - mem to mem type of transfer */ 1721 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, 1722 uc->id); 1723 1724 ret = udma_get_chan_pair(uc); 1725 if (ret) 1726 return ret; 1727 1728 ret = udma_alloc_tx_resources(uc); 1729 if (ret) 1730 return ret; 1731 1732 ret = udma_alloc_rx_resources(uc); 1733 if (ret) { 1734 udma_free_tx_resources(uc); 1735 return ret; 1736 } 1737 1738 uc->config.src_thread = ud->psil_base + uc->tchan->id; 1739 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 1740 K3_PSIL_DST_THREAD_ID_OFFSET; 1741 1742 irq_ring = uc->tchan->tc_ring; 1743 irq_udma_idx = uc->tchan->id; 1744 1745 ret = udma_tisci_m2m_channel_config(uc); 1746 break; 1747 case DMA_MEM_TO_DEV: 1748 /* Slave transfer synchronized - mem to dev (TX) trasnfer */ 1749 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, 1750 uc->id); 1751 1752 ret = udma_alloc_tx_resources(uc); 1753 if (ret) { 1754 uc->config.remote_thread_id = -1; 1755 return ret; 1756 } 1757 1758 uc->config.src_thread = ud->psil_base + uc->tchan->id; 1759 uc->config.dst_thread = uc->config.remote_thread_id; 1760 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; 1761 1762 irq_ring = uc->tchan->tc_ring; 1763 irq_udma_idx = uc->tchan->id; 1764 1765 ret = udma_tisci_tx_channel_config(uc); 1766 break; 1767 case DMA_DEV_TO_MEM: 1768 /* Slave transfer synchronized - dev to mem (RX) trasnfer */ 1769 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, 1770 uc->id); 1771 1772 ret = udma_alloc_rx_resources(uc); 1773 if (ret) { 1774 uc->config.remote_thread_id = -1; 1775 return ret; 1776 } 1777 1778 uc->config.src_thread = uc->config.remote_thread_id; 1779 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 1780 K3_PSIL_DST_THREAD_ID_OFFSET; 1781 1782 irq_ring = uc->rflow->r_ring; 1783 irq_udma_idx = match_data->rchan_oes_offset + uc->rchan->id; 1784 1785 ret = udma_tisci_rx_channel_config(uc); 1786 break; 1787 default: 1788 /* Can not happen */ 1789 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", 1790 __func__, uc->id, uc->config.dir); 1791 return -EINVAL; 1792 } 1793 1794 /* check if the channel configuration was successful */ 1795 if (ret) 1796 goto err_res_free; 1797 1798 if (udma_is_chan_running(uc)) { 1799 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); 1800 udma_stop(uc); 1801 if (udma_is_chan_running(uc)) { 1802 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); 1803 goto err_res_free; 1804 } 1805 } 1806 1807 /* PSI-L pairing */ 1808 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); 1809 if (ret) { 1810 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", 1811 uc->config.src_thread, uc->config.dst_thread); 1812 goto err_res_free; 1813 } 1814 1815 uc->psil_paired = true; 1816 1817 uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring); 1818 if (uc->irq_num_ring <= 0) { 1819 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", 1820 k3_ringacc_get_ring_id(irq_ring)); 1821 ret = -EINVAL; 1822 goto err_psi_free; 1823 } 1824 1825 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, 1826 IRQF_TRIGGER_HIGH, uc->name, uc); 1827 if (ret) { 1828 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); 1829 goto err_irq_free; 1830 } 1831 1832 /* Event from UDMA (TR events) only needed for slave TR mode channels */ 1833 if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) { 1834 uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, 1835 irq_udma_idx); 1836 if (uc->irq_num_udma <= 0) { 1837 dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", 1838 irq_udma_idx); 1839 free_irq(uc->irq_num_ring, uc); 1840 ret = -EINVAL; 1841 goto err_irq_free; 1842 } 1843 1844 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0, 1845 uc->name, uc); 1846 if (ret) { 1847 dev_err(ud->dev, "chan%d: UDMA irq request failed\n", 1848 uc->id); 1849 free_irq(uc->irq_num_ring, uc); 1850 goto err_irq_free; 1851 } 1852 } else { 1853 uc->irq_num_udma = 0; 1854 } 1855 1856 udma_reset_rings(uc); 1857 1858 INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work, 1859 udma_check_tx_completion); 1860 return 0; 1861 1862 err_irq_free: 1863 uc->irq_num_ring = 0; 1864 uc->irq_num_udma = 0; 1865 err_psi_free: 1866 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); 1867 uc->psil_paired = false; 1868 err_res_free: 1869 udma_free_tx_resources(uc); 1870 udma_free_rx_resources(uc); 1871 1872 udma_reset_uchan(uc); 1873 1874 if (uc->use_dma_pool) { 1875 dma_pool_destroy(uc->hdesc_pool); 1876 uc->use_dma_pool = false; 1877 } 1878 1879 return ret; 1880 } 1881 1882 static int udma_slave_config(struct dma_chan *chan, 1883 struct dma_slave_config *cfg) 1884 { 1885 struct udma_chan *uc = to_udma_chan(chan); 1886 1887 memcpy(&uc->cfg, cfg, sizeof(uc->cfg)); 1888 1889 return 0; 1890 } 1891 1892 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc, 1893 size_t tr_size, int tr_count, 1894 enum dma_transfer_direction dir) 1895 { 1896 struct udma_hwdesc *hwdesc; 1897 struct cppi5_desc_hdr_t *tr_desc; 1898 struct udma_desc *d; 1899 u32 reload_count = 0; 1900 u32 ring_id; 1901 1902 switch (tr_size) { 1903 case 16: 1904 case 32: 1905 case 64: 1906 case 128: 1907 break; 1908 default: 1909 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size); 1910 return NULL; 1911 } 1912 1913 /* We have only one descriptor containing multiple TRs */ 1914 d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT); 1915 if (!d) 1916 return NULL; 1917 1918 d->sglen = tr_count; 1919 1920 d->hwdesc_count = 1; 1921 hwdesc = &d->hwdesc[0]; 1922 1923 /* Allocate memory for DMA ring descriptor */ 1924 if (uc->use_dma_pool) { 1925 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 1926 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 1927 GFP_NOWAIT, 1928 &hwdesc->cppi5_desc_paddr); 1929 } else { 1930 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1931 tr_count); 1932 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, 1933 uc->ud->desc_align); 1934 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev, 1935 hwdesc->cppi5_desc_size, 1936 &hwdesc->cppi5_desc_paddr, 1937 GFP_NOWAIT); 1938 } 1939 1940 if (!hwdesc->cppi5_desc_vaddr) { 1941 kfree(d); 1942 return NULL; 1943 } 1944 1945 /* Start of the TR req records */ 1946 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; 1947 /* Start address of the TR response array */ 1948 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count; 1949 1950 tr_desc = hwdesc->cppi5_desc_vaddr; 1951 1952 if (uc->cyclic) 1953 reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE; 1954 1955 if (dir == DMA_DEV_TO_MEM) 1956 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 1957 else 1958 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 1959 1960 cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count); 1961 cppi5_desc_set_pktids(tr_desc, uc->id, 1962 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 1963 cppi5_desc_set_retpolicy(tr_desc, 0, ring_id); 1964 1965 return d; 1966 } 1967 1968 static struct udma_desc * 1969 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, 1970 unsigned int sglen, enum dma_transfer_direction dir, 1971 unsigned long tx_flags, void *context) 1972 { 1973 enum dma_slave_buswidth dev_width; 1974 struct scatterlist *sgent; 1975 struct udma_desc *d; 1976 size_t tr_size; 1977 struct cppi5_tr_type1_t *tr_req = NULL; 1978 unsigned int i; 1979 u32 burst; 1980 1981 if (dir == DMA_DEV_TO_MEM) { 1982 dev_width = uc->cfg.src_addr_width; 1983 burst = uc->cfg.src_maxburst; 1984 } else if (dir == DMA_MEM_TO_DEV) { 1985 dev_width = uc->cfg.dst_addr_width; 1986 burst = uc->cfg.dst_maxburst; 1987 } else { 1988 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 1989 return NULL; 1990 } 1991 1992 if (!burst) 1993 burst = 1; 1994 1995 /* Now allocate and setup the descriptor. */ 1996 tr_size = sizeof(struct cppi5_tr_type1_t); 1997 d = udma_alloc_tr_desc(uc, tr_size, sglen, dir); 1998 if (!d) 1999 return NULL; 2000 2001 d->sglen = sglen; 2002 2003 tr_req = d->hwdesc[0].tr_req_base; 2004 for_each_sg(sgl, sgent, sglen, i) { 2005 d->residue += sg_dma_len(sgent); 2006 2007 cppi5_tr_init(&tr_req[i].flags, CPPI5_TR_TYPE1, false, false, 2008 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2009 cppi5_tr_csf_set(&tr_req[i].flags, CPPI5_TR_CSF_SUPR_EVT); 2010 2011 tr_req[i].addr = sg_dma_address(sgent); 2012 tr_req[i].icnt0 = burst * dev_width; 2013 tr_req[i].dim1 = burst * dev_width; 2014 tr_req[i].icnt1 = sg_dma_len(sgent) / tr_req[i].icnt0; 2015 } 2016 2017 cppi5_tr_csf_set(&tr_req[i - 1].flags, CPPI5_TR_CSF_EOP); 2018 2019 return d; 2020 } 2021 2022 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d, 2023 enum dma_slave_buswidth dev_width, 2024 u16 elcnt) 2025 { 2026 if (uc->config.ep_type != PSIL_EP_PDMA_XY) 2027 return 0; 2028 2029 /* Bus width translates to the element size (ES) */ 2030 switch (dev_width) { 2031 case DMA_SLAVE_BUSWIDTH_1_BYTE: 2032 d->static_tr.elsize = 0; 2033 break; 2034 case DMA_SLAVE_BUSWIDTH_2_BYTES: 2035 d->static_tr.elsize = 1; 2036 break; 2037 case DMA_SLAVE_BUSWIDTH_3_BYTES: 2038 d->static_tr.elsize = 2; 2039 break; 2040 case DMA_SLAVE_BUSWIDTH_4_BYTES: 2041 d->static_tr.elsize = 3; 2042 break; 2043 case DMA_SLAVE_BUSWIDTH_8_BYTES: 2044 d->static_tr.elsize = 4; 2045 break; 2046 default: /* not reached */ 2047 return -EINVAL; 2048 } 2049 2050 d->static_tr.elcnt = elcnt; 2051 2052 /* 2053 * PDMA must to close the packet when the channel is in packet mode. 2054 * For TR mode when the channel is not cyclic we also need PDMA to close 2055 * the packet otherwise the transfer will stall because PDMA holds on 2056 * the data it has received from the peripheral. 2057 */ 2058 if (uc->config.pkt_mode || !uc->cyclic) { 2059 unsigned int div = dev_width * elcnt; 2060 2061 if (uc->cyclic) 2062 d->static_tr.bstcnt = d->residue / d->sglen / div; 2063 else 2064 d->static_tr.bstcnt = d->residue / div; 2065 2066 if (uc->config.dir == DMA_DEV_TO_MEM && 2067 d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) 2068 return -EINVAL; 2069 } else { 2070 d->static_tr.bstcnt = 0; 2071 } 2072 2073 return 0; 2074 } 2075 2076 static struct udma_desc * 2077 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, 2078 unsigned int sglen, enum dma_transfer_direction dir, 2079 unsigned long tx_flags, void *context) 2080 { 2081 struct scatterlist *sgent; 2082 struct cppi5_host_desc_t *h_desc = NULL; 2083 struct udma_desc *d; 2084 u32 ring_id; 2085 unsigned int i; 2086 2087 d = kzalloc(sizeof(*d) + sglen * sizeof(d->hwdesc[0]), GFP_NOWAIT); 2088 if (!d) 2089 return NULL; 2090 2091 d->sglen = sglen; 2092 d->hwdesc_count = sglen; 2093 2094 if (dir == DMA_DEV_TO_MEM) 2095 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2096 else 2097 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2098 2099 for_each_sg(sgl, sgent, sglen, i) { 2100 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 2101 dma_addr_t sg_addr = sg_dma_address(sgent); 2102 struct cppi5_host_desc_t *desc; 2103 size_t sg_len = sg_dma_len(sgent); 2104 2105 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2106 GFP_NOWAIT, 2107 &hwdesc->cppi5_desc_paddr); 2108 if (!hwdesc->cppi5_desc_vaddr) { 2109 dev_err(uc->ud->dev, 2110 "descriptor%d allocation failed\n", i); 2111 2112 udma_free_hwdesc(uc, d); 2113 kfree(d); 2114 return NULL; 2115 } 2116 2117 d->residue += sg_len; 2118 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2119 desc = hwdesc->cppi5_desc_vaddr; 2120 2121 if (i == 0) { 2122 cppi5_hdesc_init(desc, 0, 0); 2123 /* Flow and Packed ID */ 2124 cppi5_desc_set_pktids(&desc->hdr, uc->id, 2125 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2126 cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id); 2127 } else { 2128 cppi5_hdesc_reset_hbdesc(desc); 2129 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff); 2130 } 2131 2132 /* attach the sg buffer to the descriptor */ 2133 cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len); 2134 2135 /* Attach link as host buffer descriptor */ 2136 if (h_desc) 2137 cppi5_hdesc_link_hbdesc(h_desc, 2138 hwdesc->cppi5_desc_paddr); 2139 2140 if (dir == DMA_MEM_TO_DEV) 2141 h_desc = desc; 2142 } 2143 2144 if (d->residue >= SZ_4M) { 2145 dev_err(uc->ud->dev, 2146 "%s: Transfer size %u is over the supported 4M range\n", 2147 __func__, d->residue); 2148 udma_free_hwdesc(uc, d); 2149 kfree(d); 2150 return NULL; 2151 } 2152 2153 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2154 cppi5_hdesc_set_pktlen(h_desc, d->residue); 2155 2156 return d; 2157 } 2158 2159 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc, 2160 void *data, size_t len) 2161 { 2162 struct udma_desc *d = to_udma_desc(desc); 2163 struct udma_chan *uc = to_udma_chan(desc->chan); 2164 struct cppi5_host_desc_t *h_desc; 2165 u32 psd_size = len; 2166 u32 flags = 0; 2167 2168 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2169 return -ENOTSUPP; 2170 2171 if (!data || len > uc->config.metadata_size) 2172 return -EINVAL; 2173 2174 if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE) 2175 return -EINVAL; 2176 2177 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2178 if (d->dir == DMA_MEM_TO_DEV) 2179 memcpy(h_desc->epib, data, len); 2180 2181 if (uc->config.needs_epib) 2182 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 2183 2184 d->metadata = data; 2185 d->metadata_size = len; 2186 if (uc->config.needs_epib) 2187 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 2188 2189 cppi5_hdesc_update_flags(h_desc, flags); 2190 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 2191 2192 return 0; 2193 } 2194 2195 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc, 2196 size_t *payload_len, size_t *max_len) 2197 { 2198 struct udma_desc *d = to_udma_desc(desc); 2199 struct udma_chan *uc = to_udma_chan(desc->chan); 2200 struct cppi5_host_desc_t *h_desc; 2201 2202 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2203 return ERR_PTR(-ENOTSUPP); 2204 2205 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2206 2207 *max_len = uc->config.metadata_size; 2208 2209 *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ? 2210 CPPI5_INFO0_HDESC_EPIB_SIZE : 0; 2211 *payload_len += cppi5_hdesc_get_psdata_size(h_desc); 2212 2213 return h_desc->epib; 2214 } 2215 2216 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc, 2217 size_t payload_len) 2218 { 2219 struct udma_desc *d = to_udma_desc(desc); 2220 struct udma_chan *uc = to_udma_chan(desc->chan); 2221 struct cppi5_host_desc_t *h_desc; 2222 u32 psd_size = payload_len; 2223 u32 flags = 0; 2224 2225 if (!uc->config.pkt_mode || !uc->config.metadata_size) 2226 return -ENOTSUPP; 2227 2228 if (payload_len > uc->config.metadata_size) 2229 return -EINVAL; 2230 2231 if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE) 2232 return -EINVAL; 2233 2234 h_desc = d->hwdesc[0].cppi5_desc_vaddr; 2235 2236 if (uc->config.needs_epib) { 2237 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE; 2238 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT; 2239 } 2240 2241 cppi5_hdesc_update_flags(h_desc, flags); 2242 cppi5_hdesc_update_psdata_size(h_desc, psd_size); 2243 2244 return 0; 2245 } 2246 2247 static struct dma_descriptor_metadata_ops metadata_ops = { 2248 .attach = udma_attach_metadata, 2249 .get_ptr = udma_get_metadata_ptr, 2250 .set_len = udma_set_metadata_len, 2251 }; 2252 2253 static struct dma_async_tx_descriptor * 2254 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 2255 unsigned int sglen, enum dma_transfer_direction dir, 2256 unsigned long tx_flags, void *context) 2257 { 2258 struct udma_chan *uc = to_udma_chan(chan); 2259 enum dma_slave_buswidth dev_width; 2260 struct udma_desc *d; 2261 u32 burst; 2262 2263 if (dir != uc->config.dir) { 2264 dev_err(chan->device->dev, 2265 "%s: chan%d is for %s, not supporting %s\n", 2266 __func__, uc->id, 2267 dmaengine_get_direction_text(uc->config.dir), 2268 dmaengine_get_direction_text(dir)); 2269 return NULL; 2270 } 2271 2272 if (dir == DMA_DEV_TO_MEM) { 2273 dev_width = uc->cfg.src_addr_width; 2274 burst = uc->cfg.src_maxburst; 2275 } else if (dir == DMA_MEM_TO_DEV) { 2276 dev_width = uc->cfg.dst_addr_width; 2277 burst = uc->cfg.dst_maxburst; 2278 } else { 2279 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 2280 return NULL; 2281 } 2282 2283 if (!burst) 2284 burst = 1; 2285 2286 if (uc->config.pkt_mode) 2287 d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags, 2288 context); 2289 else 2290 d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags, 2291 context); 2292 2293 if (!d) 2294 return NULL; 2295 2296 d->dir = dir; 2297 d->desc_idx = 0; 2298 d->tr_idx = 0; 2299 2300 /* static TR for remote PDMA */ 2301 if (udma_configure_statictr(uc, d, dev_width, burst)) { 2302 dev_err(uc->ud->dev, 2303 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 2304 __func__, d->static_tr.bstcnt); 2305 2306 udma_free_hwdesc(uc, d); 2307 kfree(d); 2308 return NULL; 2309 } 2310 2311 if (uc->config.metadata_size) 2312 d->vd.tx.metadata_ops = &metadata_ops; 2313 2314 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 2315 } 2316 2317 static struct udma_desc * 2318 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, 2319 size_t buf_len, size_t period_len, 2320 enum dma_transfer_direction dir, unsigned long flags) 2321 { 2322 enum dma_slave_buswidth dev_width; 2323 struct udma_desc *d; 2324 size_t tr_size; 2325 struct cppi5_tr_type1_t *tr_req; 2326 unsigned int i; 2327 unsigned int periods = buf_len / period_len; 2328 u32 burst; 2329 2330 if (dir == DMA_DEV_TO_MEM) { 2331 dev_width = uc->cfg.src_addr_width; 2332 burst = uc->cfg.src_maxburst; 2333 } else if (dir == DMA_MEM_TO_DEV) { 2334 dev_width = uc->cfg.dst_addr_width; 2335 burst = uc->cfg.dst_maxburst; 2336 } else { 2337 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 2338 return NULL; 2339 } 2340 2341 if (!burst) 2342 burst = 1; 2343 2344 /* Now allocate and setup the descriptor. */ 2345 tr_size = sizeof(struct cppi5_tr_type1_t); 2346 d = udma_alloc_tr_desc(uc, tr_size, periods, dir); 2347 if (!d) 2348 return NULL; 2349 2350 tr_req = d->hwdesc[0].tr_req_base; 2351 for (i = 0; i < periods; i++) { 2352 cppi5_tr_init(&tr_req[i].flags, CPPI5_TR_TYPE1, false, false, 2353 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2354 2355 tr_req[i].addr = buf_addr + period_len * i; 2356 tr_req[i].icnt0 = dev_width; 2357 tr_req[i].icnt1 = period_len / dev_width; 2358 tr_req[i].dim1 = dev_width; 2359 2360 if (!(flags & DMA_PREP_INTERRUPT)) 2361 cppi5_tr_csf_set(&tr_req[i].flags, 2362 CPPI5_TR_CSF_SUPR_EVT); 2363 } 2364 2365 return d; 2366 } 2367 2368 static struct udma_desc * 2369 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr, 2370 size_t buf_len, size_t period_len, 2371 enum dma_transfer_direction dir, unsigned long flags) 2372 { 2373 struct udma_desc *d; 2374 u32 ring_id; 2375 int i; 2376 int periods = buf_len / period_len; 2377 2378 if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1)) 2379 return NULL; 2380 2381 if (period_len >= SZ_4M) 2382 return NULL; 2383 2384 d = kzalloc(sizeof(*d) + periods * sizeof(d->hwdesc[0]), GFP_NOWAIT); 2385 if (!d) 2386 return NULL; 2387 2388 d->hwdesc_count = periods; 2389 2390 /* TODO: re-check this... */ 2391 if (dir == DMA_DEV_TO_MEM) 2392 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); 2393 else 2394 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); 2395 2396 for (i = 0; i < periods; i++) { 2397 struct udma_hwdesc *hwdesc = &d->hwdesc[i]; 2398 dma_addr_t period_addr = buf_addr + (period_len * i); 2399 struct cppi5_host_desc_t *h_desc; 2400 2401 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, 2402 GFP_NOWAIT, 2403 &hwdesc->cppi5_desc_paddr); 2404 if (!hwdesc->cppi5_desc_vaddr) { 2405 dev_err(uc->ud->dev, 2406 "descriptor%d allocation failed\n", i); 2407 2408 udma_free_hwdesc(uc, d); 2409 kfree(d); 2410 return NULL; 2411 } 2412 2413 hwdesc->cppi5_desc_size = uc->config.hdesc_size; 2414 h_desc = hwdesc->cppi5_desc_vaddr; 2415 2416 cppi5_hdesc_init(h_desc, 0, 0); 2417 cppi5_hdesc_set_pktlen(h_desc, period_len); 2418 2419 /* Flow and Packed ID */ 2420 cppi5_desc_set_pktids(&h_desc->hdr, uc->id, 2421 CPPI5_INFO1_DESC_FLOWID_DEFAULT); 2422 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id); 2423 2424 /* attach each period to a new descriptor */ 2425 cppi5_hdesc_attach_buf(h_desc, 2426 period_addr, period_len, 2427 period_addr, period_len); 2428 } 2429 2430 return d; 2431 } 2432 2433 static struct dma_async_tx_descriptor * 2434 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 2435 size_t period_len, enum dma_transfer_direction dir, 2436 unsigned long flags) 2437 { 2438 struct udma_chan *uc = to_udma_chan(chan); 2439 enum dma_slave_buswidth dev_width; 2440 struct udma_desc *d; 2441 u32 burst; 2442 2443 if (dir != uc->config.dir) { 2444 dev_err(chan->device->dev, 2445 "%s: chan%d is for %s, not supporting %s\n", 2446 __func__, uc->id, 2447 dmaengine_get_direction_text(uc->config.dir), 2448 dmaengine_get_direction_text(dir)); 2449 return NULL; 2450 } 2451 2452 uc->cyclic = true; 2453 2454 if (dir == DMA_DEV_TO_MEM) { 2455 dev_width = uc->cfg.src_addr_width; 2456 burst = uc->cfg.src_maxburst; 2457 } else if (dir == DMA_MEM_TO_DEV) { 2458 dev_width = uc->cfg.dst_addr_width; 2459 burst = uc->cfg.dst_maxburst; 2460 } else { 2461 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); 2462 return NULL; 2463 } 2464 2465 if (!burst) 2466 burst = 1; 2467 2468 if (uc->config.pkt_mode) 2469 d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len, 2470 dir, flags); 2471 else 2472 d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len, 2473 dir, flags); 2474 2475 if (!d) 2476 return NULL; 2477 2478 d->sglen = buf_len / period_len; 2479 2480 d->dir = dir; 2481 d->residue = buf_len; 2482 2483 /* static TR for remote PDMA */ 2484 if (udma_configure_statictr(uc, d, dev_width, burst)) { 2485 dev_err(uc->ud->dev, 2486 "%s: StaticTR Z is limited to maximum 4095 (%u)\n", 2487 __func__, d->static_tr.bstcnt); 2488 2489 udma_free_hwdesc(uc, d); 2490 kfree(d); 2491 return NULL; 2492 } 2493 2494 if (uc->config.metadata_size) 2495 d->vd.tx.metadata_ops = &metadata_ops; 2496 2497 return vchan_tx_prep(&uc->vc, &d->vd, flags); 2498 } 2499 2500 static struct dma_async_tx_descriptor * 2501 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 2502 size_t len, unsigned long tx_flags) 2503 { 2504 struct udma_chan *uc = to_udma_chan(chan); 2505 struct udma_desc *d; 2506 struct cppi5_tr_type15_t *tr_req; 2507 int num_tr; 2508 size_t tr_size = sizeof(struct cppi5_tr_type15_t); 2509 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; 2510 2511 if (uc->config.dir != DMA_MEM_TO_MEM) { 2512 dev_err(chan->device->dev, 2513 "%s: chan%d is for %s, not supporting %s\n", 2514 __func__, uc->id, 2515 dmaengine_get_direction_text(uc->config.dir), 2516 dmaengine_get_direction_text(DMA_MEM_TO_MEM)); 2517 return NULL; 2518 } 2519 2520 if (len < SZ_64K) { 2521 num_tr = 1; 2522 tr0_cnt0 = len; 2523 tr0_cnt1 = 1; 2524 } else { 2525 unsigned long align_to = __ffs(src | dest); 2526 2527 if (align_to > 3) 2528 align_to = 3; 2529 /* 2530 * Keep simple: tr0: SZ_64K-alignment blocks, 2531 * tr1: the remaining 2532 */ 2533 num_tr = 2; 2534 tr0_cnt0 = (SZ_64K - BIT(align_to)); 2535 if (len / tr0_cnt0 >= SZ_64K) { 2536 dev_err(uc->ud->dev, "size %zu is not supported\n", 2537 len); 2538 return NULL; 2539 } 2540 2541 tr0_cnt1 = len / tr0_cnt0; 2542 tr1_cnt0 = len % tr0_cnt0; 2543 } 2544 2545 d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM); 2546 if (!d) 2547 return NULL; 2548 2549 d->dir = DMA_MEM_TO_MEM; 2550 d->desc_idx = 0; 2551 d->tr_idx = 0; 2552 d->residue = len; 2553 2554 tr_req = d->hwdesc[0].tr_req_base; 2555 2556 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true, 2557 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2558 cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT); 2559 2560 tr_req[0].addr = src; 2561 tr_req[0].icnt0 = tr0_cnt0; 2562 tr_req[0].icnt1 = tr0_cnt1; 2563 tr_req[0].icnt2 = 1; 2564 tr_req[0].icnt3 = 1; 2565 tr_req[0].dim1 = tr0_cnt0; 2566 2567 tr_req[0].daddr = dest; 2568 tr_req[0].dicnt0 = tr0_cnt0; 2569 tr_req[0].dicnt1 = tr0_cnt1; 2570 tr_req[0].dicnt2 = 1; 2571 tr_req[0].dicnt3 = 1; 2572 tr_req[0].ddim1 = tr0_cnt0; 2573 2574 if (num_tr == 2) { 2575 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true, 2576 CPPI5_TR_EVENT_SIZE_COMPLETION, 0); 2577 cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT); 2578 2579 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0; 2580 tr_req[1].icnt0 = tr1_cnt0; 2581 tr_req[1].icnt1 = 1; 2582 tr_req[1].icnt2 = 1; 2583 tr_req[1].icnt3 = 1; 2584 2585 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0; 2586 tr_req[1].dicnt0 = tr1_cnt0; 2587 tr_req[1].dicnt1 = 1; 2588 tr_req[1].dicnt2 = 1; 2589 tr_req[1].dicnt3 = 1; 2590 } 2591 2592 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, CPPI5_TR_CSF_EOP); 2593 2594 if (uc->config.metadata_size) 2595 d->vd.tx.metadata_ops = &metadata_ops; 2596 2597 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags); 2598 } 2599 2600 static void udma_issue_pending(struct dma_chan *chan) 2601 { 2602 struct udma_chan *uc = to_udma_chan(chan); 2603 unsigned long flags; 2604 2605 spin_lock_irqsave(&uc->vc.lock, flags); 2606 2607 /* If we have something pending and no active descriptor, then */ 2608 if (vchan_issue_pending(&uc->vc) && !uc->desc) { 2609 /* 2610 * start a descriptor if the channel is NOT [marked as 2611 * terminating _and_ it is still running (teardown has not 2612 * completed yet)]. 2613 */ 2614 if (!(uc->state == UDMA_CHAN_IS_TERMINATING && 2615 udma_is_chan_running(uc))) 2616 udma_start(uc); 2617 } 2618 2619 spin_unlock_irqrestore(&uc->vc.lock, flags); 2620 } 2621 2622 static enum dma_status udma_tx_status(struct dma_chan *chan, 2623 dma_cookie_t cookie, 2624 struct dma_tx_state *txstate) 2625 { 2626 struct udma_chan *uc = to_udma_chan(chan); 2627 enum dma_status ret; 2628 unsigned long flags; 2629 2630 spin_lock_irqsave(&uc->vc.lock, flags); 2631 2632 ret = dma_cookie_status(chan, cookie, txstate); 2633 2634 if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc)) 2635 ret = DMA_PAUSED; 2636 2637 if (ret == DMA_COMPLETE || !txstate) 2638 goto out; 2639 2640 if (uc->desc && uc->desc->vd.tx.cookie == cookie) { 2641 u32 peer_bcnt = 0; 2642 u32 bcnt = 0; 2643 u32 residue = uc->desc->residue; 2644 u32 delay = 0; 2645 2646 if (uc->desc->dir == DMA_MEM_TO_DEV) { 2647 bcnt = udma_tchanrt_read(uc->tchan, 2648 UDMA_TCHAN_RT_SBCNT_REG); 2649 2650 if (uc->config.ep_type != PSIL_EP_NATIVE) { 2651 peer_bcnt = udma_tchanrt_read(uc->tchan, 2652 UDMA_TCHAN_RT_PEER_BCNT_REG); 2653 2654 if (bcnt > peer_bcnt) 2655 delay = bcnt - peer_bcnt; 2656 } 2657 } else if (uc->desc->dir == DMA_DEV_TO_MEM) { 2658 bcnt = udma_rchanrt_read(uc->rchan, 2659 UDMA_RCHAN_RT_BCNT_REG); 2660 2661 if (uc->config.ep_type != PSIL_EP_NATIVE) { 2662 peer_bcnt = udma_rchanrt_read(uc->rchan, 2663 UDMA_RCHAN_RT_PEER_BCNT_REG); 2664 2665 if (peer_bcnt > bcnt) 2666 delay = peer_bcnt - bcnt; 2667 } 2668 } else { 2669 bcnt = udma_tchanrt_read(uc->tchan, 2670 UDMA_TCHAN_RT_BCNT_REG); 2671 } 2672 2673 bcnt -= uc->bcnt; 2674 if (bcnt && !(bcnt % uc->desc->residue)) 2675 residue = 0; 2676 else 2677 residue -= bcnt % uc->desc->residue; 2678 2679 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) { 2680 ret = DMA_COMPLETE; 2681 delay = 0; 2682 } 2683 2684 dma_set_residue(txstate, residue); 2685 dma_set_in_flight_bytes(txstate, delay); 2686 2687 } else { 2688 ret = DMA_COMPLETE; 2689 } 2690 2691 out: 2692 spin_unlock_irqrestore(&uc->vc.lock, flags); 2693 return ret; 2694 } 2695 2696 static int udma_pause(struct dma_chan *chan) 2697 { 2698 struct udma_chan *uc = to_udma_chan(chan); 2699 2700 if (!uc->desc) 2701 return -EINVAL; 2702 2703 /* pause the channel */ 2704 switch (uc->desc->dir) { 2705 case DMA_DEV_TO_MEM: 2706 udma_rchanrt_update_bits(uc->rchan, 2707 UDMA_RCHAN_RT_PEER_RT_EN_REG, 2708 UDMA_PEER_RT_EN_PAUSE, 2709 UDMA_PEER_RT_EN_PAUSE); 2710 break; 2711 case DMA_MEM_TO_DEV: 2712 udma_tchanrt_update_bits(uc->tchan, 2713 UDMA_TCHAN_RT_PEER_RT_EN_REG, 2714 UDMA_PEER_RT_EN_PAUSE, 2715 UDMA_PEER_RT_EN_PAUSE); 2716 break; 2717 case DMA_MEM_TO_MEM: 2718 udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 2719 UDMA_CHAN_RT_CTL_PAUSE, 2720 UDMA_CHAN_RT_CTL_PAUSE); 2721 break; 2722 default: 2723 return -EINVAL; 2724 } 2725 2726 return 0; 2727 } 2728 2729 static int udma_resume(struct dma_chan *chan) 2730 { 2731 struct udma_chan *uc = to_udma_chan(chan); 2732 2733 if (!uc->desc) 2734 return -EINVAL; 2735 2736 /* resume the channel */ 2737 switch (uc->desc->dir) { 2738 case DMA_DEV_TO_MEM: 2739 udma_rchanrt_update_bits(uc->rchan, 2740 UDMA_RCHAN_RT_PEER_RT_EN_REG, 2741 UDMA_PEER_RT_EN_PAUSE, 0); 2742 2743 break; 2744 case DMA_MEM_TO_DEV: 2745 udma_tchanrt_update_bits(uc->tchan, 2746 UDMA_TCHAN_RT_PEER_RT_EN_REG, 2747 UDMA_PEER_RT_EN_PAUSE, 0); 2748 break; 2749 case DMA_MEM_TO_MEM: 2750 udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 2751 UDMA_CHAN_RT_CTL_PAUSE, 0); 2752 break; 2753 default: 2754 return -EINVAL; 2755 } 2756 2757 return 0; 2758 } 2759 2760 static int udma_terminate_all(struct dma_chan *chan) 2761 { 2762 struct udma_chan *uc = to_udma_chan(chan); 2763 unsigned long flags; 2764 LIST_HEAD(head); 2765 2766 spin_lock_irqsave(&uc->vc.lock, flags); 2767 2768 if (udma_is_chan_running(uc)) 2769 udma_stop(uc); 2770 2771 if (uc->desc) { 2772 uc->terminated_desc = uc->desc; 2773 uc->desc = NULL; 2774 uc->terminated_desc->terminated = true; 2775 cancel_delayed_work(&uc->tx_drain.work); 2776 } 2777 2778 uc->paused = false; 2779 2780 vchan_get_all_descriptors(&uc->vc, &head); 2781 spin_unlock_irqrestore(&uc->vc.lock, flags); 2782 vchan_dma_desc_free_list(&uc->vc, &head); 2783 2784 return 0; 2785 } 2786 2787 static void udma_synchronize(struct dma_chan *chan) 2788 { 2789 struct udma_chan *uc = to_udma_chan(chan); 2790 unsigned long timeout = msecs_to_jiffies(1000); 2791 2792 vchan_synchronize(&uc->vc); 2793 2794 if (uc->state == UDMA_CHAN_IS_TERMINATING) { 2795 timeout = wait_for_completion_timeout(&uc->teardown_completed, 2796 timeout); 2797 if (!timeout) { 2798 dev_warn(uc->ud->dev, "chan%d teardown timeout!\n", 2799 uc->id); 2800 udma_dump_chan_stdata(uc); 2801 udma_reset_chan(uc, true); 2802 } 2803 } 2804 2805 udma_reset_chan(uc, false); 2806 if (udma_is_chan_running(uc)) 2807 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id); 2808 2809 cancel_delayed_work_sync(&uc->tx_drain.work); 2810 udma_reset_rings(uc); 2811 } 2812 2813 static void udma_desc_pre_callback(struct virt_dma_chan *vc, 2814 struct virt_dma_desc *vd, 2815 struct dmaengine_result *result) 2816 { 2817 struct udma_chan *uc = to_udma_chan(&vc->chan); 2818 struct udma_desc *d; 2819 2820 if (!vd) 2821 return; 2822 2823 d = to_udma_desc(&vd->tx); 2824 2825 if (d->metadata_size) 2826 udma_fetch_epib(uc, d); 2827 2828 /* Provide residue information for the client */ 2829 if (result) { 2830 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx); 2831 2832 if (cppi5_desc_get_type(desc_vaddr) == 2833 CPPI5_INFO0_DESC_TYPE_VAL_HOST) { 2834 result->residue = d->residue - 2835 cppi5_hdesc_get_pktlen(desc_vaddr); 2836 if (result->residue) 2837 result->result = DMA_TRANS_ABORTED; 2838 else 2839 result->result = DMA_TRANS_NOERROR; 2840 } else { 2841 result->residue = 0; 2842 result->result = DMA_TRANS_NOERROR; 2843 } 2844 } 2845 } 2846 2847 /* 2848 * This tasklet handles the completion of a DMA descriptor by 2849 * calling its callback and freeing it. 2850 */ 2851 static void udma_vchan_complete(unsigned long arg) 2852 { 2853 struct virt_dma_chan *vc = (struct virt_dma_chan *)arg; 2854 struct virt_dma_desc *vd, *_vd; 2855 struct dmaengine_desc_callback cb; 2856 LIST_HEAD(head); 2857 2858 spin_lock_irq(&vc->lock); 2859 list_splice_tail_init(&vc->desc_completed, &head); 2860 vd = vc->cyclic; 2861 if (vd) { 2862 vc->cyclic = NULL; 2863 dmaengine_desc_get_callback(&vd->tx, &cb); 2864 } else { 2865 memset(&cb, 0, sizeof(cb)); 2866 } 2867 spin_unlock_irq(&vc->lock); 2868 2869 udma_desc_pre_callback(vc, vd, NULL); 2870 dmaengine_desc_callback_invoke(&cb, NULL); 2871 2872 list_for_each_entry_safe(vd, _vd, &head, node) { 2873 struct dmaengine_result result; 2874 2875 dmaengine_desc_get_callback(&vd->tx, &cb); 2876 2877 list_del(&vd->node); 2878 2879 udma_desc_pre_callback(vc, vd, &result); 2880 dmaengine_desc_callback_invoke(&cb, &result); 2881 2882 vchan_vdesc_fini(vd); 2883 } 2884 } 2885 2886 static void udma_free_chan_resources(struct dma_chan *chan) 2887 { 2888 struct udma_chan *uc = to_udma_chan(chan); 2889 struct udma_dev *ud = to_udma_dev(chan->device); 2890 2891 udma_terminate_all(chan); 2892 if (uc->terminated_desc) { 2893 udma_reset_chan(uc, false); 2894 udma_reset_rings(uc); 2895 } 2896 2897 cancel_delayed_work_sync(&uc->tx_drain.work); 2898 destroy_delayed_work_on_stack(&uc->tx_drain.work); 2899 2900 if (uc->irq_num_ring > 0) { 2901 free_irq(uc->irq_num_ring, uc); 2902 2903 uc->irq_num_ring = 0; 2904 } 2905 if (uc->irq_num_udma > 0) { 2906 free_irq(uc->irq_num_udma, uc); 2907 2908 uc->irq_num_udma = 0; 2909 } 2910 2911 /* Release PSI-L pairing */ 2912 if (uc->psil_paired) { 2913 navss_psil_unpair(ud, uc->config.src_thread, 2914 uc->config.dst_thread); 2915 uc->psil_paired = false; 2916 } 2917 2918 vchan_free_chan_resources(&uc->vc); 2919 tasklet_kill(&uc->vc.task); 2920 2921 udma_free_tx_resources(uc); 2922 udma_free_rx_resources(uc); 2923 udma_reset_uchan(uc); 2924 2925 if (uc->use_dma_pool) { 2926 dma_pool_destroy(uc->hdesc_pool); 2927 uc->use_dma_pool = false; 2928 } 2929 } 2930 2931 static struct platform_driver udma_driver; 2932 2933 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) 2934 { 2935 struct udma_chan_config *ucc; 2936 struct psil_endpoint_config *ep_config; 2937 struct udma_chan *uc; 2938 struct udma_dev *ud; 2939 u32 *args; 2940 2941 if (chan->device->dev->driver != &udma_driver.driver) 2942 return false; 2943 2944 uc = to_udma_chan(chan); 2945 ucc = &uc->config; 2946 ud = uc->ud; 2947 args = param; 2948 2949 ucc->remote_thread_id = args[0]; 2950 2951 if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) 2952 ucc->dir = DMA_MEM_TO_DEV; 2953 else 2954 ucc->dir = DMA_DEV_TO_MEM; 2955 2956 ep_config = psil_get_ep_config(ucc->remote_thread_id); 2957 if (IS_ERR(ep_config)) { 2958 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n", 2959 ucc->remote_thread_id); 2960 ucc->dir = DMA_MEM_TO_MEM; 2961 ucc->remote_thread_id = -1; 2962 return false; 2963 } 2964 2965 ucc->pkt_mode = ep_config->pkt_mode; 2966 ucc->channel_tpl = ep_config->channel_tpl; 2967 ucc->notdpkt = ep_config->notdpkt; 2968 ucc->ep_type = ep_config->ep_type; 2969 2970 if (ucc->ep_type != PSIL_EP_NATIVE) { 2971 const struct udma_match_data *match_data = ud->match_data; 2972 2973 if (match_data->flags & UDMA_FLAG_PDMA_ACC32) 2974 ucc->enable_acc32 = ep_config->pdma_acc32; 2975 if (match_data->flags & UDMA_FLAG_PDMA_BURST) 2976 ucc->enable_burst = ep_config->pdma_burst; 2977 } 2978 2979 ucc->needs_epib = ep_config->needs_epib; 2980 ucc->psd_size = ep_config->psd_size; 2981 ucc->metadata_size = 2982 (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) + 2983 ucc->psd_size; 2984 2985 if (ucc->pkt_mode) 2986 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + 2987 ucc->metadata_size, ud->desc_align); 2988 2989 dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id, 2990 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir)); 2991 2992 return true; 2993 } 2994 2995 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, 2996 struct of_dma *ofdma) 2997 { 2998 struct udma_dev *ud = ofdma->of_dma_data; 2999 dma_cap_mask_t mask = ud->ddev.cap_mask; 3000 struct dma_chan *chan; 3001 3002 if (dma_spec->args_count != 1) 3003 return NULL; 3004 3005 chan = __dma_request_channel(&mask, udma_dma_filter_fn, 3006 &dma_spec->args[0], ofdma->of_node); 3007 if (!chan) { 3008 dev_err(ud->dev, "get channel fail in %s.\n", __func__); 3009 return ERR_PTR(-EINVAL); 3010 } 3011 3012 return chan; 3013 } 3014 3015 static struct udma_match_data am654_main_data = { 3016 .psil_base = 0x1000, 3017 .enable_memcpy_support = true, 3018 .statictr_z_mask = GENMASK(11, 0), 3019 .rchan_oes_offset = 0x2000, 3020 .tpl_levels = 2, 3021 .level_start_idx = { 3022 [0] = 8, /* Normal channels */ 3023 [1] = 0, /* High Throughput channels */ 3024 }, 3025 }; 3026 3027 static struct udma_match_data am654_mcu_data = { 3028 .psil_base = 0x6000, 3029 .enable_memcpy_support = true, /* TEST: DMA domains */ 3030 .statictr_z_mask = GENMASK(11, 0), 3031 .rchan_oes_offset = 0x2000, 3032 .tpl_levels = 2, 3033 .level_start_idx = { 3034 [0] = 2, /* Normal channels */ 3035 [1] = 0, /* High Throughput channels */ 3036 }, 3037 }; 3038 3039 static struct udma_match_data j721e_main_data = { 3040 .psil_base = 0x1000, 3041 .enable_memcpy_support = true, 3042 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, 3043 .statictr_z_mask = GENMASK(23, 0), 3044 .rchan_oes_offset = 0x400, 3045 .tpl_levels = 3, 3046 .level_start_idx = { 3047 [0] = 16, /* Normal channels */ 3048 [1] = 4, /* High Throughput channels */ 3049 [2] = 0, /* Ultra High Throughput channels */ 3050 }, 3051 }; 3052 3053 static struct udma_match_data j721e_mcu_data = { 3054 .psil_base = 0x6000, 3055 .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ 3056 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, 3057 .statictr_z_mask = GENMASK(23, 0), 3058 .rchan_oes_offset = 0x400, 3059 .tpl_levels = 2, 3060 .level_start_idx = { 3061 [0] = 2, /* Normal channels */ 3062 [1] = 0, /* High Throughput channels */ 3063 }, 3064 }; 3065 3066 static const struct of_device_id udma_of_match[] = { 3067 { 3068 .compatible = "ti,am654-navss-main-udmap", 3069 .data = &am654_main_data, 3070 }, 3071 { 3072 .compatible = "ti,am654-navss-mcu-udmap", 3073 .data = &am654_mcu_data, 3074 }, { 3075 .compatible = "ti,j721e-navss-main-udmap", 3076 .data = &j721e_main_data, 3077 }, { 3078 .compatible = "ti,j721e-navss-mcu-udmap", 3079 .data = &j721e_mcu_data, 3080 }, 3081 { /* Sentinel */ }, 3082 }; 3083 3084 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) 3085 { 3086 struct resource *res; 3087 int i; 3088 3089 for (i = 0; i < MMR_LAST; i++) { 3090 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 3091 mmr_names[i]); 3092 ud->mmrs[i] = devm_ioremap_resource(&pdev->dev, res); 3093 if (IS_ERR(ud->mmrs[i])) 3094 return PTR_ERR(ud->mmrs[i]); 3095 } 3096 3097 return 0; 3098 } 3099 3100 static int udma_setup_resources(struct udma_dev *ud) 3101 { 3102 struct device *dev = ud->dev; 3103 int ch_count, ret, i, j; 3104 u32 cap2, cap3; 3105 struct ti_sci_resource_desc *rm_desc; 3106 struct ti_sci_resource *rm_res, irq_res; 3107 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; 3108 static const char * const range_names[] = { "ti,sci-rm-range-tchan", 3109 "ti,sci-rm-range-rchan", 3110 "ti,sci-rm-range-rflow" }; 3111 3112 cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28); 3113 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); 3114 3115 ud->rflow_cnt = cap3 & 0x3fff; 3116 ud->tchan_cnt = cap2 & 0x1ff; 3117 ud->echan_cnt = (cap2 >> 9) & 0x1ff; 3118 ud->rchan_cnt = (cap2 >> 18) & 0x1ff; 3119 ch_count = ud->tchan_cnt + ud->rchan_cnt; 3120 3121 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), 3122 sizeof(unsigned long), GFP_KERNEL); 3123 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), 3124 GFP_KERNEL); 3125 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), 3126 sizeof(unsigned long), GFP_KERNEL); 3127 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), 3128 GFP_KERNEL); 3129 ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt), 3130 sizeof(unsigned long), 3131 GFP_KERNEL); 3132 ud->rflow_gp_map_allocated = devm_kcalloc(dev, 3133 BITS_TO_LONGS(ud->rflow_cnt), 3134 sizeof(unsigned long), 3135 GFP_KERNEL); 3136 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), 3137 sizeof(unsigned long), 3138 GFP_KERNEL); 3139 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), 3140 GFP_KERNEL); 3141 3142 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map || 3143 !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans || 3144 !ud->rflows || !ud->rflow_in_use) 3145 return -ENOMEM; 3146 3147 /* 3148 * RX flows with the same Ids as RX channels are reserved to be used 3149 * as default flows if remote HW can't generate flow_ids. Those 3150 * RX flows can be requested only explicitly by id. 3151 */ 3152 bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt); 3153 3154 /* by default no GP rflows are assigned to Linux */ 3155 bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt); 3156 3157 /* Get resource ranges from tisci */ 3158 for (i = 0; i < RM_RANGE_LAST; i++) 3159 tisci_rm->rm_ranges[i] = 3160 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, 3161 tisci_rm->tisci_dev_id, 3162 (char *)range_names[i]); 3163 3164 /* tchan ranges */ 3165 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 3166 if (IS_ERR(rm_res)) { 3167 bitmap_zero(ud->tchan_map, ud->tchan_cnt); 3168 } else { 3169 bitmap_fill(ud->tchan_map, ud->tchan_cnt); 3170 for (i = 0; i < rm_res->sets; i++) { 3171 rm_desc = &rm_res->desc[i]; 3172 bitmap_clear(ud->tchan_map, rm_desc->start, 3173 rm_desc->num); 3174 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n", 3175 rm_desc->start, rm_desc->num); 3176 } 3177 } 3178 irq_res.sets = rm_res->sets; 3179 3180 /* rchan and matching default flow ranges */ 3181 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 3182 if (IS_ERR(rm_res)) { 3183 bitmap_zero(ud->rchan_map, ud->rchan_cnt); 3184 } else { 3185 bitmap_fill(ud->rchan_map, ud->rchan_cnt); 3186 for (i = 0; i < rm_res->sets; i++) { 3187 rm_desc = &rm_res->desc[i]; 3188 bitmap_clear(ud->rchan_map, rm_desc->start, 3189 rm_desc->num); 3190 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n", 3191 rm_desc->start, rm_desc->num); 3192 } 3193 } 3194 3195 irq_res.sets += rm_res->sets; 3196 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); 3197 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; 3198 for (i = 0; i < rm_res->sets; i++) { 3199 irq_res.desc[i].start = rm_res->desc[i].start; 3200 irq_res.desc[i].num = rm_res->desc[i].num; 3201 } 3202 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; 3203 for (j = 0; j < rm_res->sets; j++, i++) { 3204 irq_res.desc[i].start = rm_res->desc[j].start + 3205 ud->match_data->rchan_oes_offset; 3206 irq_res.desc[i].num = rm_res->desc[j].num; 3207 } 3208 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); 3209 kfree(irq_res.desc); 3210 if (ret) { 3211 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); 3212 return ret; 3213 } 3214 3215 /* GP rflow ranges */ 3216 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; 3217 if (IS_ERR(rm_res)) { 3218 /* all gp flows are assigned exclusively to Linux */ 3219 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt, 3220 ud->rflow_cnt - ud->rchan_cnt); 3221 } else { 3222 for (i = 0; i < rm_res->sets; i++) { 3223 rm_desc = &rm_res->desc[i]; 3224 bitmap_clear(ud->rflow_gp_map, rm_desc->start, 3225 rm_desc->num); 3226 dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n", 3227 rm_desc->start, rm_desc->num); 3228 } 3229 } 3230 3231 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt); 3232 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt); 3233 if (!ch_count) 3234 return -ENODEV; 3235 3236 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels), 3237 GFP_KERNEL); 3238 if (!ud->channels) 3239 return -ENOMEM; 3240 3241 dev_info(dev, "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", 3242 ch_count, 3243 ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), 3244 ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt), 3245 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map, 3246 ud->rflow_cnt)); 3247 3248 return ch_count; 3249 } 3250 3251 #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 3252 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 3253 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ 3254 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 3255 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 3256 3257 static int udma_probe(struct platform_device *pdev) 3258 { 3259 struct device_node *navss_node = pdev->dev.parent->of_node; 3260 struct device *dev = &pdev->dev; 3261 struct udma_dev *ud; 3262 const struct of_device_id *match; 3263 int i, ret; 3264 int ch_count; 3265 3266 ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48)); 3267 if (ret) 3268 dev_err(dev, "failed to set dma mask stuff\n"); 3269 3270 ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL); 3271 if (!ud) 3272 return -ENOMEM; 3273 3274 ret = udma_get_mmrs(pdev, ud); 3275 if (ret) 3276 return ret; 3277 3278 ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci"); 3279 if (IS_ERR(ud->tisci_rm.tisci)) 3280 return PTR_ERR(ud->tisci_rm.tisci); 3281 3282 ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id", 3283 &ud->tisci_rm.tisci_dev_id); 3284 if (ret) { 3285 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret); 3286 return ret; 3287 } 3288 pdev->id = ud->tisci_rm.tisci_dev_id; 3289 3290 ret = of_property_read_u32(navss_node, "ti,sci-dev-id", 3291 &ud->tisci_rm.tisci_navss_dev_id); 3292 if (ret) { 3293 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret); 3294 return ret; 3295 } 3296 3297 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; 3298 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; 3299 3300 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc"); 3301 if (IS_ERR(ud->ringacc)) 3302 return PTR_ERR(ud->ringacc); 3303 3304 dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 3305 DOMAIN_BUS_TI_SCI_INTA_MSI); 3306 if (!dev->msi_domain) { 3307 dev_err(dev, "Failed to get MSI domain\n"); 3308 return -EPROBE_DEFER; 3309 } 3310 3311 match = of_match_node(udma_of_match, dev->of_node); 3312 if (!match) { 3313 dev_err(dev, "No compatible match found\n"); 3314 return -ENODEV; 3315 } 3316 ud->match_data = match->data; 3317 3318 dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask); 3319 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); 3320 3321 ud->ddev.device_alloc_chan_resources = udma_alloc_chan_resources; 3322 ud->ddev.device_config = udma_slave_config; 3323 ud->ddev.device_prep_slave_sg = udma_prep_slave_sg; 3324 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; 3325 ud->ddev.device_issue_pending = udma_issue_pending; 3326 ud->ddev.device_tx_status = udma_tx_status; 3327 ud->ddev.device_pause = udma_pause; 3328 ud->ddev.device_resume = udma_resume; 3329 ud->ddev.device_terminate_all = udma_terminate_all; 3330 ud->ddev.device_synchronize = udma_synchronize; 3331 3332 ud->ddev.device_free_chan_resources = udma_free_chan_resources; 3333 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; 3334 ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; 3335 ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 3336 ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 3337 ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES; 3338 ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | 3339 DESC_METADATA_ENGINE; 3340 if (ud->match_data->enable_memcpy_support) { 3341 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask); 3342 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy; 3343 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM); 3344 } 3345 3346 ud->ddev.dev = dev; 3347 ud->dev = dev; 3348 ud->psil_base = ud->match_data->psil_base; 3349 3350 INIT_LIST_HEAD(&ud->ddev.channels); 3351 INIT_LIST_HEAD(&ud->desc_to_purge); 3352 3353 ch_count = udma_setup_resources(ud); 3354 if (ch_count <= 0) 3355 return ch_count; 3356 3357 spin_lock_init(&ud->lock); 3358 INIT_WORK(&ud->purge_work, udma_purge_desc_work); 3359 3360 ud->desc_align = 64; 3361 if (ud->desc_align < dma_get_cache_alignment()) 3362 ud->desc_align = dma_get_cache_alignment(); 3363 3364 for (i = 0; i < ud->tchan_cnt; i++) { 3365 struct udma_tchan *tchan = &ud->tchans[i]; 3366 3367 tchan->id = i; 3368 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000; 3369 } 3370 3371 for (i = 0; i < ud->rchan_cnt; i++) { 3372 struct udma_rchan *rchan = &ud->rchans[i]; 3373 3374 rchan->id = i; 3375 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000; 3376 } 3377 3378 for (i = 0; i < ud->rflow_cnt; i++) { 3379 struct udma_rflow *rflow = &ud->rflows[i]; 3380 3381 rflow->id = i; 3382 } 3383 3384 for (i = 0; i < ch_count; i++) { 3385 struct udma_chan *uc = &ud->channels[i]; 3386 3387 uc->ud = ud; 3388 uc->vc.desc_free = udma_desc_free; 3389 uc->id = i; 3390 uc->tchan = NULL; 3391 uc->rchan = NULL; 3392 uc->config.remote_thread_id = -1; 3393 uc->config.dir = DMA_MEM_TO_MEM; 3394 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", 3395 dev_name(dev), i); 3396 3397 vchan_init(&uc->vc, &ud->ddev); 3398 /* Use custom vchan completion handling */ 3399 tasklet_init(&uc->vc.task, udma_vchan_complete, 3400 (unsigned long)&uc->vc); 3401 init_completion(&uc->teardown_completed); 3402 } 3403 3404 ret = dma_async_device_register(&ud->ddev); 3405 if (ret) { 3406 dev_err(dev, "failed to register slave DMA engine: %d\n", ret); 3407 return ret; 3408 } 3409 3410 platform_set_drvdata(pdev, ud); 3411 3412 ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud); 3413 if (ret) { 3414 dev_err(dev, "failed to register of_dma controller\n"); 3415 dma_async_device_unregister(&ud->ddev); 3416 } 3417 3418 return ret; 3419 } 3420 3421 static struct platform_driver udma_driver = { 3422 .driver = { 3423 .name = "ti-udma", 3424 .of_match_table = udma_of_match, 3425 .suppress_bind_attrs = true, 3426 }, 3427 .probe = udma_probe, 3428 }; 3429 builtin_platform_driver(udma_driver); 3430 3431 /* Private interfaces to UDMA */ 3432 #include "k3-udma-private.c" 3433