xref: /openbmc/linux/drivers/dma/ti/k3-udma.c (revision ca48739e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
4  *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/delay.h>
9 #include <linux/dmaengine.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmapool.h>
12 #include <linux/err.h>
13 #include <linux/init.h>
14 #include <linux/interrupt.h>
15 #include <linux/list.h>
16 #include <linux/platform_device.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/sys_soc.h>
20 #include <linux/of.h>
21 #include <linux/of_dma.h>
22 #include <linux/of_device.h>
23 #include <linux/of_irq.h>
24 #include <linux/workqueue.h>
25 #include <linux/completion.h>
26 #include <linux/soc/ti/k3-ringacc.h>
27 #include <linux/soc/ti/ti_sci_protocol.h>
28 #include <linux/soc/ti/ti_sci_inta_msi.h>
29 #include <linux/dma/k3-event-router.h>
30 #include <linux/dma/ti-cppi5.h>
31 
32 #include "../virt-dma.h"
33 #include "k3-udma.h"
34 #include "k3-psil-priv.h"
35 
36 struct udma_static_tr {
37 	u8 elsize; /* RPSTR0 */
38 	u16 elcnt; /* RPSTR0 */
39 	u16 bstcnt; /* RPSTR1 */
40 };
41 
42 #define K3_UDMA_MAX_RFLOWS		1024
43 #define K3_UDMA_DEFAULT_RING_SIZE	16
44 
45 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */
46 #define UDMA_RFLOW_SRCTAG_NONE		0
47 #define UDMA_RFLOW_SRCTAG_CFG_TAG	1
48 #define UDMA_RFLOW_SRCTAG_FLOW_ID	2
49 #define UDMA_RFLOW_SRCTAG_SRC_TAG	4
50 
51 #define UDMA_RFLOW_DSTTAG_NONE		0
52 #define UDMA_RFLOW_DSTTAG_CFG_TAG	1
53 #define UDMA_RFLOW_DSTTAG_FLOW_ID	2
54 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO	4
55 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI	5
56 
57 struct udma_chan;
58 
59 enum k3_dma_type {
60 	DMA_TYPE_UDMA = 0,
61 	DMA_TYPE_BCDMA,
62 	DMA_TYPE_PKTDMA,
63 };
64 
65 enum udma_mmr {
66 	MMR_GCFG = 0,
67 	MMR_BCHANRT,
68 	MMR_RCHANRT,
69 	MMR_TCHANRT,
70 	MMR_LAST,
71 };
72 
73 static const char * const mmr_names[] = {
74 	[MMR_GCFG] = "gcfg",
75 	[MMR_BCHANRT] = "bchanrt",
76 	[MMR_RCHANRT] = "rchanrt",
77 	[MMR_TCHANRT] = "tchanrt",
78 };
79 
80 struct udma_tchan {
81 	void __iomem *reg_rt;
82 
83 	int id;
84 	struct k3_ring *t_ring; /* Transmit ring */
85 	struct k3_ring *tc_ring; /* Transmit Completion ring */
86 	int tflow_id; /* applicable only for PKTDMA */
87 
88 };
89 
90 #define udma_bchan udma_tchan
91 
92 struct udma_rflow {
93 	int id;
94 	struct k3_ring *fd_ring; /* Free Descriptor ring */
95 	struct k3_ring *r_ring; /* Receive ring */
96 };
97 
98 struct udma_rchan {
99 	void __iomem *reg_rt;
100 
101 	int id;
102 };
103 
104 struct udma_oes_offsets {
105 	/* K3 UDMA Output Event Offset */
106 	u32 udma_rchan;
107 
108 	/* BCDMA Output Event Offsets */
109 	u32 bcdma_bchan_data;
110 	u32 bcdma_bchan_ring;
111 	u32 bcdma_tchan_data;
112 	u32 bcdma_tchan_ring;
113 	u32 bcdma_rchan_data;
114 	u32 bcdma_rchan_ring;
115 
116 	/* PKTDMA Output Event Offsets */
117 	u32 pktdma_tchan_flow;
118 	u32 pktdma_rchan_flow;
119 };
120 
121 #define UDMA_FLAG_PDMA_ACC32		BIT(0)
122 #define UDMA_FLAG_PDMA_BURST		BIT(1)
123 #define UDMA_FLAG_TDTYPE		BIT(2)
124 #define UDMA_FLAG_BURST_SIZE		BIT(3)
125 #define UDMA_FLAGS_J7_CLASS		(UDMA_FLAG_PDMA_ACC32 | \
126 					 UDMA_FLAG_PDMA_BURST | \
127 					 UDMA_FLAG_TDTYPE | \
128 					 UDMA_FLAG_BURST_SIZE)
129 
130 struct udma_match_data {
131 	enum k3_dma_type type;
132 	u32 psil_base;
133 	bool enable_memcpy_support;
134 	u32 flags;
135 	u32 statictr_z_mask;
136 	u8 burst_size[3];
137 };
138 
139 struct udma_soc_data {
140 	struct udma_oes_offsets oes;
141 	u32 bcdma_trigger_event_offset;
142 };
143 
144 struct udma_hwdesc {
145 	size_t cppi5_desc_size;
146 	void *cppi5_desc_vaddr;
147 	dma_addr_t cppi5_desc_paddr;
148 
149 	/* TR descriptor internal pointers */
150 	void *tr_req_base;
151 	struct cppi5_tr_resp_t *tr_resp_base;
152 };
153 
154 struct udma_rx_flush {
155 	struct udma_hwdesc hwdescs[2];
156 
157 	size_t buffer_size;
158 	void *buffer_vaddr;
159 	dma_addr_t buffer_paddr;
160 };
161 
162 struct udma_tpl {
163 	u8 levels;
164 	u32 start_idx[3];
165 };
166 
167 struct udma_dev {
168 	struct dma_device ddev;
169 	struct device *dev;
170 	void __iomem *mmrs[MMR_LAST];
171 	const struct udma_match_data *match_data;
172 	const struct udma_soc_data *soc_data;
173 
174 	struct udma_tpl bchan_tpl;
175 	struct udma_tpl tchan_tpl;
176 	struct udma_tpl rchan_tpl;
177 
178 	size_t desc_align; /* alignment to use for descriptors */
179 
180 	struct udma_tisci_rm tisci_rm;
181 
182 	struct k3_ringacc *ringacc;
183 
184 	struct work_struct purge_work;
185 	struct list_head desc_to_purge;
186 	spinlock_t lock;
187 
188 	struct udma_rx_flush rx_flush;
189 
190 	int bchan_cnt;
191 	int tchan_cnt;
192 	int echan_cnt;
193 	int rchan_cnt;
194 	int rflow_cnt;
195 	int tflow_cnt;
196 	unsigned long *bchan_map;
197 	unsigned long *tchan_map;
198 	unsigned long *rchan_map;
199 	unsigned long *rflow_gp_map;
200 	unsigned long *rflow_gp_map_allocated;
201 	unsigned long *rflow_in_use;
202 	unsigned long *tflow_map;
203 
204 	struct udma_bchan *bchans;
205 	struct udma_tchan *tchans;
206 	struct udma_rchan *rchans;
207 	struct udma_rflow *rflows;
208 
209 	struct udma_chan *channels;
210 	u32 psil_base;
211 	u32 atype;
212 	u32 asel;
213 };
214 
215 struct udma_desc {
216 	struct virt_dma_desc vd;
217 
218 	bool terminated;
219 
220 	enum dma_transfer_direction dir;
221 
222 	struct udma_static_tr static_tr;
223 	u32 residue;
224 
225 	unsigned int sglen;
226 	unsigned int desc_idx; /* Only used for cyclic in packet mode */
227 	unsigned int tr_idx;
228 
229 	u32 metadata_size;
230 	void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */
231 
232 	unsigned int hwdesc_count;
233 	struct udma_hwdesc hwdesc[];
234 };
235 
236 enum udma_chan_state {
237 	UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */
238 	UDMA_CHAN_IS_ACTIVE, /* Normal operation */
239 	UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */
240 };
241 
242 struct udma_tx_drain {
243 	struct delayed_work work;
244 	ktime_t tstamp;
245 	u32 residue;
246 };
247 
248 struct udma_chan_config {
249 	bool pkt_mode; /* TR or packet */
250 	bool needs_epib; /* EPIB is needed for the communication or not */
251 	u32 psd_size; /* size of Protocol Specific Data */
252 	u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */
253 	u32 hdesc_size; /* Size of a packet descriptor in packet mode */
254 	bool notdpkt; /* Suppress sending TDC packet */
255 	int remote_thread_id;
256 	u32 atype;
257 	u32 asel;
258 	u32 src_thread;
259 	u32 dst_thread;
260 	enum psil_endpoint_type ep_type;
261 	bool enable_acc32;
262 	bool enable_burst;
263 	enum udma_tp_level channel_tpl; /* Channel Throughput Level */
264 
265 	u32 tr_trigger_type;
266 
267 	/* PKDMA mapped channel */
268 	int mapped_channel_id;
269 	/* PKTDMA default tflow or rflow for mapped channel */
270 	int default_flow_id;
271 
272 	enum dma_transfer_direction dir;
273 };
274 
275 struct udma_chan {
276 	struct virt_dma_chan vc;
277 	struct dma_slave_config	cfg;
278 	struct udma_dev *ud;
279 	struct device *dma_dev;
280 	struct udma_desc *desc;
281 	struct udma_desc *terminated_desc;
282 	struct udma_static_tr static_tr;
283 	char *name;
284 
285 	struct udma_bchan *bchan;
286 	struct udma_tchan *tchan;
287 	struct udma_rchan *rchan;
288 	struct udma_rflow *rflow;
289 
290 	bool psil_paired;
291 
292 	int irq_num_ring;
293 	int irq_num_udma;
294 
295 	bool cyclic;
296 	bool paused;
297 
298 	enum udma_chan_state state;
299 	struct completion teardown_completed;
300 
301 	struct udma_tx_drain tx_drain;
302 
303 	u32 bcnt; /* number of bytes completed since the start of the channel */
304 
305 	/* Channel configuration parameters */
306 	struct udma_chan_config config;
307 
308 	/* dmapool for packet mode descriptors */
309 	bool use_dma_pool;
310 	struct dma_pool *hdesc_pool;
311 
312 	u32 id;
313 };
314 
315 static inline struct udma_dev *to_udma_dev(struct dma_device *d)
316 {
317 	return container_of(d, struct udma_dev, ddev);
318 }
319 
320 static inline struct udma_chan *to_udma_chan(struct dma_chan *c)
321 {
322 	return container_of(c, struct udma_chan, vc.chan);
323 }
324 
325 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t)
326 {
327 	return container_of(t, struct udma_desc, vd.tx);
328 }
329 
330 /* Generic register access functions */
331 static inline u32 udma_read(void __iomem *base, int reg)
332 {
333 	return readl(base + reg);
334 }
335 
336 static inline void udma_write(void __iomem *base, int reg, u32 val)
337 {
338 	writel(val, base + reg);
339 }
340 
341 static inline void udma_update_bits(void __iomem *base, int reg,
342 				    u32 mask, u32 val)
343 {
344 	u32 tmp, orig;
345 
346 	orig = readl(base + reg);
347 	tmp = orig & ~mask;
348 	tmp |= (val & mask);
349 
350 	if (tmp != orig)
351 		writel(tmp, base + reg);
352 }
353 
354 /* TCHANRT */
355 static inline u32 udma_tchanrt_read(struct udma_chan *uc, int reg)
356 {
357 	if (!uc->tchan)
358 		return 0;
359 	return udma_read(uc->tchan->reg_rt, reg);
360 }
361 
362 static inline void udma_tchanrt_write(struct udma_chan *uc, int reg, u32 val)
363 {
364 	if (!uc->tchan)
365 		return;
366 	udma_write(uc->tchan->reg_rt, reg, val);
367 }
368 
369 static inline void udma_tchanrt_update_bits(struct udma_chan *uc, int reg,
370 					    u32 mask, u32 val)
371 {
372 	if (!uc->tchan)
373 		return;
374 	udma_update_bits(uc->tchan->reg_rt, reg, mask, val);
375 }
376 
377 /* RCHANRT */
378 static inline u32 udma_rchanrt_read(struct udma_chan *uc, int reg)
379 {
380 	if (!uc->rchan)
381 		return 0;
382 	return udma_read(uc->rchan->reg_rt, reg);
383 }
384 
385 static inline void udma_rchanrt_write(struct udma_chan *uc, int reg, u32 val)
386 {
387 	if (!uc->rchan)
388 		return;
389 	udma_write(uc->rchan->reg_rt, reg, val);
390 }
391 
392 static inline void udma_rchanrt_update_bits(struct udma_chan *uc, int reg,
393 					    u32 mask, u32 val)
394 {
395 	if (!uc->rchan)
396 		return;
397 	udma_update_bits(uc->rchan->reg_rt, reg, mask, val);
398 }
399 
400 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread)
401 {
402 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
403 
404 	dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
405 	return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,
406 					      tisci_rm->tisci_navss_dev_id,
407 					      src_thread, dst_thread);
408 }
409 
410 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
411 			     u32 dst_thread)
412 {
413 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
414 
415 	dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
416 	return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,
417 						tisci_rm->tisci_navss_dev_id,
418 						src_thread, dst_thread);
419 }
420 
421 static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel)
422 {
423 	struct device *chan_dev = &chan->dev->device;
424 
425 	if (asel == 0) {
426 		/* No special handling for the channel */
427 		chan->dev->chan_dma_dev = false;
428 
429 		chan_dev->dma_coherent = false;
430 		chan_dev->dma_parms = NULL;
431 	} else if (asel == 14 || asel == 15) {
432 		chan->dev->chan_dma_dev = true;
433 
434 		chan_dev->dma_coherent = true;
435 		dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48));
436 		chan_dev->dma_parms = chan_dev->parent->dma_parms;
437 	} else {
438 		dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel);
439 
440 		chan_dev->dma_coherent = false;
441 		chan_dev->dma_parms = NULL;
442 	}
443 }
444 
445 static u8 udma_get_chan_tpl_index(struct udma_tpl *tpl_map, int chan_id)
446 {
447 	int i;
448 
449 	for (i = 0; i < tpl_map->levels; i++) {
450 		if (chan_id >= tpl_map->start_idx[i])
451 			return i;
452 	}
453 
454 	return 0;
455 }
456 
457 static void udma_reset_uchan(struct udma_chan *uc)
458 {
459 	memset(&uc->config, 0, sizeof(uc->config));
460 	uc->config.remote_thread_id = -1;
461 	uc->config.mapped_channel_id = -1;
462 	uc->config.default_flow_id = -1;
463 	uc->state = UDMA_CHAN_IS_IDLE;
464 }
465 
466 static void udma_dump_chan_stdata(struct udma_chan *uc)
467 {
468 	struct device *dev = uc->ud->dev;
469 	u32 offset;
470 	int i;
471 
472 	if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) {
473 		dev_dbg(dev, "TCHAN State data:\n");
474 		for (i = 0; i < 32; i++) {
475 			offset = UDMA_CHAN_RT_STDATA_REG + i * 4;
476 			dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i,
477 				udma_tchanrt_read(uc, offset));
478 		}
479 	}
480 
481 	if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) {
482 		dev_dbg(dev, "RCHAN State data:\n");
483 		for (i = 0; i < 32; i++) {
484 			offset = UDMA_CHAN_RT_STDATA_REG + i * 4;
485 			dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i,
486 				udma_rchanrt_read(uc, offset));
487 		}
488 	}
489 }
490 
491 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d,
492 						    int idx)
493 {
494 	return d->hwdesc[idx].cppi5_desc_paddr;
495 }
496 
497 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx)
498 {
499 	return d->hwdesc[idx].cppi5_desc_vaddr;
500 }
501 
502 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,
503 						   dma_addr_t paddr)
504 {
505 	struct udma_desc *d = uc->terminated_desc;
506 
507 	if (d) {
508 		dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
509 								   d->desc_idx);
510 
511 		if (desc_paddr != paddr)
512 			d = NULL;
513 	}
514 
515 	if (!d) {
516 		d = uc->desc;
517 		if (d) {
518 			dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
519 								d->desc_idx);
520 
521 			if (desc_paddr != paddr)
522 				d = NULL;
523 		}
524 	}
525 
526 	return d;
527 }
528 
529 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d)
530 {
531 	if (uc->use_dma_pool) {
532 		int i;
533 
534 		for (i = 0; i < d->hwdesc_count; i++) {
535 			if (!d->hwdesc[i].cppi5_desc_vaddr)
536 				continue;
537 
538 			dma_pool_free(uc->hdesc_pool,
539 				      d->hwdesc[i].cppi5_desc_vaddr,
540 				      d->hwdesc[i].cppi5_desc_paddr);
541 
542 			d->hwdesc[i].cppi5_desc_vaddr = NULL;
543 		}
544 	} else if (d->hwdesc[0].cppi5_desc_vaddr) {
545 		dma_free_coherent(uc->dma_dev, d->hwdesc[0].cppi5_desc_size,
546 				  d->hwdesc[0].cppi5_desc_vaddr,
547 				  d->hwdesc[0].cppi5_desc_paddr);
548 
549 		d->hwdesc[0].cppi5_desc_vaddr = NULL;
550 	}
551 }
552 
553 static void udma_purge_desc_work(struct work_struct *work)
554 {
555 	struct udma_dev *ud = container_of(work, typeof(*ud), purge_work);
556 	struct virt_dma_desc *vd, *_vd;
557 	unsigned long flags;
558 	LIST_HEAD(head);
559 
560 	spin_lock_irqsave(&ud->lock, flags);
561 	list_splice_tail_init(&ud->desc_to_purge, &head);
562 	spin_unlock_irqrestore(&ud->lock, flags);
563 
564 	list_for_each_entry_safe(vd, _vd, &head, node) {
565 		struct udma_chan *uc = to_udma_chan(vd->tx.chan);
566 		struct udma_desc *d = to_udma_desc(&vd->tx);
567 
568 		udma_free_hwdesc(uc, d);
569 		list_del(&vd->node);
570 		kfree(d);
571 	}
572 
573 	/* If more to purge, schedule the work again */
574 	if (!list_empty(&ud->desc_to_purge))
575 		schedule_work(&ud->purge_work);
576 }
577 
578 static void udma_desc_free(struct virt_dma_desc *vd)
579 {
580 	struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);
581 	struct udma_chan *uc = to_udma_chan(vd->tx.chan);
582 	struct udma_desc *d = to_udma_desc(&vd->tx);
583 	unsigned long flags;
584 
585 	if (uc->terminated_desc == d)
586 		uc->terminated_desc = NULL;
587 
588 	if (uc->use_dma_pool) {
589 		udma_free_hwdesc(uc, d);
590 		kfree(d);
591 		return;
592 	}
593 
594 	spin_lock_irqsave(&ud->lock, flags);
595 	list_add_tail(&vd->node, &ud->desc_to_purge);
596 	spin_unlock_irqrestore(&ud->lock, flags);
597 
598 	schedule_work(&ud->purge_work);
599 }
600 
601 static bool udma_is_chan_running(struct udma_chan *uc)
602 {
603 	u32 trt_ctl = 0;
604 	u32 rrt_ctl = 0;
605 
606 	if (uc->tchan)
607 		trt_ctl = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
608 	if (uc->rchan)
609 		rrt_ctl = udma_rchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
610 
611 	if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN)
612 		return true;
613 
614 	return false;
615 }
616 
617 static bool udma_is_chan_paused(struct udma_chan *uc)
618 {
619 	u32 val, pause_mask;
620 
621 	switch (uc->config.dir) {
622 	case DMA_DEV_TO_MEM:
623 		val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);
624 		pause_mask = UDMA_PEER_RT_EN_PAUSE;
625 		break;
626 	case DMA_MEM_TO_DEV:
627 		val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);
628 		pause_mask = UDMA_PEER_RT_EN_PAUSE;
629 		break;
630 	case DMA_MEM_TO_MEM:
631 		val = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
632 		pause_mask = UDMA_CHAN_RT_CTL_PAUSE;
633 		break;
634 	default:
635 		return false;
636 	}
637 
638 	if (val & pause_mask)
639 		return true;
640 
641 	return false;
642 }
643 
644 static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc)
645 {
646 	return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr;
647 }
648 
649 static int udma_push_to_ring(struct udma_chan *uc, int idx)
650 {
651 	struct udma_desc *d = uc->desc;
652 	struct k3_ring *ring = NULL;
653 	dma_addr_t paddr;
654 
655 	switch (uc->config.dir) {
656 	case DMA_DEV_TO_MEM:
657 		ring = uc->rflow->fd_ring;
658 		break;
659 	case DMA_MEM_TO_DEV:
660 	case DMA_MEM_TO_MEM:
661 		ring = uc->tchan->t_ring;
662 		break;
663 	default:
664 		return -EINVAL;
665 	}
666 
667 	/* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */
668 	if (idx == -1) {
669 		paddr = udma_get_rx_flush_hwdesc_paddr(uc);
670 	} else {
671 		paddr = udma_curr_cppi5_desc_paddr(d, idx);
672 
673 		wmb(); /* Ensure that writes are not moved over this point */
674 	}
675 
676 	return k3_ringacc_ring_push(ring, &paddr);
677 }
678 
679 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr)
680 {
681 	if (uc->config.dir != DMA_DEV_TO_MEM)
682 		return false;
683 
684 	if (addr == udma_get_rx_flush_hwdesc_paddr(uc))
685 		return true;
686 
687 	return false;
688 }
689 
690 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
691 {
692 	struct k3_ring *ring = NULL;
693 	int ret;
694 
695 	switch (uc->config.dir) {
696 	case DMA_DEV_TO_MEM:
697 		ring = uc->rflow->r_ring;
698 		break;
699 	case DMA_MEM_TO_DEV:
700 	case DMA_MEM_TO_MEM:
701 		ring = uc->tchan->tc_ring;
702 		break;
703 	default:
704 		return -ENOENT;
705 	}
706 
707 	ret = k3_ringacc_ring_pop(ring, addr);
708 	if (ret)
709 		return ret;
710 
711 	rmb(); /* Ensure that reads are not moved before this point */
712 
713 	/* Teardown completion */
714 	if (cppi5_desc_is_tdcm(*addr))
715 		return 0;
716 
717 	/* Check for flush descriptor */
718 	if (udma_desc_is_rx_flush(uc, *addr))
719 		return -ENOENT;
720 
721 	return 0;
722 }
723 
724 static void udma_reset_rings(struct udma_chan *uc)
725 {
726 	struct k3_ring *ring1 = NULL;
727 	struct k3_ring *ring2 = NULL;
728 
729 	switch (uc->config.dir) {
730 	case DMA_DEV_TO_MEM:
731 		if (uc->rchan) {
732 			ring1 = uc->rflow->fd_ring;
733 			ring2 = uc->rflow->r_ring;
734 		}
735 		break;
736 	case DMA_MEM_TO_DEV:
737 	case DMA_MEM_TO_MEM:
738 		if (uc->tchan) {
739 			ring1 = uc->tchan->t_ring;
740 			ring2 = uc->tchan->tc_ring;
741 		}
742 		break;
743 	default:
744 		break;
745 	}
746 
747 	if (ring1)
748 		k3_ringacc_ring_reset_dma(ring1,
749 					  k3_ringacc_ring_get_occ(ring1));
750 	if (ring2)
751 		k3_ringacc_ring_reset(ring2);
752 
753 	/* make sure we are not leaking memory by stalled descriptor */
754 	if (uc->terminated_desc) {
755 		udma_desc_free(&uc->terminated_desc->vd);
756 		uc->terminated_desc = NULL;
757 	}
758 }
759 
760 static void udma_reset_counters(struct udma_chan *uc)
761 {
762 	u32 val;
763 
764 	if (uc->tchan) {
765 		val = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
766 		udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
767 
768 		val = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
769 		udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
770 
771 		val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);
772 		udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);
773 
774 		if (!uc->bchan) {
775 			val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
776 			udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
777 		}
778 	}
779 
780 	if (uc->rchan) {
781 		val = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
782 		udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
783 
784 		val = udma_rchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
785 		udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
786 
787 		val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);
788 		udma_rchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);
789 
790 		val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
791 		udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
792 	}
793 
794 	uc->bcnt = 0;
795 }
796 
797 static int udma_reset_chan(struct udma_chan *uc, bool hard)
798 {
799 	switch (uc->config.dir) {
800 	case DMA_DEV_TO_MEM:
801 		udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
802 		udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
803 		break;
804 	case DMA_MEM_TO_DEV:
805 		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
806 		udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
807 		break;
808 	case DMA_MEM_TO_MEM:
809 		udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
810 		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
811 		break;
812 	default:
813 		return -EINVAL;
814 	}
815 
816 	/* Reset all counters */
817 	udma_reset_counters(uc);
818 
819 	/* Hard reset: re-initialize the channel to reset */
820 	if (hard) {
821 		struct udma_chan_config ucc_backup;
822 		int ret;
823 
824 		memcpy(&ucc_backup, &uc->config, sizeof(uc->config));
825 		uc->ud->ddev.device_free_chan_resources(&uc->vc.chan);
826 
827 		/* restore the channel configuration */
828 		memcpy(&uc->config, &ucc_backup, sizeof(uc->config));
829 		ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan);
830 		if (ret)
831 			return ret;
832 
833 		/*
834 		 * Setting forced teardown after forced reset helps recovering
835 		 * the rchan.
836 		 */
837 		if (uc->config.dir == DMA_DEV_TO_MEM)
838 			udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
839 					   UDMA_CHAN_RT_CTL_EN |
840 					   UDMA_CHAN_RT_CTL_TDOWN |
841 					   UDMA_CHAN_RT_CTL_FTDOWN);
842 	}
843 	uc->state = UDMA_CHAN_IS_IDLE;
844 
845 	return 0;
846 }
847 
848 static void udma_start_desc(struct udma_chan *uc)
849 {
850 	struct udma_chan_config *ucc = &uc->config;
851 
852 	if (uc->ud->match_data->type == DMA_TYPE_UDMA && ucc->pkt_mode &&
853 	    (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) {
854 		int i;
855 
856 		/*
857 		 * UDMA only: Push all descriptors to ring for packet mode
858 		 * cyclic or RX
859 		 * PKTDMA supports pre-linked descriptor and cyclic is not
860 		 * supported
861 		 */
862 		for (i = 0; i < uc->desc->sglen; i++)
863 			udma_push_to_ring(uc, i);
864 	} else {
865 		udma_push_to_ring(uc, 0);
866 	}
867 }
868 
869 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc)
870 {
871 	/* Only PDMAs have staticTR */
872 	if (uc->config.ep_type == PSIL_EP_NATIVE)
873 		return false;
874 
875 	/* Check if the staticTR configuration has changed for TX */
876 	if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr)))
877 		return true;
878 
879 	return false;
880 }
881 
882 static int udma_start(struct udma_chan *uc)
883 {
884 	struct virt_dma_desc *vd = vchan_next_desc(&uc->vc);
885 
886 	if (!vd) {
887 		uc->desc = NULL;
888 		return -ENOENT;
889 	}
890 
891 	list_del(&vd->node);
892 
893 	uc->desc = to_udma_desc(&vd->tx);
894 
895 	/* Channel is already running and does not need reconfiguration */
896 	if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) {
897 		udma_start_desc(uc);
898 		goto out;
899 	}
900 
901 	/* Make sure that we clear the teardown bit, if it is set */
902 	udma_reset_chan(uc, false);
903 
904 	/* Push descriptors before we start the channel */
905 	udma_start_desc(uc);
906 
907 	switch (uc->desc->dir) {
908 	case DMA_DEV_TO_MEM:
909 		/* Config remote TR */
910 		if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
911 			u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
912 				  PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
913 			const struct udma_match_data *match_data =
914 							uc->ud->match_data;
915 
916 			if (uc->config.enable_acc32)
917 				val |= PDMA_STATIC_TR_XY_ACC32;
918 			if (uc->config.enable_burst)
919 				val |= PDMA_STATIC_TR_XY_BURST;
920 
921 			udma_rchanrt_write(uc,
922 					   UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,
923 					   val);
924 
925 			udma_rchanrt_write(uc,
926 				UDMA_CHAN_RT_PEER_STATIC_TR_Z_REG,
927 				PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt,
928 						 match_data->statictr_z_mask));
929 
930 			/* save the current staticTR configuration */
931 			memcpy(&uc->static_tr, &uc->desc->static_tr,
932 			       sizeof(uc->static_tr));
933 		}
934 
935 		udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
936 				   UDMA_CHAN_RT_CTL_EN);
937 
938 		/* Enable remote */
939 		udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
940 				   UDMA_PEER_RT_EN_ENABLE);
941 
942 		break;
943 	case DMA_MEM_TO_DEV:
944 		/* Config remote TR */
945 		if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
946 			u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
947 				  PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
948 
949 			if (uc->config.enable_acc32)
950 				val |= PDMA_STATIC_TR_XY_ACC32;
951 			if (uc->config.enable_burst)
952 				val |= PDMA_STATIC_TR_XY_BURST;
953 
954 			udma_tchanrt_write(uc,
955 					   UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,
956 					   val);
957 
958 			/* save the current staticTR configuration */
959 			memcpy(&uc->static_tr, &uc->desc->static_tr,
960 			       sizeof(uc->static_tr));
961 		}
962 
963 		/* Enable remote */
964 		udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
965 				   UDMA_PEER_RT_EN_ENABLE);
966 
967 		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
968 				   UDMA_CHAN_RT_CTL_EN);
969 
970 		break;
971 	case DMA_MEM_TO_MEM:
972 		udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
973 				   UDMA_CHAN_RT_CTL_EN);
974 		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
975 				   UDMA_CHAN_RT_CTL_EN);
976 
977 		break;
978 	default:
979 		return -EINVAL;
980 	}
981 
982 	uc->state = UDMA_CHAN_IS_ACTIVE;
983 out:
984 
985 	return 0;
986 }
987 
988 static int udma_stop(struct udma_chan *uc)
989 {
990 	enum udma_chan_state old_state = uc->state;
991 
992 	uc->state = UDMA_CHAN_IS_TERMINATING;
993 	reinit_completion(&uc->teardown_completed);
994 
995 	switch (uc->config.dir) {
996 	case DMA_DEV_TO_MEM:
997 		if (!uc->cyclic && !uc->desc)
998 			udma_push_to_ring(uc, -1);
999 
1000 		udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
1001 				   UDMA_PEER_RT_EN_ENABLE |
1002 				   UDMA_PEER_RT_EN_TEARDOWN);
1003 		break;
1004 	case DMA_MEM_TO_DEV:
1005 		udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
1006 				   UDMA_PEER_RT_EN_ENABLE |
1007 				   UDMA_PEER_RT_EN_FLUSH);
1008 		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
1009 				   UDMA_CHAN_RT_CTL_EN |
1010 				   UDMA_CHAN_RT_CTL_TDOWN);
1011 		break;
1012 	case DMA_MEM_TO_MEM:
1013 		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
1014 				   UDMA_CHAN_RT_CTL_EN |
1015 				   UDMA_CHAN_RT_CTL_TDOWN);
1016 		break;
1017 	default:
1018 		uc->state = old_state;
1019 		complete_all(&uc->teardown_completed);
1020 		return -EINVAL;
1021 	}
1022 
1023 	return 0;
1024 }
1025 
1026 static void udma_cyclic_packet_elapsed(struct udma_chan *uc)
1027 {
1028 	struct udma_desc *d = uc->desc;
1029 	struct cppi5_host_desc_t *h_desc;
1030 
1031 	h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr;
1032 	cppi5_hdesc_reset_to_original(h_desc);
1033 	udma_push_to_ring(uc, d->desc_idx);
1034 	d->desc_idx = (d->desc_idx + 1) % d->sglen;
1035 }
1036 
1037 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d)
1038 {
1039 	struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr;
1040 
1041 	memcpy(d->metadata, h_desc->epib, d->metadata_size);
1042 }
1043 
1044 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d)
1045 {
1046 	u32 peer_bcnt, bcnt;
1047 
1048 	/* Only TX towards PDMA is affected */
1049 	if (uc->config.ep_type == PSIL_EP_NATIVE ||
1050 	    uc->config.dir != DMA_MEM_TO_DEV)
1051 		return true;
1052 
1053 	peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
1054 	bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
1055 
1056 	/* Transfer is incomplete, store current residue and time stamp */
1057 	if (peer_bcnt < bcnt) {
1058 		uc->tx_drain.residue = bcnt - peer_bcnt;
1059 		uc->tx_drain.tstamp = ktime_get();
1060 		return false;
1061 	}
1062 
1063 	return true;
1064 }
1065 
1066 static void udma_check_tx_completion(struct work_struct *work)
1067 {
1068 	struct udma_chan *uc = container_of(work, typeof(*uc),
1069 					    tx_drain.work.work);
1070 	bool desc_done = true;
1071 	u32 residue_diff;
1072 	ktime_t time_diff;
1073 	unsigned long delay;
1074 
1075 	while (1) {
1076 		if (uc->desc) {
1077 			/* Get previous residue and time stamp */
1078 			residue_diff = uc->tx_drain.residue;
1079 			time_diff = uc->tx_drain.tstamp;
1080 			/*
1081 			 * Get current residue and time stamp or see if
1082 			 * transfer is complete
1083 			 */
1084 			desc_done = udma_is_desc_really_done(uc, uc->desc);
1085 		}
1086 
1087 		if (!desc_done) {
1088 			/*
1089 			 * Find the time delta and residue delta w.r.t
1090 			 * previous poll
1091 			 */
1092 			time_diff = ktime_sub(uc->tx_drain.tstamp,
1093 					      time_diff) + 1;
1094 			residue_diff -= uc->tx_drain.residue;
1095 			if (residue_diff) {
1096 				/*
1097 				 * Try to guess when we should check
1098 				 * next time by calculating rate at
1099 				 * which data is being drained at the
1100 				 * peer device
1101 				 */
1102 				delay = (time_diff / residue_diff) *
1103 					uc->tx_drain.residue;
1104 			} else {
1105 				/* No progress, check again in 1 second  */
1106 				schedule_delayed_work(&uc->tx_drain.work, HZ);
1107 				break;
1108 			}
1109 
1110 			usleep_range(ktime_to_us(delay),
1111 				     ktime_to_us(delay) + 10);
1112 			continue;
1113 		}
1114 
1115 		if (uc->desc) {
1116 			struct udma_desc *d = uc->desc;
1117 
1118 			uc->bcnt += d->residue;
1119 			udma_start(uc);
1120 			vchan_cookie_complete(&d->vd);
1121 			break;
1122 		}
1123 
1124 		break;
1125 	}
1126 }
1127 
1128 static irqreturn_t udma_ring_irq_handler(int irq, void *data)
1129 {
1130 	struct udma_chan *uc = data;
1131 	struct udma_desc *d;
1132 	dma_addr_t paddr = 0;
1133 
1134 	if (udma_pop_from_ring(uc, &paddr) || !paddr)
1135 		return IRQ_HANDLED;
1136 
1137 	spin_lock(&uc->vc.lock);
1138 
1139 	/* Teardown completion message */
1140 	if (cppi5_desc_is_tdcm(paddr)) {
1141 		complete_all(&uc->teardown_completed);
1142 
1143 		if (uc->terminated_desc) {
1144 			udma_desc_free(&uc->terminated_desc->vd);
1145 			uc->terminated_desc = NULL;
1146 		}
1147 
1148 		if (!uc->desc)
1149 			udma_start(uc);
1150 
1151 		goto out;
1152 	}
1153 
1154 	d = udma_udma_desc_from_paddr(uc, paddr);
1155 
1156 	if (d) {
1157 		dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
1158 								   d->desc_idx);
1159 		if (desc_paddr != paddr) {
1160 			dev_err(uc->ud->dev, "not matching descriptors!\n");
1161 			goto out;
1162 		}
1163 
1164 		if (d == uc->desc) {
1165 			/* active descriptor */
1166 			if (uc->cyclic) {
1167 				udma_cyclic_packet_elapsed(uc);
1168 				vchan_cyclic_callback(&d->vd);
1169 			} else {
1170 				if (udma_is_desc_really_done(uc, d)) {
1171 					uc->bcnt += d->residue;
1172 					udma_start(uc);
1173 					vchan_cookie_complete(&d->vd);
1174 				} else {
1175 					schedule_delayed_work(&uc->tx_drain.work,
1176 							      0);
1177 				}
1178 			}
1179 		} else {
1180 			/*
1181 			 * terminated descriptor, mark the descriptor as
1182 			 * completed to update the channel's cookie marker
1183 			 */
1184 			dma_cookie_complete(&d->vd.tx);
1185 		}
1186 	}
1187 out:
1188 	spin_unlock(&uc->vc.lock);
1189 
1190 	return IRQ_HANDLED;
1191 }
1192 
1193 static irqreturn_t udma_udma_irq_handler(int irq, void *data)
1194 {
1195 	struct udma_chan *uc = data;
1196 	struct udma_desc *d;
1197 
1198 	spin_lock(&uc->vc.lock);
1199 	d = uc->desc;
1200 	if (d) {
1201 		d->tr_idx = (d->tr_idx + 1) % d->sglen;
1202 
1203 		if (uc->cyclic) {
1204 			vchan_cyclic_callback(&d->vd);
1205 		} else {
1206 			/* TODO: figure out the real amount of data */
1207 			uc->bcnt += d->residue;
1208 			udma_start(uc);
1209 			vchan_cookie_complete(&d->vd);
1210 		}
1211 	}
1212 
1213 	spin_unlock(&uc->vc.lock);
1214 
1215 	return IRQ_HANDLED;
1216 }
1217 
1218 /**
1219  * __udma_alloc_gp_rflow_range - alloc range of GP RX flows
1220  * @ud: UDMA device
1221  * @from: Start the search from this flow id number
1222  * @cnt: Number of consecutive flow ids to allocate
1223  *
1224  * Allocate range of RX flow ids for future use, those flows can be requested
1225  * only using explicit flow id number. if @from is set to -1 it will try to find
1226  * first free range. if @from is positive value it will force allocation only
1227  * of the specified range of flows.
1228  *
1229  * Returns -ENOMEM if can't find free range.
1230  * -EEXIST if requested range is busy.
1231  * -EINVAL if wrong input values passed.
1232  * Returns flow id on success.
1233  */
1234 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
1235 {
1236 	int start, tmp_from;
1237 	DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS);
1238 
1239 	tmp_from = from;
1240 	if (tmp_from < 0)
1241 		tmp_from = ud->rchan_cnt;
1242 	/* default flows can't be allocated and accessible only by id */
1243 	if (tmp_from < ud->rchan_cnt)
1244 		return -EINVAL;
1245 
1246 	if (tmp_from + cnt > ud->rflow_cnt)
1247 		return -EINVAL;
1248 
1249 	bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated,
1250 		  ud->rflow_cnt);
1251 
1252 	start = bitmap_find_next_zero_area(tmp,
1253 					   ud->rflow_cnt,
1254 					   tmp_from, cnt, 0);
1255 	if (start >= ud->rflow_cnt)
1256 		return -ENOMEM;
1257 
1258 	if (from >= 0 && start != from)
1259 		return -EEXIST;
1260 
1261 	bitmap_set(ud->rflow_gp_map_allocated, start, cnt);
1262 	return start;
1263 }
1264 
1265 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
1266 {
1267 	if (from < ud->rchan_cnt)
1268 		return -EINVAL;
1269 	if (from + cnt > ud->rflow_cnt)
1270 		return -EINVAL;
1271 
1272 	bitmap_clear(ud->rflow_gp_map_allocated, from, cnt);
1273 	return 0;
1274 }
1275 
1276 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id)
1277 {
1278 	/*
1279 	 * Attempt to request rflow by ID can be made for any rflow
1280 	 * if not in use with assumption that caller knows what's doing.
1281 	 * TI-SCI FW will perform additional permission check ant way, it's
1282 	 * safe
1283 	 */
1284 
1285 	if (id < 0 || id >= ud->rflow_cnt)
1286 		return ERR_PTR(-ENOENT);
1287 
1288 	if (test_bit(id, ud->rflow_in_use))
1289 		return ERR_PTR(-ENOENT);
1290 
1291 	if (ud->rflow_gp_map) {
1292 		/* GP rflow has to be allocated first */
1293 		if (!test_bit(id, ud->rflow_gp_map) &&
1294 		    !test_bit(id, ud->rflow_gp_map_allocated))
1295 			return ERR_PTR(-EINVAL);
1296 	}
1297 
1298 	dev_dbg(ud->dev, "get rflow%d\n", id);
1299 	set_bit(id, ud->rflow_in_use);
1300 	return &ud->rflows[id];
1301 }
1302 
1303 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow)
1304 {
1305 	if (!test_bit(rflow->id, ud->rflow_in_use)) {
1306 		dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id);
1307 		return;
1308 	}
1309 
1310 	dev_dbg(ud->dev, "put rflow%d\n", rflow->id);
1311 	clear_bit(rflow->id, ud->rflow_in_use);
1312 }
1313 
1314 #define UDMA_RESERVE_RESOURCE(res)					\
1315 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud,	\
1316 					       enum udma_tp_level tpl,	\
1317 					       int id)			\
1318 {									\
1319 	if (id >= 0) {							\
1320 		if (test_bit(id, ud->res##_map)) {			\
1321 			dev_err(ud->dev, "res##%d is in use\n", id);	\
1322 			return ERR_PTR(-ENOENT);			\
1323 		}							\
1324 	} else {							\
1325 		int start;						\
1326 									\
1327 		if (tpl >= ud->res##_tpl.levels)			\
1328 			tpl = ud->res##_tpl.levels - 1;			\
1329 									\
1330 		start = ud->res##_tpl.start_idx[tpl];			\
1331 									\
1332 		id = find_next_zero_bit(ud->res##_map, ud->res##_cnt,	\
1333 					start);				\
1334 		if (id == ud->res##_cnt) {				\
1335 			return ERR_PTR(-ENOENT);			\
1336 		}							\
1337 	}								\
1338 									\
1339 	set_bit(id, ud->res##_map);					\
1340 	return &ud->res##s[id];						\
1341 }
1342 
1343 UDMA_RESERVE_RESOURCE(bchan);
1344 UDMA_RESERVE_RESOURCE(tchan);
1345 UDMA_RESERVE_RESOURCE(rchan);
1346 
1347 static int bcdma_get_bchan(struct udma_chan *uc)
1348 {
1349 	struct udma_dev *ud = uc->ud;
1350 	enum udma_tp_level tpl;
1351 
1352 	if (uc->bchan) {
1353 		dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n",
1354 			uc->id, uc->bchan->id);
1355 		return 0;
1356 	}
1357 
1358 	/*
1359 	 * Use normal channels for peripherals, and highest TPL channel for
1360 	 * mem2mem
1361 	 */
1362 	if (uc->config.tr_trigger_type)
1363 		tpl = 0;
1364 	else
1365 		tpl = ud->bchan_tpl.levels - 1;
1366 
1367 	uc->bchan = __udma_reserve_bchan(ud, tpl, -1);
1368 	if (IS_ERR(uc->bchan))
1369 		return PTR_ERR(uc->bchan);
1370 
1371 	uc->tchan = uc->bchan;
1372 
1373 	return 0;
1374 }
1375 
1376 static int udma_get_tchan(struct udma_chan *uc)
1377 {
1378 	struct udma_dev *ud = uc->ud;
1379 
1380 	if (uc->tchan) {
1381 		dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
1382 			uc->id, uc->tchan->id);
1383 		return 0;
1384 	}
1385 
1386 	/*
1387 	 * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels.
1388 	 * For PKTDMA mapped channels it is configured to a channel which must
1389 	 * be used to service the peripheral.
1390 	 */
1391 	uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl,
1392 					 uc->config.mapped_channel_id);
1393 	if (IS_ERR(uc->tchan))
1394 		return PTR_ERR(uc->tchan);
1395 
1396 	if (ud->tflow_cnt) {
1397 		int tflow_id;
1398 
1399 		/* Only PKTDMA have support for tx flows */
1400 		if (uc->config.default_flow_id >= 0)
1401 			tflow_id = uc->config.default_flow_id;
1402 		else
1403 			tflow_id = uc->tchan->id;
1404 
1405 		if (test_bit(tflow_id, ud->tflow_map)) {
1406 			dev_err(ud->dev, "tflow%d is in use\n", tflow_id);
1407 			clear_bit(uc->tchan->id, ud->tchan_map);
1408 			uc->tchan = NULL;
1409 			return -ENOENT;
1410 		}
1411 
1412 		uc->tchan->tflow_id = tflow_id;
1413 		set_bit(tflow_id, ud->tflow_map);
1414 	} else {
1415 		uc->tchan->tflow_id = -1;
1416 	}
1417 
1418 	return 0;
1419 }
1420 
1421 static int udma_get_rchan(struct udma_chan *uc)
1422 {
1423 	struct udma_dev *ud = uc->ud;
1424 
1425 	if (uc->rchan) {
1426 		dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
1427 			uc->id, uc->rchan->id);
1428 		return 0;
1429 	}
1430 
1431 	/*
1432 	 * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels.
1433 	 * For PKTDMA mapped channels it is configured to a channel which must
1434 	 * be used to service the peripheral.
1435 	 */
1436 	uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl,
1437 					 uc->config.mapped_channel_id);
1438 
1439 	return PTR_ERR_OR_ZERO(uc->rchan);
1440 }
1441 
1442 static int udma_get_chan_pair(struct udma_chan *uc)
1443 {
1444 	struct udma_dev *ud = uc->ud;
1445 	int chan_id, end;
1446 
1447 	if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {
1448 		dev_info(ud->dev, "chan%d: already have %d pair allocated\n",
1449 			 uc->id, uc->tchan->id);
1450 		return 0;
1451 	}
1452 
1453 	if (uc->tchan) {
1454 		dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",
1455 			uc->id, uc->tchan->id);
1456 		return -EBUSY;
1457 	} else if (uc->rchan) {
1458 		dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",
1459 			uc->id, uc->rchan->id);
1460 		return -EBUSY;
1461 	}
1462 
1463 	/* Can be optimized, but let's have it like this for now */
1464 	end = min(ud->tchan_cnt, ud->rchan_cnt);
1465 	/*
1466 	 * Try to use the highest TPL channel pair for MEM_TO_MEM channels
1467 	 * Note: in UDMAP the channel TPL is symmetric between tchan and rchan
1468 	 */
1469 	chan_id = ud->tchan_tpl.start_idx[ud->tchan_tpl.levels - 1];
1470 	for (; chan_id < end; chan_id++) {
1471 		if (!test_bit(chan_id, ud->tchan_map) &&
1472 		    !test_bit(chan_id, ud->rchan_map))
1473 			break;
1474 	}
1475 
1476 	if (chan_id == end)
1477 		return -ENOENT;
1478 
1479 	set_bit(chan_id, ud->tchan_map);
1480 	set_bit(chan_id, ud->rchan_map);
1481 	uc->tchan = &ud->tchans[chan_id];
1482 	uc->rchan = &ud->rchans[chan_id];
1483 
1484 	/* UDMA does not use tx flows */
1485 	uc->tchan->tflow_id = -1;
1486 
1487 	return 0;
1488 }
1489 
1490 static int udma_get_rflow(struct udma_chan *uc, int flow_id)
1491 {
1492 	struct udma_dev *ud = uc->ud;
1493 
1494 	if (!uc->rchan) {
1495 		dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id);
1496 		return -EINVAL;
1497 	}
1498 
1499 	if (uc->rflow) {
1500 		dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",
1501 			uc->id, uc->rflow->id);
1502 		return 0;
1503 	}
1504 
1505 	uc->rflow = __udma_get_rflow(ud, flow_id);
1506 
1507 	return PTR_ERR_OR_ZERO(uc->rflow);
1508 }
1509 
1510 static void bcdma_put_bchan(struct udma_chan *uc)
1511 {
1512 	struct udma_dev *ud = uc->ud;
1513 
1514 	if (uc->bchan) {
1515 		dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id,
1516 			uc->bchan->id);
1517 		clear_bit(uc->bchan->id, ud->bchan_map);
1518 		uc->bchan = NULL;
1519 		uc->tchan = NULL;
1520 	}
1521 }
1522 
1523 static void udma_put_rchan(struct udma_chan *uc)
1524 {
1525 	struct udma_dev *ud = uc->ud;
1526 
1527 	if (uc->rchan) {
1528 		dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,
1529 			uc->rchan->id);
1530 		clear_bit(uc->rchan->id, ud->rchan_map);
1531 		uc->rchan = NULL;
1532 	}
1533 }
1534 
1535 static void udma_put_tchan(struct udma_chan *uc)
1536 {
1537 	struct udma_dev *ud = uc->ud;
1538 
1539 	if (uc->tchan) {
1540 		dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,
1541 			uc->tchan->id);
1542 		clear_bit(uc->tchan->id, ud->tchan_map);
1543 
1544 		if (uc->tchan->tflow_id >= 0)
1545 			clear_bit(uc->tchan->tflow_id, ud->tflow_map);
1546 
1547 		uc->tchan = NULL;
1548 	}
1549 }
1550 
1551 static void udma_put_rflow(struct udma_chan *uc)
1552 {
1553 	struct udma_dev *ud = uc->ud;
1554 
1555 	if (uc->rflow) {
1556 		dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,
1557 			uc->rflow->id);
1558 		__udma_put_rflow(ud, uc->rflow);
1559 		uc->rflow = NULL;
1560 	}
1561 }
1562 
1563 static void bcdma_free_bchan_resources(struct udma_chan *uc)
1564 {
1565 	if (!uc->bchan)
1566 		return;
1567 
1568 	k3_ringacc_ring_free(uc->bchan->tc_ring);
1569 	k3_ringacc_ring_free(uc->bchan->t_ring);
1570 	uc->bchan->tc_ring = NULL;
1571 	uc->bchan->t_ring = NULL;
1572 	k3_configure_chan_coherency(&uc->vc.chan, 0);
1573 
1574 	bcdma_put_bchan(uc);
1575 }
1576 
1577 static int bcdma_alloc_bchan_resources(struct udma_chan *uc)
1578 {
1579 	struct k3_ring_cfg ring_cfg;
1580 	struct udma_dev *ud = uc->ud;
1581 	int ret;
1582 
1583 	ret = bcdma_get_bchan(uc);
1584 	if (ret)
1585 		return ret;
1586 
1587 	ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1,
1588 					    &uc->bchan->t_ring,
1589 					    &uc->bchan->tc_ring);
1590 	if (ret) {
1591 		ret = -EBUSY;
1592 		goto err_ring;
1593 	}
1594 
1595 	memset(&ring_cfg, 0, sizeof(ring_cfg));
1596 	ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1597 	ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1598 	ring_cfg.mode = K3_RINGACC_RING_MODE_RING;
1599 
1600 	k3_configure_chan_coherency(&uc->vc.chan, ud->asel);
1601 	ring_cfg.asel = ud->asel;
1602 	ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1603 
1604 	ret = k3_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg);
1605 	if (ret)
1606 		goto err_ringcfg;
1607 
1608 	return 0;
1609 
1610 err_ringcfg:
1611 	k3_ringacc_ring_free(uc->bchan->tc_ring);
1612 	uc->bchan->tc_ring = NULL;
1613 	k3_ringacc_ring_free(uc->bchan->t_ring);
1614 	uc->bchan->t_ring = NULL;
1615 	k3_configure_chan_coherency(&uc->vc.chan, 0);
1616 err_ring:
1617 	bcdma_put_bchan(uc);
1618 
1619 	return ret;
1620 }
1621 
1622 static void udma_free_tx_resources(struct udma_chan *uc)
1623 {
1624 	if (!uc->tchan)
1625 		return;
1626 
1627 	k3_ringacc_ring_free(uc->tchan->t_ring);
1628 	k3_ringacc_ring_free(uc->tchan->tc_ring);
1629 	uc->tchan->t_ring = NULL;
1630 	uc->tchan->tc_ring = NULL;
1631 
1632 	udma_put_tchan(uc);
1633 }
1634 
1635 static int udma_alloc_tx_resources(struct udma_chan *uc)
1636 {
1637 	struct k3_ring_cfg ring_cfg;
1638 	struct udma_dev *ud = uc->ud;
1639 	struct udma_tchan *tchan;
1640 	int ring_idx, ret;
1641 
1642 	ret = udma_get_tchan(uc);
1643 	if (ret)
1644 		return ret;
1645 
1646 	tchan = uc->tchan;
1647 	if (tchan->tflow_id >= 0)
1648 		ring_idx = tchan->tflow_id;
1649 	else
1650 		ring_idx = ud->bchan_cnt + tchan->id;
1651 
1652 	ret = k3_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1,
1653 					    &tchan->t_ring,
1654 					    &tchan->tc_ring);
1655 	if (ret) {
1656 		ret = -EBUSY;
1657 		goto err_ring;
1658 	}
1659 
1660 	memset(&ring_cfg, 0, sizeof(ring_cfg));
1661 	ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1662 	ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1663 	if (ud->match_data->type == DMA_TYPE_UDMA) {
1664 		ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
1665 	} else {
1666 		ring_cfg.mode = K3_RINGACC_RING_MODE_RING;
1667 
1668 		k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel);
1669 		ring_cfg.asel = uc->config.asel;
1670 		ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1671 	}
1672 
1673 	ret = k3_ringacc_ring_cfg(tchan->t_ring, &ring_cfg);
1674 	ret |= k3_ringacc_ring_cfg(tchan->tc_ring, &ring_cfg);
1675 
1676 	if (ret)
1677 		goto err_ringcfg;
1678 
1679 	return 0;
1680 
1681 err_ringcfg:
1682 	k3_ringacc_ring_free(uc->tchan->tc_ring);
1683 	uc->tchan->tc_ring = NULL;
1684 	k3_ringacc_ring_free(uc->tchan->t_ring);
1685 	uc->tchan->t_ring = NULL;
1686 err_ring:
1687 	udma_put_tchan(uc);
1688 
1689 	return ret;
1690 }
1691 
1692 static void udma_free_rx_resources(struct udma_chan *uc)
1693 {
1694 	if (!uc->rchan)
1695 		return;
1696 
1697 	if (uc->rflow) {
1698 		struct udma_rflow *rflow = uc->rflow;
1699 
1700 		k3_ringacc_ring_free(rflow->fd_ring);
1701 		k3_ringacc_ring_free(rflow->r_ring);
1702 		rflow->fd_ring = NULL;
1703 		rflow->r_ring = NULL;
1704 
1705 		udma_put_rflow(uc);
1706 	}
1707 
1708 	udma_put_rchan(uc);
1709 }
1710 
1711 static int udma_alloc_rx_resources(struct udma_chan *uc)
1712 {
1713 	struct udma_dev *ud = uc->ud;
1714 	struct k3_ring_cfg ring_cfg;
1715 	struct udma_rflow *rflow;
1716 	int fd_ring_id;
1717 	int ret;
1718 
1719 	ret = udma_get_rchan(uc);
1720 	if (ret)
1721 		return ret;
1722 
1723 	/* For MEM_TO_MEM we don't need rflow or rings */
1724 	if (uc->config.dir == DMA_MEM_TO_MEM)
1725 		return 0;
1726 
1727 	if (uc->config.default_flow_id >= 0)
1728 		ret = udma_get_rflow(uc, uc->config.default_flow_id);
1729 	else
1730 		ret = udma_get_rflow(uc, uc->rchan->id);
1731 
1732 	if (ret) {
1733 		ret = -EBUSY;
1734 		goto err_rflow;
1735 	}
1736 
1737 	rflow = uc->rflow;
1738 	if (ud->tflow_cnt)
1739 		fd_ring_id = ud->tflow_cnt + rflow->id;
1740 	else
1741 		fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt +
1742 			     uc->rchan->id;
1743 
1744 	ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1,
1745 					    &rflow->fd_ring, &rflow->r_ring);
1746 	if (ret) {
1747 		ret = -EBUSY;
1748 		goto err_ring;
1749 	}
1750 
1751 	memset(&ring_cfg, 0, sizeof(ring_cfg));
1752 
1753 	ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1754 	if (ud->match_data->type == DMA_TYPE_UDMA) {
1755 		if (uc->config.pkt_mode)
1756 			ring_cfg.size = SG_MAX_SEGMENTS;
1757 		else
1758 			ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1759 
1760 		ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
1761 	} else {
1762 		ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1763 		ring_cfg.mode = K3_RINGACC_RING_MODE_RING;
1764 
1765 		k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel);
1766 		ring_cfg.asel = uc->config.asel;
1767 		ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1768 	}
1769 
1770 	ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg);
1771 
1772 	ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1773 	ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg);
1774 
1775 	if (ret)
1776 		goto err_ringcfg;
1777 
1778 	return 0;
1779 
1780 err_ringcfg:
1781 	k3_ringacc_ring_free(rflow->r_ring);
1782 	rflow->r_ring = NULL;
1783 	k3_ringacc_ring_free(rflow->fd_ring);
1784 	rflow->fd_ring = NULL;
1785 err_ring:
1786 	udma_put_rflow(uc);
1787 err_rflow:
1788 	udma_put_rchan(uc);
1789 
1790 	return ret;
1791 }
1792 
1793 #define TISCI_BCDMA_BCHAN_VALID_PARAMS (			\
1794 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |	\
1795 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_EXTENDED_CH_TYPE_VALID)
1796 
1797 #define TISCI_BCDMA_TCHAN_VALID_PARAMS (			\
1798 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |	\
1799 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID)
1800 
1801 #define TISCI_BCDMA_RCHAN_VALID_PARAMS (			\
1802 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID)
1803 
1804 #define TISCI_UDMA_TCHAN_VALID_PARAMS (				\
1805 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |	\
1806 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |	\
1807 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |	\
1808 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |		\
1809 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |	\
1810 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |		\
1811 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |		\
1812 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1813 
1814 #define TISCI_UDMA_RCHAN_VALID_PARAMS (				\
1815 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |	\
1816 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |		\
1817 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |		\
1818 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |		\
1819 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID |	\
1820 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID |	\
1821 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |	\
1822 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID |	\
1823 	TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1824 
1825 static int udma_tisci_m2m_channel_config(struct udma_chan *uc)
1826 {
1827 	struct udma_dev *ud = uc->ud;
1828 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1829 	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1830 	struct udma_tchan *tchan = uc->tchan;
1831 	struct udma_rchan *rchan = uc->rchan;
1832 	u8 burst_size = 0;
1833 	int ret;
1834 	u8 tpl;
1835 
1836 	/* Non synchronized - mem to mem type of transfer */
1837 	int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
1838 	struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1839 	struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
1840 
1841 	if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) {
1842 		tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, tchan->id);
1843 
1844 		burst_size = ud->match_data->burst_size[tpl];
1845 	}
1846 
1847 	req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS;
1848 	req_tx.nav_id = tisci_rm->tisci_dev_id;
1849 	req_tx.index = tchan->id;
1850 	req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1851 	req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1852 	req_tx.txcq_qnum = tc_ring;
1853 	req_tx.tx_atype = ud->atype;
1854 	if (burst_size) {
1855 		req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;
1856 		req_tx.tx_burst_size = burst_size;
1857 	}
1858 
1859 	ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1860 	if (ret) {
1861 		dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1862 		return ret;
1863 	}
1864 
1865 	req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS;
1866 	req_rx.nav_id = tisci_rm->tisci_dev_id;
1867 	req_rx.index = rchan->id;
1868 	req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1869 	req_rx.rxcq_qnum = tc_ring;
1870 	req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1871 	req_rx.rx_atype = ud->atype;
1872 	if (burst_size) {
1873 		req_rx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;
1874 		req_rx.rx_burst_size = burst_size;
1875 	}
1876 
1877 	ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
1878 	if (ret)
1879 		dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret);
1880 
1881 	return ret;
1882 }
1883 
1884 static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc)
1885 {
1886 	struct udma_dev *ud = uc->ud;
1887 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1888 	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1889 	struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1890 	struct udma_bchan *bchan = uc->bchan;
1891 	u8 burst_size = 0;
1892 	int ret;
1893 	u8 tpl;
1894 
1895 	if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) {
1896 		tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, bchan->id);
1897 
1898 		burst_size = ud->match_data->burst_size[tpl];
1899 	}
1900 
1901 	req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS;
1902 	req_tx.nav_id = tisci_rm->tisci_dev_id;
1903 	req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN;
1904 	req_tx.index = bchan->id;
1905 	if (burst_size) {
1906 		req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;
1907 		req_tx.tx_burst_size = burst_size;
1908 	}
1909 
1910 	ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1911 	if (ret)
1912 		dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret);
1913 
1914 	return ret;
1915 }
1916 
1917 static int udma_tisci_tx_channel_config(struct udma_chan *uc)
1918 {
1919 	struct udma_dev *ud = uc->ud;
1920 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1921 	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1922 	struct udma_tchan *tchan = uc->tchan;
1923 	int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
1924 	struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1925 	u32 mode, fetch_size;
1926 	int ret;
1927 
1928 	if (uc->config.pkt_mode) {
1929 		mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1930 		fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1931 						   uc->config.psd_size, 0);
1932 	} else {
1933 		mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
1934 		fetch_size = sizeof(struct cppi5_desc_hdr_t);
1935 	}
1936 
1937 	req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS;
1938 	req_tx.nav_id = tisci_rm->tisci_dev_id;
1939 	req_tx.index = tchan->id;
1940 	req_tx.tx_chan_type = mode;
1941 	req_tx.tx_supr_tdpkt = uc->config.notdpkt;
1942 	req_tx.tx_fetch_size = fetch_size >> 2;
1943 	req_tx.txcq_qnum = tc_ring;
1944 	req_tx.tx_atype = uc->config.atype;
1945 	if (uc->config.ep_type == PSIL_EP_PDMA_XY &&
1946 	    ud->match_data->flags & UDMA_FLAG_TDTYPE) {
1947 		/* wait for peer to complete the teardown for PDMAs */
1948 		req_tx.valid_params |=
1949 				TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;
1950 		req_tx.tx_tdtype = 1;
1951 	}
1952 
1953 	ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1954 	if (ret)
1955 		dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1956 
1957 	return ret;
1958 }
1959 
1960 static int bcdma_tisci_tx_channel_config(struct udma_chan *uc)
1961 {
1962 	struct udma_dev *ud = uc->ud;
1963 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1964 	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1965 	struct udma_tchan *tchan = uc->tchan;
1966 	struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1967 	int ret;
1968 
1969 	req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS;
1970 	req_tx.nav_id = tisci_rm->tisci_dev_id;
1971 	req_tx.index = tchan->id;
1972 	req_tx.tx_supr_tdpkt = uc->config.notdpkt;
1973 	if (ud->match_data->flags & UDMA_FLAG_TDTYPE) {
1974 		/* wait for peer to complete the teardown for PDMAs */
1975 		req_tx.valid_params |=
1976 				TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;
1977 		req_tx.tx_tdtype = 1;
1978 	}
1979 
1980 	ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1981 	if (ret)
1982 		dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1983 
1984 	return ret;
1985 }
1986 
1987 #define pktdma_tisci_tx_channel_config bcdma_tisci_tx_channel_config
1988 
1989 static int udma_tisci_rx_channel_config(struct udma_chan *uc)
1990 {
1991 	struct udma_dev *ud = uc->ud;
1992 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1993 	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1994 	struct udma_rchan *rchan = uc->rchan;
1995 	int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring);
1996 	int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring);
1997 	struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
1998 	struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
1999 	u32 mode, fetch_size;
2000 	int ret;
2001 
2002 	if (uc->config.pkt_mode) {
2003 		mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
2004 		fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
2005 						   uc->config.psd_size, 0);
2006 	} else {
2007 		mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
2008 		fetch_size = sizeof(struct cppi5_desc_hdr_t);
2009 	}
2010 
2011 	req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS;
2012 	req_rx.nav_id = tisci_rm->tisci_dev_id;
2013 	req_rx.index = rchan->id;
2014 	req_rx.rx_fetch_size =  fetch_size >> 2;
2015 	req_rx.rxcq_qnum = rx_ring;
2016 	req_rx.rx_chan_type = mode;
2017 	req_rx.rx_atype = uc->config.atype;
2018 
2019 	ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2020 	if (ret) {
2021 		dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);
2022 		return ret;
2023 	}
2024 
2025 	flow_req.valid_params =
2026 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
2027 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
2028 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
2029 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
2030 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
2031 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
2032 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
2033 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
2034 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
2035 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
2036 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
2037 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
2038 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
2039 
2040 	flow_req.nav_id = tisci_rm->tisci_dev_id;
2041 	flow_req.flow_index = rchan->id;
2042 
2043 	if (uc->config.needs_epib)
2044 		flow_req.rx_einfo_present = 1;
2045 	else
2046 		flow_req.rx_einfo_present = 0;
2047 	if (uc->config.psd_size)
2048 		flow_req.rx_psinfo_present = 1;
2049 	else
2050 		flow_req.rx_psinfo_present = 0;
2051 	flow_req.rx_error_handling = 1;
2052 	flow_req.rx_dest_qnum = rx_ring;
2053 	flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE;
2054 	flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG;
2055 	flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI;
2056 	flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO;
2057 	flow_req.rx_fdq0_sz0_qnum = fd_ring;
2058 	flow_req.rx_fdq1_qnum = fd_ring;
2059 	flow_req.rx_fdq2_qnum = fd_ring;
2060 	flow_req.rx_fdq3_qnum = fd_ring;
2061 
2062 	ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2063 
2064 	if (ret)
2065 		dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret);
2066 
2067 	return 0;
2068 }
2069 
2070 static int bcdma_tisci_rx_channel_config(struct udma_chan *uc)
2071 {
2072 	struct udma_dev *ud = uc->ud;
2073 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2074 	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2075 	struct udma_rchan *rchan = uc->rchan;
2076 	struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2077 	int ret;
2078 
2079 	req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;
2080 	req_rx.nav_id = tisci_rm->tisci_dev_id;
2081 	req_rx.index = rchan->id;
2082 
2083 	ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2084 	if (ret)
2085 		dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);
2086 
2087 	return ret;
2088 }
2089 
2090 static int pktdma_tisci_rx_channel_config(struct udma_chan *uc)
2091 {
2092 	struct udma_dev *ud = uc->ud;
2093 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2094 	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2095 	struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2096 	struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
2097 	int ret;
2098 
2099 	req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;
2100 	req_rx.nav_id = tisci_rm->tisci_dev_id;
2101 	req_rx.index = uc->rchan->id;
2102 
2103 	ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2104 	if (ret) {
2105 		dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret);
2106 		return ret;
2107 	}
2108 
2109 	flow_req.valid_params =
2110 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
2111 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
2112 		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID;
2113 
2114 	flow_req.nav_id = tisci_rm->tisci_dev_id;
2115 	flow_req.flow_index = uc->rflow->id;
2116 
2117 	if (uc->config.needs_epib)
2118 		flow_req.rx_einfo_present = 1;
2119 	else
2120 		flow_req.rx_einfo_present = 0;
2121 	if (uc->config.psd_size)
2122 		flow_req.rx_psinfo_present = 1;
2123 	else
2124 		flow_req.rx_psinfo_present = 0;
2125 	flow_req.rx_error_handling = 1;
2126 
2127 	ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2128 
2129 	if (ret)
2130 		dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id,
2131 			ret);
2132 
2133 	return ret;
2134 }
2135 
2136 static int udma_alloc_chan_resources(struct dma_chan *chan)
2137 {
2138 	struct udma_chan *uc = to_udma_chan(chan);
2139 	struct udma_dev *ud = to_udma_dev(chan->device);
2140 	const struct udma_soc_data *soc_data = ud->soc_data;
2141 	struct k3_ring *irq_ring;
2142 	u32 irq_udma_idx;
2143 	int ret;
2144 
2145 	uc->dma_dev = ud->dev;
2146 
2147 	if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) {
2148 		uc->use_dma_pool = true;
2149 		/* in case of MEM_TO_MEM we have maximum of two TRs */
2150 		if (uc->config.dir == DMA_MEM_TO_MEM) {
2151 			uc->config.hdesc_size = cppi5_trdesc_calc_size(
2152 					sizeof(struct cppi5_tr_type15_t), 2);
2153 			uc->config.pkt_mode = false;
2154 		}
2155 	}
2156 
2157 	if (uc->use_dma_pool) {
2158 		uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
2159 						 uc->config.hdesc_size,
2160 						 ud->desc_align,
2161 						 0);
2162 		if (!uc->hdesc_pool) {
2163 			dev_err(ud->ddev.dev,
2164 				"Descriptor pool allocation failed\n");
2165 			uc->use_dma_pool = false;
2166 			ret = -ENOMEM;
2167 			goto err_cleanup;
2168 		}
2169 	}
2170 
2171 	/*
2172 	 * Make sure that the completion is in a known state:
2173 	 * No teardown, the channel is idle
2174 	 */
2175 	reinit_completion(&uc->teardown_completed);
2176 	complete_all(&uc->teardown_completed);
2177 	uc->state = UDMA_CHAN_IS_IDLE;
2178 
2179 	switch (uc->config.dir) {
2180 	case DMA_MEM_TO_MEM:
2181 		/* Non synchronized - mem to mem type of transfer */
2182 		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2183 			uc->id);
2184 
2185 		ret = udma_get_chan_pair(uc);
2186 		if (ret)
2187 			goto err_cleanup;
2188 
2189 		ret = udma_alloc_tx_resources(uc);
2190 		if (ret) {
2191 			udma_put_rchan(uc);
2192 			goto err_cleanup;
2193 		}
2194 
2195 		ret = udma_alloc_rx_resources(uc);
2196 		if (ret) {
2197 			udma_free_tx_resources(uc);
2198 			goto err_cleanup;
2199 		}
2200 
2201 		uc->config.src_thread = ud->psil_base + uc->tchan->id;
2202 		uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2203 					K3_PSIL_DST_THREAD_ID_OFFSET;
2204 
2205 		irq_ring = uc->tchan->tc_ring;
2206 		irq_udma_idx = uc->tchan->id;
2207 
2208 		ret = udma_tisci_m2m_channel_config(uc);
2209 		break;
2210 	case DMA_MEM_TO_DEV:
2211 		/* Slave transfer synchronized - mem to dev (TX) trasnfer */
2212 		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2213 			uc->id);
2214 
2215 		ret = udma_alloc_tx_resources(uc);
2216 		if (ret)
2217 			goto err_cleanup;
2218 
2219 		uc->config.src_thread = ud->psil_base + uc->tchan->id;
2220 		uc->config.dst_thread = uc->config.remote_thread_id;
2221 		uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2222 
2223 		irq_ring = uc->tchan->tc_ring;
2224 		irq_udma_idx = uc->tchan->id;
2225 
2226 		ret = udma_tisci_tx_channel_config(uc);
2227 		break;
2228 	case DMA_DEV_TO_MEM:
2229 		/* Slave transfer synchronized - dev to mem (RX) trasnfer */
2230 		dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2231 			uc->id);
2232 
2233 		ret = udma_alloc_rx_resources(uc);
2234 		if (ret)
2235 			goto err_cleanup;
2236 
2237 		uc->config.src_thread = uc->config.remote_thread_id;
2238 		uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2239 					K3_PSIL_DST_THREAD_ID_OFFSET;
2240 
2241 		irq_ring = uc->rflow->r_ring;
2242 		irq_udma_idx = soc_data->oes.udma_rchan + uc->rchan->id;
2243 
2244 		ret = udma_tisci_rx_channel_config(uc);
2245 		break;
2246 	default:
2247 		/* Can not happen */
2248 		dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2249 			__func__, uc->id, uc->config.dir);
2250 		ret = -EINVAL;
2251 		goto err_cleanup;
2252 
2253 	}
2254 
2255 	/* check if the channel configuration was successful */
2256 	if (ret)
2257 		goto err_res_free;
2258 
2259 	if (udma_is_chan_running(uc)) {
2260 		dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2261 		udma_reset_chan(uc, false);
2262 		if (udma_is_chan_running(uc)) {
2263 			dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2264 			ret = -EBUSY;
2265 			goto err_res_free;
2266 		}
2267 	}
2268 
2269 	/* PSI-L pairing */
2270 	ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2271 	if (ret) {
2272 		dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2273 			uc->config.src_thread, uc->config.dst_thread);
2274 		goto err_res_free;
2275 	}
2276 
2277 	uc->psil_paired = true;
2278 
2279 	uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring);
2280 	if (uc->irq_num_ring <= 0) {
2281 		dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2282 			k3_ringacc_get_ring_id(irq_ring));
2283 		ret = -EINVAL;
2284 		goto err_psi_free;
2285 	}
2286 
2287 	ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2288 			  IRQF_TRIGGER_HIGH, uc->name, uc);
2289 	if (ret) {
2290 		dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2291 		goto err_irq_free;
2292 	}
2293 
2294 	/* Event from UDMA (TR events) only needed for slave TR mode channels */
2295 	if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) {
2296 		uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev,
2297 							    irq_udma_idx);
2298 		if (uc->irq_num_udma <= 0) {
2299 			dev_err(ud->dev, "Failed to get udma irq (index: %u)\n",
2300 				irq_udma_idx);
2301 			free_irq(uc->irq_num_ring, uc);
2302 			ret = -EINVAL;
2303 			goto err_irq_free;
2304 		}
2305 
2306 		ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
2307 				  uc->name, uc);
2308 		if (ret) {
2309 			dev_err(ud->dev, "chan%d: UDMA irq request failed\n",
2310 				uc->id);
2311 			free_irq(uc->irq_num_ring, uc);
2312 			goto err_irq_free;
2313 		}
2314 	} else {
2315 		uc->irq_num_udma = 0;
2316 	}
2317 
2318 	udma_reset_rings(uc);
2319 
2320 	return 0;
2321 
2322 err_irq_free:
2323 	uc->irq_num_ring = 0;
2324 	uc->irq_num_udma = 0;
2325 err_psi_free:
2326 	navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);
2327 	uc->psil_paired = false;
2328 err_res_free:
2329 	udma_free_tx_resources(uc);
2330 	udma_free_rx_resources(uc);
2331 err_cleanup:
2332 	udma_reset_uchan(uc);
2333 
2334 	if (uc->use_dma_pool) {
2335 		dma_pool_destroy(uc->hdesc_pool);
2336 		uc->use_dma_pool = false;
2337 	}
2338 
2339 	return ret;
2340 }
2341 
2342 static int bcdma_alloc_chan_resources(struct dma_chan *chan)
2343 {
2344 	struct udma_chan *uc = to_udma_chan(chan);
2345 	struct udma_dev *ud = to_udma_dev(chan->device);
2346 	const struct udma_oes_offsets *oes = &ud->soc_data->oes;
2347 	u32 irq_udma_idx, irq_ring_idx;
2348 	int ret;
2349 
2350 	/* Only TR mode is supported */
2351 	uc->config.pkt_mode = false;
2352 
2353 	/*
2354 	 * Make sure that the completion is in a known state:
2355 	 * No teardown, the channel is idle
2356 	 */
2357 	reinit_completion(&uc->teardown_completed);
2358 	complete_all(&uc->teardown_completed);
2359 	uc->state = UDMA_CHAN_IS_IDLE;
2360 
2361 	switch (uc->config.dir) {
2362 	case DMA_MEM_TO_MEM:
2363 		/* Non synchronized - mem to mem type of transfer */
2364 		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2365 			uc->id);
2366 
2367 		ret = bcdma_alloc_bchan_resources(uc);
2368 		if (ret)
2369 			return ret;
2370 
2371 		irq_ring_idx = uc->bchan->id + oes->bcdma_bchan_ring;
2372 		irq_udma_idx = uc->bchan->id + oes->bcdma_bchan_data;
2373 
2374 		ret = bcdma_tisci_m2m_channel_config(uc);
2375 		break;
2376 	case DMA_MEM_TO_DEV:
2377 		/* Slave transfer synchronized - mem to dev (TX) trasnfer */
2378 		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2379 			uc->id);
2380 
2381 		ret = udma_alloc_tx_resources(uc);
2382 		if (ret) {
2383 			uc->config.remote_thread_id = -1;
2384 			return ret;
2385 		}
2386 
2387 		uc->config.src_thread = ud->psil_base + uc->tchan->id;
2388 		uc->config.dst_thread = uc->config.remote_thread_id;
2389 		uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2390 
2391 		irq_ring_idx = uc->tchan->id + oes->bcdma_tchan_ring;
2392 		irq_udma_idx = uc->tchan->id + oes->bcdma_tchan_data;
2393 
2394 		ret = bcdma_tisci_tx_channel_config(uc);
2395 		break;
2396 	case DMA_DEV_TO_MEM:
2397 		/* Slave transfer synchronized - dev to mem (RX) trasnfer */
2398 		dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2399 			uc->id);
2400 
2401 		ret = udma_alloc_rx_resources(uc);
2402 		if (ret) {
2403 			uc->config.remote_thread_id = -1;
2404 			return ret;
2405 		}
2406 
2407 		uc->config.src_thread = uc->config.remote_thread_id;
2408 		uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2409 					K3_PSIL_DST_THREAD_ID_OFFSET;
2410 
2411 		irq_ring_idx = uc->rchan->id + oes->bcdma_rchan_ring;
2412 		irq_udma_idx = uc->rchan->id + oes->bcdma_rchan_data;
2413 
2414 		ret = bcdma_tisci_rx_channel_config(uc);
2415 		break;
2416 	default:
2417 		/* Can not happen */
2418 		dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2419 			__func__, uc->id, uc->config.dir);
2420 		return -EINVAL;
2421 	}
2422 
2423 	/* check if the channel configuration was successful */
2424 	if (ret)
2425 		goto err_res_free;
2426 
2427 	if (udma_is_chan_running(uc)) {
2428 		dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2429 		udma_reset_chan(uc, false);
2430 		if (udma_is_chan_running(uc)) {
2431 			dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2432 			ret = -EBUSY;
2433 			goto err_res_free;
2434 		}
2435 	}
2436 
2437 	uc->dma_dev = dmaengine_get_dma_device(chan);
2438 	if (uc->config.dir == DMA_MEM_TO_MEM  && !uc->config.tr_trigger_type) {
2439 		uc->config.hdesc_size = cppi5_trdesc_calc_size(
2440 					sizeof(struct cppi5_tr_type15_t), 2);
2441 
2442 		uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
2443 						 uc->config.hdesc_size,
2444 						 ud->desc_align,
2445 						 0);
2446 		if (!uc->hdesc_pool) {
2447 			dev_err(ud->ddev.dev,
2448 				"Descriptor pool allocation failed\n");
2449 			uc->use_dma_pool = false;
2450 			ret = -ENOMEM;
2451 			goto err_res_free;
2452 		}
2453 
2454 		uc->use_dma_pool = true;
2455 	} else if (uc->config.dir != DMA_MEM_TO_MEM) {
2456 		/* PSI-L pairing */
2457 		ret = navss_psil_pair(ud, uc->config.src_thread,
2458 				      uc->config.dst_thread);
2459 		if (ret) {
2460 			dev_err(ud->dev,
2461 				"PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2462 				uc->config.src_thread, uc->config.dst_thread);
2463 			goto err_res_free;
2464 		}
2465 
2466 		uc->psil_paired = true;
2467 	}
2468 
2469 	uc->irq_num_ring = ti_sci_inta_msi_get_virq(ud->dev, irq_ring_idx);
2470 	if (uc->irq_num_ring <= 0) {
2471 		dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2472 			irq_ring_idx);
2473 		ret = -EINVAL;
2474 		goto err_psi_free;
2475 	}
2476 
2477 	ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2478 			  IRQF_TRIGGER_HIGH, uc->name, uc);
2479 	if (ret) {
2480 		dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2481 		goto err_irq_free;
2482 	}
2483 
2484 	/* Event from BCDMA (TR events) only needed for slave channels */
2485 	if (is_slave_direction(uc->config.dir)) {
2486 		uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev,
2487 							    irq_udma_idx);
2488 		if (uc->irq_num_udma <= 0) {
2489 			dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n",
2490 				irq_udma_idx);
2491 			free_irq(uc->irq_num_ring, uc);
2492 			ret = -EINVAL;
2493 			goto err_irq_free;
2494 		}
2495 
2496 		ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
2497 				  uc->name, uc);
2498 		if (ret) {
2499 			dev_err(ud->dev, "chan%d: BCDMA irq request failed\n",
2500 				uc->id);
2501 			free_irq(uc->irq_num_ring, uc);
2502 			goto err_irq_free;
2503 		}
2504 	} else {
2505 		uc->irq_num_udma = 0;
2506 	}
2507 
2508 	udma_reset_rings(uc);
2509 
2510 	INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,
2511 				  udma_check_tx_completion);
2512 	return 0;
2513 
2514 err_irq_free:
2515 	uc->irq_num_ring = 0;
2516 	uc->irq_num_udma = 0;
2517 err_psi_free:
2518 	if (uc->psil_paired)
2519 		navss_psil_unpair(ud, uc->config.src_thread,
2520 				  uc->config.dst_thread);
2521 	uc->psil_paired = false;
2522 err_res_free:
2523 	bcdma_free_bchan_resources(uc);
2524 	udma_free_tx_resources(uc);
2525 	udma_free_rx_resources(uc);
2526 
2527 	udma_reset_uchan(uc);
2528 
2529 	if (uc->use_dma_pool) {
2530 		dma_pool_destroy(uc->hdesc_pool);
2531 		uc->use_dma_pool = false;
2532 	}
2533 
2534 	return ret;
2535 }
2536 
2537 static int bcdma_router_config(struct dma_chan *chan)
2538 {
2539 	struct k3_event_route_data *router_data = chan->route_data;
2540 	struct udma_chan *uc = to_udma_chan(chan);
2541 	u32 trigger_event;
2542 
2543 	if (!uc->bchan)
2544 		return -EINVAL;
2545 
2546 	if (uc->config.tr_trigger_type != 1 && uc->config.tr_trigger_type != 2)
2547 		return -EINVAL;
2548 
2549 	trigger_event = uc->ud->soc_data->bcdma_trigger_event_offset;
2550 	trigger_event += (uc->bchan->id * 2) + uc->config.tr_trigger_type - 1;
2551 
2552 	return router_data->set_event(router_data->priv, trigger_event);
2553 }
2554 
2555 static int pktdma_alloc_chan_resources(struct dma_chan *chan)
2556 {
2557 	struct udma_chan *uc = to_udma_chan(chan);
2558 	struct udma_dev *ud = to_udma_dev(chan->device);
2559 	const struct udma_oes_offsets *oes = &ud->soc_data->oes;
2560 	u32 irq_ring_idx;
2561 	int ret;
2562 
2563 	/*
2564 	 * Make sure that the completion is in a known state:
2565 	 * No teardown, the channel is idle
2566 	 */
2567 	reinit_completion(&uc->teardown_completed);
2568 	complete_all(&uc->teardown_completed);
2569 	uc->state = UDMA_CHAN_IS_IDLE;
2570 
2571 	switch (uc->config.dir) {
2572 	case DMA_MEM_TO_DEV:
2573 		/* Slave transfer synchronized - mem to dev (TX) trasnfer */
2574 		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2575 			uc->id);
2576 
2577 		ret = udma_alloc_tx_resources(uc);
2578 		if (ret) {
2579 			uc->config.remote_thread_id = -1;
2580 			return ret;
2581 		}
2582 
2583 		uc->config.src_thread = ud->psil_base + uc->tchan->id;
2584 		uc->config.dst_thread = uc->config.remote_thread_id;
2585 		uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2586 
2587 		irq_ring_idx = uc->tchan->tflow_id + oes->pktdma_tchan_flow;
2588 
2589 		ret = pktdma_tisci_tx_channel_config(uc);
2590 		break;
2591 	case DMA_DEV_TO_MEM:
2592 		/* Slave transfer synchronized - dev to mem (RX) trasnfer */
2593 		dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2594 			uc->id);
2595 
2596 		ret = udma_alloc_rx_resources(uc);
2597 		if (ret) {
2598 			uc->config.remote_thread_id = -1;
2599 			return ret;
2600 		}
2601 
2602 		uc->config.src_thread = uc->config.remote_thread_id;
2603 		uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2604 					K3_PSIL_DST_THREAD_ID_OFFSET;
2605 
2606 		irq_ring_idx = uc->rflow->id + oes->pktdma_rchan_flow;
2607 
2608 		ret = pktdma_tisci_rx_channel_config(uc);
2609 		break;
2610 	default:
2611 		/* Can not happen */
2612 		dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2613 			__func__, uc->id, uc->config.dir);
2614 		return -EINVAL;
2615 	}
2616 
2617 	/* check if the channel configuration was successful */
2618 	if (ret)
2619 		goto err_res_free;
2620 
2621 	if (udma_is_chan_running(uc)) {
2622 		dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2623 		udma_reset_chan(uc, false);
2624 		if (udma_is_chan_running(uc)) {
2625 			dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2626 			ret = -EBUSY;
2627 			goto err_res_free;
2628 		}
2629 	}
2630 
2631 	uc->dma_dev = dmaengine_get_dma_device(chan);
2632 	uc->hdesc_pool = dma_pool_create(uc->name, uc->dma_dev,
2633 					 uc->config.hdesc_size, ud->desc_align,
2634 					 0);
2635 	if (!uc->hdesc_pool) {
2636 		dev_err(ud->ddev.dev,
2637 			"Descriptor pool allocation failed\n");
2638 		uc->use_dma_pool = false;
2639 		ret = -ENOMEM;
2640 		goto err_res_free;
2641 	}
2642 
2643 	uc->use_dma_pool = true;
2644 
2645 	/* PSI-L pairing */
2646 	ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2647 	if (ret) {
2648 		dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2649 			uc->config.src_thread, uc->config.dst_thread);
2650 		goto err_res_free;
2651 	}
2652 
2653 	uc->psil_paired = true;
2654 
2655 	uc->irq_num_ring = ti_sci_inta_msi_get_virq(ud->dev, irq_ring_idx);
2656 	if (uc->irq_num_ring <= 0) {
2657 		dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2658 			irq_ring_idx);
2659 		ret = -EINVAL;
2660 		goto err_psi_free;
2661 	}
2662 
2663 	ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2664 			  IRQF_TRIGGER_HIGH, uc->name, uc);
2665 	if (ret) {
2666 		dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2667 		goto err_irq_free;
2668 	}
2669 
2670 	uc->irq_num_udma = 0;
2671 
2672 	udma_reset_rings(uc);
2673 
2674 	INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,
2675 				  udma_check_tx_completion);
2676 
2677 	if (uc->tchan)
2678 		dev_dbg(ud->dev,
2679 			"chan%d: tchan%d, tflow%d, Remote thread: 0x%04x\n",
2680 			uc->id, uc->tchan->id, uc->tchan->tflow_id,
2681 			uc->config.remote_thread_id);
2682 	else if (uc->rchan)
2683 		dev_dbg(ud->dev,
2684 			"chan%d: rchan%d, rflow%d, Remote thread: 0x%04x\n",
2685 			uc->id, uc->rchan->id, uc->rflow->id,
2686 			uc->config.remote_thread_id);
2687 	return 0;
2688 
2689 err_irq_free:
2690 	uc->irq_num_ring = 0;
2691 err_psi_free:
2692 	navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);
2693 	uc->psil_paired = false;
2694 err_res_free:
2695 	udma_free_tx_resources(uc);
2696 	udma_free_rx_resources(uc);
2697 
2698 	udma_reset_uchan(uc);
2699 
2700 	dma_pool_destroy(uc->hdesc_pool);
2701 	uc->use_dma_pool = false;
2702 
2703 	return ret;
2704 }
2705 
2706 static int udma_slave_config(struct dma_chan *chan,
2707 			     struct dma_slave_config *cfg)
2708 {
2709 	struct udma_chan *uc = to_udma_chan(chan);
2710 
2711 	memcpy(&uc->cfg, cfg, sizeof(uc->cfg));
2712 
2713 	return 0;
2714 }
2715 
2716 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc,
2717 					    size_t tr_size, int tr_count,
2718 					    enum dma_transfer_direction dir)
2719 {
2720 	struct udma_hwdesc *hwdesc;
2721 	struct cppi5_desc_hdr_t *tr_desc;
2722 	struct udma_desc *d;
2723 	u32 reload_count = 0;
2724 	u32 ring_id;
2725 
2726 	switch (tr_size) {
2727 	case 16:
2728 	case 32:
2729 	case 64:
2730 	case 128:
2731 		break;
2732 	default:
2733 		dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size);
2734 		return NULL;
2735 	}
2736 
2737 	/* We have only one descriptor containing multiple TRs */
2738 	d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT);
2739 	if (!d)
2740 		return NULL;
2741 
2742 	d->sglen = tr_count;
2743 
2744 	d->hwdesc_count = 1;
2745 	hwdesc = &d->hwdesc[0];
2746 
2747 	/* Allocate memory for DMA ring descriptor */
2748 	if (uc->use_dma_pool) {
2749 		hwdesc->cppi5_desc_size = uc->config.hdesc_size;
2750 		hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
2751 						GFP_NOWAIT,
2752 						&hwdesc->cppi5_desc_paddr);
2753 	} else {
2754 		hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size,
2755 								 tr_count);
2756 		hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
2757 						uc->ud->desc_align);
2758 		hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev,
2759 						hwdesc->cppi5_desc_size,
2760 						&hwdesc->cppi5_desc_paddr,
2761 						GFP_NOWAIT);
2762 	}
2763 
2764 	if (!hwdesc->cppi5_desc_vaddr) {
2765 		kfree(d);
2766 		return NULL;
2767 	}
2768 
2769 	/* Start of the TR req records */
2770 	hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
2771 	/* Start address of the TR response array */
2772 	hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count;
2773 
2774 	tr_desc = hwdesc->cppi5_desc_vaddr;
2775 
2776 	if (uc->cyclic)
2777 		reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE;
2778 
2779 	if (dir == DMA_DEV_TO_MEM)
2780 		ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
2781 	else
2782 		ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
2783 
2784 	cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count);
2785 	cppi5_desc_set_pktids(tr_desc, uc->id,
2786 			      CPPI5_INFO1_DESC_FLOWID_DEFAULT);
2787 	cppi5_desc_set_retpolicy(tr_desc, 0, ring_id);
2788 
2789 	return d;
2790 }
2791 
2792 /**
2793  * udma_get_tr_counters - calculate TR counters for a given length
2794  * @len: Length of the trasnfer
2795  * @align_to: Preferred alignment
2796  * @tr0_cnt0: First TR icnt0
2797  * @tr0_cnt1: First TR icnt1
2798  * @tr1_cnt0: Second (if used) TR icnt0
2799  *
2800  * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated
2801  * For len >= SZ_64K two TRs are used in a simple way:
2802  * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1)
2803  * Second TR: the remaining length (tr1_cnt0)
2804  *
2805  * Returns the number of TRs the length needs (1 or 2)
2806  * -EINVAL if the length can not be supported
2807  */
2808 static int udma_get_tr_counters(size_t len, unsigned long align_to,
2809 				u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0)
2810 {
2811 	if (len < SZ_64K) {
2812 		*tr0_cnt0 = len;
2813 		*tr0_cnt1 = 1;
2814 
2815 		return 1;
2816 	}
2817 
2818 	if (align_to > 3)
2819 		align_to = 3;
2820 
2821 realign:
2822 	*tr0_cnt0 = SZ_64K - BIT(align_to);
2823 	if (len / *tr0_cnt0 >= SZ_64K) {
2824 		if (align_to) {
2825 			align_to--;
2826 			goto realign;
2827 		}
2828 		return -EINVAL;
2829 	}
2830 
2831 	*tr0_cnt1 = len / *tr0_cnt0;
2832 	*tr1_cnt0 = len % *tr0_cnt0;
2833 
2834 	return 2;
2835 }
2836 
2837 static struct udma_desc *
2838 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl,
2839 		      unsigned int sglen, enum dma_transfer_direction dir,
2840 		      unsigned long tx_flags, void *context)
2841 {
2842 	struct scatterlist *sgent;
2843 	struct udma_desc *d;
2844 	struct cppi5_tr_type1_t *tr_req = NULL;
2845 	u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
2846 	unsigned int i;
2847 	size_t tr_size;
2848 	int num_tr = 0;
2849 	int tr_idx = 0;
2850 	u64 asel;
2851 
2852 	/* estimate the number of TRs we will need */
2853 	for_each_sg(sgl, sgent, sglen, i) {
2854 		if (sg_dma_len(sgent) < SZ_64K)
2855 			num_tr++;
2856 		else
2857 			num_tr += 2;
2858 	}
2859 
2860 	/* Now allocate and setup the descriptor. */
2861 	tr_size = sizeof(struct cppi5_tr_type1_t);
2862 	d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
2863 	if (!d)
2864 		return NULL;
2865 
2866 	d->sglen = sglen;
2867 
2868 	if (uc->ud->match_data->type == DMA_TYPE_UDMA)
2869 		asel = 0;
2870 	else
2871 		asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
2872 
2873 	tr_req = d->hwdesc[0].tr_req_base;
2874 	for_each_sg(sgl, sgent, sglen, i) {
2875 		dma_addr_t sg_addr = sg_dma_address(sgent);
2876 
2877 		num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr),
2878 					      &tr0_cnt0, &tr0_cnt1, &tr1_cnt0);
2879 		if (num_tr < 0) {
2880 			dev_err(uc->ud->dev, "size %u is not supported\n",
2881 				sg_dma_len(sgent));
2882 			udma_free_hwdesc(uc, d);
2883 			kfree(d);
2884 			return NULL;
2885 		}
2886 
2887 		cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
2888 			      false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2889 		cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT);
2890 
2891 		sg_addr |= asel;
2892 		tr_req[tr_idx].addr = sg_addr;
2893 		tr_req[tr_idx].icnt0 = tr0_cnt0;
2894 		tr_req[tr_idx].icnt1 = tr0_cnt1;
2895 		tr_req[tr_idx].dim1 = tr0_cnt0;
2896 		tr_idx++;
2897 
2898 		if (num_tr == 2) {
2899 			cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
2900 				      false, false,
2901 				      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2902 			cppi5_tr_csf_set(&tr_req[tr_idx].flags,
2903 					 CPPI5_TR_CSF_SUPR_EVT);
2904 
2905 			tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0;
2906 			tr_req[tr_idx].icnt0 = tr1_cnt0;
2907 			tr_req[tr_idx].icnt1 = 1;
2908 			tr_req[tr_idx].dim1 = tr1_cnt0;
2909 			tr_idx++;
2910 		}
2911 
2912 		d->residue += sg_dma_len(sgent);
2913 	}
2914 
2915 	cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
2916 			 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
2917 
2918 	return d;
2919 }
2920 
2921 static struct udma_desc *
2922 udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl,
2923 				unsigned int sglen,
2924 				enum dma_transfer_direction dir,
2925 				unsigned long tx_flags, void *context)
2926 {
2927 	struct scatterlist *sgent;
2928 	struct cppi5_tr_type15_t *tr_req = NULL;
2929 	enum dma_slave_buswidth dev_width;
2930 	u16 tr_cnt0, tr_cnt1;
2931 	dma_addr_t dev_addr;
2932 	struct udma_desc *d;
2933 	unsigned int i;
2934 	size_t tr_size, sg_len;
2935 	int num_tr = 0;
2936 	int tr_idx = 0;
2937 	u32 burst, trigger_size, port_window;
2938 	u64 asel;
2939 
2940 	if (dir == DMA_DEV_TO_MEM) {
2941 		dev_addr = uc->cfg.src_addr;
2942 		dev_width = uc->cfg.src_addr_width;
2943 		burst = uc->cfg.src_maxburst;
2944 		port_window = uc->cfg.src_port_window_size;
2945 	} else if (dir == DMA_MEM_TO_DEV) {
2946 		dev_addr = uc->cfg.dst_addr;
2947 		dev_width = uc->cfg.dst_addr_width;
2948 		burst = uc->cfg.dst_maxburst;
2949 		port_window = uc->cfg.dst_port_window_size;
2950 	} else {
2951 		dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
2952 		return NULL;
2953 	}
2954 
2955 	if (!burst)
2956 		burst = 1;
2957 
2958 	if (port_window) {
2959 		if (port_window != burst) {
2960 			dev_err(uc->ud->dev,
2961 				"The burst must be equal to port_window\n");
2962 			return NULL;
2963 		}
2964 
2965 		tr_cnt0 = dev_width * port_window;
2966 		tr_cnt1 = 1;
2967 	} else {
2968 		tr_cnt0 = dev_width;
2969 		tr_cnt1 = burst;
2970 	}
2971 	trigger_size = tr_cnt0 * tr_cnt1;
2972 
2973 	/* estimate the number of TRs we will need */
2974 	for_each_sg(sgl, sgent, sglen, i) {
2975 		sg_len = sg_dma_len(sgent);
2976 
2977 		if (sg_len % trigger_size) {
2978 			dev_err(uc->ud->dev,
2979 				"Not aligned SG entry (%zu for %u)\n", sg_len,
2980 				trigger_size);
2981 			return NULL;
2982 		}
2983 
2984 		if (sg_len / trigger_size < SZ_64K)
2985 			num_tr++;
2986 		else
2987 			num_tr += 2;
2988 	}
2989 
2990 	/* Now allocate and setup the descriptor. */
2991 	tr_size = sizeof(struct cppi5_tr_type15_t);
2992 	d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
2993 	if (!d)
2994 		return NULL;
2995 
2996 	d->sglen = sglen;
2997 
2998 	if (uc->ud->match_data->type == DMA_TYPE_UDMA) {
2999 		asel = 0;
3000 	} else {
3001 		asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3002 		dev_addr |= asel;
3003 	}
3004 
3005 	tr_req = d->hwdesc[0].tr_req_base;
3006 	for_each_sg(sgl, sgent, sglen, i) {
3007 		u16 tr0_cnt2, tr0_cnt3, tr1_cnt2;
3008 		dma_addr_t sg_addr = sg_dma_address(sgent);
3009 
3010 		sg_len = sg_dma_len(sgent);
3011 		num_tr = udma_get_tr_counters(sg_len / trigger_size, 0,
3012 					      &tr0_cnt2, &tr0_cnt3, &tr1_cnt2);
3013 		if (num_tr < 0) {
3014 			dev_err(uc->ud->dev, "size %zu is not supported\n",
3015 				sg_len);
3016 			udma_free_hwdesc(uc, d);
3017 			kfree(d);
3018 			return NULL;
3019 		}
3020 
3021 		cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false,
3022 			      true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3023 		cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT);
3024 		cppi5_tr_set_trigger(&tr_req[tr_idx].flags,
3025 				     uc->config.tr_trigger_type,
3026 				     CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0);
3027 
3028 		sg_addr |= asel;
3029 		if (dir == DMA_DEV_TO_MEM) {
3030 			tr_req[tr_idx].addr = dev_addr;
3031 			tr_req[tr_idx].icnt0 = tr_cnt0;
3032 			tr_req[tr_idx].icnt1 = tr_cnt1;
3033 			tr_req[tr_idx].icnt2 = tr0_cnt2;
3034 			tr_req[tr_idx].icnt3 = tr0_cnt3;
3035 			tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
3036 
3037 			tr_req[tr_idx].daddr = sg_addr;
3038 			tr_req[tr_idx].dicnt0 = tr_cnt0;
3039 			tr_req[tr_idx].dicnt1 = tr_cnt1;
3040 			tr_req[tr_idx].dicnt2 = tr0_cnt2;
3041 			tr_req[tr_idx].dicnt3 = tr0_cnt3;
3042 			tr_req[tr_idx].ddim1 = tr_cnt0;
3043 			tr_req[tr_idx].ddim2 = trigger_size;
3044 			tr_req[tr_idx].ddim3 = trigger_size * tr0_cnt2;
3045 		} else {
3046 			tr_req[tr_idx].addr = sg_addr;
3047 			tr_req[tr_idx].icnt0 = tr_cnt0;
3048 			tr_req[tr_idx].icnt1 = tr_cnt1;
3049 			tr_req[tr_idx].icnt2 = tr0_cnt2;
3050 			tr_req[tr_idx].icnt3 = tr0_cnt3;
3051 			tr_req[tr_idx].dim1 = tr_cnt0;
3052 			tr_req[tr_idx].dim2 = trigger_size;
3053 			tr_req[tr_idx].dim3 = trigger_size * tr0_cnt2;
3054 
3055 			tr_req[tr_idx].daddr = dev_addr;
3056 			tr_req[tr_idx].dicnt0 = tr_cnt0;
3057 			tr_req[tr_idx].dicnt1 = tr_cnt1;
3058 			tr_req[tr_idx].dicnt2 = tr0_cnt2;
3059 			tr_req[tr_idx].dicnt3 = tr0_cnt3;
3060 			tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
3061 		}
3062 
3063 		tr_idx++;
3064 
3065 		if (num_tr == 2) {
3066 			cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15,
3067 				      false, true,
3068 				      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3069 			cppi5_tr_csf_set(&tr_req[tr_idx].flags,
3070 					 CPPI5_TR_CSF_SUPR_EVT);
3071 			cppi5_tr_set_trigger(&tr_req[tr_idx].flags,
3072 					     uc->config.tr_trigger_type,
3073 					     CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC,
3074 					     0, 0);
3075 
3076 			sg_addr += trigger_size * tr0_cnt2 * tr0_cnt3;
3077 			if (dir == DMA_DEV_TO_MEM) {
3078 				tr_req[tr_idx].addr = dev_addr;
3079 				tr_req[tr_idx].icnt0 = tr_cnt0;
3080 				tr_req[tr_idx].icnt1 = tr_cnt1;
3081 				tr_req[tr_idx].icnt2 = tr1_cnt2;
3082 				tr_req[tr_idx].icnt3 = 1;
3083 				tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
3084 
3085 				tr_req[tr_idx].daddr = sg_addr;
3086 				tr_req[tr_idx].dicnt0 = tr_cnt0;
3087 				tr_req[tr_idx].dicnt1 = tr_cnt1;
3088 				tr_req[tr_idx].dicnt2 = tr1_cnt2;
3089 				tr_req[tr_idx].dicnt3 = 1;
3090 				tr_req[tr_idx].ddim1 = tr_cnt0;
3091 				tr_req[tr_idx].ddim2 = trigger_size;
3092 			} else {
3093 				tr_req[tr_idx].addr = sg_addr;
3094 				tr_req[tr_idx].icnt0 = tr_cnt0;
3095 				tr_req[tr_idx].icnt1 = tr_cnt1;
3096 				tr_req[tr_idx].icnt2 = tr1_cnt2;
3097 				tr_req[tr_idx].icnt3 = 1;
3098 				tr_req[tr_idx].dim1 = tr_cnt0;
3099 				tr_req[tr_idx].dim2 = trigger_size;
3100 
3101 				tr_req[tr_idx].daddr = dev_addr;
3102 				tr_req[tr_idx].dicnt0 = tr_cnt0;
3103 				tr_req[tr_idx].dicnt1 = tr_cnt1;
3104 				tr_req[tr_idx].dicnt2 = tr1_cnt2;
3105 				tr_req[tr_idx].dicnt3 = 1;
3106 				tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
3107 			}
3108 			tr_idx++;
3109 		}
3110 
3111 		d->residue += sg_len;
3112 	}
3113 
3114 	cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
3115 			 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
3116 
3117 	return d;
3118 }
3119 
3120 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d,
3121 				   enum dma_slave_buswidth dev_width,
3122 				   u16 elcnt)
3123 {
3124 	if (uc->config.ep_type != PSIL_EP_PDMA_XY)
3125 		return 0;
3126 
3127 	/* Bus width translates to the element size (ES) */
3128 	switch (dev_width) {
3129 	case DMA_SLAVE_BUSWIDTH_1_BYTE:
3130 		d->static_tr.elsize = 0;
3131 		break;
3132 	case DMA_SLAVE_BUSWIDTH_2_BYTES:
3133 		d->static_tr.elsize = 1;
3134 		break;
3135 	case DMA_SLAVE_BUSWIDTH_3_BYTES:
3136 		d->static_tr.elsize = 2;
3137 		break;
3138 	case DMA_SLAVE_BUSWIDTH_4_BYTES:
3139 		d->static_tr.elsize = 3;
3140 		break;
3141 	case DMA_SLAVE_BUSWIDTH_8_BYTES:
3142 		d->static_tr.elsize = 4;
3143 		break;
3144 	default: /* not reached */
3145 		return -EINVAL;
3146 	}
3147 
3148 	d->static_tr.elcnt = elcnt;
3149 
3150 	/*
3151 	 * PDMA must to close the packet when the channel is in packet mode.
3152 	 * For TR mode when the channel is not cyclic we also need PDMA to close
3153 	 * the packet otherwise the transfer will stall because PDMA holds on
3154 	 * the data it has received from the peripheral.
3155 	 */
3156 	if (uc->config.pkt_mode || !uc->cyclic) {
3157 		unsigned int div = dev_width * elcnt;
3158 
3159 		if (uc->cyclic)
3160 			d->static_tr.bstcnt = d->residue / d->sglen / div;
3161 		else
3162 			d->static_tr.bstcnt = d->residue / div;
3163 
3164 		if (uc->config.dir == DMA_DEV_TO_MEM &&
3165 		    d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask)
3166 			return -EINVAL;
3167 	} else {
3168 		d->static_tr.bstcnt = 0;
3169 	}
3170 
3171 	return 0;
3172 }
3173 
3174 static struct udma_desc *
3175 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl,
3176 		       unsigned int sglen, enum dma_transfer_direction dir,
3177 		       unsigned long tx_flags, void *context)
3178 {
3179 	struct scatterlist *sgent;
3180 	struct cppi5_host_desc_t *h_desc = NULL;
3181 	struct udma_desc *d;
3182 	u32 ring_id;
3183 	unsigned int i;
3184 	u64 asel;
3185 
3186 	d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT);
3187 	if (!d)
3188 		return NULL;
3189 
3190 	d->sglen = sglen;
3191 	d->hwdesc_count = sglen;
3192 
3193 	if (dir == DMA_DEV_TO_MEM)
3194 		ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
3195 	else
3196 		ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
3197 
3198 	if (uc->ud->match_data->type == DMA_TYPE_UDMA)
3199 		asel = 0;
3200 	else
3201 		asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3202 
3203 	for_each_sg(sgl, sgent, sglen, i) {
3204 		struct udma_hwdesc *hwdesc = &d->hwdesc[i];
3205 		dma_addr_t sg_addr = sg_dma_address(sgent);
3206 		struct cppi5_host_desc_t *desc;
3207 		size_t sg_len = sg_dma_len(sgent);
3208 
3209 		hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
3210 						GFP_NOWAIT,
3211 						&hwdesc->cppi5_desc_paddr);
3212 		if (!hwdesc->cppi5_desc_vaddr) {
3213 			dev_err(uc->ud->dev,
3214 				"descriptor%d allocation failed\n", i);
3215 
3216 			udma_free_hwdesc(uc, d);
3217 			kfree(d);
3218 			return NULL;
3219 		}
3220 
3221 		d->residue += sg_len;
3222 		hwdesc->cppi5_desc_size = uc->config.hdesc_size;
3223 		desc = hwdesc->cppi5_desc_vaddr;
3224 
3225 		if (i == 0) {
3226 			cppi5_hdesc_init(desc, 0, 0);
3227 			/* Flow and Packed ID */
3228 			cppi5_desc_set_pktids(&desc->hdr, uc->id,
3229 					      CPPI5_INFO1_DESC_FLOWID_DEFAULT);
3230 			cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id);
3231 		} else {
3232 			cppi5_hdesc_reset_hbdesc(desc);
3233 			cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff);
3234 		}
3235 
3236 		/* attach the sg buffer to the descriptor */
3237 		sg_addr |= asel;
3238 		cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len);
3239 
3240 		/* Attach link as host buffer descriptor */
3241 		if (h_desc)
3242 			cppi5_hdesc_link_hbdesc(h_desc,
3243 						hwdesc->cppi5_desc_paddr | asel);
3244 
3245 		if (uc->ud->match_data->type == DMA_TYPE_PKTDMA ||
3246 		    dir == DMA_MEM_TO_DEV)
3247 			h_desc = desc;
3248 	}
3249 
3250 	if (d->residue >= SZ_4M) {
3251 		dev_err(uc->ud->dev,
3252 			"%s: Transfer size %u is over the supported 4M range\n",
3253 			__func__, d->residue);
3254 		udma_free_hwdesc(uc, d);
3255 		kfree(d);
3256 		return NULL;
3257 	}
3258 
3259 	h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3260 	cppi5_hdesc_set_pktlen(h_desc, d->residue);
3261 
3262 	return d;
3263 }
3264 
3265 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc,
3266 				void *data, size_t len)
3267 {
3268 	struct udma_desc *d = to_udma_desc(desc);
3269 	struct udma_chan *uc = to_udma_chan(desc->chan);
3270 	struct cppi5_host_desc_t *h_desc;
3271 	u32 psd_size = len;
3272 	u32 flags = 0;
3273 
3274 	if (!uc->config.pkt_mode || !uc->config.metadata_size)
3275 		return -ENOTSUPP;
3276 
3277 	if (!data || len > uc->config.metadata_size)
3278 		return -EINVAL;
3279 
3280 	if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE)
3281 		return -EINVAL;
3282 
3283 	h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3284 	if (d->dir == DMA_MEM_TO_DEV)
3285 		memcpy(h_desc->epib, data, len);
3286 
3287 	if (uc->config.needs_epib)
3288 		psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
3289 
3290 	d->metadata = data;
3291 	d->metadata_size = len;
3292 	if (uc->config.needs_epib)
3293 		flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
3294 
3295 	cppi5_hdesc_update_flags(h_desc, flags);
3296 	cppi5_hdesc_update_psdata_size(h_desc, psd_size);
3297 
3298 	return 0;
3299 }
3300 
3301 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
3302 				   size_t *payload_len, size_t *max_len)
3303 {
3304 	struct udma_desc *d = to_udma_desc(desc);
3305 	struct udma_chan *uc = to_udma_chan(desc->chan);
3306 	struct cppi5_host_desc_t *h_desc;
3307 
3308 	if (!uc->config.pkt_mode || !uc->config.metadata_size)
3309 		return ERR_PTR(-ENOTSUPP);
3310 
3311 	h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3312 
3313 	*max_len = uc->config.metadata_size;
3314 
3315 	*payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ?
3316 		       CPPI5_INFO0_HDESC_EPIB_SIZE : 0;
3317 	*payload_len += cppi5_hdesc_get_psdata_size(h_desc);
3318 
3319 	return h_desc->epib;
3320 }
3321 
3322 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
3323 				 size_t payload_len)
3324 {
3325 	struct udma_desc *d = to_udma_desc(desc);
3326 	struct udma_chan *uc = to_udma_chan(desc->chan);
3327 	struct cppi5_host_desc_t *h_desc;
3328 	u32 psd_size = payload_len;
3329 	u32 flags = 0;
3330 
3331 	if (!uc->config.pkt_mode || !uc->config.metadata_size)
3332 		return -ENOTSUPP;
3333 
3334 	if (payload_len > uc->config.metadata_size)
3335 		return -EINVAL;
3336 
3337 	if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE)
3338 		return -EINVAL;
3339 
3340 	h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3341 
3342 	if (uc->config.needs_epib) {
3343 		psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
3344 		flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
3345 	}
3346 
3347 	cppi5_hdesc_update_flags(h_desc, flags);
3348 	cppi5_hdesc_update_psdata_size(h_desc, psd_size);
3349 
3350 	return 0;
3351 }
3352 
3353 static struct dma_descriptor_metadata_ops metadata_ops = {
3354 	.attach = udma_attach_metadata,
3355 	.get_ptr = udma_get_metadata_ptr,
3356 	.set_len = udma_set_metadata_len,
3357 };
3358 
3359 static struct dma_async_tx_descriptor *
3360 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
3361 		   unsigned int sglen, enum dma_transfer_direction dir,
3362 		   unsigned long tx_flags, void *context)
3363 {
3364 	struct udma_chan *uc = to_udma_chan(chan);
3365 	enum dma_slave_buswidth dev_width;
3366 	struct udma_desc *d;
3367 	u32 burst;
3368 
3369 	if (dir != uc->config.dir &&
3370 	    (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type)) {
3371 		dev_err(chan->device->dev,
3372 			"%s: chan%d is for %s, not supporting %s\n",
3373 			__func__, uc->id,
3374 			dmaengine_get_direction_text(uc->config.dir),
3375 			dmaengine_get_direction_text(dir));
3376 		return NULL;
3377 	}
3378 
3379 	if (dir == DMA_DEV_TO_MEM) {
3380 		dev_width = uc->cfg.src_addr_width;
3381 		burst = uc->cfg.src_maxburst;
3382 	} else if (dir == DMA_MEM_TO_DEV) {
3383 		dev_width = uc->cfg.dst_addr_width;
3384 		burst = uc->cfg.dst_maxburst;
3385 	} else {
3386 		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
3387 		return NULL;
3388 	}
3389 
3390 	if (!burst)
3391 		burst = 1;
3392 
3393 	if (uc->config.pkt_mode)
3394 		d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags,
3395 					   context);
3396 	else if (is_slave_direction(uc->config.dir))
3397 		d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags,
3398 					  context);
3399 	else
3400 		d = udma_prep_slave_sg_triggered_tr(uc, sgl, sglen, dir,
3401 						    tx_flags, context);
3402 
3403 	if (!d)
3404 		return NULL;
3405 
3406 	d->dir = dir;
3407 	d->desc_idx = 0;
3408 	d->tr_idx = 0;
3409 
3410 	/* static TR for remote PDMA */
3411 	if (udma_configure_statictr(uc, d, dev_width, burst)) {
3412 		dev_err(uc->ud->dev,
3413 			"%s: StaticTR Z is limited to maximum 4095 (%u)\n",
3414 			__func__, d->static_tr.bstcnt);
3415 
3416 		udma_free_hwdesc(uc, d);
3417 		kfree(d);
3418 		return NULL;
3419 	}
3420 
3421 	if (uc->config.metadata_size)
3422 		d->vd.tx.metadata_ops = &metadata_ops;
3423 
3424 	return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
3425 }
3426 
3427 static struct udma_desc *
3428 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr,
3429 			size_t buf_len, size_t period_len,
3430 			enum dma_transfer_direction dir, unsigned long flags)
3431 {
3432 	struct udma_desc *d;
3433 	size_t tr_size, period_addr;
3434 	struct cppi5_tr_type1_t *tr_req;
3435 	unsigned int periods = buf_len / period_len;
3436 	u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
3437 	unsigned int i;
3438 	int num_tr;
3439 
3440 	num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0,
3441 				      &tr0_cnt1, &tr1_cnt0);
3442 	if (num_tr < 0) {
3443 		dev_err(uc->ud->dev, "size %zu is not supported\n",
3444 			period_len);
3445 		return NULL;
3446 	}
3447 
3448 	/* Now allocate and setup the descriptor. */
3449 	tr_size = sizeof(struct cppi5_tr_type1_t);
3450 	d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir);
3451 	if (!d)
3452 		return NULL;
3453 
3454 	tr_req = d->hwdesc[0].tr_req_base;
3455 	if (uc->ud->match_data->type == DMA_TYPE_UDMA)
3456 		period_addr = buf_addr;
3457 	else
3458 		period_addr = buf_addr |
3459 			((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT);
3460 
3461 	for (i = 0; i < periods; i++) {
3462 		int tr_idx = i * num_tr;
3463 
3464 		cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
3465 			      false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3466 
3467 		tr_req[tr_idx].addr = period_addr;
3468 		tr_req[tr_idx].icnt0 = tr0_cnt0;
3469 		tr_req[tr_idx].icnt1 = tr0_cnt1;
3470 		tr_req[tr_idx].dim1 = tr0_cnt0;
3471 
3472 		if (num_tr == 2) {
3473 			cppi5_tr_csf_set(&tr_req[tr_idx].flags,
3474 					 CPPI5_TR_CSF_SUPR_EVT);
3475 			tr_idx++;
3476 
3477 			cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
3478 				      false, false,
3479 				      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3480 
3481 			tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0;
3482 			tr_req[tr_idx].icnt0 = tr1_cnt0;
3483 			tr_req[tr_idx].icnt1 = 1;
3484 			tr_req[tr_idx].dim1 = tr1_cnt0;
3485 		}
3486 
3487 		if (!(flags & DMA_PREP_INTERRUPT))
3488 			cppi5_tr_csf_set(&tr_req[tr_idx].flags,
3489 					 CPPI5_TR_CSF_SUPR_EVT);
3490 
3491 		period_addr += period_len;
3492 	}
3493 
3494 	return d;
3495 }
3496 
3497 static struct udma_desc *
3498 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr,
3499 			 size_t buf_len, size_t period_len,
3500 			 enum dma_transfer_direction dir, unsigned long flags)
3501 {
3502 	struct udma_desc *d;
3503 	u32 ring_id;
3504 	int i;
3505 	int periods = buf_len / period_len;
3506 
3507 	if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1))
3508 		return NULL;
3509 
3510 	if (period_len >= SZ_4M)
3511 		return NULL;
3512 
3513 	d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT);
3514 	if (!d)
3515 		return NULL;
3516 
3517 	d->hwdesc_count = periods;
3518 
3519 	/* TODO: re-check this... */
3520 	if (dir == DMA_DEV_TO_MEM)
3521 		ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
3522 	else
3523 		ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
3524 
3525 	if (uc->ud->match_data->type != DMA_TYPE_UDMA)
3526 		buf_addr |= (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3527 
3528 	for (i = 0; i < periods; i++) {
3529 		struct udma_hwdesc *hwdesc = &d->hwdesc[i];
3530 		dma_addr_t period_addr = buf_addr + (period_len * i);
3531 		struct cppi5_host_desc_t *h_desc;
3532 
3533 		hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
3534 						GFP_NOWAIT,
3535 						&hwdesc->cppi5_desc_paddr);
3536 		if (!hwdesc->cppi5_desc_vaddr) {
3537 			dev_err(uc->ud->dev,
3538 				"descriptor%d allocation failed\n", i);
3539 
3540 			udma_free_hwdesc(uc, d);
3541 			kfree(d);
3542 			return NULL;
3543 		}
3544 
3545 		hwdesc->cppi5_desc_size = uc->config.hdesc_size;
3546 		h_desc = hwdesc->cppi5_desc_vaddr;
3547 
3548 		cppi5_hdesc_init(h_desc, 0, 0);
3549 		cppi5_hdesc_set_pktlen(h_desc, period_len);
3550 
3551 		/* Flow and Packed ID */
3552 		cppi5_desc_set_pktids(&h_desc->hdr, uc->id,
3553 				      CPPI5_INFO1_DESC_FLOWID_DEFAULT);
3554 		cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id);
3555 
3556 		/* attach each period to a new descriptor */
3557 		cppi5_hdesc_attach_buf(h_desc,
3558 				       period_addr, period_len,
3559 				       period_addr, period_len);
3560 	}
3561 
3562 	return d;
3563 }
3564 
3565 static struct dma_async_tx_descriptor *
3566 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
3567 		     size_t period_len, enum dma_transfer_direction dir,
3568 		     unsigned long flags)
3569 {
3570 	struct udma_chan *uc = to_udma_chan(chan);
3571 	enum dma_slave_buswidth dev_width;
3572 	struct udma_desc *d;
3573 	u32 burst;
3574 
3575 	if (dir != uc->config.dir) {
3576 		dev_err(chan->device->dev,
3577 			"%s: chan%d is for %s, not supporting %s\n",
3578 			__func__, uc->id,
3579 			dmaengine_get_direction_text(uc->config.dir),
3580 			dmaengine_get_direction_text(dir));
3581 		return NULL;
3582 	}
3583 
3584 	uc->cyclic = true;
3585 
3586 	if (dir == DMA_DEV_TO_MEM) {
3587 		dev_width = uc->cfg.src_addr_width;
3588 		burst = uc->cfg.src_maxburst;
3589 	} else if (dir == DMA_MEM_TO_DEV) {
3590 		dev_width = uc->cfg.dst_addr_width;
3591 		burst = uc->cfg.dst_maxburst;
3592 	} else {
3593 		dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
3594 		return NULL;
3595 	}
3596 
3597 	if (!burst)
3598 		burst = 1;
3599 
3600 	if (uc->config.pkt_mode)
3601 		d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len,
3602 					     dir, flags);
3603 	else
3604 		d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len,
3605 					    dir, flags);
3606 
3607 	if (!d)
3608 		return NULL;
3609 
3610 	d->sglen = buf_len / period_len;
3611 
3612 	d->dir = dir;
3613 	d->residue = buf_len;
3614 
3615 	/* static TR for remote PDMA */
3616 	if (udma_configure_statictr(uc, d, dev_width, burst)) {
3617 		dev_err(uc->ud->dev,
3618 			"%s: StaticTR Z is limited to maximum 4095 (%u)\n",
3619 			__func__, d->static_tr.bstcnt);
3620 
3621 		udma_free_hwdesc(uc, d);
3622 		kfree(d);
3623 		return NULL;
3624 	}
3625 
3626 	if (uc->config.metadata_size)
3627 		d->vd.tx.metadata_ops = &metadata_ops;
3628 
3629 	return vchan_tx_prep(&uc->vc, &d->vd, flags);
3630 }
3631 
3632 static struct dma_async_tx_descriptor *
3633 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
3634 		     size_t len, unsigned long tx_flags)
3635 {
3636 	struct udma_chan *uc = to_udma_chan(chan);
3637 	struct udma_desc *d;
3638 	struct cppi5_tr_type15_t *tr_req;
3639 	int num_tr;
3640 	size_t tr_size = sizeof(struct cppi5_tr_type15_t);
3641 	u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
3642 
3643 	if (uc->config.dir != DMA_MEM_TO_MEM) {
3644 		dev_err(chan->device->dev,
3645 			"%s: chan%d is for %s, not supporting %s\n",
3646 			__func__, uc->id,
3647 			dmaengine_get_direction_text(uc->config.dir),
3648 			dmaengine_get_direction_text(DMA_MEM_TO_MEM));
3649 		return NULL;
3650 	}
3651 
3652 	num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0,
3653 				      &tr0_cnt1, &tr1_cnt0);
3654 	if (num_tr < 0) {
3655 		dev_err(uc->ud->dev, "size %zu is not supported\n",
3656 			len);
3657 		return NULL;
3658 	}
3659 
3660 	d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM);
3661 	if (!d)
3662 		return NULL;
3663 
3664 	d->dir = DMA_MEM_TO_MEM;
3665 	d->desc_idx = 0;
3666 	d->tr_idx = 0;
3667 	d->residue = len;
3668 
3669 	if (uc->ud->match_data->type != DMA_TYPE_UDMA) {
3670 		src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
3671 		dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
3672 	}
3673 
3674 	tr_req = d->hwdesc[0].tr_req_base;
3675 
3676 	cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
3677 		      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3678 	cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT);
3679 
3680 	tr_req[0].addr = src;
3681 	tr_req[0].icnt0 = tr0_cnt0;
3682 	tr_req[0].icnt1 = tr0_cnt1;
3683 	tr_req[0].icnt2 = 1;
3684 	tr_req[0].icnt3 = 1;
3685 	tr_req[0].dim1 = tr0_cnt0;
3686 
3687 	tr_req[0].daddr = dest;
3688 	tr_req[0].dicnt0 = tr0_cnt0;
3689 	tr_req[0].dicnt1 = tr0_cnt1;
3690 	tr_req[0].dicnt2 = 1;
3691 	tr_req[0].dicnt3 = 1;
3692 	tr_req[0].ddim1 = tr0_cnt0;
3693 
3694 	if (num_tr == 2) {
3695 		cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
3696 			      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3697 		cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT);
3698 
3699 		tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
3700 		tr_req[1].icnt0 = tr1_cnt0;
3701 		tr_req[1].icnt1 = 1;
3702 		tr_req[1].icnt2 = 1;
3703 		tr_req[1].icnt3 = 1;
3704 
3705 		tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
3706 		tr_req[1].dicnt0 = tr1_cnt0;
3707 		tr_req[1].dicnt1 = 1;
3708 		tr_req[1].dicnt2 = 1;
3709 		tr_req[1].dicnt3 = 1;
3710 	}
3711 
3712 	cppi5_tr_csf_set(&tr_req[num_tr - 1].flags,
3713 			 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
3714 
3715 	if (uc->config.metadata_size)
3716 		d->vd.tx.metadata_ops = &metadata_ops;
3717 
3718 	return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
3719 }
3720 
3721 static void udma_issue_pending(struct dma_chan *chan)
3722 {
3723 	struct udma_chan *uc = to_udma_chan(chan);
3724 	unsigned long flags;
3725 
3726 	spin_lock_irqsave(&uc->vc.lock, flags);
3727 
3728 	/* If we have something pending and no active descriptor, then */
3729 	if (vchan_issue_pending(&uc->vc) && !uc->desc) {
3730 		/*
3731 		 * start a descriptor if the channel is NOT [marked as
3732 		 * terminating _and_ it is still running (teardown has not
3733 		 * completed yet)].
3734 		 */
3735 		if (!(uc->state == UDMA_CHAN_IS_TERMINATING &&
3736 		      udma_is_chan_running(uc)))
3737 			udma_start(uc);
3738 	}
3739 
3740 	spin_unlock_irqrestore(&uc->vc.lock, flags);
3741 }
3742 
3743 static enum dma_status udma_tx_status(struct dma_chan *chan,
3744 				      dma_cookie_t cookie,
3745 				      struct dma_tx_state *txstate)
3746 {
3747 	struct udma_chan *uc = to_udma_chan(chan);
3748 	enum dma_status ret;
3749 	unsigned long flags;
3750 
3751 	spin_lock_irqsave(&uc->vc.lock, flags);
3752 
3753 	ret = dma_cookie_status(chan, cookie, txstate);
3754 
3755 	if (!udma_is_chan_running(uc))
3756 		ret = DMA_COMPLETE;
3757 
3758 	if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc))
3759 		ret = DMA_PAUSED;
3760 
3761 	if (ret == DMA_COMPLETE || !txstate)
3762 		goto out;
3763 
3764 	if (uc->desc && uc->desc->vd.tx.cookie == cookie) {
3765 		u32 peer_bcnt = 0;
3766 		u32 bcnt = 0;
3767 		u32 residue = uc->desc->residue;
3768 		u32 delay = 0;
3769 
3770 		if (uc->desc->dir == DMA_MEM_TO_DEV) {
3771 			bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
3772 
3773 			if (uc->config.ep_type != PSIL_EP_NATIVE) {
3774 				peer_bcnt = udma_tchanrt_read(uc,
3775 						UDMA_CHAN_RT_PEER_BCNT_REG);
3776 
3777 				if (bcnt > peer_bcnt)
3778 					delay = bcnt - peer_bcnt;
3779 			}
3780 		} else if (uc->desc->dir == DMA_DEV_TO_MEM) {
3781 			bcnt = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
3782 
3783 			if (uc->config.ep_type != PSIL_EP_NATIVE) {
3784 				peer_bcnt = udma_rchanrt_read(uc,
3785 						UDMA_CHAN_RT_PEER_BCNT_REG);
3786 
3787 				if (peer_bcnt > bcnt)
3788 					delay = peer_bcnt - bcnt;
3789 			}
3790 		} else {
3791 			bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
3792 		}
3793 
3794 		bcnt -= uc->bcnt;
3795 		if (bcnt && !(bcnt % uc->desc->residue))
3796 			residue = 0;
3797 		else
3798 			residue -= bcnt % uc->desc->residue;
3799 
3800 		if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) {
3801 			ret = DMA_COMPLETE;
3802 			delay = 0;
3803 		}
3804 
3805 		dma_set_residue(txstate, residue);
3806 		dma_set_in_flight_bytes(txstate, delay);
3807 
3808 	} else {
3809 		ret = DMA_COMPLETE;
3810 	}
3811 
3812 out:
3813 	spin_unlock_irqrestore(&uc->vc.lock, flags);
3814 	return ret;
3815 }
3816 
3817 static int udma_pause(struct dma_chan *chan)
3818 {
3819 	struct udma_chan *uc = to_udma_chan(chan);
3820 
3821 	/* pause the channel */
3822 	switch (uc->config.dir) {
3823 	case DMA_DEV_TO_MEM:
3824 		udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3825 					 UDMA_PEER_RT_EN_PAUSE,
3826 					 UDMA_PEER_RT_EN_PAUSE);
3827 		break;
3828 	case DMA_MEM_TO_DEV:
3829 		udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3830 					 UDMA_PEER_RT_EN_PAUSE,
3831 					 UDMA_PEER_RT_EN_PAUSE);
3832 		break;
3833 	case DMA_MEM_TO_MEM:
3834 		udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,
3835 					 UDMA_CHAN_RT_CTL_PAUSE,
3836 					 UDMA_CHAN_RT_CTL_PAUSE);
3837 		break;
3838 	default:
3839 		return -EINVAL;
3840 	}
3841 
3842 	return 0;
3843 }
3844 
3845 static int udma_resume(struct dma_chan *chan)
3846 {
3847 	struct udma_chan *uc = to_udma_chan(chan);
3848 
3849 	/* resume the channel */
3850 	switch (uc->config.dir) {
3851 	case DMA_DEV_TO_MEM:
3852 		udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3853 					 UDMA_PEER_RT_EN_PAUSE, 0);
3854 
3855 		break;
3856 	case DMA_MEM_TO_DEV:
3857 		udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3858 					 UDMA_PEER_RT_EN_PAUSE, 0);
3859 		break;
3860 	case DMA_MEM_TO_MEM:
3861 		udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,
3862 					 UDMA_CHAN_RT_CTL_PAUSE, 0);
3863 		break;
3864 	default:
3865 		return -EINVAL;
3866 	}
3867 
3868 	return 0;
3869 }
3870 
3871 static int udma_terminate_all(struct dma_chan *chan)
3872 {
3873 	struct udma_chan *uc = to_udma_chan(chan);
3874 	unsigned long flags;
3875 	LIST_HEAD(head);
3876 
3877 	spin_lock_irqsave(&uc->vc.lock, flags);
3878 
3879 	if (udma_is_chan_running(uc))
3880 		udma_stop(uc);
3881 
3882 	if (uc->desc) {
3883 		uc->terminated_desc = uc->desc;
3884 		uc->desc = NULL;
3885 		uc->terminated_desc->terminated = true;
3886 		cancel_delayed_work(&uc->tx_drain.work);
3887 	}
3888 
3889 	uc->paused = false;
3890 
3891 	vchan_get_all_descriptors(&uc->vc, &head);
3892 	spin_unlock_irqrestore(&uc->vc.lock, flags);
3893 	vchan_dma_desc_free_list(&uc->vc, &head);
3894 
3895 	return 0;
3896 }
3897 
3898 static void udma_synchronize(struct dma_chan *chan)
3899 {
3900 	struct udma_chan *uc = to_udma_chan(chan);
3901 	unsigned long timeout = msecs_to_jiffies(1000);
3902 
3903 	vchan_synchronize(&uc->vc);
3904 
3905 	if (uc->state == UDMA_CHAN_IS_TERMINATING) {
3906 		timeout = wait_for_completion_timeout(&uc->teardown_completed,
3907 						      timeout);
3908 		if (!timeout) {
3909 			dev_warn(uc->ud->dev, "chan%d teardown timeout!\n",
3910 				 uc->id);
3911 			udma_dump_chan_stdata(uc);
3912 			udma_reset_chan(uc, true);
3913 		}
3914 	}
3915 
3916 	udma_reset_chan(uc, false);
3917 	if (udma_is_chan_running(uc))
3918 		dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id);
3919 
3920 	cancel_delayed_work_sync(&uc->tx_drain.work);
3921 	udma_reset_rings(uc);
3922 }
3923 
3924 static void udma_desc_pre_callback(struct virt_dma_chan *vc,
3925 				   struct virt_dma_desc *vd,
3926 				   struct dmaengine_result *result)
3927 {
3928 	struct udma_chan *uc = to_udma_chan(&vc->chan);
3929 	struct udma_desc *d;
3930 
3931 	if (!vd)
3932 		return;
3933 
3934 	d = to_udma_desc(&vd->tx);
3935 
3936 	if (d->metadata_size)
3937 		udma_fetch_epib(uc, d);
3938 
3939 	/* Provide residue information for the client */
3940 	if (result) {
3941 		void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx);
3942 
3943 		if (cppi5_desc_get_type(desc_vaddr) ==
3944 		    CPPI5_INFO0_DESC_TYPE_VAL_HOST) {
3945 			result->residue = d->residue -
3946 					  cppi5_hdesc_get_pktlen(desc_vaddr);
3947 			if (result->residue)
3948 				result->result = DMA_TRANS_ABORTED;
3949 			else
3950 				result->result = DMA_TRANS_NOERROR;
3951 		} else {
3952 			result->residue = 0;
3953 			result->result = DMA_TRANS_NOERROR;
3954 		}
3955 	}
3956 }
3957 
3958 /*
3959  * This tasklet handles the completion of a DMA descriptor by
3960  * calling its callback and freeing it.
3961  */
3962 static void udma_vchan_complete(struct tasklet_struct *t)
3963 {
3964 	struct virt_dma_chan *vc = from_tasklet(vc, t, task);
3965 	struct virt_dma_desc *vd, *_vd;
3966 	struct dmaengine_desc_callback cb;
3967 	LIST_HEAD(head);
3968 
3969 	spin_lock_irq(&vc->lock);
3970 	list_splice_tail_init(&vc->desc_completed, &head);
3971 	vd = vc->cyclic;
3972 	if (vd) {
3973 		vc->cyclic = NULL;
3974 		dmaengine_desc_get_callback(&vd->tx, &cb);
3975 	} else {
3976 		memset(&cb, 0, sizeof(cb));
3977 	}
3978 	spin_unlock_irq(&vc->lock);
3979 
3980 	udma_desc_pre_callback(vc, vd, NULL);
3981 	dmaengine_desc_callback_invoke(&cb, NULL);
3982 
3983 	list_for_each_entry_safe(vd, _vd, &head, node) {
3984 		struct dmaengine_result result;
3985 
3986 		dmaengine_desc_get_callback(&vd->tx, &cb);
3987 
3988 		list_del(&vd->node);
3989 
3990 		udma_desc_pre_callback(vc, vd, &result);
3991 		dmaengine_desc_callback_invoke(&cb, &result);
3992 
3993 		vchan_vdesc_fini(vd);
3994 	}
3995 }
3996 
3997 static void udma_free_chan_resources(struct dma_chan *chan)
3998 {
3999 	struct udma_chan *uc = to_udma_chan(chan);
4000 	struct udma_dev *ud = to_udma_dev(chan->device);
4001 
4002 	udma_terminate_all(chan);
4003 	if (uc->terminated_desc) {
4004 		udma_reset_chan(uc, false);
4005 		udma_reset_rings(uc);
4006 	}
4007 
4008 	cancel_delayed_work_sync(&uc->tx_drain.work);
4009 
4010 	if (uc->irq_num_ring > 0) {
4011 		free_irq(uc->irq_num_ring, uc);
4012 
4013 		uc->irq_num_ring = 0;
4014 	}
4015 	if (uc->irq_num_udma > 0) {
4016 		free_irq(uc->irq_num_udma, uc);
4017 
4018 		uc->irq_num_udma = 0;
4019 	}
4020 
4021 	/* Release PSI-L pairing */
4022 	if (uc->psil_paired) {
4023 		navss_psil_unpair(ud, uc->config.src_thread,
4024 				  uc->config.dst_thread);
4025 		uc->psil_paired = false;
4026 	}
4027 
4028 	vchan_free_chan_resources(&uc->vc);
4029 	tasklet_kill(&uc->vc.task);
4030 
4031 	bcdma_free_bchan_resources(uc);
4032 	udma_free_tx_resources(uc);
4033 	udma_free_rx_resources(uc);
4034 	udma_reset_uchan(uc);
4035 
4036 	if (uc->use_dma_pool) {
4037 		dma_pool_destroy(uc->hdesc_pool);
4038 		uc->use_dma_pool = false;
4039 	}
4040 }
4041 
4042 static struct platform_driver udma_driver;
4043 static struct platform_driver bcdma_driver;
4044 static struct platform_driver pktdma_driver;
4045 
4046 struct udma_filter_param {
4047 	int remote_thread_id;
4048 	u32 atype;
4049 	u32 asel;
4050 	u32 tr_trigger_type;
4051 };
4052 
4053 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param)
4054 {
4055 	struct udma_chan_config *ucc;
4056 	struct psil_endpoint_config *ep_config;
4057 	struct udma_filter_param *filter_param;
4058 	struct udma_chan *uc;
4059 	struct udma_dev *ud;
4060 
4061 	if (chan->device->dev->driver != &udma_driver.driver &&
4062 	    chan->device->dev->driver != &bcdma_driver.driver &&
4063 	    chan->device->dev->driver != &pktdma_driver.driver)
4064 		return false;
4065 
4066 	uc = to_udma_chan(chan);
4067 	ucc = &uc->config;
4068 	ud = uc->ud;
4069 	filter_param = param;
4070 
4071 	if (filter_param->atype > 2) {
4072 		dev_err(ud->dev, "Invalid channel atype: %u\n",
4073 			filter_param->atype);
4074 		return false;
4075 	}
4076 
4077 	if (filter_param->asel > 15) {
4078 		dev_err(ud->dev, "Invalid channel asel: %u\n",
4079 			filter_param->asel);
4080 		return false;
4081 	}
4082 
4083 	ucc->remote_thread_id = filter_param->remote_thread_id;
4084 	ucc->atype = filter_param->atype;
4085 	ucc->asel = filter_param->asel;
4086 	ucc->tr_trigger_type = filter_param->tr_trigger_type;
4087 
4088 	if (ucc->tr_trigger_type) {
4089 		ucc->dir = DMA_MEM_TO_MEM;
4090 		goto triggered_bchan;
4091 	} else if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) {
4092 		ucc->dir = DMA_MEM_TO_DEV;
4093 	} else {
4094 		ucc->dir = DMA_DEV_TO_MEM;
4095 	}
4096 
4097 	ep_config = psil_get_ep_config(ucc->remote_thread_id);
4098 	if (IS_ERR(ep_config)) {
4099 		dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n",
4100 			ucc->remote_thread_id);
4101 		ucc->dir = DMA_MEM_TO_MEM;
4102 		ucc->remote_thread_id = -1;
4103 		ucc->atype = 0;
4104 		ucc->asel = 0;
4105 		return false;
4106 	}
4107 
4108 	if (ud->match_data->type == DMA_TYPE_BCDMA &&
4109 	    ep_config->pkt_mode) {
4110 		dev_err(ud->dev,
4111 			"Only TR mode is supported (psi-l thread 0x%04x)\n",
4112 			ucc->remote_thread_id);
4113 		ucc->dir = DMA_MEM_TO_MEM;
4114 		ucc->remote_thread_id = -1;
4115 		ucc->atype = 0;
4116 		ucc->asel = 0;
4117 		return false;
4118 	}
4119 
4120 	ucc->pkt_mode = ep_config->pkt_mode;
4121 	ucc->channel_tpl = ep_config->channel_tpl;
4122 	ucc->notdpkt = ep_config->notdpkt;
4123 	ucc->ep_type = ep_config->ep_type;
4124 
4125 	if (ud->match_data->type == DMA_TYPE_PKTDMA &&
4126 	    ep_config->mapped_channel_id >= 0) {
4127 		ucc->mapped_channel_id = ep_config->mapped_channel_id;
4128 		ucc->default_flow_id = ep_config->default_flow_id;
4129 	} else {
4130 		ucc->mapped_channel_id = -1;
4131 		ucc->default_flow_id = -1;
4132 	}
4133 
4134 	if (ucc->ep_type != PSIL_EP_NATIVE) {
4135 		const struct udma_match_data *match_data = ud->match_data;
4136 
4137 		if (match_data->flags & UDMA_FLAG_PDMA_ACC32)
4138 			ucc->enable_acc32 = ep_config->pdma_acc32;
4139 		if (match_data->flags & UDMA_FLAG_PDMA_BURST)
4140 			ucc->enable_burst = ep_config->pdma_burst;
4141 	}
4142 
4143 	ucc->needs_epib = ep_config->needs_epib;
4144 	ucc->psd_size = ep_config->psd_size;
4145 	ucc->metadata_size =
4146 			(ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) +
4147 			ucc->psd_size;
4148 
4149 	if (ucc->pkt_mode)
4150 		ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
4151 				 ucc->metadata_size, ud->desc_align);
4152 
4153 	dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id,
4154 		ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir));
4155 
4156 	return true;
4157 
4158 triggered_bchan:
4159 	dev_dbg(ud->dev, "chan%d: triggered channel (type: %u)\n", uc->id,
4160 		ucc->tr_trigger_type);
4161 
4162 	return true;
4163 
4164 }
4165 
4166 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec,
4167 				      struct of_dma *ofdma)
4168 {
4169 	struct udma_dev *ud = ofdma->of_dma_data;
4170 	dma_cap_mask_t mask = ud->ddev.cap_mask;
4171 	struct udma_filter_param filter_param;
4172 	struct dma_chan *chan;
4173 
4174 	if (ud->match_data->type == DMA_TYPE_BCDMA) {
4175 		if (dma_spec->args_count != 3)
4176 			return NULL;
4177 
4178 		filter_param.tr_trigger_type = dma_spec->args[0];
4179 		filter_param.remote_thread_id = dma_spec->args[1];
4180 		filter_param.asel = dma_spec->args[2];
4181 		filter_param.atype = 0;
4182 	} else {
4183 		if (dma_spec->args_count != 1 && dma_spec->args_count != 2)
4184 			return NULL;
4185 
4186 		filter_param.remote_thread_id = dma_spec->args[0];
4187 		filter_param.tr_trigger_type = 0;
4188 		if (dma_spec->args_count == 2) {
4189 			if (ud->match_data->type == DMA_TYPE_UDMA) {
4190 				filter_param.atype = dma_spec->args[1];
4191 				filter_param.asel = 0;
4192 			} else {
4193 				filter_param.atype = 0;
4194 				filter_param.asel = dma_spec->args[1];
4195 			}
4196 		} else {
4197 			filter_param.atype = 0;
4198 			filter_param.asel = 0;
4199 		}
4200 	}
4201 
4202 	chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param,
4203 				     ofdma->of_node);
4204 	if (!chan) {
4205 		dev_err(ud->dev, "get channel fail in %s.\n", __func__);
4206 		return ERR_PTR(-EINVAL);
4207 	}
4208 
4209 	return chan;
4210 }
4211 
4212 static struct udma_match_data am654_main_data = {
4213 	.type = DMA_TYPE_UDMA,
4214 	.psil_base = 0x1000,
4215 	.enable_memcpy_support = true,
4216 	.statictr_z_mask = GENMASK(11, 0),
4217 	.burst_size = {
4218 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4219 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */
4220 		0, /* No UH Channels */
4221 	},
4222 };
4223 
4224 static struct udma_match_data am654_mcu_data = {
4225 	.type = DMA_TYPE_UDMA,
4226 	.psil_base = 0x6000,
4227 	.enable_memcpy_support = false,
4228 	.statictr_z_mask = GENMASK(11, 0),
4229 	.burst_size = {
4230 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4231 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */
4232 		0, /* No UH Channels */
4233 	},
4234 };
4235 
4236 static struct udma_match_data j721e_main_data = {
4237 	.type = DMA_TYPE_UDMA,
4238 	.psil_base = 0x1000,
4239 	.enable_memcpy_support = true,
4240 	.flags = UDMA_FLAGS_J7_CLASS,
4241 	.statictr_z_mask = GENMASK(23, 0),
4242 	.burst_size = {
4243 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4244 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* H Channels */
4245 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* UH Channels */
4246 	},
4247 };
4248 
4249 static struct udma_match_data j721e_mcu_data = {
4250 	.type = DMA_TYPE_UDMA,
4251 	.psil_base = 0x6000,
4252 	.enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */
4253 	.flags = UDMA_FLAGS_J7_CLASS,
4254 	.statictr_z_mask = GENMASK(23, 0),
4255 	.burst_size = {
4256 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4257 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES, /* H Channels */
4258 		0, /* No UH Channels */
4259 	},
4260 };
4261 
4262 static struct udma_match_data am64_bcdma_data = {
4263 	.type = DMA_TYPE_BCDMA,
4264 	.psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */
4265 	.enable_memcpy_support = true, /* Supported via bchan */
4266 	.flags = UDMA_FLAGS_J7_CLASS,
4267 	.statictr_z_mask = GENMASK(23, 0),
4268 	.burst_size = {
4269 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4270 		0, /* No H Channels */
4271 		0, /* No UH Channels */
4272 	},
4273 };
4274 
4275 static struct udma_match_data am64_pktdma_data = {
4276 	.type = DMA_TYPE_PKTDMA,
4277 	.psil_base = 0x1000,
4278 	.enable_memcpy_support = false, /* PKTDMA does not support MEM_TO_MEM */
4279 	.flags = UDMA_FLAGS_J7_CLASS,
4280 	.statictr_z_mask = GENMASK(23, 0),
4281 	.burst_size = {
4282 		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4283 		0, /* No H Channels */
4284 		0, /* No UH Channels */
4285 	},
4286 };
4287 
4288 static const struct of_device_id udma_of_match[] = {
4289 	{
4290 		.compatible = "ti,am654-navss-main-udmap",
4291 		.data = &am654_main_data,
4292 	},
4293 	{
4294 		.compatible = "ti,am654-navss-mcu-udmap",
4295 		.data = &am654_mcu_data,
4296 	}, {
4297 		.compatible = "ti,j721e-navss-main-udmap",
4298 		.data = &j721e_main_data,
4299 	}, {
4300 		.compatible = "ti,j721e-navss-mcu-udmap",
4301 		.data = &j721e_mcu_data,
4302 	},
4303 	{ /* Sentinel */ },
4304 };
4305 
4306 static const struct of_device_id bcdma_of_match[] = {
4307 	{
4308 		.compatible = "ti,am64-dmss-bcdma",
4309 		.data = &am64_bcdma_data,
4310 	},
4311 	{ /* Sentinel */ },
4312 };
4313 
4314 static const struct of_device_id pktdma_of_match[] = {
4315 	{
4316 		.compatible = "ti,am64-dmss-pktdma",
4317 		.data = &am64_pktdma_data,
4318 	},
4319 	{ /* Sentinel */ },
4320 };
4321 
4322 static struct udma_soc_data am654_soc_data = {
4323 	.oes = {
4324 		.udma_rchan = 0x200,
4325 	},
4326 };
4327 
4328 static struct udma_soc_data j721e_soc_data = {
4329 	.oes = {
4330 		.udma_rchan = 0x400,
4331 	},
4332 };
4333 
4334 static struct udma_soc_data j7200_soc_data = {
4335 	.oes = {
4336 		.udma_rchan = 0x80,
4337 	},
4338 };
4339 
4340 static struct udma_soc_data am64_soc_data = {
4341 	.oes = {
4342 		.bcdma_bchan_data = 0x2200,
4343 		.bcdma_bchan_ring = 0x2400,
4344 		.bcdma_tchan_data = 0x2800,
4345 		.bcdma_tchan_ring = 0x2a00,
4346 		.bcdma_rchan_data = 0x2e00,
4347 		.bcdma_rchan_ring = 0x3000,
4348 		.pktdma_tchan_flow = 0x1200,
4349 		.pktdma_rchan_flow = 0x1600,
4350 	},
4351 	.bcdma_trigger_event_offset = 0xc400,
4352 };
4353 
4354 static const struct soc_device_attribute k3_soc_devices[] = {
4355 	{ .family = "AM65X", .data = &am654_soc_data },
4356 	{ .family = "J721E", .data = &j721e_soc_data },
4357 	{ .family = "J7200", .data = &j7200_soc_data },
4358 	{ .family = "AM64X", .data = &am64_soc_data },
4359 	{ /* sentinel */ }
4360 };
4361 
4362 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud)
4363 {
4364 	u32 cap2, cap3, cap4;
4365 	int i;
4366 
4367 	ud->mmrs[MMR_GCFG] = devm_platform_ioremap_resource_byname(pdev, mmr_names[MMR_GCFG]);
4368 	if (IS_ERR(ud->mmrs[MMR_GCFG]))
4369 		return PTR_ERR(ud->mmrs[MMR_GCFG]);
4370 
4371 	cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28);
4372 	cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4373 
4374 	switch (ud->match_data->type) {
4375 	case DMA_TYPE_UDMA:
4376 		ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);
4377 		ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);
4378 		ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2);
4379 		ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);
4380 		break;
4381 	case DMA_TYPE_BCDMA:
4382 		ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2);
4383 		ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2);
4384 		ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2);
4385 		ud->rflow_cnt = ud->rchan_cnt;
4386 		break;
4387 	case DMA_TYPE_PKTDMA:
4388 		cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30);
4389 		ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);
4390 		ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);
4391 		ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);
4392 		ud->tflow_cnt = PKTDMA_CAP4_TFLOW_CNT(cap4);
4393 		break;
4394 	default:
4395 		return -EINVAL;
4396 	}
4397 
4398 	for (i = 1; i < MMR_LAST; i++) {
4399 		if (i == MMR_BCHANRT && ud->bchan_cnt == 0)
4400 			continue;
4401 		if (i == MMR_TCHANRT && ud->tchan_cnt == 0)
4402 			continue;
4403 		if (i == MMR_RCHANRT && ud->rchan_cnt == 0)
4404 			continue;
4405 
4406 		ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]);
4407 		if (IS_ERR(ud->mmrs[i]))
4408 			return PTR_ERR(ud->mmrs[i]);
4409 	}
4410 
4411 	return 0;
4412 }
4413 
4414 static void udma_mark_resource_ranges(struct udma_dev *ud, unsigned long *map,
4415 				      struct ti_sci_resource_desc *rm_desc,
4416 				      char *name)
4417 {
4418 	bitmap_clear(map, rm_desc->start, rm_desc->num);
4419 	bitmap_clear(map, rm_desc->start_sec, rm_desc->num_sec);
4420 	dev_dbg(ud->dev, "ti_sci resource range for %s: %d:%d | %d:%d\n", name,
4421 		rm_desc->start, rm_desc->num, rm_desc->start_sec,
4422 		rm_desc->num_sec);
4423 }
4424 
4425 static const char * const range_names[] = {
4426 	[RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan",
4427 	[RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan",
4428 	[RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan",
4429 	[RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow",
4430 	[RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow",
4431 };
4432 
4433 static int udma_setup_resources(struct udma_dev *ud)
4434 {
4435 	int ret, i, j;
4436 	struct device *dev = ud->dev;
4437 	struct ti_sci_resource *rm_res, irq_res;
4438 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4439 	u32 cap3;
4440 
4441 	/* Set up the throughput level start indexes */
4442 	cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4443 	if (of_device_is_compatible(dev->of_node,
4444 				    "ti,am654-navss-main-udmap")) {
4445 		ud->tchan_tpl.levels = 2;
4446 		ud->tchan_tpl.start_idx[0] = 8;
4447 	} else if (of_device_is_compatible(dev->of_node,
4448 					   "ti,am654-navss-mcu-udmap")) {
4449 		ud->tchan_tpl.levels = 2;
4450 		ud->tchan_tpl.start_idx[0] = 2;
4451 	} else if (UDMA_CAP3_UCHAN_CNT(cap3)) {
4452 		ud->tchan_tpl.levels = 3;
4453 		ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);
4454 		ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4455 	} else if (UDMA_CAP3_HCHAN_CNT(cap3)) {
4456 		ud->tchan_tpl.levels = 2;
4457 		ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4458 	} else {
4459 		ud->tchan_tpl.levels = 1;
4460 	}
4461 
4462 	ud->rchan_tpl.levels = ud->tchan_tpl.levels;
4463 	ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0];
4464 	ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1];
4465 
4466 	ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4467 					   sizeof(unsigned long), GFP_KERNEL);
4468 	ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4469 				  GFP_KERNEL);
4470 	ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4471 					   sizeof(unsigned long), GFP_KERNEL);
4472 	ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4473 				  GFP_KERNEL);
4474 	ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),
4475 					      sizeof(unsigned long),
4476 					      GFP_KERNEL);
4477 	ud->rflow_gp_map_allocated = devm_kcalloc(dev,
4478 						  BITS_TO_LONGS(ud->rflow_cnt),
4479 						  sizeof(unsigned long),
4480 						  GFP_KERNEL);
4481 	ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
4482 					sizeof(unsigned long),
4483 					GFP_KERNEL);
4484 	ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
4485 				  GFP_KERNEL);
4486 
4487 	if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map ||
4488 	    !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans ||
4489 	    !ud->rflows || !ud->rflow_in_use)
4490 		return -ENOMEM;
4491 
4492 	/*
4493 	 * RX flows with the same Ids as RX channels are reserved to be used
4494 	 * as default flows if remote HW can't generate flow_ids. Those
4495 	 * RX flows can be requested only explicitly by id.
4496 	 */
4497 	bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt);
4498 
4499 	/* by default no GP rflows are assigned to Linux */
4500 	bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt);
4501 
4502 	/* Get resource ranges from tisci */
4503 	for (i = 0; i < RM_RANGE_LAST; i++) {
4504 		if (i == RM_RANGE_BCHAN || i == RM_RANGE_TFLOW)
4505 			continue;
4506 
4507 		tisci_rm->rm_ranges[i] =
4508 			devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4509 						    tisci_rm->tisci_dev_id,
4510 						    (char *)range_names[i]);
4511 	}
4512 
4513 	/* tchan ranges */
4514 	rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4515 	if (IS_ERR(rm_res)) {
4516 		bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4517 	} else {
4518 		bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4519 		for (i = 0; i < rm_res->sets; i++)
4520 			udma_mark_resource_ranges(ud, ud->tchan_map,
4521 						  &rm_res->desc[i], "tchan");
4522 	}
4523 	irq_res.sets = rm_res->sets;
4524 
4525 	/* rchan and matching default flow ranges */
4526 	rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4527 	if (IS_ERR(rm_res)) {
4528 		bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4529 	} else {
4530 		bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4531 		for (i = 0; i < rm_res->sets; i++)
4532 			udma_mark_resource_ranges(ud, ud->rchan_map,
4533 						  &rm_res->desc[i], "rchan");
4534 	}
4535 
4536 	irq_res.sets += rm_res->sets;
4537 	irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
4538 	rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4539 	for (i = 0; i < rm_res->sets; i++) {
4540 		irq_res.desc[i].start = rm_res->desc[i].start;
4541 		irq_res.desc[i].num = rm_res->desc[i].num;
4542 		irq_res.desc[i].start_sec = rm_res->desc[i].start_sec;
4543 		irq_res.desc[i].num_sec = rm_res->desc[i].num_sec;
4544 	}
4545 	rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4546 	for (j = 0; j < rm_res->sets; j++, i++) {
4547 		if (rm_res->desc[j].num) {
4548 			irq_res.desc[i].start = rm_res->desc[j].start +
4549 					ud->soc_data->oes.udma_rchan;
4550 			irq_res.desc[i].num = rm_res->desc[j].num;
4551 		}
4552 		if (rm_res->desc[j].num_sec) {
4553 			irq_res.desc[i].start_sec = rm_res->desc[j].start_sec +
4554 					ud->soc_data->oes.udma_rchan;
4555 			irq_res.desc[i].num_sec = rm_res->desc[j].num_sec;
4556 		}
4557 	}
4558 	ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4559 	kfree(irq_res.desc);
4560 	if (ret) {
4561 		dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
4562 		return ret;
4563 	}
4564 
4565 	/* GP rflow ranges */
4566 	rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
4567 	if (IS_ERR(rm_res)) {
4568 		/* all gp flows are assigned exclusively to Linux */
4569 		bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt,
4570 			     ud->rflow_cnt - ud->rchan_cnt);
4571 	} else {
4572 		for (i = 0; i < rm_res->sets; i++)
4573 			udma_mark_resource_ranges(ud, ud->rflow_gp_map,
4574 						  &rm_res->desc[i], "gp-rflow");
4575 	}
4576 
4577 	return 0;
4578 }
4579 
4580 static int bcdma_setup_resources(struct udma_dev *ud)
4581 {
4582 	int ret, i, j;
4583 	struct device *dev = ud->dev;
4584 	struct ti_sci_resource *rm_res, irq_res;
4585 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4586 	const struct udma_oes_offsets *oes = &ud->soc_data->oes;
4587 	u32 cap;
4588 
4589 	/* Set up the throughput level start indexes */
4590 	cap = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4591 	if (BCDMA_CAP3_UBCHAN_CNT(cap)) {
4592 		ud->bchan_tpl.levels = 3;
4593 		ud->bchan_tpl.start_idx[1] = BCDMA_CAP3_UBCHAN_CNT(cap);
4594 		ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap);
4595 	} else if (BCDMA_CAP3_HBCHAN_CNT(cap)) {
4596 		ud->bchan_tpl.levels = 2;
4597 		ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap);
4598 	} else {
4599 		ud->bchan_tpl.levels = 1;
4600 	}
4601 
4602 	cap = udma_read(ud->mmrs[MMR_GCFG], 0x30);
4603 	if (BCDMA_CAP4_URCHAN_CNT(cap)) {
4604 		ud->rchan_tpl.levels = 3;
4605 		ud->rchan_tpl.start_idx[1] = BCDMA_CAP4_URCHAN_CNT(cap);
4606 		ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap);
4607 	} else if (BCDMA_CAP4_HRCHAN_CNT(cap)) {
4608 		ud->rchan_tpl.levels = 2;
4609 		ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap);
4610 	} else {
4611 		ud->rchan_tpl.levels = 1;
4612 	}
4613 
4614 	if (BCDMA_CAP4_UTCHAN_CNT(cap)) {
4615 		ud->tchan_tpl.levels = 3;
4616 		ud->tchan_tpl.start_idx[1] = BCDMA_CAP4_UTCHAN_CNT(cap);
4617 		ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap);
4618 	} else if (BCDMA_CAP4_HTCHAN_CNT(cap)) {
4619 		ud->tchan_tpl.levels = 2;
4620 		ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap);
4621 	} else {
4622 		ud->tchan_tpl.levels = 1;
4623 	}
4624 
4625 	ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt),
4626 					   sizeof(unsigned long), GFP_KERNEL);
4627 	ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans),
4628 				  GFP_KERNEL);
4629 	ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4630 					   sizeof(unsigned long), GFP_KERNEL);
4631 	ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4632 				  GFP_KERNEL);
4633 	ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4634 					   sizeof(unsigned long), GFP_KERNEL);
4635 	ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4636 				  GFP_KERNEL);
4637 	/* BCDMA do not really have flows, but the driver expect it */
4638 	ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rchan_cnt),
4639 					sizeof(unsigned long),
4640 					GFP_KERNEL);
4641 	ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows),
4642 				  GFP_KERNEL);
4643 
4644 	if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map ||
4645 	    !ud->rflow_in_use || !ud->bchans || !ud->tchans || !ud->rchans ||
4646 	    !ud->rflows)
4647 		return -ENOMEM;
4648 
4649 	/* Get resource ranges from tisci */
4650 	for (i = 0; i < RM_RANGE_LAST; i++) {
4651 		if (i == RM_RANGE_RFLOW || i == RM_RANGE_TFLOW)
4652 			continue;
4653 		if (i == RM_RANGE_BCHAN && ud->bchan_cnt == 0)
4654 			continue;
4655 		if (i == RM_RANGE_TCHAN && ud->tchan_cnt == 0)
4656 			continue;
4657 		if (i == RM_RANGE_RCHAN && ud->rchan_cnt == 0)
4658 			continue;
4659 
4660 		tisci_rm->rm_ranges[i] =
4661 			devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4662 						    tisci_rm->tisci_dev_id,
4663 						    (char *)range_names[i]);
4664 	}
4665 
4666 	irq_res.sets = 0;
4667 
4668 	/* bchan ranges */
4669 	if (ud->bchan_cnt) {
4670 		rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
4671 		if (IS_ERR(rm_res)) {
4672 			bitmap_zero(ud->bchan_map, ud->bchan_cnt);
4673 		} else {
4674 			bitmap_fill(ud->bchan_map, ud->bchan_cnt);
4675 			for (i = 0; i < rm_res->sets; i++)
4676 				udma_mark_resource_ranges(ud, ud->bchan_map,
4677 							  &rm_res->desc[i],
4678 							  "bchan");
4679 		}
4680 		irq_res.sets += rm_res->sets;
4681 	}
4682 
4683 	/* tchan ranges */
4684 	if (ud->tchan_cnt) {
4685 		rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4686 		if (IS_ERR(rm_res)) {
4687 			bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4688 		} else {
4689 			bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4690 			for (i = 0; i < rm_res->sets; i++)
4691 				udma_mark_resource_ranges(ud, ud->tchan_map,
4692 							  &rm_res->desc[i],
4693 							  "tchan");
4694 		}
4695 		irq_res.sets += rm_res->sets * 2;
4696 	}
4697 
4698 	/* rchan ranges */
4699 	if (ud->rchan_cnt) {
4700 		rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4701 		if (IS_ERR(rm_res)) {
4702 			bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4703 		} else {
4704 			bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4705 			for (i = 0; i < rm_res->sets; i++)
4706 				udma_mark_resource_ranges(ud, ud->rchan_map,
4707 							  &rm_res->desc[i],
4708 							  "rchan");
4709 		}
4710 		irq_res.sets += rm_res->sets * 2;
4711 	}
4712 
4713 	irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
4714 	if (ud->bchan_cnt) {
4715 		rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
4716 		for (i = 0; i < rm_res->sets; i++) {
4717 			irq_res.desc[i].start = rm_res->desc[i].start +
4718 						oes->bcdma_bchan_ring;
4719 			irq_res.desc[i].num = rm_res->desc[i].num;
4720 		}
4721 	}
4722 	if (ud->tchan_cnt) {
4723 		rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4724 		for (j = 0; j < rm_res->sets; j++, i += 2) {
4725 			irq_res.desc[i].start = rm_res->desc[j].start +
4726 						oes->bcdma_tchan_data;
4727 			irq_res.desc[i].num = rm_res->desc[j].num;
4728 
4729 			irq_res.desc[i + 1].start = rm_res->desc[j].start +
4730 						oes->bcdma_tchan_ring;
4731 			irq_res.desc[i + 1].num = rm_res->desc[j].num;
4732 		}
4733 	}
4734 	if (ud->rchan_cnt) {
4735 		rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4736 		for (j = 0; j < rm_res->sets; j++, i += 2) {
4737 			irq_res.desc[i].start = rm_res->desc[j].start +
4738 						oes->bcdma_rchan_data;
4739 			irq_res.desc[i].num = rm_res->desc[j].num;
4740 
4741 			irq_res.desc[i + 1].start = rm_res->desc[j].start +
4742 						oes->bcdma_rchan_ring;
4743 			irq_res.desc[i + 1].num = rm_res->desc[j].num;
4744 		}
4745 	}
4746 
4747 	ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4748 	kfree(irq_res.desc);
4749 	if (ret) {
4750 		dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
4751 		return ret;
4752 	}
4753 
4754 	return 0;
4755 }
4756 
4757 static int pktdma_setup_resources(struct udma_dev *ud)
4758 {
4759 	int ret, i, j;
4760 	struct device *dev = ud->dev;
4761 	struct ti_sci_resource *rm_res, irq_res;
4762 	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4763 	const struct udma_oes_offsets *oes = &ud->soc_data->oes;
4764 	u32 cap3;
4765 
4766 	/* Set up the throughput level start indexes */
4767 	cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4768 	if (UDMA_CAP3_UCHAN_CNT(cap3)) {
4769 		ud->tchan_tpl.levels = 3;
4770 		ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);
4771 		ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4772 	} else if (UDMA_CAP3_HCHAN_CNT(cap3)) {
4773 		ud->tchan_tpl.levels = 2;
4774 		ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4775 	} else {
4776 		ud->tchan_tpl.levels = 1;
4777 	}
4778 
4779 	ud->rchan_tpl.levels = ud->tchan_tpl.levels;
4780 	ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0];
4781 	ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1];
4782 
4783 	ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4784 					   sizeof(unsigned long), GFP_KERNEL);
4785 	ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4786 				  GFP_KERNEL);
4787 	ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4788 					   sizeof(unsigned long), GFP_KERNEL);
4789 	ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4790 				  GFP_KERNEL);
4791 	ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
4792 					sizeof(unsigned long),
4793 					GFP_KERNEL);
4794 	ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
4795 				  GFP_KERNEL);
4796 	ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt),
4797 					   sizeof(unsigned long), GFP_KERNEL);
4798 
4799 	if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans ||
4800 	    !ud->rchans || !ud->rflows || !ud->rflow_in_use)
4801 		return -ENOMEM;
4802 
4803 	/* Get resource ranges from tisci */
4804 	for (i = 0; i < RM_RANGE_LAST; i++) {
4805 		if (i == RM_RANGE_BCHAN)
4806 			continue;
4807 
4808 		tisci_rm->rm_ranges[i] =
4809 			devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4810 						    tisci_rm->tisci_dev_id,
4811 						    (char *)range_names[i]);
4812 	}
4813 
4814 	/* tchan ranges */
4815 	rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4816 	if (IS_ERR(rm_res)) {
4817 		bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4818 	} else {
4819 		bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4820 		for (i = 0; i < rm_res->sets; i++)
4821 			udma_mark_resource_ranges(ud, ud->tchan_map,
4822 						  &rm_res->desc[i], "tchan");
4823 	}
4824 
4825 	/* rchan ranges */
4826 	rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4827 	if (IS_ERR(rm_res)) {
4828 		bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4829 	} else {
4830 		bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4831 		for (i = 0; i < rm_res->sets; i++)
4832 			udma_mark_resource_ranges(ud, ud->rchan_map,
4833 						  &rm_res->desc[i], "rchan");
4834 	}
4835 
4836 	/* rflow ranges */
4837 	rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
4838 	if (IS_ERR(rm_res)) {
4839 		/* all rflows are assigned exclusively to Linux */
4840 		bitmap_zero(ud->rflow_in_use, ud->rflow_cnt);
4841 	} else {
4842 		bitmap_fill(ud->rflow_in_use, ud->rflow_cnt);
4843 		for (i = 0; i < rm_res->sets; i++)
4844 			udma_mark_resource_ranges(ud, ud->rflow_in_use,
4845 						  &rm_res->desc[i], "rflow");
4846 	}
4847 	irq_res.sets = rm_res->sets;
4848 
4849 	/* tflow ranges */
4850 	rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
4851 	if (IS_ERR(rm_res)) {
4852 		/* all tflows are assigned exclusively to Linux */
4853 		bitmap_zero(ud->tflow_map, ud->tflow_cnt);
4854 	} else {
4855 		bitmap_fill(ud->tflow_map, ud->tflow_cnt);
4856 		for (i = 0; i < rm_res->sets; i++)
4857 			udma_mark_resource_ranges(ud, ud->tflow_map,
4858 						  &rm_res->desc[i], "tflow");
4859 	}
4860 	irq_res.sets += rm_res->sets;
4861 
4862 	irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
4863 	rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
4864 	for (i = 0; i < rm_res->sets; i++) {
4865 		irq_res.desc[i].start = rm_res->desc[i].start +
4866 					oes->pktdma_tchan_flow;
4867 		irq_res.desc[i].num = rm_res->desc[i].num;
4868 	}
4869 	rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
4870 	for (j = 0; j < rm_res->sets; j++, i++) {
4871 		irq_res.desc[i].start = rm_res->desc[j].start +
4872 					oes->pktdma_rchan_flow;
4873 		irq_res.desc[i].num = rm_res->desc[j].num;
4874 	}
4875 	ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4876 	kfree(irq_res.desc);
4877 	if (ret) {
4878 		dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
4879 		return ret;
4880 	}
4881 
4882 	return 0;
4883 }
4884 
4885 static int setup_resources(struct udma_dev *ud)
4886 {
4887 	struct device *dev = ud->dev;
4888 	int ch_count, ret;
4889 
4890 	switch (ud->match_data->type) {
4891 	case DMA_TYPE_UDMA:
4892 		ret = udma_setup_resources(ud);
4893 		break;
4894 	case DMA_TYPE_BCDMA:
4895 		ret = bcdma_setup_resources(ud);
4896 		break;
4897 	case DMA_TYPE_PKTDMA:
4898 		ret = pktdma_setup_resources(ud);
4899 		break;
4900 	default:
4901 		return -EINVAL;
4902 	}
4903 
4904 	if (ret)
4905 		return ret;
4906 
4907 	ch_count  = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt;
4908 	if (ud->bchan_cnt)
4909 		ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt);
4910 	ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);
4911 	ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);
4912 	if (!ch_count)
4913 		return -ENODEV;
4914 
4915 	ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),
4916 				    GFP_KERNEL);
4917 	if (!ud->channels)
4918 		return -ENOMEM;
4919 
4920 	switch (ud->match_data->type) {
4921 	case DMA_TYPE_UDMA:
4922 		dev_info(dev,
4923 			 "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n",
4924 			 ch_count,
4925 			 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
4926 						       ud->tchan_cnt),
4927 			 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
4928 						       ud->rchan_cnt),
4929 			 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map,
4930 						       ud->rflow_cnt));
4931 		break;
4932 	case DMA_TYPE_BCDMA:
4933 		dev_info(dev,
4934 			 "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n",
4935 			 ch_count,
4936 			 ud->bchan_cnt - bitmap_weight(ud->bchan_map,
4937 						       ud->bchan_cnt),
4938 			 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
4939 						       ud->tchan_cnt),
4940 			 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
4941 						       ud->rchan_cnt));
4942 		break;
4943 	case DMA_TYPE_PKTDMA:
4944 		dev_info(dev,
4945 			 "Channels: %d (tchan: %u, rchan: %u)\n",
4946 			 ch_count,
4947 			 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
4948 						       ud->tchan_cnt),
4949 			 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
4950 						       ud->rchan_cnt));
4951 	default:
4952 		break;
4953 	}
4954 
4955 	return ch_count;
4956 }
4957 
4958 static int udma_setup_rx_flush(struct udma_dev *ud)
4959 {
4960 	struct udma_rx_flush *rx_flush = &ud->rx_flush;
4961 	struct cppi5_desc_hdr_t *tr_desc;
4962 	struct cppi5_tr_type1_t *tr_req;
4963 	struct cppi5_host_desc_t *desc;
4964 	struct device *dev = ud->dev;
4965 	struct udma_hwdesc *hwdesc;
4966 	size_t tr_size;
4967 
4968 	/* Allocate 1K buffer for discarded data on RX channel teardown */
4969 	rx_flush->buffer_size = SZ_1K;
4970 	rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size,
4971 					      GFP_KERNEL);
4972 	if (!rx_flush->buffer_vaddr)
4973 		return -ENOMEM;
4974 
4975 	rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr,
4976 						rx_flush->buffer_size,
4977 						DMA_TO_DEVICE);
4978 	if (dma_mapping_error(dev, rx_flush->buffer_paddr))
4979 		return -ENOMEM;
4980 
4981 	/* Set up descriptor to be used for TR mode */
4982 	hwdesc = &rx_flush->hwdescs[0];
4983 	tr_size = sizeof(struct cppi5_tr_type1_t);
4984 	hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1);
4985 	hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
4986 					ud->desc_align);
4987 
4988 	hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
4989 						GFP_KERNEL);
4990 	if (!hwdesc->cppi5_desc_vaddr)
4991 		return -ENOMEM;
4992 
4993 	hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
4994 						  hwdesc->cppi5_desc_size,
4995 						  DMA_TO_DEVICE);
4996 	if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
4997 		return -ENOMEM;
4998 
4999 	/* Start of the TR req records */
5000 	hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
5001 	/* Start address of the TR response array */
5002 	hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size;
5003 
5004 	tr_desc = hwdesc->cppi5_desc_vaddr;
5005 	cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0);
5006 	cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
5007 	cppi5_desc_set_retpolicy(tr_desc, 0, 0);
5008 
5009 	tr_req = hwdesc->tr_req_base;
5010 	cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false,
5011 		      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
5012 	cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT);
5013 
5014 	tr_req->addr = rx_flush->buffer_paddr;
5015 	tr_req->icnt0 = rx_flush->buffer_size;
5016 	tr_req->icnt1 = 1;
5017 
5018 	dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
5019 				   hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
5020 
5021 	/* Set up descriptor to be used for packet mode */
5022 	hwdesc = &rx_flush->hwdescs[1];
5023 	hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
5024 					CPPI5_INFO0_HDESC_EPIB_SIZE +
5025 					CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE,
5026 					ud->desc_align);
5027 
5028 	hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
5029 						GFP_KERNEL);
5030 	if (!hwdesc->cppi5_desc_vaddr)
5031 		return -ENOMEM;
5032 
5033 	hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
5034 						  hwdesc->cppi5_desc_size,
5035 						  DMA_TO_DEVICE);
5036 	if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
5037 		return -ENOMEM;
5038 
5039 	desc = hwdesc->cppi5_desc_vaddr;
5040 	cppi5_hdesc_init(desc, 0, 0);
5041 	cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
5042 	cppi5_desc_set_retpolicy(&desc->hdr, 0, 0);
5043 
5044 	cppi5_hdesc_attach_buf(desc,
5045 			       rx_flush->buffer_paddr, rx_flush->buffer_size,
5046 			       rx_flush->buffer_paddr, rx_flush->buffer_size);
5047 
5048 	dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
5049 				   hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
5050 	return 0;
5051 }
5052 
5053 #ifdef CONFIG_DEBUG_FS
5054 static void udma_dbg_summary_show_chan(struct seq_file *s,
5055 				       struct dma_chan *chan)
5056 {
5057 	struct udma_chan *uc = to_udma_chan(chan);
5058 	struct udma_chan_config *ucc = &uc->config;
5059 
5060 	seq_printf(s, " %-13s| %s", dma_chan_name(chan),
5061 		   chan->dbg_client_name ?: "in-use");
5062 	if (ucc->tr_trigger_type)
5063 		seq_puts(s, " (triggered, ");
5064 	else
5065 		seq_printf(s, " (%s, ",
5066 			   dmaengine_get_direction_text(uc->config.dir));
5067 
5068 	switch (uc->config.dir) {
5069 	case DMA_MEM_TO_MEM:
5070 		if (uc->ud->match_data->type == DMA_TYPE_BCDMA) {
5071 			seq_printf(s, "bchan%d)\n", uc->bchan->id);
5072 			return;
5073 		}
5074 
5075 		seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id,
5076 			   ucc->src_thread, ucc->dst_thread);
5077 		break;
5078 	case DMA_DEV_TO_MEM:
5079 		seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id,
5080 			   ucc->src_thread, ucc->dst_thread);
5081 		if (uc->ud->match_data->type == DMA_TYPE_PKTDMA)
5082 			seq_printf(s, "rflow%d, ", uc->rflow->id);
5083 		break;
5084 	case DMA_MEM_TO_DEV:
5085 		seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id,
5086 			   ucc->src_thread, ucc->dst_thread);
5087 		if (uc->ud->match_data->type == DMA_TYPE_PKTDMA)
5088 			seq_printf(s, "tflow%d, ", uc->tchan->tflow_id);
5089 		break;
5090 	default:
5091 		seq_printf(s, ")\n");
5092 		return;
5093 	}
5094 
5095 	if (ucc->ep_type == PSIL_EP_NATIVE) {
5096 		seq_printf(s, "PSI-L Native");
5097 		if (ucc->metadata_size) {
5098 			seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : "");
5099 			if (ucc->psd_size)
5100 				seq_printf(s, " PSDsize:%u", ucc->psd_size);
5101 			seq_printf(s, " ]");
5102 		}
5103 	} else {
5104 		seq_printf(s, "PDMA");
5105 		if (ucc->enable_acc32 || ucc->enable_burst)
5106 			seq_printf(s, "[%s%s ]",
5107 				   ucc->enable_acc32 ? " ACC32" : "",
5108 				   ucc->enable_burst ? " BURST" : "");
5109 	}
5110 
5111 	seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode");
5112 }
5113 
5114 static void udma_dbg_summary_show(struct seq_file *s,
5115 				  struct dma_device *dma_dev)
5116 {
5117 	struct dma_chan *chan;
5118 
5119 	list_for_each_entry(chan, &dma_dev->channels, device_node) {
5120 		if (chan->client_count)
5121 			udma_dbg_summary_show_chan(s, chan);
5122 	}
5123 }
5124 #endif /* CONFIG_DEBUG_FS */
5125 
5126 static enum dmaengine_alignment udma_get_copy_align(struct udma_dev *ud)
5127 {
5128 	const struct udma_match_data *match_data = ud->match_data;
5129 	u8 tpl;
5130 
5131 	if (!match_data->enable_memcpy_support)
5132 		return DMAENGINE_ALIGN_8_BYTES;
5133 
5134 	/* Get the highest TPL level the device supports for memcpy */
5135 	if (ud->bchan_cnt)
5136 		tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, 0);
5137 	else if (ud->tchan_cnt)
5138 		tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, 0);
5139 	else
5140 		return DMAENGINE_ALIGN_8_BYTES;
5141 
5142 	switch (match_data->burst_size[tpl]) {
5143 	case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES:
5144 		return DMAENGINE_ALIGN_256_BYTES;
5145 	case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES:
5146 		return DMAENGINE_ALIGN_128_BYTES;
5147 	case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES:
5148 	fallthrough;
5149 	default:
5150 		return DMAENGINE_ALIGN_64_BYTES;
5151 	}
5152 }
5153 
5154 #define TI_UDMAC_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
5155 				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
5156 				 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
5157 				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
5158 				 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
5159 
5160 static int udma_probe(struct platform_device *pdev)
5161 {
5162 	struct device_node *navss_node = pdev->dev.parent->of_node;
5163 	const struct soc_device_attribute *soc;
5164 	struct device *dev = &pdev->dev;
5165 	struct udma_dev *ud;
5166 	const struct of_device_id *match;
5167 	int i, ret;
5168 	int ch_count;
5169 
5170 	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48));
5171 	if (ret)
5172 		dev_err(dev, "failed to set dma mask stuff\n");
5173 
5174 	ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL);
5175 	if (!ud)
5176 		return -ENOMEM;
5177 
5178 	match = of_match_node(udma_of_match, dev->of_node);
5179 	if (!match)
5180 		match = of_match_node(bcdma_of_match, dev->of_node);
5181 	if (!match) {
5182 		match = of_match_node(pktdma_of_match, dev->of_node);
5183 		if (!match) {
5184 			dev_err(dev, "No compatible match found\n");
5185 			return -ENODEV;
5186 		}
5187 	}
5188 	ud->match_data = match->data;
5189 
5190 	soc = soc_device_match(k3_soc_devices);
5191 	if (!soc) {
5192 		dev_err(dev, "No compatible SoC found\n");
5193 		return -ENODEV;
5194 	}
5195 	ud->soc_data = soc->data;
5196 
5197 	ret = udma_get_mmrs(pdev, ud);
5198 	if (ret)
5199 		return ret;
5200 
5201 	ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci");
5202 	if (IS_ERR(ud->tisci_rm.tisci))
5203 		return PTR_ERR(ud->tisci_rm.tisci);
5204 
5205 	ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id",
5206 				   &ud->tisci_rm.tisci_dev_id);
5207 	if (ret) {
5208 		dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
5209 		return ret;
5210 	}
5211 	pdev->id = ud->tisci_rm.tisci_dev_id;
5212 
5213 	ret = of_property_read_u32(navss_node, "ti,sci-dev-id",
5214 				   &ud->tisci_rm.tisci_navss_dev_id);
5215 	if (ret) {
5216 		dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret);
5217 		return ret;
5218 	}
5219 
5220 	if (ud->match_data->type == DMA_TYPE_UDMA) {
5221 		ret = of_property_read_u32(dev->of_node, "ti,udma-atype",
5222 					   &ud->atype);
5223 		if (!ret && ud->atype > 2) {
5224 			dev_err(dev, "Invalid atype: %u\n", ud->atype);
5225 			return -EINVAL;
5226 		}
5227 	} else {
5228 		ret = of_property_read_u32(dev->of_node, "ti,asel",
5229 					   &ud->asel);
5230 		if (!ret && ud->asel > 15) {
5231 			dev_err(dev, "Invalid asel: %u\n", ud->asel);
5232 			return -EINVAL;
5233 		}
5234 	}
5235 
5236 	ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops;
5237 	ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops;
5238 
5239 	if (ud->match_data->type == DMA_TYPE_UDMA) {
5240 		ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc");
5241 	} else {
5242 		struct k3_ringacc_init_data ring_init_data;
5243 
5244 		ring_init_data.tisci = ud->tisci_rm.tisci;
5245 		ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id;
5246 		if (ud->match_data->type == DMA_TYPE_BCDMA) {
5247 			ring_init_data.num_rings = ud->bchan_cnt +
5248 						   ud->tchan_cnt +
5249 						   ud->rchan_cnt;
5250 		} else {
5251 			ring_init_data.num_rings = ud->rflow_cnt +
5252 						   ud->tflow_cnt;
5253 		}
5254 
5255 		ud->ringacc = k3_ringacc_dmarings_init(pdev, &ring_init_data);
5256 	}
5257 
5258 	if (IS_ERR(ud->ringacc))
5259 		return PTR_ERR(ud->ringacc);
5260 
5261 	dev->msi_domain = of_msi_get_domain(dev, dev->of_node,
5262 					    DOMAIN_BUS_TI_SCI_INTA_MSI);
5263 	if (!dev->msi_domain) {
5264 		dev_err(dev, "Failed to get MSI domain\n");
5265 		return -EPROBE_DEFER;
5266 	}
5267 
5268 	dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask);
5269 	/* cyclic operation is not supported via PKTDMA */
5270 	if (ud->match_data->type != DMA_TYPE_PKTDMA) {
5271 		dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask);
5272 		ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic;
5273 	}
5274 
5275 	ud->ddev.device_config = udma_slave_config;
5276 	ud->ddev.device_prep_slave_sg = udma_prep_slave_sg;
5277 	ud->ddev.device_issue_pending = udma_issue_pending;
5278 	ud->ddev.device_tx_status = udma_tx_status;
5279 	ud->ddev.device_pause = udma_pause;
5280 	ud->ddev.device_resume = udma_resume;
5281 	ud->ddev.device_terminate_all = udma_terminate_all;
5282 	ud->ddev.device_synchronize = udma_synchronize;
5283 #ifdef CONFIG_DEBUG_FS
5284 	ud->ddev.dbg_summary_show = udma_dbg_summary_show;
5285 #endif
5286 
5287 	switch (ud->match_data->type) {
5288 	case DMA_TYPE_UDMA:
5289 		ud->ddev.device_alloc_chan_resources =
5290 					udma_alloc_chan_resources;
5291 		break;
5292 	case DMA_TYPE_BCDMA:
5293 		ud->ddev.device_alloc_chan_resources =
5294 					bcdma_alloc_chan_resources;
5295 		ud->ddev.device_router_config = bcdma_router_config;
5296 		break;
5297 	case DMA_TYPE_PKTDMA:
5298 		ud->ddev.device_alloc_chan_resources =
5299 					pktdma_alloc_chan_resources;
5300 		break;
5301 	default:
5302 		return -EINVAL;
5303 	}
5304 	ud->ddev.device_free_chan_resources = udma_free_chan_resources;
5305 
5306 	ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS;
5307 	ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS;
5308 	ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
5309 	ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
5310 	ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT |
5311 				       DESC_METADATA_ENGINE;
5312 	if (ud->match_data->enable_memcpy_support &&
5313 	    !(ud->match_data->type == DMA_TYPE_BCDMA && ud->bchan_cnt == 0)) {
5314 		dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask);
5315 		ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy;
5316 		ud->ddev.directions |= BIT(DMA_MEM_TO_MEM);
5317 	}
5318 
5319 	ud->ddev.dev = dev;
5320 	ud->dev = dev;
5321 	ud->psil_base = ud->match_data->psil_base;
5322 
5323 	INIT_LIST_HEAD(&ud->ddev.channels);
5324 	INIT_LIST_HEAD(&ud->desc_to_purge);
5325 
5326 	ch_count = setup_resources(ud);
5327 	if (ch_count <= 0)
5328 		return ch_count;
5329 
5330 	spin_lock_init(&ud->lock);
5331 	INIT_WORK(&ud->purge_work, udma_purge_desc_work);
5332 
5333 	ud->desc_align = 64;
5334 	if (ud->desc_align < dma_get_cache_alignment())
5335 		ud->desc_align = dma_get_cache_alignment();
5336 
5337 	ret = udma_setup_rx_flush(ud);
5338 	if (ret)
5339 		return ret;
5340 
5341 	for (i = 0; i < ud->bchan_cnt; i++) {
5342 		struct udma_bchan *bchan = &ud->bchans[i];
5343 
5344 		bchan->id = i;
5345 		bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000;
5346 	}
5347 
5348 	for (i = 0; i < ud->tchan_cnt; i++) {
5349 		struct udma_tchan *tchan = &ud->tchans[i];
5350 
5351 		tchan->id = i;
5352 		tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000;
5353 	}
5354 
5355 	for (i = 0; i < ud->rchan_cnt; i++) {
5356 		struct udma_rchan *rchan = &ud->rchans[i];
5357 
5358 		rchan->id = i;
5359 		rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000;
5360 	}
5361 
5362 	for (i = 0; i < ud->rflow_cnt; i++) {
5363 		struct udma_rflow *rflow = &ud->rflows[i];
5364 
5365 		rflow->id = i;
5366 	}
5367 
5368 	for (i = 0; i < ch_count; i++) {
5369 		struct udma_chan *uc = &ud->channels[i];
5370 
5371 		uc->ud = ud;
5372 		uc->vc.desc_free = udma_desc_free;
5373 		uc->id = i;
5374 		uc->bchan = NULL;
5375 		uc->tchan = NULL;
5376 		uc->rchan = NULL;
5377 		uc->config.remote_thread_id = -1;
5378 		uc->config.mapped_channel_id = -1;
5379 		uc->config.default_flow_id = -1;
5380 		uc->config.dir = DMA_MEM_TO_MEM;
5381 		uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d",
5382 					  dev_name(dev), i);
5383 
5384 		vchan_init(&uc->vc, &ud->ddev);
5385 		/* Use custom vchan completion handling */
5386 		tasklet_setup(&uc->vc.task, udma_vchan_complete);
5387 		init_completion(&uc->teardown_completed);
5388 		INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion);
5389 	}
5390 
5391 	/* Configure the copy_align to the maximum burst size the device supports */
5392 	ud->ddev.copy_align = udma_get_copy_align(ud);
5393 
5394 	ret = dma_async_device_register(&ud->ddev);
5395 	if (ret) {
5396 		dev_err(dev, "failed to register slave DMA engine: %d\n", ret);
5397 		return ret;
5398 	}
5399 
5400 	platform_set_drvdata(pdev, ud);
5401 
5402 	ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud);
5403 	if (ret) {
5404 		dev_err(dev, "failed to register of_dma controller\n");
5405 		dma_async_device_unregister(&ud->ddev);
5406 	}
5407 
5408 	return ret;
5409 }
5410 
5411 static struct platform_driver udma_driver = {
5412 	.driver = {
5413 		.name	= "ti-udma",
5414 		.of_match_table = udma_of_match,
5415 		.suppress_bind_attrs = true,
5416 	},
5417 	.probe		= udma_probe,
5418 };
5419 builtin_platform_driver(udma_driver);
5420 
5421 static struct platform_driver bcdma_driver = {
5422 	.driver = {
5423 		.name	= "ti-bcdma",
5424 		.of_match_table = bcdma_of_match,
5425 		.suppress_bind_attrs = true,
5426 	},
5427 	.probe		= udma_probe,
5428 };
5429 builtin_platform_driver(bcdma_driver);
5430 
5431 static struct platform_driver pktdma_driver = {
5432 	.driver = {
5433 		.name	= "ti-pktdma",
5434 		.of_match_table = pktdma_of_match,
5435 		.suppress_bind_attrs = true,
5436 	},
5437 	.probe		= udma_probe,
5438 };
5439 builtin_platform_driver(pktdma_driver);
5440 
5441 /* Private interfaces to UDMA */
5442 #include "k3-udma-private.c"
5443