xref: /openbmc/linux/drivers/soc/ti/k3-ringacc.c (revision 5497b23e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * TI K3 NAVSS Ring Accelerator subsystem driver
4  *
5  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
6  */
7 
8 #include <linux/dma-mapping.h>
9 #include <linux/io.h>
10 #include <linux/init.h>
11 #include <linux/of.h>
12 #include <linux/of_device.h>
13 #include <linux/platform_device.h>
14 #include <linux/sys_soc.h>
15 #include <linux/dma/ti-cppi5.h>
16 #include <linux/soc/ti/k3-ringacc.h>
17 #include <linux/soc/ti/ti_sci_protocol.h>
18 #include <linux/soc/ti/ti_sci_inta_msi.h>
19 #include <linux/of_irq.h>
20 #include <linux/irqdomain.h>
21 
22 static LIST_HEAD(k3_ringacc_list);
23 static DEFINE_MUTEX(k3_ringacc_list_lock);
24 
25 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK		GENMASK(19, 0)
26 #define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK		GENMASK(15, 0)
27 
28 /**
29  * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region
30  *
31  * @resv_16: Reserved
32  * @db: Ring Doorbell Register
33  * @resv_4: Reserved
34  * @occ: Ring Occupancy Register
35  * @indx: Ring Current Index Register
36  * @hwocc: Ring Hardware Occupancy Register
37  * @hwindx: Ring Hardware Current Index Register
38  */
39 struct k3_ring_rt_regs {
40 	u32	resv_16[4];
41 	u32	db;
42 	u32	resv_4[1];
43 	u32	occ;
44 	u32	indx;
45 	u32	hwocc;
46 	u32	hwindx;
47 };
48 
49 #define K3_RINGACC_RT_REGS_STEP			0x1000
50 #define K3_DMARING_RT_REGS_STEP			0x2000
51 #define K3_DMARING_RT_REGS_REVERSE_OFS		0x1000
52 #define K3_RINGACC_RT_OCC_MASK			GENMASK(20, 0)
53 #define K3_DMARING_RT_OCC_TDOWN_COMPLETE	BIT(31)
54 #define K3_DMARING_RT_DB_ENTRY_MASK		GENMASK(7, 0)
55 #define K3_DMARING_RT_DB_TDOWN_ACK		BIT(31)
56 
57 /**
58  * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region
59  *
60  * @head_data: Ring Head Entry Data Registers
61  * @tail_data: Ring Tail Entry Data Registers
62  * @peek_head_data: Ring Peek Head Entry Data Regs
63  * @peek_tail_data: Ring Peek Tail Entry Data Regs
64  */
65 struct k3_ring_fifo_regs {
66 	u32	head_data[128];
67 	u32	tail_data[128];
68 	u32	peek_head_data[128];
69 	u32	peek_tail_data[128];
70 };
71 
72 /**
73  * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
74  *
75  * @revision: Revision Register
76  * @config: Config Register
77  */
78 struct k3_ringacc_proxy_gcfg_regs {
79 	u32	revision;
80 	u32	config;
81 };
82 
83 #define K3_RINGACC_PROXY_CFG_THREADS_MASK		GENMASK(15, 0)
84 
85 /**
86  * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region
87  *
88  * @control: Proxy Control Register
89  * @status: Proxy Status Register
90  * @resv_512: Reserved
91  * @data: Proxy Data Register
92  */
93 struct k3_ringacc_proxy_target_regs {
94 	u32	control;
95 	u32	status;
96 	u8	resv_512[504];
97 	u32	data[128];
98 };
99 
100 #define K3_RINGACC_PROXY_TARGET_STEP	0x1000
101 #define K3_RINGACC_PROXY_NOT_USED	(-1)
102 
103 enum k3_ringacc_proxy_access_mode {
104 	PROXY_ACCESS_MODE_HEAD = 0,
105 	PROXY_ACCESS_MODE_TAIL = 1,
106 	PROXY_ACCESS_MODE_PEEK_HEAD = 2,
107 	PROXY_ACCESS_MODE_PEEK_TAIL = 3,
108 };
109 
110 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES  (512U)
111 #define K3_RINGACC_FIFO_REGS_STEP	0x1000
112 #define K3_RINGACC_MAX_DB_RING_CNT    (127U)
113 
114 struct k3_ring_ops {
115 	int (*push_tail)(struct k3_ring *ring, void *elm);
116 	int (*push_head)(struct k3_ring *ring, void *elm);
117 	int (*pop_tail)(struct k3_ring *ring, void *elm);
118 	int (*pop_head)(struct k3_ring *ring, void *elm);
119 };
120 
121 /**
122  * struct k3_ring_state - Internal state tracking structure
123  *
124  * @free: Number of free entries
125  * @occ: Occupancy
126  * @windex: Write index
127  * @rindex: Read index
128  */
129 struct k3_ring_state {
130 	u32 free;
131 	u32 occ;
132 	u32 windex;
133 	u32 rindex;
134 	u32 tdown_complete:1;
135 };
136 
137 /**
138  * struct k3_ring - RA Ring descriptor
139  *
140  * @rt: Ring control/status registers
141  * @fifos: Ring queues registers
142  * @proxy: Ring Proxy Datapath registers
143  * @ring_mem_dma: Ring buffer dma address
144  * @ring_mem_virt: Ring buffer virt address
145  * @ops: Ring operations
146  * @size: Ring size in elements
147  * @elm_size: Size of the ring element
148  * @mode: Ring mode
149  * @flags: flags
150  * @state: Ring state
151  * @ring_id: Ring Id
152  * @parent: Pointer on struct @k3_ringacc
153  * @use_count: Use count for shared rings
154  * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY)
155  * @dma_dev: device to be used for DMA API (allocation, mapping)
156  * @asel: Address Space Select value for physical addresses
157  */
158 struct k3_ring {
159 	struct k3_ring_rt_regs __iomem *rt;
160 	struct k3_ring_fifo_regs __iomem *fifos;
161 	struct k3_ringacc_proxy_target_regs  __iomem *proxy;
162 	dma_addr_t	ring_mem_dma;
163 	void		*ring_mem_virt;
164 	struct k3_ring_ops *ops;
165 	u32		size;
166 	enum k3_ring_size elm_size;
167 	enum k3_ring_mode mode;
168 	u32		flags;
169 #define K3_RING_FLAG_BUSY	BIT(1)
170 #define K3_RING_FLAG_SHARED	BIT(2)
171 #define K3_RING_FLAG_REVERSE	BIT(3)
172 	struct k3_ring_state state;
173 	u32		ring_id;
174 	struct k3_ringacc	*parent;
175 	u32		use_count;
176 	int		proxy_id;
177 	struct device	*dma_dev;
178 	u32		asel;
179 #define K3_ADDRESS_ASEL_SHIFT	48
180 };
181 
182 struct k3_ringacc_ops {
183 	int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc);
184 };
185 
186 /**
187  * struct k3_ringacc - Rings accelerator descriptor
188  *
189  * @dev: pointer on RA device
190  * @proxy_gcfg: RA proxy global config registers
191  * @proxy_target_base: RA proxy datapath region
192  * @num_rings: number of ring in RA
193  * @rings_inuse: bitfield for ring usage tracking
194  * @rm_gp_range: general purpose rings range from tisci
195  * @dma_ring_reset_quirk: DMA reset w/a enable
196  * @num_proxies: number of RA proxies
197  * @proxy_inuse: bitfield for proxy usage tracking
198  * @rings: array of rings descriptors (struct @k3_ring)
199  * @list: list of RAs in the system
200  * @req_lock: protect rings allocation
201  * @tisci: pointer ti-sci handle
202  * @tisci_ring_ops: ti-sci rings ops
203  * @tisci_dev_id: ti-sci device id
204  * @ops: SoC specific ringacc operation
205  * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA)
206  */
207 struct k3_ringacc {
208 	struct device *dev;
209 	struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
210 	void __iomem *proxy_target_base;
211 	u32 num_rings; /* number of rings in Ringacc module */
212 	unsigned long *rings_inuse;
213 	struct ti_sci_resource *rm_gp_range;
214 
215 	bool dma_ring_reset_quirk;
216 	u32 num_proxies;
217 	unsigned long *proxy_inuse;
218 
219 	struct k3_ring *rings;
220 	struct list_head list;
221 	struct mutex req_lock; /* protect rings allocation */
222 
223 	const struct ti_sci_handle *tisci;
224 	const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
225 	u32 tisci_dev_id;
226 
227 	const struct k3_ringacc_ops *ops;
228 	bool dma_rings;
229 };
230 
231 /**
232  * struct k3_ringacc - Rings accelerator SoC data
233  *
234  * @dma_ring_reset_quirk:  DMA reset w/a enable
235  */
236 struct k3_ringacc_soc_data {
237 	unsigned dma_ring_reset_quirk:1;
238 };
239 
240 static int k3_ringacc_ring_read_occ(struct k3_ring *ring)
241 {
242 	return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK;
243 }
244 
245 static void k3_ringacc_ring_update_occ(struct k3_ring *ring)
246 {
247 	u32 val;
248 
249 	val = readl(&ring->rt->occ);
250 
251 	ring->state.occ = val & K3_RINGACC_RT_OCC_MASK;
252 	ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE);
253 }
254 
255 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
256 {
257 	return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
258 	       (4 << ring->elm_size);
259 }
260 
261 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
262 {
263 	return (ring->ring_mem_virt + idx * (4 << ring->elm_size));
264 }
265 
266 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
267 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
268 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem);
269 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem);
270 
271 static struct k3_ring_ops k3_ring_mode_ring_ops = {
272 		.push_tail = k3_ringacc_ring_push_mem,
273 		.pop_head = k3_ringacc_ring_pop_mem,
274 };
275 
276 static struct k3_ring_ops k3_dmaring_fwd_ops = {
277 		.push_tail = k3_ringacc_ring_push_mem,
278 		.pop_head = k3_dmaring_fwd_pop,
279 };
280 
281 static struct k3_ring_ops k3_dmaring_reverse_ops = {
282 		/* Reverse side of the DMA ring can only be popped by SW */
283 		.pop_head = k3_dmaring_reverse_pop,
284 };
285 
286 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
287 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
288 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
289 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
290 
291 static struct k3_ring_ops k3_ring_mode_msg_ops = {
292 		.push_tail = k3_ringacc_ring_push_io,
293 		.push_head = k3_ringacc_ring_push_head_io,
294 		.pop_tail = k3_ringacc_ring_pop_tail_io,
295 		.pop_head = k3_ringacc_ring_pop_io,
296 };
297 
298 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
299 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
300 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
301 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
302 
303 static struct k3_ring_ops k3_ring_mode_proxy_ops = {
304 		.push_tail = k3_ringacc_ring_push_tail_proxy,
305 		.push_head = k3_ringacc_ring_push_head_proxy,
306 		.pop_tail = k3_ringacc_ring_pop_tail_proxy,
307 		.pop_head = k3_ringacc_ring_pop_head_proxy,
308 };
309 
310 static void k3_ringacc_ring_dump(struct k3_ring *ring)
311 {
312 	struct device *dev = ring->parent->dev;
313 
314 	dev_dbg(dev, "dump ring: %d\n", ring->ring_id);
315 	dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt,
316 		&ring->ring_mem_dma);
317 	dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n",
318 		ring->elm_size, ring->size, ring->mode, ring->proxy_id);
319 	dev_dbg(dev, "dump flags %08X\n", ring->flags);
320 
321 	dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db));
322 	dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ));
323 	dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx));
324 	dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc));
325 	dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx));
326 
327 	if (ring->ring_mem_virt)
328 		print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE,
329 				     16, 1, ring->ring_mem_virt, 16 * 8, false);
330 }
331 
332 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
333 					int id, u32 flags)
334 {
335 	int proxy_id = K3_RINGACC_PROXY_NOT_USED;
336 
337 	mutex_lock(&ringacc->req_lock);
338 
339 	if (id == K3_RINGACC_RING_ID_ANY) {
340 		/* Request for any general purpose ring */
341 		struct ti_sci_resource_desc *gp_rings =
342 						&ringacc->rm_gp_range->desc[0];
343 		unsigned long size;
344 
345 		size = gp_rings->start + gp_rings->num;
346 		id = find_next_zero_bit(ringacc->rings_inuse, size,
347 					gp_rings->start);
348 		if (id == size)
349 			goto error;
350 	} else if (id < 0) {
351 		goto error;
352 	}
353 
354 	if (test_bit(id, ringacc->rings_inuse) &&
355 	    !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED))
356 		goto error;
357 	else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED)
358 		goto out;
359 
360 	if (flags & K3_RINGACC_RING_USE_PROXY) {
361 		proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
362 					      ringacc->num_proxies, 0);
363 		if (proxy_id == ringacc->num_proxies)
364 			goto error;
365 	}
366 
367 	if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
368 		set_bit(proxy_id, ringacc->proxy_inuse);
369 		ringacc->rings[id].proxy_id = proxy_id;
370 		dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id,
371 			proxy_id);
372 	} else {
373 		dev_dbg(ringacc->dev, "Giving ring#%d\n", id);
374 	}
375 
376 	set_bit(id, ringacc->rings_inuse);
377 out:
378 	ringacc->rings[id].use_count++;
379 	mutex_unlock(&ringacc->req_lock);
380 	return &ringacc->rings[id];
381 
382 error:
383 	mutex_unlock(&ringacc->req_lock);
384 	return NULL;
385 }
386 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
387 
388 static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id,
389 					struct k3_ring **fwd_ring,
390 					struct k3_ring **compl_ring)
391 {
392 	int ret = 0;
393 
394 	/*
395 	 * DMA rings must be requested by ID, completion ring is the reverse
396 	 * side of the forward ring
397 	 */
398 	if (fwd_id < 0)
399 		return -EINVAL;
400 
401 	mutex_lock(&ringacc->req_lock);
402 
403 	if (test_bit(fwd_id, ringacc->rings_inuse)) {
404 		ret = -EBUSY;
405 		goto error;
406 	}
407 
408 	*fwd_ring = &ringacc->rings[fwd_id];
409 	*compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];
410 	set_bit(fwd_id, ringacc->rings_inuse);
411 	ringacc->rings[fwd_id].use_count++;
412 	dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);
413 
414 	mutex_unlock(&ringacc->req_lock);
415 	return 0;
416 
417 error:
418 	mutex_unlock(&ringacc->req_lock);
419 	return ret;
420 }
421 
422 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
423 				  int fwd_id, int compl_id,
424 				  struct k3_ring **fwd_ring,
425 				  struct k3_ring **compl_ring)
426 {
427 	int ret = 0;
428 
429 	if (!fwd_ring || !compl_ring)
430 		return -EINVAL;
431 
432 	if (ringacc->dma_rings)
433 		return k3_dmaring_request_dual_ring(ringacc, fwd_id,
434 						    fwd_ring, compl_ring);
435 
436 	*fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0);
437 	if (!(*fwd_ring))
438 		return -ENODEV;
439 
440 	*compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0);
441 	if (!(*compl_ring)) {
442 		k3_ringacc_ring_free(*fwd_ring);
443 		ret = -ENODEV;
444 	}
445 
446 	return ret;
447 }
448 EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair);
449 
450 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring)
451 {
452 	struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
453 	struct k3_ringacc *ringacc = ring->parent;
454 	int ret;
455 
456 	ring_cfg.nav_id = ringacc->tisci_dev_id;
457 	ring_cfg.index = ring->ring_id;
458 	ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID;
459 	ring_cfg.count = ring->size;
460 
461 	ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
462 	if (ret)
463 		dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
464 			ret, ring->ring_id);
465 }
466 
467 void k3_ringacc_ring_reset(struct k3_ring *ring)
468 {
469 	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
470 		return;
471 
472 	memset(&ring->state, 0, sizeof(ring->state));
473 
474 	k3_ringacc_ring_reset_sci(ring);
475 }
476 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset);
477 
478 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring,
479 					       enum k3_ring_mode mode)
480 {
481 	struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
482 	struct k3_ringacc *ringacc = ring->parent;
483 	int ret;
484 
485 	ring_cfg.nav_id = ringacc->tisci_dev_id;
486 	ring_cfg.index = ring->ring_id;
487 	ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID;
488 	ring_cfg.mode = mode;
489 
490 	ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
491 	if (ret)
492 		dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
493 			ret, ring->ring_id);
494 }
495 
496 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
497 {
498 	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
499 		return;
500 
501 	if (!ring->parent->dma_ring_reset_quirk)
502 		goto reset;
503 
504 	if (!occ)
505 		occ = k3_ringacc_ring_read_occ(ring);
506 
507 	if (occ) {
508 		u32 db_ring_cnt, db_ring_cnt_cur;
509 
510 		dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__,
511 			ring->ring_id, occ);
512 		/* TI-SCI ring reset */
513 		k3_ringacc_ring_reset_sci(ring);
514 
515 		/*
516 		 * Setup the ring in ring/doorbell mode (if not already in this
517 		 * mode)
518 		 */
519 		if (ring->mode != K3_RINGACC_RING_MODE_RING)
520 			k3_ringacc_ring_reconfig_qmode_sci(
521 					ring, K3_RINGACC_RING_MODE_RING);
522 		/*
523 		 * Ring the doorbell 2**22 – ringOcc times.
524 		 * This will wrap the internal UDMAP ring state occupancy
525 		 * counter (which is 21-bits wide) to 0.
526 		 */
527 		db_ring_cnt = (1U << 22) - occ;
528 
529 		while (db_ring_cnt != 0) {
530 			/*
531 			 * Ring the doorbell with the maximum count each
532 			 * iteration if possible to minimize the total
533 			 * of writes
534 			 */
535 			if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT)
536 				db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT;
537 			else
538 				db_ring_cnt_cur = db_ring_cnt;
539 
540 			writel(db_ring_cnt_cur, &ring->rt->db);
541 			db_ring_cnt -= db_ring_cnt_cur;
542 		}
543 
544 		/* Restore the original ring mode (if not ring mode) */
545 		if (ring->mode != K3_RINGACC_RING_MODE_RING)
546 			k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
547 	}
548 
549 reset:
550 	/* Reset the ring */
551 	k3_ringacc_ring_reset(ring);
552 }
553 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma);
554 
555 static void k3_ringacc_ring_free_sci(struct k3_ring *ring)
556 {
557 	struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
558 	struct k3_ringacc *ringacc = ring->parent;
559 	int ret;
560 
561 	ring_cfg.nav_id = ringacc->tisci_dev_id;
562 	ring_cfg.index = ring->ring_id;
563 	ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER;
564 
565 	ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
566 	if (ret)
567 		dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
568 			ret, ring->ring_id);
569 }
570 
571 int k3_ringacc_ring_free(struct k3_ring *ring)
572 {
573 	struct k3_ringacc *ringacc;
574 
575 	if (!ring)
576 		return -EINVAL;
577 
578 	ringacc = ring->parent;
579 
580 	/*
581 	 * DMA rings: rings shared memory and configuration, only forward ring
582 	 * is configured and reverse ring considered as slave.
583 	 */
584 	if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
585 		return 0;
586 
587 	dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
588 
589 	if (!test_bit(ring->ring_id, ringacc->rings_inuse))
590 		return -EINVAL;
591 
592 	mutex_lock(&ringacc->req_lock);
593 
594 	if (--ring->use_count)
595 		goto out;
596 
597 	if (!(ring->flags & K3_RING_FLAG_BUSY))
598 		goto no_init;
599 
600 	k3_ringacc_ring_free_sci(ring);
601 
602 	dma_free_coherent(ring->dma_dev,
603 			  ring->size * (4 << ring->elm_size),
604 			  ring->ring_mem_virt, ring->ring_mem_dma);
605 	ring->flags = 0;
606 	ring->ops = NULL;
607 	ring->dma_dev = NULL;
608 	ring->asel = 0;
609 
610 	if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
611 		clear_bit(ring->proxy_id, ringacc->proxy_inuse);
612 		ring->proxy = NULL;
613 		ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
614 	}
615 
616 no_init:
617 	clear_bit(ring->ring_id, ringacc->rings_inuse);
618 
619 out:
620 	mutex_unlock(&ringacc->req_lock);
621 	return 0;
622 }
623 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free);
624 
625 u32 k3_ringacc_get_ring_id(struct k3_ring *ring)
626 {
627 	if (!ring)
628 		return -EINVAL;
629 
630 	return ring->ring_id;
631 }
632 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id);
633 
634 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring)
635 {
636 	if (!ring)
637 		return -EINVAL;
638 
639 	return ring->parent->tisci_dev_id;
640 }
641 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id);
642 
643 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring)
644 {
645 	int irq_num;
646 
647 	if (!ring)
648 		return -EINVAL;
649 
650 	irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id);
651 	if (irq_num <= 0)
652 		irq_num = -EINVAL;
653 	return irq_num;
654 }
655 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num);
656 
657 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
658 {
659 	struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
660 	struct k3_ringacc *ringacc = ring->parent;
661 	int ret;
662 
663 	if (!ringacc->tisci)
664 		return -EINVAL;
665 
666 	ring_cfg.nav_id = ringacc->tisci_dev_id;
667 	ring_cfg.index = ring->ring_id;
668 	ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER;
669 	ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma);
670 	ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma);
671 	ring_cfg.count = ring->size;
672 	ring_cfg.mode = ring->mode;
673 	ring_cfg.size = ring->elm_size;
674 	ring_cfg.asel = ring->asel;
675 
676 	ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
677 	if (ret)
678 		dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
679 			ret, ring->ring_id);
680 
681 	return ret;
682 }
683 
684 static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
685 {
686 	struct k3_ringacc *ringacc;
687 	struct k3_ring *reverse_ring;
688 	int ret = 0;
689 
690 	if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 ||
691 	    cfg->mode != K3_RINGACC_RING_MODE_RING ||
692 	    cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK)
693 		return -EINVAL;
694 
695 	ringacc = ring->parent;
696 
697 	/*
698 	 * DMA rings: rings shared memory and configuration, only forward ring
699 	 * is configured and reverse ring considered as slave.
700 	 */
701 	if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
702 		return 0;
703 
704 	if (!test_bit(ring->ring_id, ringacc->rings_inuse))
705 		return -EINVAL;
706 
707 	ring->size = cfg->size;
708 	ring->elm_size = cfg->elm_size;
709 	ring->mode = cfg->mode;
710 	ring->asel = cfg->asel;
711 	ring->dma_dev = cfg->dma_dev;
712 	if (!ring->dma_dev) {
713 		dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n",
714 			 ring->ring_id);
715 		ring->dma_dev = ringacc->dev;
716 	}
717 
718 	memset(&ring->state, 0, sizeof(ring->state));
719 
720 	ring->ops = &k3_dmaring_fwd_ops;
721 
722 	ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
723 						 ring->size * (4 << ring->elm_size),
724 						 &ring->ring_mem_dma, GFP_KERNEL);
725 	if (!ring->ring_mem_virt) {
726 		dev_err(ringacc->dev, "Failed to alloc ring mem\n");
727 		ret = -ENOMEM;
728 		goto err_free_ops;
729 	}
730 
731 	ret = k3_ringacc_ring_cfg_sci(ring);
732 	if (ret)
733 		goto err_free_mem;
734 
735 	ring->flags |= K3_RING_FLAG_BUSY;
736 
737 	k3_ringacc_ring_dump(ring);
738 
739 	/* DMA rings: configure reverse ring */
740 	reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
741 	reverse_ring->size = cfg->size;
742 	reverse_ring->elm_size = cfg->elm_size;
743 	reverse_ring->mode = cfg->mode;
744 	reverse_ring->asel = cfg->asel;
745 	memset(&reverse_ring->state, 0, sizeof(reverse_ring->state));
746 	reverse_ring->ops = &k3_dmaring_reverse_ops;
747 
748 	reverse_ring->ring_mem_virt = ring->ring_mem_virt;
749 	reverse_ring->ring_mem_dma = ring->ring_mem_dma;
750 	reverse_ring->flags |= K3_RING_FLAG_BUSY;
751 	k3_ringacc_ring_dump(reverse_ring);
752 
753 	return 0;
754 
755 err_free_mem:
756 	dma_free_coherent(ring->dma_dev,
757 			  ring->size * (4 << ring->elm_size),
758 			  ring->ring_mem_virt,
759 			  ring->ring_mem_dma);
760 err_free_ops:
761 	ring->ops = NULL;
762 	ring->proxy = NULL;
763 	ring->dma_dev = NULL;
764 	ring->asel = 0;
765 	return ret;
766 }
767 
768 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
769 {
770 	struct k3_ringacc *ringacc;
771 	int ret = 0;
772 
773 	if (!ring || !cfg)
774 		return -EINVAL;
775 
776 	ringacc = ring->parent;
777 
778 	if (ringacc->dma_rings)
779 		return k3_dmaring_cfg(ring, cfg);
780 
781 	if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
782 	    cfg->mode >= K3_RINGACC_RING_MODE_INVALID ||
783 	    cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
784 	    !test_bit(ring->ring_id, ringacc->rings_inuse))
785 		return -EINVAL;
786 
787 	if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE &&
788 	    ring->proxy_id == K3_RINGACC_PROXY_NOT_USED &&
789 	    cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) {
790 		dev_err(ringacc->dev,
791 			"Message mode must use proxy for %u element size\n",
792 			4 << ring->elm_size);
793 		return -EINVAL;
794 	}
795 
796 	/*
797 	 * In case of shared ring only the first user (master user) can
798 	 * configure the ring. The sequence should be by the client:
799 	 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user
800 	 * k3_ringacc_ring_cfg(ring, cfg); # master configuration
801 	 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
802 	 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
803 	 */
804 	if (ring->use_count != 1)
805 		return 0;
806 
807 	ring->size = cfg->size;
808 	ring->elm_size = cfg->elm_size;
809 	ring->mode = cfg->mode;
810 	memset(&ring->state, 0, sizeof(ring->state));
811 
812 	if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
813 		ring->proxy = ringacc->proxy_target_base +
814 			      ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
815 
816 	switch (ring->mode) {
817 	case K3_RINGACC_RING_MODE_RING:
818 		ring->ops = &k3_ring_mode_ring_ops;
819 		ring->dma_dev = cfg->dma_dev;
820 		if (!ring->dma_dev)
821 			ring->dma_dev = ringacc->dev;
822 		break;
823 	case K3_RINGACC_RING_MODE_MESSAGE:
824 		ring->dma_dev = ringacc->dev;
825 		if (ring->proxy)
826 			ring->ops = &k3_ring_mode_proxy_ops;
827 		else
828 			ring->ops = &k3_ring_mode_msg_ops;
829 		break;
830 	default:
831 		ring->ops = NULL;
832 		ret = -EINVAL;
833 		goto err_free_proxy;
834 	}
835 
836 	ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
837 						 ring->size * (4 << ring->elm_size),
838 						 &ring->ring_mem_dma, GFP_KERNEL);
839 	if (!ring->ring_mem_virt) {
840 		dev_err(ringacc->dev, "Failed to alloc ring mem\n");
841 		ret = -ENOMEM;
842 		goto err_free_ops;
843 	}
844 
845 	ret = k3_ringacc_ring_cfg_sci(ring);
846 
847 	if (ret)
848 		goto err_free_mem;
849 
850 	ring->flags |= K3_RING_FLAG_BUSY;
851 	ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ?
852 			K3_RING_FLAG_SHARED : 0;
853 
854 	k3_ringacc_ring_dump(ring);
855 
856 	return 0;
857 
858 err_free_mem:
859 	dma_free_coherent(ring->dma_dev,
860 			  ring->size * (4 << ring->elm_size),
861 			  ring->ring_mem_virt,
862 			  ring->ring_mem_dma);
863 err_free_ops:
864 	ring->ops = NULL;
865 	ring->dma_dev = NULL;
866 err_free_proxy:
867 	ring->proxy = NULL;
868 	return ret;
869 }
870 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg);
871 
872 u32 k3_ringacc_ring_get_size(struct k3_ring *ring)
873 {
874 	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
875 		return -EINVAL;
876 
877 	return ring->size;
878 }
879 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size);
880 
881 u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
882 {
883 	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
884 		return -EINVAL;
885 
886 	if (!ring->state.free)
887 		ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring);
888 
889 	return ring->state.free;
890 }
891 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free);
892 
893 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
894 {
895 	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
896 		return -EINVAL;
897 
898 	return k3_ringacc_ring_read_occ(ring);
899 }
900 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
901 
902 u32 k3_ringacc_ring_is_full(struct k3_ring *ring)
903 {
904 	return !k3_ringacc_ring_get_free(ring);
905 }
906 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full);
907 
908 enum k3_ringacc_access_mode {
909 	K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
910 	K3_RINGACC_ACCESS_MODE_POP_HEAD,
911 	K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
912 	K3_RINGACC_ACCESS_MODE_POP_TAIL,
913 	K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
914 	K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
915 };
916 
917 #define K3_RINGACC_PROXY_MODE(x)	(((x) & 0x3) << 16)
918 #define K3_RINGACC_PROXY_ELSIZE(x)	(((x) & 0x7) << 24)
919 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring,
920 				     enum k3_ringacc_proxy_access_mode mode)
921 {
922 	u32 val;
923 
924 	val = ring->ring_id;
925 	val |= K3_RINGACC_PROXY_MODE(mode);
926 	val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size);
927 	writel(val, &ring->proxy->control);
928 	return 0;
929 }
930 
931 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem,
932 					enum k3_ringacc_access_mode access_mode)
933 {
934 	void __iomem *ptr;
935 
936 	ptr = (void __iomem *)&ring->proxy->data;
937 
938 	switch (access_mode) {
939 	case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
940 	case K3_RINGACC_ACCESS_MODE_POP_HEAD:
941 		k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
942 		break;
943 	case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
944 	case K3_RINGACC_ACCESS_MODE_POP_TAIL:
945 		k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
946 		break;
947 	default:
948 		return -EINVAL;
949 	}
950 
951 	ptr += k3_ringacc_ring_get_fifo_pos(ring);
952 
953 	switch (access_mode) {
954 	case K3_RINGACC_ACCESS_MODE_POP_HEAD:
955 	case K3_RINGACC_ACCESS_MODE_POP_TAIL:
956 		dev_dbg(ring->parent->dev,
957 			"proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
958 			access_mode);
959 		memcpy_fromio(elem, ptr, (4 << ring->elm_size));
960 		ring->state.occ--;
961 		break;
962 	case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
963 	case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
964 		dev_dbg(ring->parent->dev,
965 			"proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
966 			access_mode);
967 		memcpy_toio(ptr, elem, (4 << ring->elm_size));
968 		ring->state.free--;
969 		break;
970 	default:
971 		return -EINVAL;
972 	}
973 
974 	dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free,
975 		ring->state.occ);
976 	return 0;
977 }
978 
979 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem)
980 {
981 	return k3_ringacc_ring_access_proxy(ring, elem,
982 					    K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
983 }
984 
985 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem)
986 {
987 	return k3_ringacc_ring_access_proxy(ring, elem,
988 					    K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
989 }
990 
991 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
992 {
993 	return k3_ringacc_ring_access_proxy(ring, elem,
994 					    K3_RINGACC_ACCESS_MODE_POP_HEAD);
995 }
996 
997 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
998 {
999 	return k3_ringacc_ring_access_proxy(ring, elem,
1000 					    K3_RINGACC_ACCESS_MODE_POP_HEAD);
1001 }
1002 
1003 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
1004 				     enum k3_ringacc_access_mode access_mode)
1005 {
1006 	void __iomem *ptr;
1007 
1008 	switch (access_mode) {
1009 	case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
1010 	case K3_RINGACC_ACCESS_MODE_POP_HEAD:
1011 		ptr = (void __iomem *)&ring->fifos->head_data;
1012 		break;
1013 	case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
1014 	case K3_RINGACC_ACCESS_MODE_POP_TAIL:
1015 		ptr = (void __iomem *)&ring->fifos->tail_data;
1016 		break;
1017 	default:
1018 		return -EINVAL;
1019 	}
1020 
1021 	ptr += k3_ringacc_ring_get_fifo_pos(ring);
1022 
1023 	switch (access_mode) {
1024 	case K3_RINGACC_ACCESS_MODE_POP_HEAD:
1025 	case K3_RINGACC_ACCESS_MODE_POP_TAIL:
1026 		dev_dbg(ring->parent->dev,
1027 			"memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
1028 			access_mode);
1029 		memcpy_fromio(elem, ptr, (4 << ring->elm_size));
1030 		ring->state.occ--;
1031 		break;
1032 	case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
1033 	case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
1034 		dev_dbg(ring->parent->dev,
1035 			"memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
1036 			access_mode);
1037 		memcpy_toio(ptr, elem, (4 << ring->elm_size));
1038 		ring->state.free--;
1039 		break;
1040 	default:
1041 		return -EINVAL;
1042 	}
1043 
1044 	dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n",
1045 		ring->state.free, ring->state.windex, ring->state.occ,
1046 		ring->state.rindex);
1047 	return 0;
1048 }
1049 
1050 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem)
1051 {
1052 	return k3_ringacc_ring_access_io(ring, elem,
1053 					 K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
1054 }
1055 
1056 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem)
1057 {
1058 	return k3_ringacc_ring_access_io(ring, elem,
1059 					 K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
1060 }
1061 
1062 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
1063 {
1064 	return k3_ringacc_ring_access_io(ring, elem,
1065 					 K3_RINGACC_ACCESS_MODE_POP_HEAD);
1066 }
1067 
1068 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
1069 {
1070 	return k3_ringacc_ring_access_io(ring, elem,
1071 					 K3_RINGACC_ACCESS_MODE_POP_HEAD);
1072 }
1073 
1074 /*
1075  * The element is 48 bits of address + ASEL bits in the ring.
1076  * ASEL is used by the DMAs and should be removed for the kernel as it is not
1077  * part of the physical memory address.
1078  */
1079 static void k3_dmaring_remove_asel_from_elem(u64 *elem)
1080 {
1081 	*elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1082 }
1083 
1084 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem)
1085 {
1086 	void *elem_ptr;
1087 	u32 elem_idx;
1088 
1089 	/*
1090 	 * DMA rings: forward ring is always tied DMA channel and HW does not
1091 	 * maintain any state data required for POP operation and its unknown
1092 	 * how much elements were consumed by HW. So, to actually
1093 	 * do POP, the read pointer has to be recalculated every time.
1094 	 */
1095 	ring->state.occ = k3_ringacc_ring_read_occ(ring);
1096 	if (ring->state.windex >= ring->state.occ)
1097 		elem_idx = ring->state.windex - ring->state.occ;
1098 	else
1099 		elem_idx = ring->size - (ring->state.occ - ring->state.windex);
1100 
1101 	elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx);
1102 	memcpy(elem, elem_ptr, (4 << ring->elm_size));
1103 	k3_dmaring_remove_asel_from_elem(elem);
1104 
1105 	ring->state.occ--;
1106 	writel(-1, &ring->rt->db);
1107 
1108 	dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
1109 		__func__, ring->state.occ, ring->state.windex, elem_idx,
1110 		elem_ptr);
1111 	return 0;
1112 }
1113 
1114 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem)
1115 {
1116 	void *elem_ptr;
1117 
1118 	elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1119 
1120 	if (ring->state.occ) {
1121 		memcpy(elem, elem_ptr, (4 << ring->elm_size));
1122 		k3_dmaring_remove_asel_from_elem(elem);
1123 
1124 		ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1125 		ring->state.occ--;
1126 		writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db);
1127 	} else if (ring->state.tdown_complete) {
1128 		dma_addr_t *value = elem;
1129 
1130 		*value = CPPI5_TDCM_MARKER;
1131 		writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db);
1132 		ring->state.tdown_complete = false;
1133 	}
1134 
1135 	dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
1136 		__func__, ring->state.occ, ring->state.rindex, elem_ptr);
1137 	return 0;
1138 }
1139 
1140 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
1141 {
1142 	void *elem_ptr;
1143 
1144 	elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex);
1145 
1146 	memcpy(elem_ptr, elem, (4 << ring->elm_size));
1147 	if (ring->parent->dma_rings) {
1148 		u64 *addr = elem_ptr;
1149 
1150 		*addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT);
1151 	}
1152 
1153 	ring->state.windex = (ring->state.windex + 1) % ring->size;
1154 	ring->state.free--;
1155 	writel(1, &ring->rt->db);
1156 
1157 	dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n",
1158 		ring->state.free, ring->state.windex);
1159 
1160 	return 0;
1161 }
1162 
1163 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem)
1164 {
1165 	void *elem_ptr;
1166 
1167 	elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1168 
1169 	memcpy(elem, elem_ptr, (4 << ring->elm_size));
1170 
1171 	ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1172 	ring->state.occ--;
1173 	writel(-1, &ring->rt->db);
1174 
1175 	dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n",
1176 		ring->state.occ, ring->state.rindex, elem_ptr);
1177 	return 0;
1178 }
1179 
1180 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem)
1181 {
1182 	int ret = -EOPNOTSUPP;
1183 
1184 	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1185 		return -EINVAL;
1186 
1187 	dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n",
1188 		ring->state.free, ring->state.windex);
1189 
1190 	if (k3_ringacc_ring_is_full(ring))
1191 		return -ENOMEM;
1192 
1193 	if (ring->ops && ring->ops->push_tail)
1194 		ret = ring->ops->push_tail(ring, elem);
1195 
1196 	return ret;
1197 }
1198 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push);
1199 
1200 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem)
1201 {
1202 	int ret = -EOPNOTSUPP;
1203 
1204 	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1205 		return -EINVAL;
1206 
1207 	dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n",
1208 		ring->state.free, ring->state.windex);
1209 
1210 	if (k3_ringacc_ring_is_full(ring))
1211 		return -ENOMEM;
1212 
1213 	if (ring->ops && ring->ops->push_head)
1214 		ret = ring->ops->push_head(ring, elem);
1215 
1216 	return ret;
1217 }
1218 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head);
1219 
1220 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
1221 {
1222 	int ret = -EOPNOTSUPP;
1223 
1224 	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1225 		return -EINVAL;
1226 
1227 	if (!ring->state.occ)
1228 		k3_ringacc_ring_update_occ(ring);
1229 
1230 	dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ,
1231 		ring->state.rindex);
1232 
1233 	if (!ring->state.occ && !ring->state.tdown_complete)
1234 		return -ENODATA;
1235 
1236 	if (ring->ops && ring->ops->pop_head)
1237 		ret = ring->ops->pop_head(ring, elem);
1238 
1239 	return ret;
1240 }
1241 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop);
1242 
1243 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
1244 {
1245 	int ret = -EOPNOTSUPP;
1246 
1247 	if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1248 		return -EINVAL;
1249 
1250 	if (!ring->state.occ)
1251 		k3_ringacc_ring_update_occ(ring);
1252 
1253 	dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
1254 		ring->state.occ, ring->state.rindex);
1255 
1256 	if (!ring->state.occ)
1257 		return -ENODATA;
1258 
1259 	if (ring->ops && ring->ops->pop_tail)
1260 		ret = ring->ops->pop_tail(ring, elem);
1261 
1262 	return ret;
1263 }
1264 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail);
1265 
1266 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
1267 						const char *property)
1268 {
1269 	struct device_node *ringacc_np;
1270 	struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER);
1271 	struct k3_ringacc *entry;
1272 
1273 	ringacc_np = of_parse_phandle(np, property, 0);
1274 	if (!ringacc_np)
1275 		return ERR_PTR(-ENODEV);
1276 
1277 	mutex_lock(&k3_ringacc_list_lock);
1278 	list_for_each_entry(entry, &k3_ringacc_list, list)
1279 		if (entry->dev->of_node == ringacc_np) {
1280 			ringacc = entry;
1281 			break;
1282 		}
1283 	mutex_unlock(&k3_ringacc_list_lock);
1284 	of_node_put(ringacc_np);
1285 
1286 	return ringacc;
1287 }
1288 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle);
1289 
1290 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc)
1291 {
1292 	struct device_node *node = ringacc->dev->of_node;
1293 	struct device *dev = ringacc->dev;
1294 	struct platform_device *pdev = to_platform_device(dev);
1295 	int ret;
1296 
1297 	if (!node) {
1298 		dev_err(dev, "device tree info unavailable\n");
1299 		return -ENODEV;
1300 	}
1301 
1302 	ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings);
1303 	if (ret) {
1304 		dev_err(dev, "ti,num-rings read failure %d\n", ret);
1305 		return ret;
1306 	}
1307 
1308 	ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci");
1309 	if (IS_ERR(ringacc->tisci)) {
1310 		ret = PTR_ERR(ringacc->tisci);
1311 		if (ret != -EPROBE_DEFER)
1312 			dev_err(dev, "ti,sci read fail %d\n", ret);
1313 		ringacc->tisci = NULL;
1314 		return ret;
1315 	}
1316 
1317 	ret = of_property_read_u32(node, "ti,sci-dev-id",
1318 				   &ringacc->tisci_dev_id);
1319 	if (ret) {
1320 		dev_err(dev, "ti,sci-dev-id read fail %d\n", ret);
1321 		return ret;
1322 	}
1323 
1324 	pdev->id = ringacc->tisci_dev_id;
1325 
1326 	ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev,
1327 						ringacc->tisci_dev_id,
1328 						"ti,sci-rm-range-gp-rings");
1329 	if (IS_ERR(ringacc->rm_gp_range)) {
1330 		dev_err(dev, "Failed to allocate MSI interrupts\n");
1331 		return PTR_ERR(ringacc->rm_gp_range);
1332 	}
1333 
1334 	return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev,
1335 						 ringacc->rm_gp_range);
1336 }
1337 
1338 static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = {
1339 	.dma_ring_reset_quirk = 1,
1340 };
1341 
1342 static const struct soc_device_attribute k3_ringacc_socinfo[] = {
1343 	{ .family = "AM65X",
1344 	  .revision = "SR1.0",
1345 	  .data = &k3_ringacc_soc_data_sr1
1346 	},
1347 	{/* sentinel */}
1348 };
1349 
1350 static int k3_ringacc_init(struct platform_device *pdev,
1351 			   struct k3_ringacc *ringacc)
1352 {
1353 	const struct soc_device_attribute *soc;
1354 	void __iomem *base_fifo, *base_rt;
1355 	struct device *dev = &pdev->dev;
1356 	struct resource *res;
1357 	int ret, i;
1358 
1359 	dev->msi_domain = of_msi_get_domain(dev, dev->of_node,
1360 					    DOMAIN_BUS_TI_SCI_INTA_MSI);
1361 	if (!dev->msi_domain) {
1362 		dev_err(dev, "Failed to get MSI domain\n");
1363 		return -EPROBE_DEFER;
1364 	}
1365 
1366 	ret = k3_ringacc_probe_dt(ringacc);
1367 	if (ret)
1368 		return ret;
1369 
1370 	soc = soc_device_match(k3_ringacc_socinfo);
1371 	if (soc && soc->data) {
1372 		const struct k3_ringacc_soc_data *soc_data = soc->data;
1373 
1374 		ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk;
1375 	}
1376 
1377 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt");
1378 	base_rt = devm_ioremap_resource(dev, res);
1379 	if (IS_ERR(base_rt))
1380 		return PTR_ERR(base_rt);
1381 
1382 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos");
1383 	base_fifo = devm_ioremap_resource(dev, res);
1384 	if (IS_ERR(base_fifo))
1385 		return PTR_ERR(base_fifo);
1386 
1387 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg");
1388 	ringacc->proxy_gcfg = devm_ioremap_resource(dev, res);
1389 	if (IS_ERR(ringacc->proxy_gcfg))
1390 		return PTR_ERR(ringacc->proxy_gcfg);
1391 
1392 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
1393 					   "proxy_target");
1394 	ringacc->proxy_target_base = devm_ioremap_resource(dev, res);
1395 	if (IS_ERR(ringacc->proxy_target_base))
1396 		return PTR_ERR(ringacc->proxy_target_base);
1397 
1398 	ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) &
1399 				     K3_RINGACC_PROXY_CFG_THREADS_MASK;
1400 
1401 	ringacc->rings = devm_kzalloc(dev,
1402 				      sizeof(*ringacc->rings) *
1403 				      ringacc->num_rings,
1404 				      GFP_KERNEL);
1405 	ringacc->rings_inuse = devm_kcalloc(dev,
1406 					    BITS_TO_LONGS(ringacc->num_rings),
1407 					    sizeof(unsigned long), GFP_KERNEL);
1408 	ringacc->proxy_inuse = devm_kcalloc(dev,
1409 					    BITS_TO_LONGS(ringacc->num_proxies),
1410 					    sizeof(unsigned long), GFP_KERNEL);
1411 
1412 	if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1413 		return -ENOMEM;
1414 
1415 	for (i = 0; i < ringacc->num_rings; i++) {
1416 		ringacc->rings[i].rt = base_rt +
1417 				       K3_RINGACC_RT_REGS_STEP * i;
1418 		ringacc->rings[i].fifos = base_fifo +
1419 					  K3_RINGACC_FIFO_REGS_STEP * i;
1420 		ringacc->rings[i].parent = ringacc;
1421 		ringacc->rings[i].ring_id = i;
1422 		ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1423 	}
1424 
1425 	ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1426 
1427 	dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1428 		 ringacc->num_rings,
1429 		 ringacc->rm_gp_range->desc[0].start,
1430 		 ringacc->rm_gp_range->desc[0].num,
1431 		 ringacc->tisci_dev_id);
1432 	dev_info(dev, "dma-ring-reset-quirk: %s\n",
1433 		 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1434 	dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1435 		 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies);
1436 
1437 	return 0;
1438 }
1439 
1440 struct ringacc_match_data {
1441 	struct k3_ringacc_ops ops;
1442 };
1443 
1444 static struct ringacc_match_data k3_ringacc_data = {
1445 	.ops = {
1446 		.init = k3_ringacc_init,
1447 	},
1448 };
1449 
1450 /* Match table for of_platform binding */
1451 static const struct of_device_id k3_ringacc_of_match[] = {
1452 	{ .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, },
1453 	{},
1454 };
1455 
1456 struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev,
1457 					    struct k3_ringacc_init_data *data)
1458 {
1459 	struct device *dev = &pdev->dev;
1460 	struct k3_ringacc *ringacc;
1461 	void __iomem *base_rt;
1462 	struct resource *res;
1463 	int i;
1464 
1465 	ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1466 	if (!ringacc)
1467 		return ERR_PTR(-ENOMEM);
1468 
1469 	ringacc->dev = dev;
1470 	ringacc->dma_rings = true;
1471 	ringacc->num_rings = data->num_rings;
1472 	ringacc->tisci = data->tisci;
1473 	ringacc->tisci_dev_id = data->tisci_dev_id;
1474 
1475 	mutex_init(&ringacc->req_lock);
1476 
1477 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt");
1478 	base_rt = devm_ioremap_resource(dev, res);
1479 	if (IS_ERR(base_rt))
1480 		return ERR_CAST(base_rt);
1481 
1482 	ringacc->rings = devm_kzalloc(dev,
1483 				      sizeof(*ringacc->rings) *
1484 				      ringacc->num_rings * 2,
1485 				      GFP_KERNEL);
1486 	ringacc->rings_inuse = devm_kcalloc(dev,
1487 					    BITS_TO_LONGS(ringacc->num_rings),
1488 					    sizeof(unsigned long), GFP_KERNEL);
1489 
1490 	if (!ringacc->rings || !ringacc->rings_inuse)
1491 		return ERR_PTR(-ENOMEM);
1492 
1493 	for (i = 0; i < ringacc->num_rings; i++) {
1494 		struct k3_ring *ring = &ringacc->rings[i];
1495 
1496 		ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i;
1497 		ring->parent = ringacc;
1498 		ring->ring_id = i;
1499 		ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1500 
1501 		ring = &ringacc->rings[ringacc->num_rings + i];
1502 		ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i +
1503 			   K3_DMARING_RT_REGS_REVERSE_OFS;
1504 		ring->parent = ringacc;
1505 		ring->ring_id = i;
1506 		ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1507 		ring->flags = K3_RING_FLAG_REVERSE;
1508 	}
1509 
1510 	ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1511 
1512 	dev_info(dev, "Number of rings: %u\n", ringacc->num_rings);
1513 
1514 	return ringacc;
1515 }
1516 EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init);
1517 
1518 static int k3_ringacc_probe(struct platform_device *pdev)
1519 {
1520 	const struct ringacc_match_data *match_data;
1521 	struct device *dev = &pdev->dev;
1522 	struct k3_ringacc *ringacc;
1523 	int ret;
1524 
1525 	match_data = of_device_get_match_data(&pdev->dev);
1526 	if (!match_data)
1527 		return -ENODEV;
1528 
1529 	ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1530 	if (!ringacc)
1531 		return -ENOMEM;
1532 
1533 	ringacc->dev = dev;
1534 	mutex_init(&ringacc->req_lock);
1535 	ringacc->ops = &match_data->ops;
1536 
1537 	ret = ringacc->ops->init(pdev, ringacc);
1538 	if (ret)
1539 		return ret;
1540 
1541 	dev_set_drvdata(dev, ringacc);
1542 
1543 	mutex_lock(&k3_ringacc_list_lock);
1544 	list_add_tail(&ringacc->list, &k3_ringacc_list);
1545 	mutex_unlock(&k3_ringacc_list_lock);
1546 
1547 	return 0;
1548 }
1549 
1550 static struct platform_driver k3_ringacc_driver = {
1551 	.probe		= k3_ringacc_probe,
1552 	.driver		= {
1553 		.name	= "k3-ringacc",
1554 		.of_match_table = k3_ringacc_of_match,
1555 		.suppress_bind_attrs = true,
1556 	},
1557 };
1558 builtin_platform_driver(k3_ringacc_driver);
1559