13277e8aaSGrygorii Strashko // SPDX-License-Identifier: GPL-2.0 23277e8aaSGrygorii Strashko /* 33277e8aaSGrygorii Strashko * TI K3 NAVSS Ring Accelerator subsystem driver 43277e8aaSGrygorii Strashko * 53277e8aaSGrygorii Strashko * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 63277e8aaSGrygorii Strashko */ 73277e8aaSGrygorii Strashko 83277e8aaSGrygorii Strashko #include <linux/dma-mapping.h> 93277e8aaSGrygorii Strashko #include <linux/io.h> 103277e8aaSGrygorii Strashko #include <linux/init.h> 113277e8aaSGrygorii Strashko #include <linux/of.h> 123277e8aaSGrygorii Strashko #include <linux/platform_device.h> 1395e7be06SGrygorii Strashko #include <linux/sys_soc.h> 14d782298cSGrygorii Strashko #include <linux/dma/ti-cppi5.h> 153277e8aaSGrygorii Strashko #include <linux/soc/ti/k3-ringacc.h> 163277e8aaSGrygorii Strashko #include <linux/soc/ti/ti_sci_protocol.h> 173277e8aaSGrygorii Strashko #include <linux/soc/ti/ti_sci_inta_msi.h> 183277e8aaSGrygorii Strashko #include <linux/of_irq.h> 193277e8aaSGrygorii Strashko #include <linux/irqdomain.h> 203277e8aaSGrygorii Strashko 213277e8aaSGrygorii Strashko static LIST_HEAD(k3_ringacc_list); 223277e8aaSGrygorii Strashko static DEFINE_MUTEX(k3_ringacc_list_lock); 233277e8aaSGrygorii Strashko 243277e8aaSGrygorii Strashko #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) 25d782298cSGrygorii Strashko #define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0) 263277e8aaSGrygorii Strashko 273277e8aaSGrygorii Strashko /** 283277e8aaSGrygorii Strashko * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region 293277e8aaSGrygorii Strashko * 303277e8aaSGrygorii Strashko * @resv_16: Reserved 313277e8aaSGrygorii Strashko * @db: Ring Doorbell Register 323277e8aaSGrygorii Strashko * @resv_4: Reserved 333277e8aaSGrygorii Strashko * @occ: Ring Occupancy Register 343277e8aaSGrygorii Strashko * @indx: Ring Current Index Register 353277e8aaSGrygorii Strashko * @hwocc: Ring Hardware Occupancy Register 363277e8aaSGrygorii Strashko * @hwindx: Ring Hardware Current Index Register 373277e8aaSGrygorii Strashko */ 383277e8aaSGrygorii Strashko struct k3_ring_rt_regs { 393277e8aaSGrygorii Strashko u32 resv_16[4]; 403277e8aaSGrygorii Strashko u32 db; 413277e8aaSGrygorii Strashko u32 resv_4[1]; 423277e8aaSGrygorii Strashko u32 occ; 433277e8aaSGrygorii Strashko u32 indx; 443277e8aaSGrygorii Strashko u32 hwocc; 453277e8aaSGrygorii Strashko u32 hwindx; 463277e8aaSGrygorii Strashko }; 473277e8aaSGrygorii Strashko 483277e8aaSGrygorii Strashko #define K3_RINGACC_RT_REGS_STEP 0x1000 49d782298cSGrygorii Strashko #define K3_DMARING_RT_REGS_STEP 0x2000 50d782298cSGrygorii Strashko #define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000 51d782298cSGrygorii Strashko #define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0) 52d782298cSGrygorii Strashko #define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31) 53d782298cSGrygorii Strashko #define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0) 54d782298cSGrygorii Strashko #define K3_DMARING_RT_DB_TDOWN_ACK BIT(31) 553277e8aaSGrygorii Strashko 563277e8aaSGrygorii Strashko /** 573277e8aaSGrygorii Strashko * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region 583277e8aaSGrygorii Strashko * 593277e8aaSGrygorii Strashko * @head_data: Ring Head Entry Data Registers 603277e8aaSGrygorii Strashko * @tail_data: Ring Tail Entry Data Registers 613277e8aaSGrygorii Strashko * @peek_head_data: Ring Peek Head Entry Data Regs 623277e8aaSGrygorii Strashko * @peek_tail_data: Ring Peek Tail Entry Data Regs 633277e8aaSGrygorii Strashko */ 643277e8aaSGrygorii Strashko struct k3_ring_fifo_regs { 653277e8aaSGrygorii Strashko u32 head_data[128]; 663277e8aaSGrygorii Strashko u32 tail_data[128]; 673277e8aaSGrygorii Strashko u32 peek_head_data[128]; 683277e8aaSGrygorii Strashko u32 peek_tail_data[128]; 693277e8aaSGrygorii Strashko }; 703277e8aaSGrygorii Strashko 713277e8aaSGrygorii Strashko /** 723277e8aaSGrygorii Strashko * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region 733277e8aaSGrygorii Strashko * 743277e8aaSGrygorii Strashko * @revision: Revision Register 753277e8aaSGrygorii Strashko * @config: Config Register 763277e8aaSGrygorii Strashko */ 773277e8aaSGrygorii Strashko struct k3_ringacc_proxy_gcfg_regs { 783277e8aaSGrygorii Strashko u32 revision; 793277e8aaSGrygorii Strashko u32 config; 803277e8aaSGrygorii Strashko }; 813277e8aaSGrygorii Strashko 823277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) 833277e8aaSGrygorii Strashko 843277e8aaSGrygorii Strashko /** 853277e8aaSGrygorii Strashko * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region 863277e8aaSGrygorii Strashko * 873277e8aaSGrygorii Strashko * @control: Proxy Control Register 883277e8aaSGrygorii Strashko * @status: Proxy Status Register 893277e8aaSGrygorii Strashko * @resv_512: Reserved 903277e8aaSGrygorii Strashko * @data: Proxy Data Register 913277e8aaSGrygorii Strashko */ 923277e8aaSGrygorii Strashko struct k3_ringacc_proxy_target_regs { 933277e8aaSGrygorii Strashko u32 control; 943277e8aaSGrygorii Strashko u32 status; 953277e8aaSGrygorii Strashko u8 resv_512[504]; 963277e8aaSGrygorii Strashko u32 data[128]; 973277e8aaSGrygorii Strashko }; 983277e8aaSGrygorii Strashko 993277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_TARGET_STEP 0x1000 1003277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_NOT_USED (-1) 1013277e8aaSGrygorii Strashko 1023277e8aaSGrygorii Strashko enum k3_ringacc_proxy_access_mode { 1033277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_HEAD = 0, 1043277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_TAIL = 1, 1053277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_PEEK_HEAD = 2, 1063277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_PEEK_TAIL = 3, 1073277e8aaSGrygorii Strashko }; 1083277e8aaSGrygorii Strashko 1093277e8aaSGrygorii Strashko #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) 1103277e8aaSGrygorii Strashko #define K3_RINGACC_FIFO_REGS_STEP 0x1000 1113277e8aaSGrygorii Strashko #define K3_RINGACC_MAX_DB_RING_CNT (127U) 1123277e8aaSGrygorii Strashko 1133277e8aaSGrygorii Strashko struct k3_ring_ops { 1143277e8aaSGrygorii Strashko int (*push_tail)(struct k3_ring *ring, void *elm); 1153277e8aaSGrygorii Strashko int (*push_head)(struct k3_ring *ring, void *elm); 1163277e8aaSGrygorii Strashko int (*pop_tail)(struct k3_ring *ring, void *elm); 1173277e8aaSGrygorii Strashko int (*pop_head)(struct k3_ring *ring, void *elm); 1183277e8aaSGrygorii Strashko }; 1193277e8aaSGrygorii Strashko 1203277e8aaSGrygorii Strashko /** 1216b3da0b4SPeter Ujfalusi * struct k3_ring_state - Internal state tracking structure 1226b3da0b4SPeter Ujfalusi * 1236b3da0b4SPeter Ujfalusi * @free: Number of free entries 1246b3da0b4SPeter Ujfalusi * @occ: Occupancy 1256b3da0b4SPeter Ujfalusi * @windex: Write index 1266b3da0b4SPeter Ujfalusi * @rindex: Read index 1276b3da0b4SPeter Ujfalusi */ 1286b3da0b4SPeter Ujfalusi struct k3_ring_state { 1296b3da0b4SPeter Ujfalusi u32 free; 1306b3da0b4SPeter Ujfalusi u32 occ; 1316b3da0b4SPeter Ujfalusi u32 windex; 1326b3da0b4SPeter Ujfalusi u32 rindex; 133d782298cSGrygorii Strashko u32 tdown_complete:1; 1346b3da0b4SPeter Ujfalusi }; 1356b3da0b4SPeter Ujfalusi 1366b3da0b4SPeter Ujfalusi /** 1373277e8aaSGrygorii Strashko * struct k3_ring - RA Ring descriptor 1383277e8aaSGrygorii Strashko * 1393277e8aaSGrygorii Strashko * @rt: Ring control/status registers 1403277e8aaSGrygorii Strashko * @fifos: Ring queues registers 1413277e8aaSGrygorii Strashko * @proxy: Ring Proxy Datapath registers 1423277e8aaSGrygorii Strashko * @ring_mem_dma: Ring buffer dma address 1433277e8aaSGrygorii Strashko * @ring_mem_virt: Ring buffer virt address 1443277e8aaSGrygorii Strashko * @ops: Ring operations 1453277e8aaSGrygorii Strashko * @size: Ring size in elements 1463277e8aaSGrygorii Strashko * @elm_size: Size of the ring element 1473277e8aaSGrygorii Strashko * @mode: Ring mode 1483277e8aaSGrygorii Strashko * @flags: flags 14950883affSLee Jones * @state: Ring state 1503277e8aaSGrygorii Strashko * @ring_id: Ring Id 1513277e8aaSGrygorii Strashko * @parent: Pointer on struct @k3_ringacc 1523277e8aaSGrygorii Strashko * @use_count: Use count for shared rings 1533277e8aaSGrygorii Strashko * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) 1548c42379eSPeter Ujfalusi * @dma_dev: device to be used for DMA API (allocation, mapping) 155d782298cSGrygorii Strashko * @asel: Address Space Select value for physical addresses 1563277e8aaSGrygorii Strashko */ 1573277e8aaSGrygorii Strashko struct k3_ring { 1583277e8aaSGrygorii Strashko struct k3_ring_rt_regs __iomem *rt; 1593277e8aaSGrygorii Strashko struct k3_ring_fifo_regs __iomem *fifos; 1603277e8aaSGrygorii Strashko struct k3_ringacc_proxy_target_regs __iomem *proxy; 1613277e8aaSGrygorii Strashko dma_addr_t ring_mem_dma; 1623277e8aaSGrygorii Strashko void *ring_mem_virt; 1633277e8aaSGrygorii Strashko struct k3_ring_ops *ops; 1643277e8aaSGrygorii Strashko u32 size; 1653277e8aaSGrygorii Strashko enum k3_ring_size elm_size; 1663277e8aaSGrygorii Strashko enum k3_ring_mode mode; 1673277e8aaSGrygorii Strashko u32 flags; 1683277e8aaSGrygorii Strashko #define K3_RING_FLAG_BUSY BIT(1) 1693277e8aaSGrygorii Strashko #define K3_RING_FLAG_SHARED BIT(2) 170d782298cSGrygorii Strashko #define K3_RING_FLAG_REVERSE BIT(3) 1716b3da0b4SPeter Ujfalusi struct k3_ring_state state; 1723277e8aaSGrygorii Strashko u32 ring_id; 1733277e8aaSGrygorii Strashko struct k3_ringacc *parent; 1743277e8aaSGrygorii Strashko u32 use_count; 1753277e8aaSGrygorii Strashko int proxy_id; 1768c42379eSPeter Ujfalusi struct device *dma_dev; 177d782298cSGrygorii Strashko u32 asel; 178d782298cSGrygorii Strashko #define K3_ADDRESS_ASEL_SHIFT 48 1793277e8aaSGrygorii Strashko }; 1803277e8aaSGrygorii Strashko 18140a2a7c3SGrygorii Strashko struct k3_ringacc_ops { 18240a2a7c3SGrygorii Strashko int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc); 18340a2a7c3SGrygorii Strashko }; 18440a2a7c3SGrygorii Strashko 1853277e8aaSGrygorii Strashko /** 1863277e8aaSGrygorii Strashko * struct k3_ringacc - Rings accelerator descriptor 1873277e8aaSGrygorii Strashko * 1883277e8aaSGrygorii Strashko * @dev: pointer on RA device 1893277e8aaSGrygorii Strashko * @proxy_gcfg: RA proxy global config registers 1903277e8aaSGrygorii Strashko * @proxy_target_base: RA proxy datapath region 1913277e8aaSGrygorii Strashko * @num_rings: number of ring in RA 1923277e8aaSGrygorii Strashko * @rings_inuse: bitfield for ring usage tracking 1933277e8aaSGrygorii Strashko * @rm_gp_range: general purpose rings range from tisci 1943277e8aaSGrygorii Strashko * @dma_ring_reset_quirk: DMA reset w/a enable 1953277e8aaSGrygorii Strashko * @num_proxies: number of RA proxies 1963277e8aaSGrygorii Strashko * @proxy_inuse: bitfield for proxy usage tracking 1973277e8aaSGrygorii Strashko * @rings: array of rings descriptors (struct @k3_ring) 1983277e8aaSGrygorii Strashko * @list: list of RAs in the system 1993277e8aaSGrygorii Strashko * @req_lock: protect rings allocation 2003277e8aaSGrygorii Strashko * @tisci: pointer ti-sci handle 2013277e8aaSGrygorii Strashko * @tisci_ring_ops: ti-sci rings ops 2023277e8aaSGrygorii Strashko * @tisci_dev_id: ti-sci device id 20340a2a7c3SGrygorii Strashko * @ops: SoC specific ringacc operation 204d782298cSGrygorii Strashko * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA) 2053277e8aaSGrygorii Strashko */ 2063277e8aaSGrygorii Strashko struct k3_ringacc { 2073277e8aaSGrygorii Strashko struct device *dev; 2083277e8aaSGrygorii Strashko struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; 2093277e8aaSGrygorii Strashko void __iomem *proxy_target_base; 2103277e8aaSGrygorii Strashko u32 num_rings; /* number of rings in Ringacc module */ 2113277e8aaSGrygorii Strashko unsigned long *rings_inuse; 2123277e8aaSGrygorii Strashko struct ti_sci_resource *rm_gp_range; 2133277e8aaSGrygorii Strashko 2143277e8aaSGrygorii Strashko bool dma_ring_reset_quirk; 2153277e8aaSGrygorii Strashko u32 num_proxies; 2163277e8aaSGrygorii Strashko unsigned long *proxy_inuse; 2173277e8aaSGrygorii Strashko 2183277e8aaSGrygorii Strashko struct k3_ring *rings; 2193277e8aaSGrygorii Strashko struct list_head list; 2203277e8aaSGrygorii Strashko struct mutex req_lock; /* protect rings allocation */ 2213277e8aaSGrygorii Strashko 2223277e8aaSGrygorii Strashko const struct ti_sci_handle *tisci; 2233277e8aaSGrygorii Strashko const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; 2243277e8aaSGrygorii Strashko u32 tisci_dev_id; 22540a2a7c3SGrygorii Strashko 22640a2a7c3SGrygorii Strashko const struct k3_ringacc_ops *ops; 227d782298cSGrygorii Strashko bool dma_rings; 2283277e8aaSGrygorii Strashko }; 2293277e8aaSGrygorii Strashko 23095e7be06SGrygorii Strashko /** 23195e7be06SGrygorii Strashko * struct k3_ringacc - Rings accelerator SoC data 23295e7be06SGrygorii Strashko * 23395e7be06SGrygorii Strashko * @dma_ring_reset_quirk: DMA reset w/a enable 23495e7be06SGrygorii Strashko */ 23595e7be06SGrygorii Strashko struct k3_ringacc_soc_data { 23695e7be06SGrygorii Strashko unsigned dma_ring_reset_quirk:1; 23795e7be06SGrygorii Strashko }; 23895e7be06SGrygorii Strashko 239d782298cSGrygorii Strashko static int k3_ringacc_ring_read_occ(struct k3_ring *ring) 240d782298cSGrygorii Strashko { 241d782298cSGrygorii Strashko return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK; 242d782298cSGrygorii Strashko } 243d782298cSGrygorii Strashko 244d782298cSGrygorii Strashko static void k3_ringacc_ring_update_occ(struct k3_ring *ring) 245d782298cSGrygorii Strashko { 246d782298cSGrygorii Strashko u32 val; 247d782298cSGrygorii Strashko 248d782298cSGrygorii Strashko val = readl(&ring->rt->occ); 249d782298cSGrygorii Strashko 250d782298cSGrygorii Strashko ring->state.occ = val & K3_RINGACC_RT_OCC_MASK; 251d782298cSGrygorii Strashko ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE); 252d782298cSGrygorii Strashko } 253d782298cSGrygorii Strashko 2543277e8aaSGrygorii Strashko static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) 2553277e8aaSGrygorii Strashko { 2563277e8aaSGrygorii Strashko return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - 2573277e8aaSGrygorii Strashko (4 << ring->elm_size); 2583277e8aaSGrygorii Strashko } 2593277e8aaSGrygorii Strashko 2603277e8aaSGrygorii Strashko static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) 2613277e8aaSGrygorii Strashko { 2623277e8aaSGrygorii Strashko return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); 2633277e8aaSGrygorii Strashko } 2643277e8aaSGrygorii Strashko 2653277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); 2663277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); 267d782298cSGrygorii Strashko static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem); 268d782298cSGrygorii Strashko static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem); 2693277e8aaSGrygorii Strashko 2703277e8aaSGrygorii Strashko static struct k3_ring_ops k3_ring_mode_ring_ops = { 2713277e8aaSGrygorii Strashko .push_tail = k3_ringacc_ring_push_mem, 2723277e8aaSGrygorii Strashko .pop_head = k3_ringacc_ring_pop_mem, 2733277e8aaSGrygorii Strashko }; 2743277e8aaSGrygorii Strashko 275d782298cSGrygorii Strashko static struct k3_ring_ops k3_dmaring_fwd_ops = { 276d782298cSGrygorii Strashko .push_tail = k3_ringacc_ring_push_mem, 277d782298cSGrygorii Strashko .pop_head = k3_dmaring_fwd_pop, 278d782298cSGrygorii Strashko }; 279d782298cSGrygorii Strashko 280d782298cSGrygorii Strashko static struct k3_ring_ops k3_dmaring_reverse_ops = { 281d782298cSGrygorii Strashko /* Reverse side of the DMA ring can only be popped by SW */ 282d782298cSGrygorii Strashko .pop_head = k3_dmaring_reverse_pop, 283d782298cSGrygorii Strashko }; 284d782298cSGrygorii Strashko 2853277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); 2863277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); 2873277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); 2883277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); 2893277e8aaSGrygorii Strashko 2903277e8aaSGrygorii Strashko static struct k3_ring_ops k3_ring_mode_msg_ops = { 2913277e8aaSGrygorii Strashko .push_tail = k3_ringacc_ring_push_io, 2923277e8aaSGrygorii Strashko .push_head = k3_ringacc_ring_push_head_io, 2933277e8aaSGrygorii Strashko .pop_tail = k3_ringacc_ring_pop_tail_io, 2943277e8aaSGrygorii Strashko .pop_head = k3_ringacc_ring_pop_io, 2953277e8aaSGrygorii Strashko }; 2963277e8aaSGrygorii Strashko 2973277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); 2983277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); 2993277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); 3003277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); 3013277e8aaSGrygorii Strashko 3023277e8aaSGrygorii Strashko static struct k3_ring_ops k3_ring_mode_proxy_ops = { 3033277e8aaSGrygorii Strashko .push_tail = k3_ringacc_ring_push_tail_proxy, 3043277e8aaSGrygorii Strashko .push_head = k3_ringacc_ring_push_head_proxy, 3053277e8aaSGrygorii Strashko .pop_tail = k3_ringacc_ring_pop_tail_proxy, 3063277e8aaSGrygorii Strashko .pop_head = k3_ringacc_ring_pop_head_proxy, 3073277e8aaSGrygorii Strashko }; 3083277e8aaSGrygorii Strashko 3093277e8aaSGrygorii Strashko static void k3_ringacc_ring_dump(struct k3_ring *ring) 3103277e8aaSGrygorii Strashko { 3113277e8aaSGrygorii Strashko struct device *dev = ring->parent->dev; 3123277e8aaSGrygorii Strashko 3133277e8aaSGrygorii Strashko dev_dbg(dev, "dump ring: %d\n", ring->ring_id); 3143277e8aaSGrygorii Strashko dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, 3153277e8aaSGrygorii Strashko &ring->ring_mem_dma); 3163277e8aaSGrygorii Strashko dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", 3173277e8aaSGrygorii Strashko ring->elm_size, ring->size, ring->mode, ring->proxy_id); 318175e663fSGrygorii Strashko dev_dbg(dev, "dump flags %08X\n", ring->flags); 3193277e8aaSGrygorii Strashko 3203277e8aaSGrygorii Strashko dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); 3213277e8aaSGrygorii Strashko dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); 3223277e8aaSGrygorii Strashko dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); 3233277e8aaSGrygorii Strashko dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); 3243277e8aaSGrygorii Strashko dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); 3253277e8aaSGrygorii Strashko 3263277e8aaSGrygorii Strashko if (ring->ring_mem_virt) 3273277e8aaSGrygorii Strashko print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, 3283277e8aaSGrygorii Strashko 16, 1, ring->ring_mem_virt, 16 * 8, false); 3293277e8aaSGrygorii Strashko } 3303277e8aaSGrygorii Strashko 3313277e8aaSGrygorii Strashko struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, 3323277e8aaSGrygorii Strashko int id, u32 flags) 3333277e8aaSGrygorii Strashko { 3343277e8aaSGrygorii Strashko int proxy_id = K3_RINGACC_PROXY_NOT_USED; 3353277e8aaSGrygorii Strashko 3363277e8aaSGrygorii Strashko mutex_lock(&ringacc->req_lock); 3373277e8aaSGrygorii Strashko 3383277e8aaSGrygorii Strashko if (id == K3_RINGACC_RING_ID_ANY) { 3393277e8aaSGrygorii Strashko /* Request for any general purpose ring */ 3403277e8aaSGrygorii Strashko struct ti_sci_resource_desc *gp_rings = 3413277e8aaSGrygorii Strashko &ringacc->rm_gp_range->desc[0]; 3423277e8aaSGrygorii Strashko unsigned long size; 3433277e8aaSGrygorii Strashko 3443277e8aaSGrygorii Strashko size = gp_rings->start + gp_rings->num; 3453277e8aaSGrygorii Strashko id = find_next_zero_bit(ringacc->rings_inuse, size, 3463277e8aaSGrygorii Strashko gp_rings->start); 3473277e8aaSGrygorii Strashko if (id == size) 3483277e8aaSGrygorii Strashko goto error; 3493277e8aaSGrygorii Strashko } else if (id < 0) { 3503277e8aaSGrygorii Strashko goto error; 3513277e8aaSGrygorii Strashko } 3523277e8aaSGrygorii Strashko 3533277e8aaSGrygorii Strashko if (test_bit(id, ringacc->rings_inuse) && 3543277e8aaSGrygorii Strashko !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) 3553277e8aaSGrygorii Strashko goto error; 3563277e8aaSGrygorii Strashko else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) 3573277e8aaSGrygorii Strashko goto out; 3583277e8aaSGrygorii Strashko 3593277e8aaSGrygorii Strashko if (flags & K3_RINGACC_RING_USE_PROXY) { 3603277e8aaSGrygorii Strashko proxy_id = find_next_zero_bit(ringacc->proxy_inuse, 3613277e8aaSGrygorii Strashko ringacc->num_proxies, 0); 3623277e8aaSGrygorii Strashko if (proxy_id == ringacc->num_proxies) 3633277e8aaSGrygorii Strashko goto error; 3643277e8aaSGrygorii Strashko } 3653277e8aaSGrygorii Strashko 3663277e8aaSGrygorii Strashko if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { 3673277e8aaSGrygorii Strashko set_bit(proxy_id, ringacc->proxy_inuse); 3683277e8aaSGrygorii Strashko ringacc->rings[id].proxy_id = proxy_id; 3693277e8aaSGrygorii Strashko dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, 3703277e8aaSGrygorii Strashko proxy_id); 3713277e8aaSGrygorii Strashko } else { 3723277e8aaSGrygorii Strashko dev_dbg(ringacc->dev, "Giving ring#%d\n", id); 3733277e8aaSGrygorii Strashko } 3743277e8aaSGrygorii Strashko 3753277e8aaSGrygorii Strashko set_bit(id, ringacc->rings_inuse); 3763277e8aaSGrygorii Strashko out: 3773277e8aaSGrygorii Strashko ringacc->rings[id].use_count++; 3783277e8aaSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 3793277e8aaSGrygorii Strashko return &ringacc->rings[id]; 3803277e8aaSGrygorii Strashko 3813277e8aaSGrygorii Strashko error: 3823277e8aaSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 3833277e8aaSGrygorii Strashko return NULL; 3843277e8aaSGrygorii Strashko } 3853277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); 3863277e8aaSGrygorii Strashko 387d782298cSGrygorii Strashko static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id, 388d782298cSGrygorii Strashko struct k3_ring **fwd_ring, 389d782298cSGrygorii Strashko struct k3_ring **compl_ring) 390d782298cSGrygorii Strashko { 391d782298cSGrygorii Strashko int ret = 0; 392d782298cSGrygorii Strashko 393d782298cSGrygorii Strashko /* 394d782298cSGrygorii Strashko * DMA rings must be requested by ID, completion ring is the reverse 395d782298cSGrygorii Strashko * side of the forward ring 396d782298cSGrygorii Strashko */ 397d782298cSGrygorii Strashko if (fwd_id < 0) 398d782298cSGrygorii Strashko return -EINVAL; 399d782298cSGrygorii Strashko 400d782298cSGrygorii Strashko mutex_lock(&ringacc->req_lock); 401d782298cSGrygorii Strashko 402d782298cSGrygorii Strashko if (test_bit(fwd_id, ringacc->rings_inuse)) { 403d782298cSGrygorii Strashko ret = -EBUSY; 404d782298cSGrygorii Strashko goto error; 405d782298cSGrygorii Strashko } 406d782298cSGrygorii Strashko 407d782298cSGrygorii Strashko *fwd_ring = &ringacc->rings[fwd_id]; 408d782298cSGrygorii Strashko *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings]; 409d782298cSGrygorii Strashko set_bit(fwd_id, ringacc->rings_inuse); 410d782298cSGrygorii Strashko ringacc->rings[fwd_id].use_count++; 411d782298cSGrygorii Strashko dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id); 412d782298cSGrygorii Strashko 413d782298cSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 414d782298cSGrygorii Strashko return 0; 415d782298cSGrygorii Strashko 416d782298cSGrygorii Strashko error: 417d782298cSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 418d782298cSGrygorii Strashko return ret; 419d782298cSGrygorii Strashko } 420d782298cSGrygorii Strashko 42143148b1cSGrygorii Strashko int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, 42243148b1cSGrygorii Strashko int fwd_id, int compl_id, 42343148b1cSGrygorii Strashko struct k3_ring **fwd_ring, 42443148b1cSGrygorii Strashko struct k3_ring **compl_ring) 42543148b1cSGrygorii Strashko { 42643148b1cSGrygorii Strashko int ret = 0; 42743148b1cSGrygorii Strashko 42843148b1cSGrygorii Strashko if (!fwd_ring || !compl_ring) 42943148b1cSGrygorii Strashko return -EINVAL; 43043148b1cSGrygorii Strashko 431d782298cSGrygorii Strashko if (ringacc->dma_rings) 432d782298cSGrygorii Strashko return k3_dmaring_request_dual_ring(ringacc, fwd_id, 433d782298cSGrygorii Strashko fwd_ring, compl_ring); 434d782298cSGrygorii Strashko 43543148b1cSGrygorii Strashko *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); 43643148b1cSGrygorii Strashko if (!(*fwd_ring)) 43743148b1cSGrygorii Strashko return -ENODEV; 43843148b1cSGrygorii Strashko 43943148b1cSGrygorii Strashko *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0); 44043148b1cSGrygorii Strashko if (!(*compl_ring)) { 44143148b1cSGrygorii Strashko k3_ringacc_ring_free(*fwd_ring); 44243148b1cSGrygorii Strashko ret = -ENODEV; 44343148b1cSGrygorii Strashko } 44443148b1cSGrygorii Strashko 44543148b1cSGrygorii Strashko return ret; 44643148b1cSGrygorii Strashko } 44743148b1cSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair); 44843148b1cSGrygorii Strashko 4493277e8aaSGrygorii Strashko static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) 4503277e8aaSGrygorii Strashko { 451bb49ca00SPeter Ujfalusi struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 4523277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 4533277e8aaSGrygorii Strashko int ret; 4543277e8aaSGrygorii Strashko 455bb49ca00SPeter Ujfalusi ring_cfg.nav_id = ringacc->tisci_dev_id; 456bb49ca00SPeter Ujfalusi ring_cfg.index = ring->ring_id; 457bb49ca00SPeter Ujfalusi ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID; 458bb49ca00SPeter Ujfalusi ring_cfg.count = ring->size; 459bb49ca00SPeter Ujfalusi 460bb49ca00SPeter Ujfalusi ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 4613277e8aaSGrygorii Strashko if (ret) 4623277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", 4633277e8aaSGrygorii Strashko ret, ring->ring_id); 4643277e8aaSGrygorii Strashko } 4653277e8aaSGrygorii Strashko 4663277e8aaSGrygorii Strashko void k3_ringacc_ring_reset(struct k3_ring *ring) 4673277e8aaSGrygorii Strashko { 4683277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 4693277e8aaSGrygorii Strashko return; 4703277e8aaSGrygorii Strashko 4716b3da0b4SPeter Ujfalusi memset(&ring->state, 0, sizeof(ring->state)); 4723277e8aaSGrygorii Strashko 4733277e8aaSGrygorii Strashko k3_ringacc_ring_reset_sci(ring); 4743277e8aaSGrygorii Strashko } 4753277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); 4763277e8aaSGrygorii Strashko 4773277e8aaSGrygorii Strashko static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, 4783277e8aaSGrygorii Strashko enum k3_ring_mode mode) 4793277e8aaSGrygorii Strashko { 480bb49ca00SPeter Ujfalusi struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 4813277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 4823277e8aaSGrygorii Strashko int ret; 4833277e8aaSGrygorii Strashko 484bb49ca00SPeter Ujfalusi ring_cfg.nav_id = ringacc->tisci_dev_id; 485bb49ca00SPeter Ujfalusi ring_cfg.index = ring->ring_id; 486bb49ca00SPeter Ujfalusi ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID; 487bb49ca00SPeter Ujfalusi ring_cfg.mode = mode; 488bb49ca00SPeter Ujfalusi 489bb49ca00SPeter Ujfalusi ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 4903277e8aaSGrygorii Strashko if (ret) 4913277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", 4923277e8aaSGrygorii Strashko ret, ring->ring_id); 4933277e8aaSGrygorii Strashko } 4943277e8aaSGrygorii Strashko 4953277e8aaSGrygorii Strashko void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) 4963277e8aaSGrygorii Strashko { 4973277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 4983277e8aaSGrygorii Strashko return; 4993277e8aaSGrygorii Strashko 5003277e8aaSGrygorii Strashko if (!ring->parent->dma_ring_reset_quirk) 5013277e8aaSGrygorii Strashko goto reset; 5023277e8aaSGrygorii Strashko 5033277e8aaSGrygorii Strashko if (!occ) 504d782298cSGrygorii Strashko occ = k3_ringacc_ring_read_occ(ring); 5053277e8aaSGrygorii Strashko 5063277e8aaSGrygorii Strashko if (occ) { 5073277e8aaSGrygorii Strashko u32 db_ring_cnt, db_ring_cnt_cur; 5083277e8aaSGrygorii Strashko 5093277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, 5103277e8aaSGrygorii Strashko ring->ring_id, occ); 5113277e8aaSGrygorii Strashko /* TI-SCI ring reset */ 5123277e8aaSGrygorii Strashko k3_ringacc_ring_reset_sci(ring); 5133277e8aaSGrygorii Strashko 5143277e8aaSGrygorii Strashko /* 5153277e8aaSGrygorii Strashko * Setup the ring in ring/doorbell mode (if not already in this 5163277e8aaSGrygorii Strashko * mode) 5173277e8aaSGrygorii Strashko */ 5183277e8aaSGrygorii Strashko if (ring->mode != K3_RINGACC_RING_MODE_RING) 5193277e8aaSGrygorii Strashko k3_ringacc_ring_reconfig_qmode_sci( 5203277e8aaSGrygorii Strashko ring, K3_RINGACC_RING_MODE_RING); 5213277e8aaSGrygorii Strashko /* 5223277e8aaSGrygorii Strashko * Ring the doorbell 2**22 – ringOcc times. 5233277e8aaSGrygorii Strashko * This will wrap the internal UDMAP ring state occupancy 5243277e8aaSGrygorii Strashko * counter (which is 21-bits wide) to 0. 5253277e8aaSGrygorii Strashko */ 5263277e8aaSGrygorii Strashko db_ring_cnt = (1U << 22) - occ; 5273277e8aaSGrygorii Strashko 5283277e8aaSGrygorii Strashko while (db_ring_cnt != 0) { 5293277e8aaSGrygorii Strashko /* 5303277e8aaSGrygorii Strashko * Ring the doorbell with the maximum count each 5313277e8aaSGrygorii Strashko * iteration if possible to minimize the total 5323277e8aaSGrygorii Strashko * of writes 5333277e8aaSGrygorii Strashko */ 5343277e8aaSGrygorii Strashko if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) 5353277e8aaSGrygorii Strashko db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; 5363277e8aaSGrygorii Strashko else 5373277e8aaSGrygorii Strashko db_ring_cnt_cur = db_ring_cnt; 5383277e8aaSGrygorii Strashko 5393277e8aaSGrygorii Strashko writel(db_ring_cnt_cur, &ring->rt->db); 5403277e8aaSGrygorii Strashko db_ring_cnt -= db_ring_cnt_cur; 5413277e8aaSGrygorii Strashko } 5423277e8aaSGrygorii Strashko 5433277e8aaSGrygorii Strashko /* Restore the original ring mode (if not ring mode) */ 5443277e8aaSGrygorii Strashko if (ring->mode != K3_RINGACC_RING_MODE_RING) 5453277e8aaSGrygorii Strashko k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); 5463277e8aaSGrygorii Strashko } 5473277e8aaSGrygorii Strashko 5483277e8aaSGrygorii Strashko reset: 5493277e8aaSGrygorii Strashko /* Reset the ring */ 5503277e8aaSGrygorii Strashko k3_ringacc_ring_reset(ring); 5513277e8aaSGrygorii Strashko } 5523277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); 5533277e8aaSGrygorii Strashko 5543277e8aaSGrygorii Strashko static void k3_ringacc_ring_free_sci(struct k3_ring *ring) 5553277e8aaSGrygorii Strashko { 556bb49ca00SPeter Ujfalusi struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 5573277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 5583277e8aaSGrygorii Strashko int ret; 5593277e8aaSGrygorii Strashko 560bb49ca00SPeter Ujfalusi ring_cfg.nav_id = ringacc->tisci_dev_id; 561bb49ca00SPeter Ujfalusi ring_cfg.index = ring->ring_id; 562bb49ca00SPeter Ujfalusi ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 563bb49ca00SPeter Ujfalusi 564bb49ca00SPeter Ujfalusi ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 5653277e8aaSGrygorii Strashko if (ret) 5663277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", 5673277e8aaSGrygorii Strashko ret, ring->ring_id); 5683277e8aaSGrygorii Strashko } 5693277e8aaSGrygorii Strashko 5703277e8aaSGrygorii Strashko int k3_ringacc_ring_free(struct k3_ring *ring) 5713277e8aaSGrygorii Strashko { 5723277e8aaSGrygorii Strashko struct k3_ringacc *ringacc; 5733277e8aaSGrygorii Strashko 5743277e8aaSGrygorii Strashko if (!ring) 5753277e8aaSGrygorii Strashko return -EINVAL; 5763277e8aaSGrygorii Strashko 5773277e8aaSGrygorii Strashko ringacc = ring->parent; 5783277e8aaSGrygorii Strashko 579d782298cSGrygorii Strashko /* 580d782298cSGrygorii Strashko * DMA rings: rings shared memory and configuration, only forward ring 581d782298cSGrygorii Strashko * is configured and reverse ring considered as slave. 582d782298cSGrygorii Strashko */ 583d782298cSGrygorii Strashko if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) 584d782298cSGrygorii Strashko return 0; 585d782298cSGrygorii Strashko 5863277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); 5873277e8aaSGrygorii Strashko 5883277e8aaSGrygorii Strashko if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 5893277e8aaSGrygorii Strashko return -EINVAL; 5903277e8aaSGrygorii Strashko 5913277e8aaSGrygorii Strashko mutex_lock(&ringacc->req_lock); 5923277e8aaSGrygorii Strashko 5933277e8aaSGrygorii Strashko if (--ring->use_count) 5943277e8aaSGrygorii Strashko goto out; 5953277e8aaSGrygorii Strashko 5963277e8aaSGrygorii Strashko if (!(ring->flags & K3_RING_FLAG_BUSY)) 5973277e8aaSGrygorii Strashko goto no_init; 5983277e8aaSGrygorii Strashko 5993277e8aaSGrygorii Strashko k3_ringacc_ring_free_sci(ring); 6003277e8aaSGrygorii Strashko 6018c42379eSPeter Ujfalusi dma_free_coherent(ring->dma_dev, 6023277e8aaSGrygorii Strashko ring->size * (4 << ring->elm_size), 6033277e8aaSGrygorii Strashko ring->ring_mem_virt, ring->ring_mem_dma); 6043277e8aaSGrygorii Strashko ring->flags = 0; 6053277e8aaSGrygorii Strashko ring->ops = NULL; 6068c42379eSPeter Ujfalusi ring->dma_dev = NULL; 607d782298cSGrygorii Strashko ring->asel = 0; 608d782298cSGrygorii Strashko 6093277e8aaSGrygorii Strashko if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { 6103277e8aaSGrygorii Strashko clear_bit(ring->proxy_id, ringacc->proxy_inuse); 6113277e8aaSGrygorii Strashko ring->proxy = NULL; 6123277e8aaSGrygorii Strashko ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 6133277e8aaSGrygorii Strashko } 6143277e8aaSGrygorii Strashko 6153277e8aaSGrygorii Strashko no_init: 6163277e8aaSGrygorii Strashko clear_bit(ring->ring_id, ringacc->rings_inuse); 6173277e8aaSGrygorii Strashko 6183277e8aaSGrygorii Strashko out: 6193277e8aaSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 6203277e8aaSGrygorii Strashko return 0; 6213277e8aaSGrygorii Strashko } 6223277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); 6233277e8aaSGrygorii Strashko 6243277e8aaSGrygorii Strashko u32 k3_ringacc_get_ring_id(struct k3_ring *ring) 6253277e8aaSGrygorii Strashko { 6263277e8aaSGrygorii Strashko if (!ring) 6273277e8aaSGrygorii Strashko return -EINVAL; 6283277e8aaSGrygorii Strashko 6293277e8aaSGrygorii Strashko return ring->ring_id; 6303277e8aaSGrygorii Strashko } 6313277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); 6323277e8aaSGrygorii Strashko 6333277e8aaSGrygorii Strashko u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) 6343277e8aaSGrygorii Strashko { 6353277e8aaSGrygorii Strashko if (!ring) 6363277e8aaSGrygorii Strashko return -EINVAL; 6373277e8aaSGrygorii Strashko 6383277e8aaSGrygorii Strashko return ring->parent->tisci_dev_id; 6393277e8aaSGrygorii Strashko } 6403277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); 6413277e8aaSGrygorii Strashko 6423277e8aaSGrygorii Strashko int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) 6433277e8aaSGrygorii Strashko { 6443277e8aaSGrygorii Strashko int irq_num; 6453277e8aaSGrygorii Strashko 6463277e8aaSGrygorii Strashko if (!ring) 6473277e8aaSGrygorii Strashko return -EINVAL; 6483277e8aaSGrygorii Strashko 6493277e8aaSGrygorii Strashko irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); 6503277e8aaSGrygorii Strashko if (irq_num <= 0) 6513277e8aaSGrygorii Strashko irq_num = -EINVAL; 6523277e8aaSGrygorii Strashko return irq_num; 6533277e8aaSGrygorii Strashko } 6543277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); 6553277e8aaSGrygorii Strashko 6563277e8aaSGrygorii Strashko static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) 6573277e8aaSGrygorii Strashko { 658bb49ca00SPeter Ujfalusi struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 6593277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 6603277e8aaSGrygorii Strashko int ret; 6613277e8aaSGrygorii Strashko 6623277e8aaSGrygorii Strashko if (!ringacc->tisci) 6633277e8aaSGrygorii Strashko return -EINVAL; 6643277e8aaSGrygorii Strashko 665bb49ca00SPeter Ujfalusi ring_cfg.nav_id = ringacc->tisci_dev_id; 666bb49ca00SPeter Ujfalusi ring_cfg.index = ring->ring_id; 667bb49ca00SPeter Ujfalusi ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 668bb49ca00SPeter Ujfalusi ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma); 669bb49ca00SPeter Ujfalusi ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma); 670bb49ca00SPeter Ujfalusi ring_cfg.count = ring->size; 671bb49ca00SPeter Ujfalusi ring_cfg.mode = ring->mode; 672bb49ca00SPeter Ujfalusi ring_cfg.size = ring->elm_size; 673d782298cSGrygorii Strashko ring_cfg.asel = ring->asel; 674bb49ca00SPeter Ujfalusi 675bb49ca00SPeter Ujfalusi ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 6763277e8aaSGrygorii Strashko if (ret) 6773277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", 678bb49ca00SPeter Ujfalusi ret, ring->ring_id); 6793277e8aaSGrygorii Strashko 6803277e8aaSGrygorii Strashko return ret; 6813277e8aaSGrygorii Strashko } 6823277e8aaSGrygorii Strashko 683d782298cSGrygorii Strashko static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 684d782298cSGrygorii Strashko { 685d782298cSGrygorii Strashko struct k3_ringacc *ringacc; 686d782298cSGrygorii Strashko struct k3_ring *reverse_ring; 687d782298cSGrygorii Strashko int ret = 0; 688d782298cSGrygorii Strashko 689d782298cSGrygorii Strashko if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 || 690d782298cSGrygorii Strashko cfg->mode != K3_RINGACC_RING_MODE_RING || 691d782298cSGrygorii Strashko cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK) 692d782298cSGrygorii Strashko return -EINVAL; 693d782298cSGrygorii Strashko 694d782298cSGrygorii Strashko ringacc = ring->parent; 695d782298cSGrygorii Strashko 696d782298cSGrygorii Strashko /* 697d782298cSGrygorii Strashko * DMA rings: rings shared memory and configuration, only forward ring 698d782298cSGrygorii Strashko * is configured and reverse ring considered as slave. 699d782298cSGrygorii Strashko */ 700d782298cSGrygorii Strashko if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) 701d782298cSGrygorii Strashko return 0; 702d782298cSGrygorii Strashko 703d782298cSGrygorii Strashko if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 704d782298cSGrygorii Strashko return -EINVAL; 705d782298cSGrygorii Strashko 706d782298cSGrygorii Strashko ring->size = cfg->size; 707d782298cSGrygorii Strashko ring->elm_size = cfg->elm_size; 708d782298cSGrygorii Strashko ring->mode = cfg->mode; 709d782298cSGrygorii Strashko ring->asel = cfg->asel; 710d782298cSGrygorii Strashko ring->dma_dev = cfg->dma_dev; 711d782298cSGrygorii Strashko if (!ring->dma_dev) { 712d782298cSGrygorii Strashko dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n", 713d782298cSGrygorii Strashko ring->ring_id); 714d782298cSGrygorii Strashko ring->dma_dev = ringacc->dev; 715d782298cSGrygorii Strashko } 716d782298cSGrygorii Strashko 717d782298cSGrygorii Strashko memset(&ring->state, 0, sizeof(ring->state)); 718d782298cSGrygorii Strashko 719d782298cSGrygorii Strashko ring->ops = &k3_dmaring_fwd_ops; 720d782298cSGrygorii Strashko 721d782298cSGrygorii Strashko ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, 722d782298cSGrygorii Strashko ring->size * (4 << ring->elm_size), 723d782298cSGrygorii Strashko &ring->ring_mem_dma, GFP_KERNEL); 724d782298cSGrygorii Strashko if (!ring->ring_mem_virt) { 725d782298cSGrygorii Strashko dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 726d782298cSGrygorii Strashko ret = -ENOMEM; 727d782298cSGrygorii Strashko goto err_free_ops; 728d782298cSGrygorii Strashko } 729d782298cSGrygorii Strashko 730d782298cSGrygorii Strashko ret = k3_ringacc_ring_cfg_sci(ring); 731d782298cSGrygorii Strashko if (ret) 732d782298cSGrygorii Strashko goto err_free_mem; 733d782298cSGrygorii Strashko 734d782298cSGrygorii Strashko ring->flags |= K3_RING_FLAG_BUSY; 735d782298cSGrygorii Strashko 736d782298cSGrygorii Strashko k3_ringacc_ring_dump(ring); 737d782298cSGrygorii Strashko 738d782298cSGrygorii Strashko /* DMA rings: configure reverse ring */ 739d782298cSGrygorii Strashko reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings]; 740d782298cSGrygorii Strashko reverse_ring->size = cfg->size; 741d782298cSGrygorii Strashko reverse_ring->elm_size = cfg->elm_size; 742d782298cSGrygorii Strashko reverse_ring->mode = cfg->mode; 743d782298cSGrygorii Strashko reverse_ring->asel = cfg->asel; 744d782298cSGrygorii Strashko memset(&reverse_ring->state, 0, sizeof(reverse_ring->state)); 745d782298cSGrygorii Strashko reverse_ring->ops = &k3_dmaring_reverse_ops; 746d782298cSGrygorii Strashko 747d782298cSGrygorii Strashko reverse_ring->ring_mem_virt = ring->ring_mem_virt; 748d782298cSGrygorii Strashko reverse_ring->ring_mem_dma = ring->ring_mem_dma; 749d782298cSGrygorii Strashko reverse_ring->flags |= K3_RING_FLAG_BUSY; 750d782298cSGrygorii Strashko k3_ringacc_ring_dump(reverse_ring); 751d782298cSGrygorii Strashko 752d782298cSGrygorii Strashko return 0; 753d782298cSGrygorii Strashko 754d782298cSGrygorii Strashko err_free_mem: 755d782298cSGrygorii Strashko dma_free_coherent(ring->dma_dev, 756d782298cSGrygorii Strashko ring->size * (4 << ring->elm_size), 757d782298cSGrygorii Strashko ring->ring_mem_virt, 758d782298cSGrygorii Strashko ring->ring_mem_dma); 759d782298cSGrygorii Strashko err_free_ops: 760d782298cSGrygorii Strashko ring->ops = NULL; 761d782298cSGrygorii Strashko ring->proxy = NULL; 762d782298cSGrygorii Strashko ring->dma_dev = NULL; 763d782298cSGrygorii Strashko ring->asel = 0; 764d782298cSGrygorii Strashko return ret; 765d782298cSGrygorii Strashko } 766d782298cSGrygorii Strashko 7673277e8aaSGrygorii Strashko int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 7683277e8aaSGrygorii Strashko { 76980ff73f7SGrygorii Strashko struct k3_ringacc *ringacc; 7703277e8aaSGrygorii Strashko int ret = 0; 7713277e8aaSGrygorii Strashko 7723277e8aaSGrygorii Strashko if (!ring || !cfg) 7733277e8aaSGrygorii Strashko return -EINVAL; 774d782298cSGrygorii Strashko 77580ff73f7SGrygorii Strashko ringacc = ring->parent; 77680ff73f7SGrygorii Strashko 777d782298cSGrygorii Strashko if (ringacc->dma_rings) 778d782298cSGrygorii Strashko return k3_dmaring_cfg(ring, cfg); 779d782298cSGrygorii Strashko 7803277e8aaSGrygorii Strashko if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || 7813277e8aaSGrygorii Strashko cfg->mode >= K3_RINGACC_RING_MODE_INVALID || 7823277e8aaSGrygorii Strashko cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || 7833277e8aaSGrygorii Strashko !test_bit(ring->ring_id, ringacc->rings_inuse)) 7843277e8aaSGrygorii Strashko return -EINVAL; 7853277e8aaSGrygorii Strashko 7863277e8aaSGrygorii Strashko if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && 7873277e8aaSGrygorii Strashko ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && 7883277e8aaSGrygorii Strashko cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { 7893277e8aaSGrygorii Strashko dev_err(ringacc->dev, 7903277e8aaSGrygorii Strashko "Message mode must use proxy for %u element size\n", 7913277e8aaSGrygorii Strashko 4 << ring->elm_size); 7923277e8aaSGrygorii Strashko return -EINVAL; 7933277e8aaSGrygorii Strashko } 7943277e8aaSGrygorii Strashko 7953277e8aaSGrygorii Strashko /* 7963277e8aaSGrygorii Strashko * In case of shared ring only the first user (master user) can 7973277e8aaSGrygorii Strashko * configure the ring. The sequence should be by the client: 7983277e8aaSGrygorii Strashko * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user 7993277e8aaSGrygorii Strashko * k3_ringacc_ring_cfg(ring, cfg); # master configuration 8003277e8aaSGrygorii Strashko * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 8013277e8aaSGrygorii Strashko * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 8023277e8aaSGrygorii Strashko */ 8033277e8aaSGrygorii Strashko if (ring->use_count != 1) 8043277e8aaSGrygorii Strashko return 0; 8053277e8aaSGrygorii Strashko 8063277e8aaSGrygorii Strashko ring->size = cfg->size; 8073277e8aaSGrygorii Strashko ring->elm_size = cfg->elm_size; 8083277e8aaSGrygorii Strashko ring->mode = cfg->mode; 8096b3da0b4SPeter Ujfalusi memset(&ring->state, 0, sizeof(ring->state)); 8103277e8aaSGrygorii Strashko 8113277e8aaSGrygorii Strashko if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) 8123277e8aaSGrygorii Strashko ring->proxy = ringacc->proxy_target_base + 8133277e8aaSGrygorii Strashko ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; 8143277e8aaSGrygorii Strashko 8153277e8aaSGrygorii Strashko switch (ring->mode) { 8163277e8aaSGrygorii Strashko case K3_RINGACC_RING_MODE_RING: 8173277e8aaSGrygorii Strashko ring->ops = &k3_ring_mode_ring_ops; 8188c42379eSPeter Ujfalusi ring->dma_dev = cfg->dma_dev; 8198c42379eSPeter Ujfalusi if (!ring->dma_dev) 8208c42379eSPeter Ujfalusi ring->dma_dev = ringacc->dev; 8213277e8aaSGrygorii Strashko break; 8223277e8aaSGrygorii Strashko case K3_RINGACC_RING_MODE_MESSAGE: 8238c42379eSPeter Ujfalusi ring->dma_dev = ringacc->dev; 8243277e8aaSGrygorii Strashko if (ring->proxy) 8253277e8aaSGrygorii Strashko ring->ops = &k3_ring_mode_proxy_ops; 8263277e8aaSGrygorii Strashko else 8273277e8aaSGrygorii Strashko ring->ops = &k3_ring_mode_msg_ops; 8283277e8aaSGrygorii Strashko break; 8293277e8aaSGrygorii Strashko default: 8303277e8aaSGrygorii Strashko ring->ops = NULL; 8313277e8aaSGrygorii Strashko ret = -EINVAL; 8323277e8aaSGrygorii Strashko goto err_free_proxy; 8331d036016Skernel test robot } 8343277e8aaSGrygorii Strashko 8358c42379eSPeter Ujfalusi ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, 8363277e8aaSGrygorii Strashko ring->size * (4 << ring->elm_size), 8373277e8aaSGrygorii Strashko &ring->ring_mem_dma, GFP_KERNEL); 8383277e8aaSGrygorii Strashko if (!ring->ring_mem_virt) { 8393277e8aaSGrygorii Strashko dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 8403277e8aaSGrygorii Strashko ret = -ENOMEM; 8413277e8aaSGrygorii Strashko goto err_free_ops; 8423277e8aaSGrygorii Strashko } 8433277e8aaSGrygorii Strashko 8443277e8aaSGrygorii Strashko ret = k3_ringacc_ring_cfg_sci(ring); 8453277e8aaSGrygorii Strashko 8463277e8aaSGrygorii Strashko if (ret) 8473277e8aaSGrygorii Strashko goto err_free_mem; 8483277e8aaSGrygorii Strashko 8493277e8aaSGrygorii Strashko ring->flags |= K3_RING_FLAG_BUSY; 8503277e8aaSGrygorii Strashko ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? 8513277e8aaSGrygorii Strashko K3_RING_FLAG_SHARED : 0; 8523277e8aaSGrygorii Strashko 8533277e8aaSGrygorii Strashko k3_ringacc_ring_dump(ring); 8543277e8aaSGrygorii Strashko 8553277e8aaSGrygorii Strashko return 0; 8563277e8aaSGrygorii Strashko 8573277e8aaSGrygorii Strashko err_free_mem: 8588c42379eSPeter Ujfalusi dma_free_coherent(ring->dma_dev, 8593277e8aaSGrygorii Strashko ring->size * (4 << ring->elm_size), 8603277e8aaSGrygorii Strashko ring->ring_mem_virt, 8613277e8aaSGrygorii Strashko ring->ring_mem_dma); 8623277e8aaSGrygorii Strashko err_free_ops: 8633277e8aaSGrygorii Strashko ring->ops = NULL; 8648c42379eSPeter Ujfalusi ring->dma_dev = NULL; 8653277e8aaSGrygorii Strashko err_free_proxy: 8663277e8aaSGrygorii Strashko ring->proxy = NULL; 8673277e8aaSGrygorii Strashko return ret; 8683277e8aaSGrygorii Strashko } 8693277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); 8703277e8aaSGrygorii Strashko 8713277e8aaSGrygorii Strashko u32 k3_ringacc_ring_get_size(struct k3_ring *ring) 8723277e8aaSGrygorii Strashko { 8733277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 8743277e8aaSGrygorii Strashko return -EINVAL; 8753277e8aaSGrygorii Strashko 8763277e8aaSGrygorii Strashko return ring->size; 8773277e8aaSGrygorii Strashko } 8783277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); 8793277e8aaSGrygorii Strashko 8803277e8aaSGrygorii Strashko u32 k3_ringacc_ring_get_free(struct k3_ring *ring) 8813277e8aaSGrygorii Strashko { 8823277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 8833277e8aaSGrygorii Strashko return -EINVAL; 8843277e8aaSGrygorii Strashko 8856b3da0b4SPeter Ujfalusi if (!ring->state.free) 886d782298cSGrygorii Strashko ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring); 8873277e8aaSGrygorii Strashko 8886b3da0b4SPeter Ujfalusi return ring->state.free; 8893277e8aaSGrygorii Strashko } 8903277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); 8913277e8aaSGrygorii Strashko 8923277e8aaSGrygorii Strashko u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) 8933277e8aaSGrygorii Strashko { 8943277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 8953277e8aaSGrygorii Strashko return -EINVAL; 8963277e8aaSGrygorii Strashko 897d782298cSGrygorii Strashko return k3_ringacc_ring_read_occ(ring); 8983277e8aaSGrygorii Strashko } 8993277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); 9003277e8aaSGrygorii Strashko 9013277e8aaSGrygorii Strashko u32 k3_ringacc_ring_is_full(struct k3_ring *ring) 9023277e8aaSGrygorii Strashko { 9033277e8aaSGrygorii Strashko return !k3_ringacc_ring_get_free(ring); 9043277e8aaSGrygorii Strashko } 9053277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); 9063277e8aaSGrygorii Strashko 9073277e8aaSGrygorii Strashko enum k3_ringacc_access_mode { 9083277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_HEAD, 9093277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD, 9103277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_TAIL, 9113277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_TAIL, 9123277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PEEK_HEAD, 9133277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PEEK_TAIL, 9143277e8aaSGrygorii Strashko }; 9153277e8aaSGrygorii Strashko 9163277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) 9173277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) 9183277e8aaSGrygorii Strashko static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, 9193277e8aaSGrygorii Strashko enum k3_ringacc_proxy_access_mode mode) 9203277e8aaSGrygorii Strashko { 9213277e8aaSGrygorii Strashko u32 val; 9223277e8aaSGrygorii Strashko 9233277e8aaSGrygorii Strashko val = ring->ring_id; 9243277e8aaSGrygorii Strashko val |= K3_RINGACC_PROXY_MODE(mode); 9253277e8aaSGrygorii Strashko val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); 9263277e8aaSGrygorii Strashko writel(val, &ring->proxy->control); 9273277e8aaSGrygorii Strashko return 0; 9283277e8aaSGrygorii Strashko } 9293277e8aaSGrygorii Strashko 9303277e8aaSGrygorii Strashko static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, 9313277e8aaSGrygorii Strashko enum k3_ringacc_access_mode access_mode) 9323277e8aaSGrygorii Strashko { 9333277e8aaSGrygorii Strashko void __iomem *ptr; 9343277e8aaSGrygorii Strashko 9353277e8aaSGrygorii Strashko ptr = (void __iomem *)&ring->proxy->data; 9363277e8aaSGrygorii Strashko 9373277e8aaSGrygorii Strashko switch (access_mode) { 9383277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 9393277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 9403277e8aaSGrygorii Strashko k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); 9413277e8aaSGrygorii Strashko break; 9423277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 9433277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 9443277e8aaSGrygorii Strashko k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); 9453277e8aaSGrygorii Strashko break; 9463277e8aaSGrygorii Strashko default: 9473277e8aaSGrygorii Strashko return -EINVAL; 9483277e8aaSGrygorii Strashko } 9493277e8aaSGrygorii Strashko 9503277e8aaSGrygorii Strashko ptr += k3_ringacc_ring_get_fifo_pos(ring); 9513277e8aaSGrygorii Strashko 9523277e8aaSGrygorii Strashko switch (access_mode) { 9533277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 9543277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 9553277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 9563277e8aaSGrygorii Strashko "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 9573277e8aaSGrygorii Strashko access_mode); 9583277e8aaSGrygorii Strashko memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 9596b3da0b4SPeter Ujfalusi ring->state.occ--; 9603277e8aaSGrygorii Strashko break; 9613277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 9623277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 9633277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 9643277e8aaSGrygorii Strashko "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 9653277e8aaSGrygorii Strashko access_mode); 9663277e8aaSGrygorii Strashko memcpy_toio(ptr, elem, (4 << ring->elm_size)); 9676b3da0b4SPeter Ujfalusi ring->state.free--; 9683277e8aaSGrygorii Strashko break; 9693277e8aaSGrygorii Strashko default: 9703277e8aaSGrygorii Strashko return -EINVAL; 9713277e8aaSGrygorii Strashko } 9723277e8aaSGrygorii Strashko 9736b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, 9746b3da0b4SPeter Ujfalusi ring->state.occ); 9753277e8aaSGrygorii Strashko return 0; 9763277e8aaSGrygorii Strashko } 9773277e8aaSGrygorii Strashko 9783277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) 9793277e8aaSGrygorii Strashko { 9803277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 9813277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 9823277e8aaSGrygorii Strashko } 9833277e8aaSGrygorii Strashko 9843277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) 9853277e8aaSGrygorii Strashko { 9863277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 9873277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 9883277e8aaSGrygorii Strashko } 9893277e8aaSGrygorii Strashko 9903277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) 9913277e8aaSGrygorii Strashko { 9923277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 9933277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 9943277e8aaSGrygorii Strashko } 9953277e8aaSGrygorii Strashko 9963277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) 9973277e8aaSGrygorii Strashko { 9983277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 9993277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 10003277e8aaSGrygorii Strashko } 10013277e8aaSGrygorii Strashko 10023277e8aaSGrygorii Strashko static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, 10033277e8aaSGrygorii Strashko enum k3_ringacc_access_mode access_mode) 10043277e8aaSGrygorii Strashko { 10053277e8aaSGrygorii Strashko void __iomem *ptr; 10063277e8aaSGrygorii Strashko 10073277e8aaSGrygorii Strashko switch (access_mode) { 10083277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 10093277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 10103277e8aaSGrygorii Strashko ptr = (void __iomem *)&ring->fifos->head_data; 10113277e8aaSGrygorii Strashko break; 10123277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 10133277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 10143277e8aaSGrygorii Strashko ptr = (void __iomem *)&ring->fifos->tail_data; 10153277e8aaSGrygorii Strashko break; 10163277e8aaSGrygorii Strashko default: 10173277e8aaSGrygorii Strashko return -EINVAL; 10183277e8aaSGrygorii Strashko } 10193277e8aaSGrygorii Strashko 10203277e8aaSGrygorii Strashko ptr += k3_ringacc_ring_get_fifo_pos(ring); 10213277e8aaSGrygorii Strashko 10223277e8aaSGrygorii Strashko switch (access_mode) { 10233277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 10243277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 10253277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 10263277e8aaSGrygorii Strashko "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 10273277e8aaSGrygorii Strashko access_mode); 10283277e8aaSGrygorii Strashko memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 10296b3da0b4SPeter Ujfalusi ring->state.occ--; 10303277e8aaSGrygorii Strashko break; 10313277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 10323277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 10333277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 10343277e8aaSGrygorii Strashko "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 10353277e8aaSGrygorii Strashko access_mode); 10363277e8aaSGrygorii Strashko memcpy_toio(ptr, elem, (4 << ring->elm_size)); 10376b3da0b4SPeter Ujfalusi ring->state.free--; 10383277e8aaSGrygorii Strashko break; 10393277e8aaSGrygorii Strashko default: 10403277e8aaSGrygorii Strashko return -EINVAL; 10413277e8aaSGrygorii Strashko } 10423277e8aaSGrygorii Strashko 10436b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", 10446b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex, ring->state.occ, 10456b3da0b4SPeter Ujfalusi ring->state.rindex); 10463277e8aaSGrygorii Strashko return 0; 10473277e8aaSGrygorii Strashko } 10483277e8aaSGrygorii Strashko 10493277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) 10503277e8aaSGrygorii Strashko { 10513277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 10523277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 10533277e8aaSGrygorii Strashko } 10543277e8aaSGrygorii Strashko 10553277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) 10563277e8aaSGrygorii Strashko { 10573277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 10583277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 10593277e8aaSGrygorii Strashko } 10603277e8aaSGrygorii Strashko 10613277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) 10623277e8aaSGrygorii Strashko { 10633277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 10643277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 10653277e8aaSGrygorii Strashko } 10663277e8aaSGrygorii Strashko 10673277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) 10683277e8aaSGrygorii Strashko { 10693277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 10703277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 10713277e8aaSGrygorii Strashko } 10723277e8aaSGrygorii Strashko 1073d782298cSGrygorii Strashko /* 1074d782298cSGrygorii Strashko * The element is 48 bits of address + ASEL bits in the ring. 1075d782298cSGrygorii Strashko * ASEL is used by the DMAs and should be removed for the kernel as it is not 1076d782298cSGrygorii Strashko * part of the physical memory address. 1077d782298cSGrygorii Strashko */ 1078d782298cSGrygorii Strashko static void k3_dmaring_remove_asel_from_elem(u64 *elem) 1079d782298cSGrygorii Strashko { 1080d782298cSGrygorii Strashko *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0); 1081d782298cSGrygorii Strashko } 1082d782298cSGrygorii Strashko 1083d782298cSGrygorii Strashko static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem) 1084d782298cSGrygorii Strashko { 1085d782298cSGrygorii Strashko void *elem_ptr; 1086d782298cSGrygorii Strashko u32 elem_idx; 1087d782298cSGrygorii Strashko 1088d782298cSGrygorii Strashko /* 1089d782298cSGrygorii Strashko * DMA rings: forward ring is always tied DMA channel and HW does not 1090d782298cSGrygorii Strashko * maintain any state data required for POP operation and its unknown 1091d782298cSGrygorii Strashko * how much elements were consumed by HW. So, to actually 1092d782298cSGrygorii Strashko * do POP, the read pointer has to be recalculated every time. 1093d782298cSGrygorii Strashko */ 1094d782298cSGrygorii Strashko ring->state.occ = k3_ringacc_ring_read_occ(ring); 1095d782298cSGrygorii Strashko if (ring->state.windex >= ring->state.occ) 1096d782298cSGrygorii Strashko elem_idx = ring->state.windex - ring->state.occ; 1097d782298cSGrygorii Strashko else 1098d782298cSGrygorii Strashko elem_idx = ring->size - (ring->state.occ - ring->state.windex); 1099d782298cSGrygorii Strashko 1100d782298cSGrygorii Strashko elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx); 1101d782298cSGrygorii Strashko memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1102d782298cSGrygorii Strashko k3_dmaring_remove_asel_from_elem(elem); 1103d782298cSGrygorii Strashko 1104d782298cSGrygorii Strashko ring->state.occ--; 1105d782298cSGrygorii Strashko writel(-1, &ring->rt->db); 1106d782298cSGrygorii Strashko 1107d782298cSGrygorii Strashko dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n", 1108d782298cSGrygorii Strashko __func__, ring->state.occ, ring->state.windex, elem_idx, 1109d782298cSGrygorii Strashko elem_ptr); 1110d782298cSGrygorii Strashko return 0; 1111d782298cSGrygorii Strashko } 1112d782298cSGrygorii Strashko 1113d782298cSGrygorii Strashko static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem) 1114d782298cSGrygorii Strashko { 1115d782298cSGrygorii Strashko void *elem_ptr; 1116d782298cSGrygorii Strashko 1117d782298cSGrygorii Strashko elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 1118d782298cSGrygorii Strashko 1119d782298cSGrygorii Strashko if (ring->state.occ) { 1120d782298cSGrygorii Strashko memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1121d782298cSGrygorii Strashko k3_dmaring_remove_asel_from_elem(elem); 1122d782298cSGrygorii Strashko 1123d782298cSGrygorii Strashko ring->state.rindex = (ring->state.rindex + 1) % ring->size; 1124d782298cSGrygorii Strashko ring->state.occ--; 1125d782298cSGrygorii Strashko writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db); 1126d782298cSGrygorii Strashko } else if (ring->state.tdown_complete) { 1127d782298cSGrygorii Strashko dma_addr_t *value = elem; 1128d782298cSGrygorii Strashko 1129d782298cSGrygorii Strashko *value = CPPI5_TDCM_MARKER; 1130d782298cSGrygorii Strashko writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db); 1131d782298cSGrygorii Strashko ring->state.tdown_complete = false; 1132d782298cSGrygorii Strashko } 1133d782298cSGrygorii Strashko 1134d782298cSGrygorii Strashko dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n", 1135d782298cSGrygorii Strashko __func__, ring->state.occ, ring->state.rindex, elem_ptr); 1136d782298cSGrygorii Strashko return 0; 1137d782298cSGrygorii Strashko } 1138d782298cSGrygorii Strashko 11393277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) 11403277e8aaSGrygorii Strashko { 11413277e8aaSGrygorii Strashko void *elem_ptr; 11423277e8aaSGrygorii Strashko 11436b3da0b4SPeter Ujfalusi elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); 11443277e8aaSGrygorii Strashko 11453277e8aaSGrygorii Strashko memcpy(elem_ptr, elem, (4 << ring->elm_size)); 1146d782298cSGrygorii Strashko if (ring->parent->dma_rings) { 1147d782298cSGrygorii Strashko u64 *addr = elem_ptr; 1148d782298cSGrygorii Strashko 1149d782298cSGrygorii Strashko *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT); 1150d782298cSGrygorii Strashko } 11513277e8aaSGrygorii Strashko 11526b3da0b4SPeter Ujfalusi ring->state.windex = (ring->state.windex + 1) % ring->size; 11536b3da0b4SPeter Ujfalusi ring->state.free--; 11543277e8aaSGrygorii Strashko writel(1, &ring->rt->db); 11553277e8aaSGrygorii Strashko 11563277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", 11576b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex); 11583277e8aaSGrygorii Strashko 11593277e8aaSGrygorii Strashko return 0; 11603277e8aaSGrygorii Strashko } 11613277e8aaSGrygorii Strashko 11623277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) 11633277e8aaSGrygorii Strashko { 11643277e8aaSGrygorii Strashko void *elem_ptr; 11653277e8aaSGrygorii Strashko 11666b3da0b4SPeter Ujfalusi elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 11673277e8aaSGrygorii Strashko 11683277e8aaSGrygorii Strashko memcpy(elem, elem_ptr, (4 << ring->elm_size)); 11693277e8aaSGrygorii Strashko 11706b3da0b4SPeter Ujfalusi ring->state.rindex = (ring->state.rindex + 1) % ring->size; 11716b3da0b4SPeter Ujfalusi ring->state.occ--; 11723277e8aaSGrygorii Strashko writel(-1, &ring->rt->db); 11733277e8aaSGrygorii Strashko 11743277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", 11756b3da0b4SPeter Ujfalusi ring->state.occ, ring->state.rindex, elem_ptr); 11763277e8aaSGrygorii Strashko return 0; 11773277e8aaSGrygorii Strashko } 11783277e8aaSGrygorii Strashko 11793277e8aaSGrygorii Strashko int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) 11803277e8aaSGrygorii Strashko { 11813277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 11823277e8aaSGrygorii Strashko 11833277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 11843277e8aaSGrygorii Strashko return -EINVAL; 11853277e8aaSGrygorii Strashko 11866b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", 11876b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex); 11883277e8aaSGrygorii Strashko 11893277e8aaSGrygorii Strashko if (k3_ringacc_ring_is_full(ring)) 11903277e8aaSGrygorii Strashko return -ENOMEM; 11913277e8aaSGrygorii Strashko 11923277e8aaSGrygorii Strashko if (ring->ops && ring->ops->push_tail) 11933277e8aaSGrygorii Strashko ret = ring->ops->push_tail(ring, elem); 11943277e8aaSGrygorii Strashko 11953277e8aaSGrygorii Strashko return ret; 11963277e8aaSGrygorii Strashko } 11973277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); 11983277e8aaSGrygorii Strashko 11993277e8aaSGrygorii Strashko int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) 12003277e8aaSGrygorii Strashko { 12013277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 12023277e8aaSGrygorii Strashko 12033277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 12043277e8aaSGrygorii Strashko return -EINVAL; 12053277e8aaSGrygorii Strashko 12063277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", 12076b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex); 12083277e8aaSGrygorii Strashko 12093277e8aaSGrygorii Strashko if (k3_ringacc_ring_is_full(ring)) 12103277e8aaSGrygorii Strashko return -ENOMEM; 12113277e8aaSGrygorii Strashko 12123277e8aaSGrygorii Strashko if (ring->ops && ring->ops->push_head) 12133277e8aaSGrygorii Strashko ret = ring->ops->push_head(ring, elem); 12143277e8aaSGrygorii Strashko 12153277e8aaSGrygorii Strashko return ret; 12163277e8aaSGrygorii Strashko } 12173277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); 12183277e8aaSGrygorii Strashko 12193277e8aaSGrygorii Strashko int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) 12203277e8aaSGrygorii Strashko { 12213277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 12223277e8aaSGrygorii Strashko 12233277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 12243277e8aaSGrygorii Strashko return -EINVAL; 12253277e8aaSGrygorii Strashko 12266b3da0b4SPeter Ujfalusi if (!ring->state.occ) 1227d782298cSGrygorii Strashko k3_ringacc_ring_update_occ(ring); 12283277e8aaSGrygorii Strashko 12296b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, 12306b3da0b4SPeter Ujfalusi ring->state.rindex); 12313277e8aaSGrygorii Strashko 1232d782298cSGrygorii Strashko if (!ring->state.occ && !ring->state.tdown_complete) 12333277e8aaSGrygorii Strashko return -ENODATA; 12343277e8aaSGrygorii Strashko 12353277e8aaSGrygorii Strashko if (ring->ops && ring->ops->pop_head) 12363277e8aaSGrygorii Strashko ret = ring->ops->pop_head(ring, elem); 12373277e8aaSGrygorii Strashko 12383277e8aaSGrygorii Strashko return ret; 12393277e8aaSGrygorii Strashko } 12403277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); 12413277e8aaSGrygorii Strashko 12423277e8aaSGrygorii Strashko int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) 12433277e8aaSGrygorii Strashko { 12443277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 12453277e8aaSGrygorii Strashko 12463277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 12473277e8aaSGrygorii Strashko return -EINVAL; 12483277e8aaSGrygorii Strashko 12496b3da0b4SPeter Ujfalusi if (!ring->state.occ) 1250d782298cSGrygorii Strashko k3_ringacc_ring_update_occ(ring); 12513277e8aaSGrygorii Strashko 12526b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", 12536b3da0b4SPeter Ujfalusi ring->state.occ, ring->state.rindex); 12543277e8aaSGrygorii Strashko 12556b3da0b4SPeter Ujfalusi if (!ring->state.occ) 12563277e8aaSGrygorii Strashko return -ENODATA; 12573277e8aaSGrygorii Strashko 12583277e8aaSGrygorii Strashko if (ring->ops && ring->ops->pop_tail) 12593277e8aaSGrygorii Strashko ret = ring->ops->pop_tail(ring, elem); 12603277e8aaSGrygorii Strashko 12613277e8aaSGrygorii Strashko return ret; 12623277e8aaSGrygorii Strashko } 12633277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); 12643277e8aaSGrygorii Strashko 12653277e8aaSGrygorii Strashko struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, 12663277e8aaSGrygorii Strashko const char *property) 12673277e8aaSGrygorii Strashko { 12683277e8aaSGrygorii Strashko struct device_node *ringacc_np; 12693277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); 12703277e8aaSGrygorii Strashko struct k3_ringacc *entry; 12713277e8aaSGrygorii Strashko 12723277e8aaSGrygorii Strashko ringacc_np = of_parse_phandle(np, property, 0); 12733277e8aaSGrygorii Strashko if (!ringacc_np) 12743277e8aaSGrygorii Strashko return ERR_PTR(-ENODEV); 12753277e8aaSGrygorii Strashko 12763277e8aaSGrygorii Strashko mutex_lock(&k3_ringacc_list_lock); 12773277e8aaSGrygorii Strashko list_for_each_entry(entry, &k3_ringacc_list, list) 12783277e8aaSGrygorii Strashko if (entry->dev->of_node == ringacc_np) { 12793277e8aaSGrygorii Strashko ringacc = entry; 12803277e8aaSGrygorii Strashko break; 12813277e8aaSGrygorii Strashko } 12823277e8aaSGrygorii Strashko mutex_unlock(&k3_ringacc_list_lock); 12833277e8aaSGrygorii Strashko of_node_put(ringacc_np); 12843277e8aaSGrygorii Strashko 12853277e8aaSGrygorii Strashko return ringacc; 12863277e8aaSGrygorii Strashko } 12873277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); 12883277e8aaSGrygorii Strashko 12893277e8aaSGrygorii Strashko static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) 12903277e8aaSGrygorii Strashko { 12913277e8aaSGrygorii Strashko struct device_node *node = ringacc->dev->of_node; 12923277e8aaSGrygorii Strashko struct device *dev = ringacc->dev; 12933277e8aaSGrygorii Strashko struct platform_device *pdev = to_platform_device(dev); 12943277e8aaSGrygorii Strashko int ret; 12953277e8aaSGrygorii Strashko 12963277e8aaSGrygorii Strashko if (!node) { 12973277e8aaSGrygorii Strashko dev_err(dev, "device tree info unavailable\n"); 12983277e8aaSGrygorii Strashko return -ENODEV; 12993277e8aaSGrygorii Strashko } 13003277e8aaSGrygorii Strashko 13013277e8aaSGrygorii Strashko ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); 13023277e8aaSGrygorii Strashko if (ret) { 13033277e8aaSGrygorii Strashko dev_err(dev, "ti,num-rings read failure %d\n", ret); 13043277e8aaSGrygorii Strashko return ret; 13053277e8aaSGrygorii Strashko } 13063277e8aaSGrygorii Strashko 13073277e8aaSGrygorii Strashko ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); 13083277e8aaSGrygorii Strashko if (IS_ERR(ringacc->tisci)) { 13093277e8aaSGrygorii Strashko ret = PTR_ERR(ringacc->tisci); 13103277e8aaSGrygorii Strashko if (ret != -EPROBE_DEFER) 13113277e8aaSGrygorii Strashko dev_err(dev, "ti,sci read fail %d\n", ret); 13123277e8aaSGrygorii Strashko ringacc->tisci = NULL; 13133277e8aaSGrygorii Strashko return ret; 13143277e8aaSGrygorii Strashko } 13153277e8aaSGrygorii Strashko 13163277e8aaSGrygorii Strashko ret = of_property_read_u32(node, "ti,sci-dev-id", 13173277e8aaSGrygorii Strashko &ringacc->tisci_dev_id); 13183277e8aaSGrygorii Strashko if (ret) { 13193277e8aaSGrygorii Strashko dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); 13203277e8aaSGrygorii Strashko return ret; 13213277e8aaSGrygorii Strashko } 13223277e8aaSGrygorii Strashko 13233277e8aaSGrygorii Strashko pdev->id = ringacc->tisci_dev_id; 13243277e8aaSGrygorii Strashko 13253277e8aaSGrygorii Strashko ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, 13263277e8aaSGrygorii Strashko ringacc->tisci_dev_id, 13273277e8aaSGrygorii Strashko "ti,sci-rm-range-gp-rings"); 13283277e8aaSGrygorii Strashko if (IS_ERR(ringacc->rm_gp_range)) { 13293277e8aaSGrygorii Strashko dev_err(dev, "Failed to allocate MSI interrupts\n"); 13303277e8aaSGrygorii Strashko return PTR_ERR(ringacc->rm_gp_range); 13313277e8aaSGrygorii Strashko } 13323277e8aaSGrygorii Strashko 13333277e8aaSGrygorii Strashko return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, 13343277e8aaSGrygorii Strashko ringacc->rm_gp_range); 13353277e8aaSGrygorii Strashko } 13363277e8aaSGrygorii Strashko 133795e7be06SGrygorii Strashko static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = { 133895e7be06SGrygorii Strashko .dma_ring_reset_quirk = 1, 133995e7be06SGrygorii Strashko }; 134095e7be06SGrygorii Strashko 134195e7be06SGrygorii Strashko static const struct soc_device_attribute k3_ringacc_socinfo[] = { 134295e7be06SGrygorii Strashko { .family = "AM65X", 134395e7be06SGrygorii Strashko .revision = "SR1.0", 134495e7be06SGrygorii Strashko .data = &k3_ringacc_soc_data_sr1 134595e7be06SGrygorii Strashko }, 134695e7be06SGrygorii Strashko {/* sentinel */} 134795e7be06SGrygorii Strashko }; 134895e7be06SGrygorii Strashko 134940a2a7c3SGrygorii Strashko static int k3_ringacc_init(struct platform_device *pdev, 135040a2a7c3SGrygorii Strashko struct k3_ringacc *ringacc) 13513277e8aaSGrygorii Strashko { 135295e7be06SGrygorii Strashko const struct soc_device_attribute *soc; 13533277e8aaSGrygorii Strashko void __iomem *base_fifo, *base_rt; 13543277e8aaSGrygorii Strashko struct device *dev = &pdev->dev; 13553277e8aaSGrygorii Strashko struct resource *res; 13563277e8aaSGrygorii Strashko int ret, i; 13573277e8aaSGrygorii Strashko 13583277e8aaSGrygorii Strashko dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 13593277e8aaSGrygorii Strashko DOMAIN_BUS_TI_SCI_INTA_MSI); 13603277e8aaSGrygorii Strashko if (!dev->msi_domain) { 13613277e8aaSGrygorii Strashko dev_err(dev, "Failed to get MSI domain\n"); 13623277e8aaSGrygorii Strashko return -EPROBE_DEFER; 13633277e8aaSGrygorii Strashko } 13643277e8aaSGrygorii Strashko 13653277e8aaSGrygorii Strashko ret = k3_ringacc_probe_dt(ringacc); 13663277e8aaSGrygorii Strashko if (ret) 13673277e8aaSGrygorii Strashko return ret; 13683277e8aaSGrygorii Strashko 136995e7be06SGrygorii Strashko soc = soc_device_match(k3_ringacc_socinfo); 137095e7be06SGrygorii Strashko if (soc && soc->data) { 137195e7be06SGrygorii Strashko const struct k3_ringacc_soc_data *soc_data = soc->data; 137295e7be06SGrygorii Strashko 137395e7be06SGrygorii Strashko ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk; 137495e7be06SGrygorii Strashko } 137595e7be06SGrygorii Strashko 13763277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); 13773277e8aaSGrygorii Strashko base_rt = devm_ioremap_resource(dev, res); 13783277e8aaSGrygorii Strashko if (IS_ERR(base_rt)) 13793277e8aaSGrygorii Strashko return PTR_ERR(base_rt); 13803277e8aaSGrygorii Strashko 13813277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos"); 13823277e8aaSGrygorii Strashko base_fifo = devm_ioremap_resource(dev, res); 13833277e8aaSGrygorii Strashko if (IS_ERR(base_fifo)) 13843277e8aaSGrygorii Strashko return PTR_ERR(base_fifo); 13853277e8aaSGrygorii Strashko 13863277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg"); 13873277e8aaSGrygorii Strashko ringacc->proxy_gcfg = devm_ioremap_resource(dev, res); 13883277e8aaSGrygorii Strashko if (IS_ERR(ringacc->proxy_gcfg)) 13893277e8aaSGrygorii Strashko return PTR_ERR(ringacc->proxy_gcfg); 13903277e8aaSGrygorii Strashko 13913277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 13923277e8aaSGrygorii Strashko "proxy_target"); 13933277e8aaSGrygorii Strashko ringacc->proxy_target_base = devm_ioremap_resource(dev, res); 13943277e8aaSGrygorii Strashko if (IS_ERR(ringacc->proxy_target_base)) 13953277e8aaSGrygorii Strashko return PTR_ERR(ringacc->proxy_target_base); 13963277e8aaSGrygorii Strashko 13973277e8aaSGrygorii Strashko ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & 13983277e8aaSGrygorii Strashko K3_RINGACC_PROXY_CFG_THREADS_MASK; 13993277e8aaSGrygorii Strashko 14003277e8aaSGrygorii Strashko ringacc->rings = devm_kzalloc(dev, 14013277e8aaSGrygorii Strashko sizeof(*ringacc->rings) * 14023277e8aaSGrygorii Strashko ringacc->num_rings, 14033277e8aaSGrygorii Strashko GFP_KERNEL); 14043277e8aaSGrygorii Strashko ringacc->rings_inuse = devm_kcalloc(dev, 14053277e8aaSGrygorii Strashko BITS_TO_LONGS(ringacc->num_rings), 14063277e8aaSGrygorii Strashko sizeof(unsigned long), GFP_KERNEL); 14073277e8aaSGrygorii Strashko ringacc->proxy_inuse = devm_kcalloc(dev, 14083277e8aaSGrygorii Strashko BITS_TO_LONGS(ringacc->num_proxies), 14093277e8aaSGrygorii Strashko sizeof(unsigned long), GFP_KERNEL); 14103277e8aaSGrygorii Strashko 14113277e8aaSGrygorii Strashko if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) 14123277e8aaSGrygorii Strashko return -ENOMEM; 14133277e8aaSGrygorii Strashko 14143277e8aaSGrygorii Strashko for (i = 0; i < ringacc->num_rings; i++) { 14153277e8aaSGrygorii Strashko ringacc->rings[i].rt = base_rt + 14163277e8aaSGrygorii Strashko K3_RINGACC_RT_REGS_STEP * i; 14173277e8aaSGrygorii Strashko ringacc->rings[i].fifos = base_fifo + 14183277e8aaSGrygorii Strashko K3_RINGACC_FIFO_REGS_STEP * i; 14193277e8aaSGrygorii Strashko ringacc->rings[i].parent = ringacc; 14203277e8aaSGrygorii Strashko ringacc->rings[i].ring_id = i; 14213277e8aaSGrygorii Strashko ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; 14223277e8aaSGrygorii Strashko } 14233277e8aaSGrygorii Strashko 14243277e8aaSGrygorii Strashko ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 14253277e8aaSGrygorii Strashko 14263277e8aaSGrygorii Strashko dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", 14273277e8aaSGrygorii Strashko ringacc->num_rings, 14283277e8aaSGrygorii Strashko ringacc->rm_gp_range->desc[0].start, 14293277e8aaSGrygorii Strashko ringacc->rm_gp_range->desc[0].num, 14303277e8aaSGrygorii Strashko ringacc->tisci_dev_id); 14313277e8aaSGrygorii Strashko dev_info(dev, "dma-ring-reset-quirk: %s\n", 14323277e8aaSGrygorii Strashko ringacc->dma_ring_reset_quirk ? "enabled" : "disabled"); 14333277e8aaSGrygorii Strashko dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", 14343277e8aaSGrygorii Strashko readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); 143540a2a7c3SGrygorii Strashko 14363277e8aaSGrygorii Strashko return 0; 14373277e8aaSGrygorii Strashko } 14383277e8aaSGrygorii Strashko 143940a2a7c3SGrygorii Strashko struct ringacc_match_data { 144040a2a7c3SGrygorii Strashko struct k3_ringacc_ops ops; 144140a2a7c3SGrygorii Strashko }; 144240a2a7c3SGrygorii Strashko 144340a2a7c3SGrygorii Strashko static struct ringacc_match_data k3_ringacc_data = { 144440a2a7c3SGrygorii Strashko .ops = { 144540a2a7c3SGrygorii Strashko .init = k3_ringacc_init, 144640a2a7c3SGrygorii Strashko }, 144740a2a7c3SGrygorii Strashko }; 144840a2a7c3SGrygorii Strashko 14493277e8aaSGrygorii Strashko /* Match table for of_platform binding */ 14503277e8aaSGrygorii Strashko static const struct of_device_id k3_ringacc_of_match[] = { 145140a2a7c3SGrygorii Strashko { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, }, 14523277e8aaSGrygorii Strashko {}, 14533277e8aaSGrygorii Strashko }; 14543277e8aaSGrygorii Strashko 1455d782298cSGrygorii Strashko struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev, 1456d782298cSGrygorii Strashko struct k3_ringacc_init_data *data) 1457d782298cSGrygorii Strashko { 1458d782298cSGrygorii Strashko struct device *dev = &pdev->dev; 1459d782298cSGrygorii Strashko struct k3_ringacc *ringacc; 1460d782298cSGrygorii Strashko void __iomem *base_rt; 1461d782298cSGrygorii Strashko struct resource *res; 1462d782298cSGrygorii Strashko int i; 1463d782298cSGrygorii Strashko 1464d782298cSGrygorii Strashko ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1465d782298cSGrygorii Strashko if (!ringacc) 1466d782298cSGrygorii Strashko return ERR_PTR(-ENOMEM); 1467d782298cSGrygorii Strashko 1468d782298cSGrygorii Strashko ringacc->dev = dev; 1469d782298cSGrygorii Strashko ringacc->dma_rings = true; 1470d782298cSGrygorii Strashko ringacc->num_rings = data->num_rings; 1471d782298cSGrygorii Strashko ringacc->tisci = data->tisci; 1472d782298cSGrygorii Strashko ringacc->tisci_dev_id = data->tisci_dev_id; 1473d782298cSGrygorii Strashko 1474d782298cSGrygorii Strashko mutex_init(&ringacc->req_lock); 1475d782298cSGrygorii Strashko 1476d782298cSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt"); 1477d782298cSGrygorii Strashko base_rt = devm_ioremap_resource(dev, res); 1478d782298cSGrygorii Strashko if (IS_ERR(base_rt)) 1479*115ff12aSPeter Ujfalusi return ERR_CAST(base_rt); 1480d782298cSGrygorii Strashko 1481d782298cSGrygorii Strashko ringacc->rings = devm_kzalloc(dev, 1482d782298cSGrygorii Strashko sizeof(*ringacc->rings) * 1483d782298cSGrygorii Strashko ringacc->num_rings * 2, 1484d782298cSGrygorii Strashko GFP_KERNEL); 1485d782298cSGrygorii Strashko ringacc->rings_inuse = devm_kcalloc(dev, 1486d782298cSGrygorii Strashko BITS_TO_LONGS(ringacc->num_rings), 1487d782298cSGrygorii Strashko sizeof(unsigned long), GFP_KERNEL); 1488d782298cSGrygorii Strashko 1489d782298cSGrygorii Strashko if (!ringacc->rings || !ringacc->rings_inuse) 1490d782298cSGrygorii Strashko return ERR_PTR(-ENOMEM); 1491d782298cSGrygorii Strashko 1492d782298cSGrygorii Strashko for (i = 0; i < ringacc->num_rings; i++) { 1493d782298cSGrygorii Strashko struct k3_ring *ring = &ringacc->rings[i]; 1494d782298cSGrygorii Strashko 1495d782298cSGrygorii Strashko ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i; 1496d782298cSGrygorii Strashko ring->parent = ringacc; 1497d782298cSGrygorii Strashko ring->ring_id = i; 1498d782298cSGrygorii Strashko ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 1499d782298cSGrygorii Strashko 1500d782298cSGrygorii Strashko ring = &ringacc->rings[ringacc->num_rings + i]; 1501d782298cSGrygorii Strashko ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i + 1502d782298cSGrygorii Strashko K3_DMARING_RT_REGS_REVERSE_OFS; 1503d782298cSGrygorii Strashko ring->parent = ringacc; 1504d782298cSGrygorii Strashko ring->ring_id = i; 1505d782298cSGrygorii Strashko ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 1506d782298cSGrygorii Strashko ring->flags = K3_RING_FLAG_REVERSE; 1507d782298cSGrygorii Strashko } 1508d782298cSGrygorii Strashko 1509d782298cSGrygorii Strashko ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1510d782298cSGrygorii Strashko 1511d782298cSGrygorii Strashko dev_info(dev, "Number of rings: %u\n", ringacc->num_rings); 1512d782298cSGrygorii Strashko 1513d782298cSGrygorii Strashko return ringacc; 1514d782298cSGrygorii Strashko } 1515d782298cSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init); 1516d782298cSGrygorii Strashko 151740a2a7c3SGrygorii Strashko static int k3_ringacc_probe(struct platform_device *pdev) 151840a2a7c3SGrygorii Strashko { 151940a2a7c3SGrygorii Strashko const struct ringacc_match_data *match_data; 152040a2a7c3SGrygorii Strashko const struct of_device_id *match; 152140a2a7c3SGrygorii Strashko struct device *dev = &pdev->dev; 152240a2a7c3SGrygorii Strashko struct k3_ringacc *ringacc; 152340a2a7c3SGrygorii Strashko int ret; 152440a2a7c3SGrygorii Strashko 152540a2a7c3SGrygorii Strashko match = of_match_node(k3_ringacc_of_match, dev->of_node); 152640a2a7c3SGrygorii Strashko if (!match) 152740a2a7c3SGrygorii Strashko return -ENODEV; 152840a2a7c3SGrygorii Strashko match_data = match->data; 152940a2a7c3SGrygorii Strashko 153040a2a7c3SGrygorii Strashko ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 153140a2a7c3SGrygorii Strashko if (!ringacc) 153240a2a7c3SGrygorii Strashko return -ENOMEM; 153340a2a7c3SGrygorii Strashko 153440a2a7c3SGrygorii Strashko ringacc->dev = dev; 153540a2a7c3SGrygorii Strashko mutex_init(&ringacc->req_lock); 153640a2a7c3SGrygorii Strashko ringacc->ops = &match_data->ops; 153740a2a7c3SGrygorii Strashko 153840a2a7c3SGrygorii Strashko ret = ringacc->ops->init(pdev, ringacc); 153940a2a7c3SGrygorii Strashko if (ret) 154040a2a7c3SGrygorii Strashko return ret; 154140a2a7c3SGrygorii Strashko 154240a2a7c3SGrygorii Strashko dev_set_drvdata(dev, ringacc); 154340a2a7c3SGrygorii Strashko 154440a2a7c3SGrygorii Strashko mutex_lock(&k3_ringacc_list_lock); 154540a2a7c3SGrygorii Strashko list_add_tail(&ringacc->list, &k3_ringacc_list); 154640a2a7c3SGrygorii Strashko mutex_unlock(&k3_ringacc_list_lock); 154740a2a7c3SGrygorii Strashko 154840a2a7c3SGrygorii Strashko return 0; 154940a2a7c3SGrygorii Strashko } 155040a2a7c3SGrygorii Strashko 15513277e8aaSGrygorii Strashko static struct platform_driver k3_ringacc_driver = { 15523277e8aaSGrygorii Strashko .probe = k3_ringacc_probe, 15533277e8aaSGrygorii Strashko .driver = { 15543277e8aaSGrygorii Strashko .name = "k3-ringacc", 15553277e8aaSGrygorii Strashko .of_match_table = k3_ringacc_of_match, 15563277e8aaSGrygorii Strashko .suppress_bind_attrs = true, 15573277e8aaSGrygorii Strashko }, 15583277e8aaSGrygorii Strashko }; 15593277e8aaSGrygorii Strashko builtin_platform_driver(k3_ringacc_driver); 1560