13277e8aaSGrygorii Strashko // SPDX-License-Identifier: GPL-2.0 23277e8aaSGrygorii Strashko /* 33277e8aaSGrygorii Strashko * TI K3 NAVSS Ring Accelerator subsystem driver 43277e8aaSGrygorii Strashko * 53277e8aaSGrygorii Strashko * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 63277e8aaSGrygorii Strashko */ 73277e8aaSGrygorii Strashko 83277e8aaSGrygorii Strashko #include <linux/dma-mapping.h> 93277e8aaSGrygorii Strashko #include <linux/io.h> 103277e8aaSGrygorii Strashko #include <linux/init.h> 113277e8aaSGrygorii Strashko #include <linux/of.h> 123277e8aaSGrygorii Strashko #include <linux/platform_device.h> 133277e8aaSGrygorii Strashko #include <linux/soc/ti/k3-ringacc.h> 143277e8aaSGrygorii Strashko #include <linux/soc/ti/ti_sci_protocol.h> 153277e8aaSGrygorii Strashko #include <linux/soc/ti/ti_sci_inta_msi.h> 163277e8aaSGrygorii Strashko #include <linux/of_irq.h> 173277e8aaSGrygorii Strashko #include <linux/irqdomain.h> 183277e8aaSGrygorii Strashko 193277e8aaSGrygorii Strashko static LIST_HEAD(k3_ringacc_list); 203277e8aaSGrygorii Strashko static DEFINE_MUTEX(k3_ringacc_list_lock); 213277e8aaSGrygorii Strashko 223277e8aaSGrygorii Strashko #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) 233277e8aaSGrygorii Strashko 243277e8aaSGrygorii Strashko /** 253277e8aaSGrygorii Strashko * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region 263277e8aaSGrygorii Strashko * 273277e8aaSGrygorii Strashko * @resv_16: Reserved 283277e8aaSGrygorii Strashko * @db: Ring Doorbell Register 293277e8aaSGrygorii Strashko * @resv_4: Reserved 303277e8aaSGrygorii Strashko * @occ: Ring Occupancy Register 313277e8aaSGrygorii Strashko * @indx: Ring Current Index Register 323277e8aaSGrygorii Strashko * @hwocc: Ring Hardware Occupancy Register 333277e8aaSGrygorii Strashko * @hwindx: Ring Hardware Current Index Register 343277e8aaSGrygorii Strashko */ 353277e8aaSGrygorii Strashko struct k3_ring_rt_regs { 363277e8aaSGrygorii Strashko u32 resv_16[4]; 373277e8aaSGrygorii Strashko u32 db; 383277e8aaSGrygorii Strashko u32 resv_4[1]; 393277e8aaSGrygorii Strashko u32 occ; 403277e8aaSGrygorii Strashko u32 indx; 413277e8aaSGrygorii Strashko u32 hwocc; 423277e8aaSGrygorii Strashko u32 hwindx; 433277e8aaSGrygorii Strashko }; 443277e8aaSGrygorii Strashko 453277e8aaSGrygorii Strashko #define K3_RINGACC_RT_REGS_STEP 0x1000 463277e8aaSGrygorii Strashko 473277e8aaSGrygorii Strashko /** 483277e8aaSGrygorii Strashko * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region 493277e8aaSGrygorii Strashko * 503277e8aaSGrygorii Strashko * @head_data: Ring Head Entry Data Registers 513277e8aaSGrygorii Strashko * @tail_data: Ring Tail Entry Data Registers 523277e8aaSGrygorii Strashko * @peek_head_data: Ring Peek Head Entry Data Regs 533277e8aaSGrygorii Strashko * @peek_tail_data: Ring Peek Tail Entry Data Regs 543277e8aaSGrygorii Strashko */ 553277e8aaSGrygorii Strashko struct k3_ring_fifo_regs { 563277e8aaSGrygorii Strashko u32 head_data[128]; 573277e8aaSGrygorii Strashko u32 tail_data[128]; 583277e8aaSGrygorii Strashko u32 peek_head_data[128]; 593277e8aaSGrygorii Strashko u32 peek_tail_data[128]; 603277e8aaSGrygorii Strashko }; 613277e8aaSGrygorii Strashko 623277e8aaSGrygorii Strashko /** 633277e8aaSGrygorii Strashko * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region 643277e8aaSGrygorii Strashko * 653277e8aaSGrygorii Strashko * @revision: Revision Register 663277e8aaSGrygorii Strashko * @config: Config Register 673277e8aaSGrygorii Strashko */ 683277e8aaSGrygorii Strashko struct k3_ringacc_proxy_gcfg_regs { 693277e8aaSGrygorii Strashko u32 revision; 703277e8aaSGrygorii Strashko u32 config; 713277e8aaSGrygorii Strashko }; 723277e8aaSGrygorii Strashko 733277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) 743277e8aaSGrygorii Strashko 753277e8aaSGrygorii Strashko /** 763277e8aaSGrygorii Strashko * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region 773277e8aaSGrygorii Strashko * 783277e8aaSGrygorii Strashko * @control: Proxy Control Register 793277e8aaSGrygorii Strashko * @status: Proxy Status Register 803277e8aaSGrygorii Strashko * @resv_512: Reserved 813277e8aaSGrygorii Strashko * @data: Proxy Data Register 823277e8aaSGrygorii Strashko */ 833277e8aaSGrygorii Strashko struct k3_ringacc_proxy_target_regs { 843277e8aaSGrygorii Strashko u32 control; 853277e8aaSGrygorii Strashko u32 status; 863277e8aaSGrygorii Strashko u8 resv_512[504]; 873277e8aaSGrygorii Strashko u32 data[128]; 883277e8aaSGrygorii Strashko }; 893277e8aaSGrygorii Strashko 903277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_TARGET_STEP 0x1000 913277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_NOT_USED (-1) 923277e8aaSGrygorii Strashko 933277e8aaSGrygorii Strashko enum k3_ringacc_proxy_access_mode { 943277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_HEAD = 0, 953277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_TAIL = 1, 963277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_PEEK_HEAD = 2, 973277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_PEEK_TAIL = 3, 983277e8aaSGrygorii Strashko }; 993277e8aaSGrygorii Strashko 1003277e8aaSGrygorii Strashko #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) 1013277e8aaSGrygorii Strashko #define K3_RINGACC_FIFO_REGS_STEP 0x1000 1023277e8aaSGrygorii Strashko #define K3_RINGACC_MAX_DB_RING_CNT (127U) 1033277e8aaSGrygorii Strashko 1043277e8aaSGrygorii Strashko struct k3_ring_ops { 1053277e8aaSGrygorii Strashko int (*push_tail)(struct k3_ring *ring, void *elm); 1063277e8aaSGrygorii Strashko int (*push_head)(struct k3_ring *ring, void *elm); 1073277e8aaSGrygorii Strashko int (*pop_tail)(struct k3_ring *ring, void *elm); 1083277e8aaSGrygorii Strashko int (*pop_head)(struct k3_ring *ring, void *elm); 1093277e8aaSGrygorii Strashko }; 1103277e8aaSGrygorii Strashko 1113277e8aaSGrygorii Strashko /** 1126b3da0b4SPeter Ujfalusi * struct k3_ring_state - Internal state tracking structure 1136b3da0b4SPeter Ujfalusi * 1146b3da0b4SPeter Ujfalusi * @free: Number of free entries 1156b3da0b4SPeter Ujfalusi * @occ: Occupancy 1166b3da0b4SPeter Ujfalusi * @windex: Write index 1176b3da0b4SPeter Ujfalusi * @rindex: Read index 1186b3da0b4SPeter Ujfalusi */ 1196b3da0b4SPeter Ujfalusi struct k3_ring_state { 1206b3da0b4SPeter Ujfalusi u32 free; 1216b3da0b4SPeter Ujfalusi u32 occ; 1226b3da0b4SPeter Ujfalusi u32 windex; 1236b3da0b4SPeter Ujfalusi u32 rindex; 1246b3da0b4SPeter Ujfalusi }; 1256b3da0b4SPeter Ujfalusi 1266b3da0b4SPeter Ujfalusi /** 1273277e8aaSGrygorii Strashko * struct k3_ring - RA Ring descriptor 1283277e8aaSGrygorii Strashko * 1293277e8aaSGrygorii Strashko * @rt: Ring control/status registers 1303277e8aaSGrygorii Strashko * @fifos: Ring queues registers 1313277e8aaSGrygorii Strashko * @proxy: Ring Proxy Datapath registers 1323277e8aaSGrygorii Strashko * @ring_mem_dma: Ring buffer dma address 1333277e8aaSGrygorii Strashko * @ring_mem_virt: Ring buffer virt address 1343277e8aaSGrygorii Strashko * @ops: Ring operations 1353277e8aaSGrygorii Strashko * @size: Ring size in elements 1363277e8aaSGrygorii Strashko * @elm_size: Size of the ring element 1373277e8aaSGrygorii Strashko * @mode: Ring mode 1383277e8aaSGrygorii Strashko * @flags: flags 1393277e8aaSGrygorii Strashko * @ring_id: Ring Id 1403277e8aaSGrygorii Strashko * @parent: Pointer on struct @k3_ringacc 1413277e8aaSGrygorii Strashko * @use_count: Use count for shared rings 1423277e8aaSGrygorii Strashko * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) 1433277e8aaSGrygorii Strashko */ 1443277e8aaSGrygorii Strashko struct k3_ring { 1453277e8aaSGrygorii Strashko struct k3_ring_rt_regs __iomem *rt; 1463277e8aaSGrygorii Strashko struct k3_ring_fifo_regs __iomem *fifos; 1473277e8aaSGrygorii Strashko struct k3_ringacc_proxy_target_regs __iomem *proxy; 1483277e8aaSGrygorii Strashko dma_addr_t ring_mem_dma; 1493277e8aaSGrygorii Strashko void *ring_mem_virt; 1503277e8aaSGrygorii Strashko struct k3_ring_ops *ops; 1513277e8aaSGrygorii Strashko u32 size; 1523277e8aaSGrygorii Strashko enum k3_ring_size elm_size; 1533277e8aaSGrygorii Strashko enum k3_ring_mode mode; 1543277e8aaSGrygorii Strashko u32 flags; 1553277e8aaSGrygorii Strashko #define K3_RING_FLAG_BUSY BIT(1) 1563277e8aaSGrygorii Strashko #define K3_RING_FLAG_SHARED BIT(2) 1576b3da0b4SPeter Ujfalusi struct k3_ring_state state; 1583277e8aaSGrygorii Strashko u32 ring_id; 1593277e8aaSGrygorii Strashko struct k3_ringacc *parent; 1603277e8aaSGrygorii Strashko u32 use_count; 1613277e8aaSGrygorii Strashko int proxy_id; 1623277e8aaSGrygorii Strashko }; 1633277e8aaSGrygorii Strashko 1643277e8aaSGrygorii Strashko /** 1653277e8aaSGrygorii Strashko * struct k3_ringacc - Rings accelerator descriptor 1663277e8aaSGrygorii Strashko * 1673277e8aaSGrygorii Strashko * @dev: pointer on RA device 1683277e8aaSGrygorii Strashko * @proxy_gcfg: RA proxy global config registers 1693277e8aaSGrygorii Strashko * @proxy_target_base: RA proxy datapath region 1703277e8aaSGrygorii Strashko * @num_rings: number of ring in RA 1713277e8aaSGrygorii Strashko * @rings_inuse: bitfield for ring usage tracking 1723277e8aaSGrygorii Strashko * @rm_gp_range: general purpose rings range from tisci 1733277e8aaSGrygorii Strashko * @dma_ring_reset_quirk: DMA reset w/a enable 1743277e8aaSGrygorii Strashko * @num_proxies: number of RA proxies 1753277e8aaSGrygorii Strashko * @proxy_inuse: bitfield for proxy usage tracking 1763277e8aaSGrygorii Strashko * @rings: array of rings descriptors (struct @k3_ring) 1773277e8aaSGrygorii Strashko * @list: list of RAs in the system 1783277e8aaSGrygorii Strashko * @req_lock: protect rings allocation 1793277e8aaSGrygorii Strashko * @tisci: pointer ti-sci handle 1803277e8aaSGrygorii Strashko * @tisci_ring_ops: ti-sci rings ops 1813277e8aaSGrygorii Strashko * @tisci_dev_id: ti-sci device id 1823277e8aaSGrygorii Strashko */ 1833277e8aaSGrygorii Strashko struct k3_ringacc { 1843277e8aaSGrygorii Strashko struct device *dev; 1853277e8aaSGrygorii Strashko struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; 1863277e8aaSGrygorii Strashko void __iomem *proxy_target_base; 1873277e8aaSGrygorii Strashko u32 num_rings; /* number of rings in Ringacc module */ 1883277e8aaSGrygorii Strashko unsigned long *rings_inuse; 1893277e8aaSGrygorii Strashko struct ti_sci_resource *rm_gp_range; 1903277e8aaSGrygorii Strashko 1913277e8aaSGrygorii Strashko bool dma_ring_reset_quirk; 1923277e8aaSGrygorii Strashko u32 num_proxies; 1933277e8aaSGrygorii Strashko unsigned long *proxy_inuse; 1943277e8aaSGrygorii Strashko 1953277e8aaSGrygorii Strashko struct k3_ring *rings; 1963277e8aaSGrygorii Strashko struct list_head list; 1973277e8aaSGrygorii Strashko struct mutex req_lock; /* protect rings allocation */ 1983277e8aaSGrygorii Strashko 1993277e8aaSGrygorii Strashko const struct ti_sci_handle *tisci; 2003277e8aaSGrygorii Strashko const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; 2013277e8aaSGrygorii Strashko u32 tisci_dev_id; 2023277e8aaSGrygorii Strashko }; 2033277e8aaSGrygorii Strashko 2043277e8aaSGrygorii Strashko static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) 2053277e8aaSGrygorii Strashko { 2063277e8aaSGrygorii Strashko return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - 2073277e8aaSGrygorii Strashko (4 << ring->elm_size); 2083277e8aaSGrygorii Strashko } 2093277e8aaSGrygorii Strashko 2103277e8aaSGrygorii Strashko static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) 2113277e8aaSGrygorii Strashko { 2123277e8aaSGrygorii Strashko return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); 2133277e8aaSGrygorii Strashko } 2143277e8aaSGrygorii Strashko 2153277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); 2163277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); 2173277e8aaSGrygorii Strashko 2183277e8aaSGrygorii Strashko static struct k3_ring_ops k3_ring_mode_ring_ops = { 2193277e8aaSGrygorii Strashko .push_tail = k3_ringacc_ring_push_mem, 2203277e8aaSGrygorii Strashko .pop_head = k3_ringacc_ring_pop_mem, 2213277e8aaSGrygorii Strashko }; 2223277e8aaSGrygorii Strashko 2233277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); 2243277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); 2253277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); 2263277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); 2273277e8aaSGrygorii Strashko 2283277e8aaSGrygorii Strashko static struct k3_ring_ops k3_ring_mode_msg_ops = { 2293277e8aaSGrygorii Strashko .push_tail = k3_ringacc_ring_push_io, 2303277e8aaSGrygorii Strashko .push_head = k3_ringacc_ring_push_head_io, 2313277e8aaSGrygorii Strashko .pop_tail = k3_ringacc_ring_pop_tail_io, 2323277e8aaSGrygorii Strashko .pop_head = k3_ringacc_ring_pop_io, 2333277e8aaSGrygorii Strashko }; 2343277e8aaSGrygorii Strashko 2353277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); 2363277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); 2373277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); 2383277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); 2393277e8aaSGrygorii Strashko 2403277e8aaSGrygorii Strashko static struct k3_ring_ops k3_ring_mode_proxy_ops = { 2413277e8aaSGrygorii Strashko .push_tail = k3_ringacc_ring_push_tail_proxy, 2423277e8aaSGrygorii Strashko .push_head = k3_ringacc_ring_push_head_proxy, 2433277e8aaSGrygorii Strashko .pop_tail = k3_ringacc_ring_pop_tail_proxy, 2443277e8aaSGrygorii Strashko .pop_head = k3_ringacc_ring_pop_head_proxy, 2453277e8aaSGrygorii Strashko }; 2463277e8aaSGrygorii Strashko 2473277e8aaSGrygorii Strashko static void k3_ringacc_ring_dump(struct k3_ring *ring) 2483277e8aaSGrygorii Strashko { 2493277e8aaSGrygorii Strashko struct device *dev = ring->parent->dev; 2503277e8aaSGrygorii Strashko 2513277e8aaSGrygorii Strashko dev_dbg(dev, "dump ring: %d\n", ring->ring_id); 2523277e8aaSGrygorii Strashko dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, 2533277e8aaSGrygorii Strashko &ring->ring_mem_dma); 2543277e8aaSGrygorii Strashko dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", 2553277e8aaSGrygorii Strashko ring->elm_size, ring->size, ring->mode, ring->proxy_id); 256175e663fSGrygorii Strashko dev_dbg(dev, "dump flags %08X\n", ring->flags); 2573277e8aaSGrygorii Strashko 2583277e8aaSGrygorii Strashko dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); 2593277e8aaSGrygorii Strashko dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); 2603277e8aaSGrygorii Strashko dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); 2613277e8aaSGrygorii Strashko dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); 2623277e8aaSGrygorii Strashko dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); 2633277e8aaSGrygorii Strashko 2643277e8aaSGrygorii Strashko if (ring->ring_mem_virt) 2653277e8aaSGrygorii Strashko print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, 2663277e8aaSGrygorii Strashko 16, 1, ring->ring_mem_virt, 16 * 8, false); 2673277e8aaSGrygorii Strashko } 2683277e8aaSGrygorii Strashko 2693277e8aaSGrygorii Strashko struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, 2703277e8aaSGrygorii Strashko int id, u32 flags) 2713277e8aaSGrygorii Strashko { 2723277e8aaSGrygorii Strashko int proxy_id = K3_RINGACC_PROXY_NOT_USED; 2733277e8aaSGrygorii Strashko 2743277e8aaSGrygorii Strashko mutex_lock(&ringacc->req_lock); 2753277e8aaSGrygorii Strashko 2763277e8aaSGrygorii Strashko if (id == K3_RINGACC_RING_ID_ANY) { 2773277e8aaSGrygorii Strashko /* Request for any general purpose ring */ 2783277e8aaSGrygorii Strashko struct ti_sci_resource_desc *gp_rings = 2793277e8aaSGrygorii Strashko &ringacc->rm_gp_range->desc[0]; 2803277e8aaSGrygorii Strashko unsigned long size; 2813277e8aaSGrygorii Strashko 2823277e8aaSGrygorii Strashko size = gp_rings->start + gp_rings->num; 2833277e8aaSGrygorii Strashko id = find_next_zero_bit(ringacc->rings_inuse, size, 2843277e8aaSGrygorii Strashko gp_rings->start); 2853277e8aaSGrygorii Strashko if (id == size) 2863277e8aaSGrygorii Strashko goto error; 2873277e8aaSGrygorii Strashko } else if (id < 0) { 2883277e8aaSGrygorii Strashko goto error; 2893277e8aaSGrygorii Strashko } 2903277e8aaSGrygorii Strashko 2913277e8aaSGrygorii Strashko if (test_bit(id, ringacc->rings_inuse) && 2923277e8aaSGrygorii Strashko !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) 2933277e8aaSGrygorii Strashko goto error; 2943277e8aaSGrygorii Strashko else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) 2953277e8aaSGrygorii Strashko goto out; 2963277e8aaSGrygorii Strashko 2973277e8aaSGrygorii Strashko if (flags & K3_RINGACC_RING_USE_PROXY) { 2983277e8aaSGrygorii Strashko proxy_id = find_next_zero_bit(ringacc->proxy_inuse, 2993277e8aaSGrygorii Strashko ringacc->num_proxies, 0); 3003277e8aaSGrygorii Strashko if (proxy_id == ringacc->num_proxies) 3013277e8aaSGrygorii Strashko goto error; 3023277e8aaSGrygorii Strashko } 3033277e8aaSGrygorii Strashko 3043277e8aaSGrygorii Strashko if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { 3053277e8aaSGrygorii Strashko set_bit(proxy_id, ringacc->proxy_inuse); 3063277e8aaSGrygorii Strashko ringacc->rings[id].proxy_id = proxy_id; 3073277e8aaSGrygorii Strashko dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, 3083277e8aaSGrygorii Strashko proxy_id); 3093277e8aaSGrygorii Strashko } else { 3103277e8aaSGrygorii Strashko dev_dbg(ringacc->dev, "Giving ring#%d\n", id); 3113277e8aaSGrygorii Strashko } 3123277e8aaSGrygorii Strashko 3133277e8aaSGrygorii Strashko set_bit(id, ringacc->rings_inuse); 3143277e8aaSGrygorii Strashko out: 3153277e8aaSGrygorii Strashko ringacc->rings[id].use_count++; 3163277e8aaSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 3173277e8aaSGrygorii Strashko return &ringacc->rings[id]; 3183277e8aaSGrygorii Strashko 3193277e8aaSGrygorii Strashko error: 3203277e8aaSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 3213277e8aaSGrygorii Strashko return NULL; 3223277e8aaSGrygorii Strashko } 3233277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); 3243277e8aaSGrygorii Strashko 32543148b1cSGrygorii Strashko int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, 32643148b1cSGrygorii Strashko int fwd_id, int compl_id, 32743148b1cSGrygorii Strashko struct k3_ring **fwd_ring, 32843148b1cSGrygorii Strashko struct k3_ring **compl_ring) 32943148b1cSGrygorii Strashko { 33043148b1cSGrygorii Strashko int ret = 0; 33143148b1cSGrygorii Strashko 33243148b1cSGrygorii Strashko if (!fwd_ring || !compl_ring) 33343148b1cSGrygorii Strashko return -EINVAL; 33443148b1cSGrygorii Strashko 33543148b1cSGrygorii Strashko *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); 33643148b1cSGrygorii Strashko if (!(*fwd_ring)) 33743148b1cSGrygorii Strashko return -ENODEV; 33843148b1cSGrygorii Strashko 33943148b1cSGrygorii Strashko *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0); 34043148b1cSGrygorii Strashko if (!(*compl_ring)) { 34143148b1cSGrygorii Strashko k3_ringacc_ring_free(*fwd_ring); 34243148b1cSGrygorii Strashko ret = -ENODEV; 34343148b1cSGrygorii Strashko } 34443148b1cSGrygorii Strashko 34543148b1cSGrygorii Strashko return ret; 34643148b1cSGrygorii Strashko } 34743148b1cSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair); 34843148b1cSGrygorii Strashko 3493277e8aaSGrygorii Strashko static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) 3503277e8aaSGrygorii Strashko { 3513277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 3523277e8aaSGrygorii Strashko int ret; 3533277e8aaSGrygorii Strashko 3543277e8aaSGrygorii Strashko ret = ringacc->tisci_ring_ops->config( 3553277e8aaSGrygorii Strashko ringacc->tisci, 3563277e8aaSGrygorii Strashko TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID, 3573277e8aaSGrygorii Strashko ringacc->tisci_dev_id, 3583277e8aaSGrygorii Strashko ring->ring_id, 3593277e8aaSGrygorii Strashko 0, 3603277e8aaSGrygorii Strashko 0, 3613277e8aaSGrygorii Strashko ring->size, 3623277e8aaSGrygorii Strashko 0, 3633277e8aaSGrygorii Strashko 0, 3643277e8aaSGrygorii Strashko 0); 3653277e8aaSGrygorii Strashko if (ret) 3663277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", 3673277e8aaSGrygorii Strashko ret, ring->ring_id); 3683277e8aaSGrygorii Strashko } 3693277e8aaSGrygorii Strashko 3703277e8aaSGrygorii Strashko void k3_ringacc_ring_reset(struct k3_ring *ring) 3713277e8aaSGrygorii Strashko { 3723277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 3733277e8aaSGrygorii Strashko return; 3743277e8aaSGrygorii Strashko 3756b3da0b4SPeter Ujfalusi memset(&ring->state, 0, sizeof(ring->state)); 3763277e8aaSGrygorii Strashko 3773277e8aaSGrygorii Strashko k3_ringacc_ring_reset_sci(ring); 3783277e8aaSGrygorii Strashko } 3793277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); 3803277e8aaSGrygorii Strashko 3813277e8aaSGrygorii Strashko static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, 3823277e8aaSGrygorii Strashko enum k3_ring_mode mode) 3833277e8aaSGrygorii Strashko { 3843277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 3853277e8aaSGrygorii Strashko int ret; 3863277e8aaSGrygorii Strashko 3873277e8aaSGrygorii Strashko ret = ringacc->tisci_ring_ops->config( 3883277e8aaSGrygorii Strashko ringacc->tisci, 3893277e8aaSGrygorii Strashko TI_SCI_MSG_VALUE_RM_RING_MODE_VALID, 3903277e8aaSGrygorii Strashko ringacc->tisci_dev_id, 3913277e8aaSGrygorii Strashko ring->ring_id, 3923277e8aaSGrygorii Strashko 0, 3933277e8aaSGrygorii Strashko 0, 3943277e8aaSGrygorii Strashko 0, 3953277e8aaSGrygorii Strashko mode, 3963277e8aaSGrygorii Strashko 0, 3973277e8aaSGrygorii Strashko 0); 3983277e8aaSGrygorii Strashko if (ret) 3993277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", 4003277e8aaSGrygorii Strashko ret, ring->ring_id); 4013277e8aaSGrygorii Strashko } 4023277e8aaSGrygorii Strashko 4033277e8aaSGrygorii Strashko void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) 4043277e8aaSGrygorii Strashko { 4053277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 4063277e8aaSGrygorii Strashko return; 4073277e8aaSGrygorii Strashko 4083277e8aaSGrygorii Strashko if (!ring->parent->dma_ring_reset_quirk) 4093277e8aaSGrygorii Strashko goto reset; 4103277e8aaSGrygorii Strashko 4113277e8aaSGrygorii Strashko if (!occ) 4123277e8aaSGrygorii Strashko occ = readl(&ring->rt->occ); 4133277e8aaSGrygorii Strashko 4143277e8aaSGrygorii Strashko if (occ) { 4153277e8aaSGrygorii Strashko u32 db_ring_cnt, db_ring_cnt_cur; 4163277e8aaSGrygorii Strashko 4173277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, 4183277e8aaSGrygorii Strashko ring->ring_id, occ); 4193277e8aaSGrygorii Strashko /* TI-SCI ring reset */ 4203277e8aaSGrygorii Strashko k3_ringacc_ring_reset_sci(ring); 4213277e8aaSGrygorii Strashko 4223277e8aaSGrygorii Strashko /* 4233277e8aaSGrygorii Strashko * Setup the ring in ring/doorbell mode (if not already in this 4243277e8aaSGrygorii Strashko * mode) 4253277e8aaSGrygorii Strashko */ 4263277e8aaSGrygorii Strashko if (ring->mode != K3_RINGACC_RING_MODE_RING) 4273277e8aaSGrygorii Strashko k3_ringacc_ring_reconfig_qmode_sci( 4283277e8aaSGrygorii Strashko ring, K3_RINGACC_RING_MODE_RING); 4293277e8aaSGrygorii Strashko /* 4303277e8aaSGrygorii Strashko * Ring the doorbell 2**22 – ringOcc times. 4313277e8aaSGrygorii Strashko * This will wrap the internal UDMAP ring state occupancy 4323277e8aaSGrygorii Strashko * counter (which is 21-bits wide) to 0. 4333277e8aaSGrygorii Strashko */ 4343277e8aaSGrygorii Strashko db_ring_cnt = (1U << 22) - occ; 4353277e8aaSGrygorii Strashko 4363277e8aaSGrygorii Strashko while (db_ring_cnt != 0) { 4373277e8aaSGrygorii Strashko /* 4383277e8aaSGrygorii Strashko * Ring the doorbell with the maximum count each 4393277e8aaSGrygorii Strashko * iteration if possible to minimize the total 4403277e8aaSGrygorii Strashko * of writes 4413277e8aaSGrygorii Strashko */ 4423277e8aaSGrygorii Strashko if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) 4433277e8aaSGrygorii Strashko db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; 4443277e8aaSGrygorii Strashko else 4453277e8aaSGrygorii Strashko db_ring_cnt_cur = db_ring_cnt; 4463277e8aaSGrygorii Strashko 4473277e8aaSGrygorii Strashko writel(db_ring_cnt_cur, &ring->rt->db); 4483277e8aaSGrygorii Strashko db_ring_cnt -= db_ring_cnt_cur; 4493277e8aaSGrygorii Strashko } 4503277e8aaSGrygorii Strashko 4513277e8aaSGrygorii Strashko /* Restore the original ring mode (if not ring mode) */ 4523277e8aaSGrygorii Strashko if (ring->mode != K3_RINGACC_RING_MODE_RING) 4533277e8aaSGrygorii Strashko k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); 4543277e8aaSGrygorii Strashko } 4553277e8aaSGrygorii Strashko 4563277e8aaSGrygorii Strashko reset: 4573277e8aaSGrygorii Strashko /* Reset the ring */ 4583277e8aaSGrygorii Strashko k3_ringacc_ring_reset(ring); 4593277e8aaSGrygorii Strashko } 4603277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); 4613277e8aaSGrygorii Strashko 4623277e8aaSGrygorii Strashko static void k3_ringacc_ring_free_sci(struct k3_ring *ring) 4633277e8aaSGrygorii Strashko { 4643277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 4653277e8aaSGrygorii Strashko int ret; 4663277e8aaSGrygorii Strashko 4673277e8aaSGrygorii Strashko ret = ringacc->tisci_ring_ops->config( 4683277e8aaSGrygorii Strashko ringacc->tisci, 4693277e8aaSGrygorii Strashko TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER, 4703277e8aaSGrygorii Strashko ringacc->tisci_dev_id, 4713277e8aaSGrygorii Strashko ring->ring_id, 4723277e8aaSGrygorii Strashko 0, 4733277e8aaSGrygorii Strashko 0, 4743277e8aaSGrygorii Strashko 0, 4753277e8aaSGrygorii Strashko 0, 4763277e8aaSGrygorii Strashko 0, 4773277e8aaSGrygorii Strashko 0); 4783277e8aaSGrygorii Strashko if (ret) 4793277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", 4803277e8aaSGrygorii Strashko ret, ring->ring_id); 4813277e8aaSGrygorii Strashko } 4823277e8aaSGrygorii Strashko 4833277e8aaSGrygorii Strashko int k3_ringacc_ring_free(struct k3_ring *ring) 4843277e8aaSGrygorii Strashko { 4853277e8aaSGrygorii Strashko struct k3_ringacc *ringacc; 4863277e8aaSGrygorii Strashko 4873277e8aaSGrygorii Strashko if (!ring) 4883277e8aaSGrygorii Strashko return -EINVAL; 4893277e8aaSGrygorii Strashko 4903277e8aaSGrygorii Strashko ringacc = ring->parent; 4913277e8aaSGrygorii Strashko 4923277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); 4933277e8aaSGrygorii Strashko 4943277e8aaSGrygorii Strashko if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 4953277e8aaSGrygorii Strashko return -EINVAL; 4963277e8aaSGrygorii Strashko 4973277e8aaSGrygorii Strashko mutex_lock(&ringacc->req_lock); 4983277e8aaSGrygorii Strashko 4993277e8aaSGrygorii Strashko if (--ring->use_count) 5003277e8aaSGrygorii Strashko goto out; 5013277e8aaSGrygorii Strashko 5023277e8aaSGrygorii Strashko if (!(ring->flags & K3_RING_FLAG_BUSY)) 5033277e8aaSGrygorii Strashko goto no_init; 5043277e8aaSGrygorii Strashko 5053277e8aaSGrygorii Strashko k3_ringacc_ring_free_sci(ring); 5063277e8aaSGrygorii Strashko 5073277e8aaSGrygorii Strashko dma_free_coherent(ringacc->dev, 5083277e8aaSGrygorii Strashko ring->size * (4 << ring->elm_size), 5093277e8aaSGrygorii Strashko ring->ring_mem_virt, ring->ring_mem_dma); 5103277e8aaSGrygorii Strashko ring->flags = 0; 5113277e8aaSGrygorii Strashko ring->ops = NULL; 5123277e8aaSGrygorii Strashko if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { 5133277e8aaSGrygorii Strashko clear_bit(ring->proxy_id, ringacc->proxy_inuse); 5143277e8aaSGrygorii Strashko ring->proxy = NULL; 5153277e8aaSGrygorii Strashko ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 5163277e8aaSGrygorii Strashko } 5173277e8aaSGrygorii Strashko 5183277e8aaSGrygorii Strashko no_init: 5193277e8aaSGrygorii Strashko clear_bit(ring->ring_id, ringacc->rings_inuse); 5203277e8aaSGrygorii Strashko 5213277e8aaSGrygorii Strashko out: 5223277e8aaSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 5233277e8aaSGrygorii Strashko return 0; 5243277e8aaSGrygorii Strashko } 5253277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); 5263277e8aaSGrygorii Strashko 5273277e8aaSGrygorii Strashko u32 k3_ringacc_get_ring_id(struct k3_ring *ring) 5283277e8aaSGrygorii Strashko { 5293277e8aaSGrygorii Strashko if (!ring) 5303277e8aaSGrygorii Strashko return -EINVAL; 5313277e8aaSGrygorii Strashko 5323277e8aaSGrygorii Strashko return ring->ring_id; 5333277e8aaSGrygorii Strashko } 5343277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); 5353277e8aaSGrygorii Strashko 5363277e8aaSGrygorii Strashko u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) 5373277e8aaSGrygorii Strashko { 5383277e8aaSGrygorii Strashko if (!ring) 5393277e8aaSGrygorii Strashko return -EINVAL; 5403277e8aaSGrygorii Strashko 5413277e8aaSGrygorii Strashko return ring->parent->tisci_dev_id; 5423277e8aaSGrygorii Strashko } 5433277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); 5443277e8aaSGrygorii Strashko 5453277e8aaSGrygorii Strashko int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) 5463277e8aaSGrygorii Strashko { 5473277e8aaSGrygorii Strashko int irq_num; 5483277e8aaSGrygorii Strashko 5493277e8aaSGrygorii Strashko if (!ring) 5503277e8aaSGrygorii Strashko return -EINVAL; 5513277e8aaSGrygorii Strashko 5523277e8aaSGrygorii Strashko irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); 5533277e8aaSGrygorii Strashko if (irq_num <= 0) 5543277e8aaSGrygorii Strashko irq_num = -EINVAL; 5553277e8aaSGrygorii Strashko return irq_num; 5563277e8aaSGrygorii Strashko } 5573277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); 5583277e8aaSGrygorii Strashko 5593277e8aaSGrygorii Strashko static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) 5603277e8aaSGrygorii Strashko { 5613277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 5623277e8aaSGrygorii Strashko u32 ring_idx; 5633277e8aaSGrygorii Strashko int ret; 5643277e8aaSGrygorii Strashko 5653277e8aaSGrygorii Strashko if (!ringacc->tisci) 5663277e8aaSGrygorii Strashko return -EINVAL; 5673277e8aaSGrygorii Strashko 5683277e8aaSGrygorii Strashko ring_idx = ring->ring_id; 5693277e8aaSGrygorii Strashko ret = ringacc->tisci_ring_ops->config( 5703277e8aaSGrygorii Strashko ringacc->tisci, 5713277e8aaSGrygorii Strashko TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER, 5723277e8aaSGrygorii Strashko ringacc->tisci_dev_id, 5733277e8aaSGrygorii Strashko ring_idx, 5743277e8aaSGrygorii Strashko lower_32_bits(ring->ring_mem_dma), 5753277e8aaSGrygorii Strashko upper_32_bits(ring->ring_mem_dma), 5763277e8aaSGrygorii Strashko ring->size, 5773277e8aaSGrygorii Strashko ring->mode, 5783277e8aaSGrygorii Strashko ring->elm_size, 5793277e8aaSGrygorii Strashko 0); 5803277e8aaSGrygorii Strashko if (ret) 5813277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", 5823277e8aaSGrygorii Strashko ret, ring_idx); 5833277e8aaSGrygorii Strashko 5843277e8aaSGrygorii Strashko return ret; 5853277e8aaSGrygorii Strashko } 5863277e8aaSGrygorii Strashko 5873277e8aaSGrygorii Strashko int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 5883277e8aaSGrygorii Strashko { 5893277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 5903277e8aaSGrygorii Strashko int ret = 0; 5913277e8aaSGrygorii Strashko 5923277e8aaSGrygorii Strashko if (!ring || !cfg) 5933277e8aaSGrygorii Strashko return -EINVAL; 5943277e8aaSGrygorii Strashko if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || 5953277e8aaSGrygorii Strashko cfg->mode >= K3_RINGACC_RING_MODE_INVALID || 5963277e8aaSGrygorii Strashko cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || 5973277e8aaSGrygorii Strashko !test_bit(ring->ring_id, ringacc->rings_inuse)) 5983277e8aaSGrygorii Strashko return -EINVAL; 5993277e8aaSGrygorii Strashko 6003277e8aaSGrygorii Strashko if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && 6013277e8aaSGrygorii Strashko ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && 6023277e8aaSGrygorii Strashko cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { 6033277e8aaSGrygorii Strashko dev_err(ringacc->dev, 6043277e8aaSGrygorii Strashko "Message mode must use proxy for %u element size\n", 6053277e8aaSGrygorii Strashko 4 << ring->elm_size); 6063277e8aaSGrygorii Strashko return -EINVAL; 6073277e8aaSGrygorii Strashko } 6083277e8aaSGrygorii Strashko 6093277e8aaSGrygorii Strashko /* 6103277e8aaSGrygorii Strashko * In case of shared ring only the first user (master user) can 6113277e8aaSGrygorii Strashko * configure the ring. The sequence should be by the client: 6123277e8aaSGrygorii Strashko * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user 6133277e8aaSGrygorii Strashko * k3_ringacc_ring_cfg(ring, cfg); # master configuration 6143277e8aaSGrygorii Strashko * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 6153277e8aaSGrygorii Strashko * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 6163277e8aaSGrygorii Strashko */ 6173277e8aaSGrygorii Strashko if (ring->use_count != 1) 6183277e8aaSGrygorii Strashko return 0; 6193277e8aaSGrygorii Strashko 6203277e8aaSGrygorii Strashko ring->size = cfg->size; 6213277e8aaSGrygorii Strashko ring->elm_size = cfg->elm_size; 6223277e8aaSGrygorii Strashko ring->mode = cfg->mode; 6236b3da0b4SPeter Ujfalusi memset(&ring->state, 0, sizeof(ring->state)); 6243277e8aaSGrygorii Strashko 6253277e8aaSGrygorii Strashko if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) 6263277e8aaSGrygorii Strashko ring->proxy = ringacc->proxy_target_base + 6273277e8aaSGrygorii Strashko ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; 6283277e8aaSGrygorii Strashko 6293277e8aaSGrygorii Strashko switch (ring->mode) { 6303277e8aaSGrygorii Strashko case K3_RINGACC_RING_MODE_RING: 6313277e8aaSGrygorii Strashko ring->ops = &k3_ring_mode_ring_ops; 6323277e8aaSGrygorii Strashko break; 6333277e8aaSGrygorii Strashko case K3_RINGACC_RING_MODE_MESSAGE: 6343277e8aaSGrygorii Strashko if (ring->proxy) 6353277e8aaSGrygorii Strashko ring->ops = &k3_ring_mode_proxy_ops; 6363277e8aaSGrygorii Strashko else 6373277e8aaSGrygorii Strashko ring->ops = &k3_ring_mode_msg_ops; 6383277e8aaSGrygorii Strashko break; 6393277e8aaSGrygorii Strashko default: 6403277e8aaSGrygorii Strashko ring->ops = NULL; 6413277e8aaSGrygorii Strashko ret = -EINVAL; 6423277e8aaSGrygorii Strashko goto err_free_proxy; 6433277e8aaSGrygorii Strashko }; 6443277e8aaSGrygorii Strashko 6453277e8aaSGrygorii Strashko ring->ring_mem_virt = dma_alloc_coherent(ringacc->dev, 6463277e8aaSGrygorii Strashko ring->size * (4 << ring->elm_size), 6473277e8aaSGrygorii Strashko &ring->ring_mem_dma, GFP_KERNEL); 6483277e8aaSGrygorii Strashko if (!ring->ring_mem_virt) { 6493277e8aaSGrygorii Strashko dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 6503277e8aaSGrygorii Strashko ret = -ENOMEM; 6513277e8aaSGrygorii Strashko goto err_free_ops; 6523277e8aaSGrygorii Strashko } 6533277e8aaSGrygorii Strashko 6543277e8aaSGrygorii Strashko ret = k3_ringacc_ring_cfg_sci(ring); 6553277e8aaSGrygorii Strashko 6563277e8aaSGrygorii Strashko if (ret) 6573277e8aaSGrygorii Strashko goto err_free_mem; 6583277e8aaSGrygorii Strashko 6593277e8aaSGrygorii Strashko ring->flags |= K3_RING_FLAG_BUSY; 6603277e8aaSGrygorii Strashko ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? 6613277e8aaSGrygorii Strashko K3_RING_FLAG_SHARED : 0; 6623277e8aaSGrygorii Strashko 6633277e8aaSGrygorii Strashko k3_ringacc_ring_dump(ring); 6643277e8aaSGrygorii Strashko 6653277e8aaSGrygorii Strashko return 0; 6663277e8aaSGrygorii Strashko 6673277e8aaSGrygorii Strashko err_free_mem: 6683277e8aaSGrygorii Strashko dma_free_coherent(ringacc->dev, 6693277e8aaSGrygorii Strashko ring->size * (4 << ring->elm_size), 6703277e8aaSGrygorii Strashko ring->ring_mem_virt, 6713277e8aaSGrygorii Strashko ring->ring_mem_dma); 6723277e8aaSGrygorii Strashko err_free_ops: 6733277e8aaSGrygorii Strashko ring->ops = NULL; 6743277e8aaSGrygorii Strashko err_free_proxy: 6753277e8aaSGrygorii Strashko ring->proxy = NULL; 6763277e8aaSGrygorii Strashko return ret; 6773277e8aaSGrygorii Strashko } 6783277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); 6793277e8aaSGrygorii Strashko 6803277e8aaSGrygorii Strashko u32 k3_ringacc_ring_get_size(struct k3_ring *ring) 6813277e8aaSGrygorii Strashko { 6823277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 6833277e8aaSGrygorii Strashko return -EINVAL; 6843277e8aaSGrygorii Strashko 6853277e8aaSGrygorii Strashko return ring->size; 6863277e8aaSGrygorii Strashko } 6873277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); 6883277e8aaSGrygorii Strashko 6893277e8aaSGrygorii Strashko u32 k3_ringacc_ring_get_free(struct k3_ring *ring) 6903277e8aaSGrygorii Strashko { 6913277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 6923277e8aaSGrygorii Strashko return -EINVAL; 6933277e8aaSGrygorii Strashko 6946b3da0b4SPeter Ujfalusi if (!ring->state.free) 6956b3da0b4SPeter Ujfalusi ring->state.free = ring->size - readl(&ring->rt->occ); 6963277e8aaSGrygorii Strashko 6976b3da0b4SPeter Ujfalusi return ring->state.free; 6983277e8aaSGrygorii Strashko } 6993277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); 7003277e8aaSGrygorii Strashko 7013277e8aaSGrygorii Strashko u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) 7023277e8aaSGrygorii Strashko { 7033277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 7043277e8aaSGrygorii Strashko return -EINVAL; 7053277e8aaSGrygorii Strashko 7063277e8aaSGrygorii Strashko return readl(&ring->rt->occ); 7073277e8aaSGrygorii Strashko } 7083277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); 7093277e8aaSGrygorii Strashko 7103277e8aaSGrygorii Strashko u32 k3_ringacc_ring_is_full(struct k3_ring *ring) 7113277e8aaSGrygorii Strashko { 7123277e8aaSGrygorii Strashko return !k3_ringacc_ring_get_free(ring); 7133277e8aaSGrygorii Strashko } 7143277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); 7153277e8aaSGrygorii Strashko 7163277e8aaSGrygorii Strashko enum k3_ringacc_access_mode { 7173277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_HEAD, 7183277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD, 7193277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_TAIL, 7203277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_TAIL, 7213277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PEEK_HEAD, 7223277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PEEK_TAIL, 7233277e8aaSGrygorii Strashko }; 7243277e8aaSGrygorii Strashko 7253277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) 7263277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) 7273277e8aaSGrygorii Strashko static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, 7283277e8aaSGrygorii Strashko enum k3_ringacc_proxy_access_mode mode) 7293277e8aaSGrygorii Strashko { 7303277e8aaSGrygorii Strashko u32 val; 7313277e8aaSGrygorii Strashko 7323277e8aaSGrygorii Strashko val = ring->ring_id; 7333277e8aaSGrygorii Strashko val |= K3_RINGACC_PROXY_MODE(mode); 7343277e8aaSGrygorii Strashko val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); 7353277e8aaSGrygorii Strashko writel(val, &ring->proxy->control); 7363277e8aaSGrygorii Strashko return 0; 7373277e8aaSGrygorii Strashko } 7383277e8aaSGrygorii Strashko 7393277e8aaSGrygorii Strashko static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, 7403277e8aaSGrygorii Strashko enum k3_ringacc_access_mode access_mode) 7413277e8aaSGrygorii Strashko { 7423277e8aaSGrygorii Strashko void __iomem *ptr; 7433277e8aaSGrygorii Strashko 7443277e8aaSGrygorii Strashko ptr = (void __iomem *)&ring->proxy->data; 7453277e8aaSGrygorii Strashko 7463277e8aaSGrygorii Strashko switch (access_mode) { 7473277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 7483277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 7493277e8aaSGrygorii Strashko k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); 7503277e8aaSGrygorii Strashko break; 7513277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 7523277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 7533277e8aaSGrygorii Strashko k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); 7543277e8aaSGrygorii Strashko break; 7553277e8aaSGrygorii Strashko default: 7563277e8aaSGrygorii Strashko return -EINVAL; 7573277e8aaSGrygorii Strashko } 7583277e8aaSGrygorii Strashko 7593277e8aaSGrygorii Strashko ptr += k3_ringacc_ring_get_fifo_pos(ring); 7603277e8aaSGrygorii Strashko 7613277e8aaSGrygorii Strashko switch (access_mode) { 7623277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 7633277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 7643277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 7653277e8aaSGrygorii Strashko "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 7663277e8aaSGrygorii Strashko access_mode); 7673277e8aaSGrygorii Strashko memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 7686b3da0b4SPeter Ujfalusi ring->state.occ--; 7693277e8aaSGrygorii Strashko break; 7703277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 7713277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 7723277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 7733277e8aaSGrygorii Strashko "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 7743277e8aaSGrygorii Strashko access_mode); 7753277e8aaSGrygorii Strashko memcpy_toio(ptr, elem, (4 << ring->elm_size)); 7766b3da0b4SPeter Ujfalusi ring->state.free--; 7773277e8aaSGrygorii Strashko break; 7783277e8aaSGrygorii Strashko default: 7793277e8aaSGrygorii Strashko return -EINVAL; 7803277e8aaSGrygorii Strashko } 7813277e8aaSGrygorii Strashko 7826b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, 7836b3da0b4SPeter Ujfalusi ring->state.occ); 7843277e8aaSGrygorii Strashko return 0; 7853277e8aaSGrygorii Strashko } 7863277e8aaSGrygorii Strashko 7873277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) 7883277e8aaSGrygorii Strashko { 7893277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 7903277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 7913277e8aaSGrygorii Strashko } 7923277e8aaSGrygorii Strashko 7933277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) 7943277e8aaSGrygorii Strashko { 7953277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 7963277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 7973277e8aaSGrygorii Strashko } 7983277e8aaSGrygorii Strashko 7993277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) 8003277e8aaSGrygorii Strashko { 8013277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 8023277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 8033277e8aaSGrygorii Strashko } 8043277e8aaSGrygorii Strashko 8053277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) 8063277e8aaSGrygorii Strashko { 8073277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 8083277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 8093277e8aaSGrygorii Strashko } 8103277e8aaSGrygorii Strashko 8113277e8aaSGrygorii Strashko static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, 8123277e8aaSGrygorii Strashko enum k3_ringacc_access_mode access_mode) 8133277e8aaSGrygorii Strashko { 8143277e8aaSGrygorii Strashko void __iomem *ptr; 8153277e8aaSGrygorii Strashko 8163277e8aaSGrygorii Strashko switch (access_mode) { 8173277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 8183277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 8193277e8aaSGrygorii Strashko ptr = (void __iomem *)&ring->fifos->head_data; 8203277e8aaSGrygorii Strashko break; 8213277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 8223277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 8233277e8aaSGrygorii Strashko ptr = (void __iomem *)&ring->fifos->tail_data; 8243277e8aaSGrygorii Strashko break; 8253277e8aaSGrygorii Strashko default: 8263277e8aaSGrygorii Strashko return -EINVAL; 8273277e8aaSGrygorii Strashko } 8283277e8aaSGrygorii Strashko 8293277e8aaSGrygorii Strashko ptr += k3_ringacc_ring_get_fifo_pos(ring); 8303277e8aaSGrygorii Strashko 8313277e8aaSGrygorii Strashko switch (access_mode) { 8323277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 8333277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 8343277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 8353277e8aaSGrygorii Strashko "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 8363277e8aaSGrygorii Strashko access_mode); 8373277e8aaSGrygorii Strashko memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 8386b3da0b4SPeter Ujfalusi ring->state.occ--; 8393277e8aaSGrygorii Strashko break; 8403277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 8413277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 8423277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 8433277e8aaSGrygorii Strashko "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 8443277e8aaSGrygorii Strashko access_mode); 8453277e8aaSGrygorii Strashko memcpy_toio(ptr, elem, (4 << ring->elm_size)); 8466b3da0b4SPeter Ujfalusi ring->state.free--; 8473277e8aaSGrygorii Strashko break; 8483277e8aaSGrygorii Strashko default: 8493277e8aaSGrygorii Strashko return -EINVAL; 8503277e8aaSGrygorii Strashko } 8513277e8aaSGrygorii Strashko 8526b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", 8536b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex, ring->state.occ, 8546b3da0b4SPeter Ujfalusi ring->state.rindex); 8553277e8aaSGrygorii Strashko return 0; 8563277e8aaSGrygorii Strashko } 8573277e8aaSGrygorii Strashko 8583277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) 8593277e8aaSGrygorii Strashko { 8603277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 8613277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 8623277e8aaSGrygorii Strashko } 8633277e8aaSGrygorii Strashko 8643277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) 8653277e8aaSGrygorii Strashko { 8663277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 8673277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 8683277e8aaSGrygorii Strashko } 8693277e8aaSGrygorii Strashko 8703277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) 8713277e8aaSGrygorii Strashko { 8723277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 8733277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 8743277e8aaSGrygorii Strashko } 8753277e8aaSGrygorii Strashko 8763277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) 8773277e8aaSGrygorii Strashko { 8783277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 8793277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 8803277e8aaSGrygorii Strashko } 8813277e8aaSGrygorii Strashko 8823277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) 8833277e8aaSGrygorii Strashko { 8843277e8aaSGrygorii Strashko void *elem_ptr; 8853277e8aaSGrygorii Strashko 8866b3da0b4SPeter Ujfalusi elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); 8873277e8aaSGrygorii Strashko 8883277e8aaSGrygorii Strashko memcpy(elem_ptr, elem, (4 << ring->elm_size)); 8893277e8aaSGrygorii Strashko 8906b3da0b4SPeter Ujfalusi ring->state.windex = (ring->state.windex + 1) % ring->size; 8916b3da0b4SPeter Ujfalusi ring->state.free--; 8923277e8aaSGrygorii Strashko writel(1, &ring->rt->db); 8933277e8aaSGrygorii Strashko 8943277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", 8956b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex); 8963277e8aaSGrygorii Strashko 8973277e8aaSGrygorii Strashko return 0; 8983277e8aaSGrygorii Strashko } 8993277e8aaSGrygorii Strashko 9003277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) 9013277e8aaSGrygorii Strashko { 9023277e8aaSGrygorii Strashko void *elem_ptr; 9033277e8aaSGrygorii Strashko 9046b3da0b4SPeter Ujfalusi elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 9053277e8aaSGrygorii Strashko 9063277e8aaSGrygorii Strashko memcpy(elem, elem_ptr, (4 << ring->elm_size)); 9073277e8aaSGrygorii Strashko 9086b3da0b4SPeter Ujfalusi ring->state.rindex = (ring->state.rindex + 1) % ring->size; 9096b3da0b4SPeter Ujfalusi ring->state.occ--; 9103277e8aaSGrygorii Strashko writel(-1, &ring->rt->db); 9113277e8aaSGrygorii Strashko 9123277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", 9136b3da0b4SPeter Ujfalusi ring->state.occ, ring->state.rindex, elem_ptr); 9143277e8aaSGrygorii Strashko return 0; 9153277e8aaSGrygorii Strashko } 9163277e8aaSGrygorii Strashko 9173277e8aaSGrygorii Strashko int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) 9183277e8aaSGrygorii Strashko { 9193277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 9203277e8aaSGrygorii Strashko 9213277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 9223277e8aaSGrygorii Strashko return -EINVAL; 9233277e8aaSGrygorii Strashko 9246b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", 9256b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex); 9263277e8aaSGrygorii Strashko 9273277e8aaSGrygorii Strashko if (k3_ringacc_ring_is_full(ring)) 9283277e8aaSGrygorii Strashko return -ENOMEM; 9293277e8aaSGrygorii Strashko 9303277e8aaSGrygorii Strashko if (ring->ops && ring->ops->push_tail) 9313277e8aaSGrygorii Strashko ret = ring->ops->push_tail(ring, elem); 9323277e8aaSGrygorii Strashko 9333277e8aaSGrygorii Strashko return ret; 9343277e8aaSGrygorii Strashko } 9353277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); 9363277e8aaSGrygorii Strashko 9373277e8aaSGrygorii Strashko int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) 9383277e8aaSGrygorii Strashko { 9393277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 9403277e8aaSGrygorii Strashko 9413277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 9423277e8aaSGrygorii Strashko return -EINVAL; 9433277e8aaSGrygorii Strashko 9443277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", 9456b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex); 9463277e8aaSGrygorii Strashko 9473277e8aaSGrygorii Strashko if (k3_ringacc_ring_is_full(ring)) 9483277e8aaSGrygorii Strashko return -ENOMEM; 9493277e8aaSGrygorii Strashko 9503277e8aaSGrygorii Strashko if (ring->ops && ring->ops->push_head) 9513277e8aaSGrygorii Strashko ret = ring->ops->push_head(ring, elem); 9523277e8aaSGrygorii Strashko 9533277e8aaSGrygorii Strashko return ret; 9543277e8aaSGrygorii Strashko } 9553277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); 9563277e8aaSGrygorii Strashko 9573277e8aaSGrygorii Strashko int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) 9583277e8aaSGrygorii Strashko { 9593277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 9603277e8aaSGrygorii Strashko 9613277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 9623277e8aaSGrygorii Strashko return -EINVAL; 9633277e8aaSGrygorii Strashko 9646b3da0b4SPeter Ujfalusi if (!ring->state.occ) 9656b3da0b4SPeter Ujfalusi ring->state.occ = k3_ringacc_ring_get_occ(ring); 9663277e8aaSGrygorii Strashko 9676b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, 9686b3da0b4SPeter Ujfalusi ring->state.rindex); 9693277e8aaSGrygorii Strashko 9706b3da0b4SPeter Ujfalusi if (!ring->state.occ) 9713277e8aaSGrygorii Strashko return -ENODATA; 9723277e8aaSGrygorii Strashko 9733277e8aaSGrygorii Strashko if (ring->ops && ring->ops->pop_head) 9743277e8aaSGrygorii Strashko ret = ring->ops->pop_head(ring, elem); 9753277e8aaSGrygorii Strashko 9763277e8aaSGrygorii Strashko return ret; 9773277e8aaSGrygorii Strashko } 9783277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); 9793277e8aaSGrygorii Strashko 9803277e8aaSGrygorii Strashko int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) 9813277e8aaSGrygorii Strashko { 9823277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 9833277e8aaSGrygorii Strashko 9843277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 9853277e8aaSGrygorii Strashko return -EINVAL; 9863277e8aaSGrygorii Strashko 9876b3da0b4SPeter Ujfalusi if (!ring->state.occ) 9886b3da0b4SPeter Ujfalusi ring->state.occ = k3_ringacc_ring_get_occ(ring); 9893277e8aaSGrygorii Strashko 9906b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", 9916b3da0b4SPeter Ujfalusi ring->state.occ, ring->state.rindex); 9923277e8aaSGrygorii Strashko 9936b3da0b4SPeter Ujfalusi if (!ring->state.occ) 9943277e8aaSGrygorii Strashko return -ENODATA; 9953277e8aaSGrygorii Strashko 9963277e8aaSGrygorii Strashko if (ring->ops && ring->ops->pop_tail) 9973277e8aaSGrygorii Strashko ret = ring->ops->pop_tail(ring, elem); 9983277e8aaSGrygorii Strashko 9993277e8aaSGrygorii Strashko return ret; 10003277e8aaSGrygorii Strashko } 10013277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); 10023277e8aaSGrygorii Strashko 10033277e8aaSGrygorii Strashko struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, 10043277e8aaSGrygorii Strashko const char *property) 10053277e8aaSGrygorii Strashko { 10063277e8aaSGrygorii Strashko struct device_node *ringacc_np; 10073277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); 10083277e8aaSGrygorii Strashko struct k3_ringacc *entry; 10093277e8aaSGrygorii Strashko 10103277e8aaSGrygorii Strashko ringacc_np = of_parse_phandle(np, property, 0); 10113277e8aaSGrygorii Strashko if (!ringacc_np) 10123277e8aaSGrygorii Strashko return ERR_PTR(-ENODEV); 10133277e8aaSGrygorii Strashko 10143277e8aaSGrygorii Strashko mutex_lock(&k3_ringacc_list_lock); 10153277e8aaSGrygorii Strashko list_for_each_entry(entry, &k3_ringacc_list, list) 10163277e8aaSGrygorii Strashko if (entry->dev->of_node == ringacc_np) { 10173277e8aaSGrygorii Strashko ringacc = entry; 10183277e8aaSGrygorii Strashko break; 10193277e8aaSGrygorii Strashko } 10203277e8aaSGrygorii Strashko mutex_unlock(&k3_ringacc_list_lock); 10213277e8aaSGrygorii Strashko of_node_put(ringacc_np); 10223277e8aaSGrygorii Strashko 10233277e8aaSGrygorii Strashko return ringacc; 10243277e8aaSGrygorii Strashko } 10253277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); 10263277e8aaSGrygorii Strashko 10273277e8aaSGrygorii Strashko static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) 10283277e8aaSGrygorii Strashko { 10293277e8aaSGrygorii Strashko struct device_node *node = ringacc->dev->of_node; 10303277e8aaSGrygorii Strashko struct device *dev = ringacc->dev; 10313277e8aaSGrygorii Strashko struct platform_device *pdev = to_platform_device(dev); 10323277e8aaSGrygorii Strashko int ret; 10333277e8aaSGrygorii Strashko 10343277e8aaSGrygorii Strashko if (!node) { 10353277e8aaSGrygorii Strashko dev_err(dev, "device tree info unavailable\n"); 10363277e8aaSGrygorii Strashko return -ENODEV; 10373277e8aaSGrygorii Strashko } 10383277e8aaSGrygorii Strashko 10393277e8aaSGrygorii Strashko ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); 10403277e8aaSGrygorii Strashko if (ret) { 10413277e8aaSGrygorii Strashko dev_err(dev, "ti,num-rings read failure %d\n", ret); 10423277e8aaSGrygorii Strashko return ret; 10433277e8aaSGrygorii Strashko } 10443277e8aaSGrygorii Strashko 10453277e8aaSGrygorii Strashko ringacc->dma_ring_reset_quirk = 10463277e8aaSGrygorii Strashko of_property_read_bool(node, "ti,dma-ring-reset-quirk"); 10473277e8aaSGrygorii Strashko 10483277e8aaSGrygorii Strashko ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); 10493277e8aaSGrygorii Strashko if (IS_ERR(ringacc->tisci)) { 10503277e8aaSGrygorii Strashko ret = PTR_ERR(ringacc->tisci); 10513277e8aaSGrygorii Strashko if (ret != -EPROBE_DEFER) 10523277e8aaSGrygorii Strashko dev_err(dev, "ti,sci read fail %d\n", ret); 10533277e8aaSGrygorii Strashko ringacc->tisci = NULL; 10543277e8aaSGrygorii Strashko return ret; 10553277e8aaSGrygorii Strashko } 10563277e8aaSGrygorii Strashko 10573277e8aaSGrygorii Strashko ret = of_property_read_u32(node, "ti,sci-dev-id", 10583277e8aaSGrygorii Strashko &ringacc->tisci_dev_id); 10593277e8aaSGrygorii Strashko if (ret) { 10603277e8aaSGrygorii Strashko dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); 10613277e8aaSGrygorii Strashko return ret; 10623277e8aaSGrygorii Strashko } 10633277e8aaSGrygorii Strashko 10643277e8aaSGrygorii Strashko pdev->id = ringacc->tisci_dev_id; 10653277e8aaSGrygorii Strashko 10663277e8aaSGrygorii Strashko ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, 10673277e8aaSGrygorii Strashko ringacc->tisci_dev_id, 10683277e8aaSGrygorii Strashko "ti,sci-rm-range-gp-rings"); 10693277e8aaSGrygorii Strashko if (IS_ERR(ringacc->rm_gp_range)) { 10703277e8aaSGrygorii Strashko dev_err(dev, "Failed to allocate MSI interrupts\n"); 10713277e8aaSGrygorii Strashko return PTR_ERR(ringacc->rm_gp_range); 10723277e8aaSGrygorii Strashko } 10733277e8aaSGrygorii Strashko 10743277e8aaSGrygorii Strashko return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, 10753277e8aaSGrygorii Strashko ringacc->rm_gp_range); 10763277e8aaSGrygorii Strashko } 10773277e8aaSGrygorii Strashko 10783277e8aaSGrygorii Strashko static int k3_ringacc_probe(struct platform_device *pdev) 10793277e8aaSGrygorii Strashko { 10803277e8aaSGrygorii Strashko struct k3_ringacc *ringacc; 10813277e8aaSGrygorii Strashko void __iomem *base_fifo, *base_rt; 10823277e8aaSGrygorii Strashko struct device *dev = &pdev->dev; 10833277e8aaSGrygorii Strashko struct resource *res; 10843277e8aaSGrygorii Strashko int ret, i; 10853277e8aaSGrygorii Strashko 10863277e8aaSGrygorii Strashko ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 10873277e8aaSGrygorii Strashko if (!ringacc) 10883277e8aaSGrygorii Strashko return -ENOMEM; 10893277e8aaSGrygorii Strashko 10903277e8aaSGrygorii Strashko ringacc->dev = dev; 10913277e8aaSGrygorii Strashko mutex_init(&ringacc->req_lock); 10923277e8aaSGrygorii Strashko 10933277e8aaSGrygorii Strashko dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 10943277e8aaSGrygorii Strashko DOMAIN_BUS_TI_SCI_INTA_MSI); 10953277e8aaSGrygorii Strashko if (!dev->msi_domain) { 10963277e8aaSGrygorii Strashko dev_err(dev, "Failed to get MSI domain\n"); 10973277e8aaSGrygorii Strashko return -EPROBE_DEFER; 10983277e8aaSGrygorii Strashko } 10993277e8aaSGrygorii Strashko 11003277e8aaSGrygorii Strashko ret = k3_ringacc_probe_dt(ringacc); 11013277e8aaSGrygorii Strashko if (ret) 11023277e8aaSGrygorii Strashko return ret; 11033277e8aaSGrygorii Strashko 11043277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); 11053277e8aaSGrygorii Strashko base_rt = devm_ioremap_resource(dev, res); 11063277e8aaSGrygorii Strashko if (IS_ERR(base_rt)) 11073277e8aaSGrygorii Strashko return PTR_ERR(base_rt); 11083277e8aaSGrygorii Strashko 11093277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos"); 11103277e8aaSGrygorii Strashko base_fifo = devm_ioremap_resource(dev, res); 11113277e8aaSGrygorii Strashko if (IS_ERR(base_fifo)) 11123277e8aaSGrygorii Strashko return PTR_ERR(base_fifo); 11133277e8aaSGrygorii Strashko 11143277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg"); 11153277e8aaSGrygorii Strashko ringacc->proxy_gcfg = devm_ioremap_resource(dev, res); 11163277e8aaSGrygorii Strashko if (IS_ERR(ringacc->proxy_gcfg)) 11173277e8aaSGrygorii Strashko return PTR_ERR(ringacc->proxy_gcfg); 11183277e8aaSGrygorii Strashko 11193277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 11203277e8aaSGrygorii Strashko "proxy_target"); 11213277e8aaSGrygorii Strashko ringacc->proxy_target_base = devm_ioremap_resource(dev, res); 11223277e8aaSGrygorii Strashko if (IS_ERR(ringacc->proxy_target_base)) 11233277e8aaSGrygorii Strashko return PTR_ERR(ringacc->proxy_target_base); 11243277e8aaSGrygorii Strashko 11253277e8aaSGrygorii Strashko ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & 11263277e8aaSGrygorii Strashko K3_RINGACC_PROXY_CFG_THREADS_MASK; 11273277e8aaSGrygorii Strashko 11283277e8aaSGrygorii Strashko ringacc->rings = devm_kzalloc(dev, 11293277e8aaSGrygorii Strashko sizeof(*ringacc->rings) * 11303277e8aaSGrygorii Strashko ringacc->num_rings, 11313277e8aaSGrygorii Strashko GFP_KERNEL); 11323277e8aaSGrygorii Strashko ringacc->rings_inuse = devm_kcalloc(dev, 11333277e8aaSGrygorii Strashko BITS_TO_LONGS(ringacc->num_rings), 11343277e8aaSGrygorii Strashko sizeof(unsigned long), GFP_KERNEL); 11353277e8aaSGrygorii Strashko ringacc->proxy_inuse = devm_kcalloc(dev, 11363277e8aaSGrygorii Strashko BITS_TO_LONGS(ringacc->num_proxies), 11373277e8aaSGrygorii Strashko sizeof(unsigned long), GFP_KERNEL); 11383277e8aaSGrygorii Strashko 11393277e8aaSGrygorii Strashko if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) 11403277e8aaSGrygorii Strashko return -ENOMEM; 11413277e8aaSGrygorii Strashko 11423277e8aaSGrygorii Strashko for (i = 0; i < ringacc->num_rings; i++) { 11433277e8aaSGrygorii Strashko ringacc->rings[i].rt = base_rt + 11443277e8aaSGrygorii Strashko K3_RINGACC_RT_REGS_STEP * i; 11453277e8aaSGrygorii Strashko ringacc->rings[i].fifos = base_fifo + 11463277e8aaSGrygorii Strashko K3_RINGACC_FIFO_REGS_STEP * i; 11473277e8aaSGrygorii Strashko ringacc->rings[i].parent = ringacc; 11483277e8aaSGrygorii Strashko ringacc->rings[i].ring_id = i; 11493277e8aaSGrygorii Strashko ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; 11503277e8aaSGrygorii Strashko } 11513277e8aaSGrygorii Strashko dev_set_drvdata(dev, ringacc); 11523277e8aaSGrygorii Strashko 11533277e8aaSGrygorii Strashko ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 11543277e8aaSGrygorii Strashko 11553277e8aaSGrygorii Strashko mutex_lock(&k3_ringacc_list_lock); 11563277e8aaSGrygorii Strashko list_add_tail(&ringacc->list, &k3_ringacc_list); 11573277e8aaSGrygorii Strashko mutex_unlock(&k3_ringacc_list_lock); 11583277e8aaSGrygorii Strashko 11593277e8aaSGrygorii Strashko dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", 11603277e8aaSGrygorii Strashko ringacc->num_rings, 11613277e8aaSGrygorii Strashko ringacc->rm_gp_range->desc[0].start, 11623277e8aaSGrygorii Strashko ringacc->rm_gp_range->desc[0].num, 11633277e8aaSGrygorii Strashko ringacc->tisci_dev_id); 11643277e8aaSGrygorii Strashko dev_info(dev, "dma-ring-reset-quirk: %s\n", 11653277e8aaSGrygorii Strashko ringacc->dma_ring_reset_quirk ? "enabled" : "disabled"); 11663277e8aaSGrygorii Strashko dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", 11673277e8aaSGrygorii Strashko readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); 11683277e8aaSGrygorii Strashko return 0; 11693277e8aaSGrygorii Strashko } 11703277e8aaSGrygorii Strashko 11713277e8aaSGrygorii Strashko /* Match table for of_platform binding */ 11723277e8aaSGrygorii Strashko static const struct of_device_id k3_ringacc_of_match[] = { 11733277e8aaSGrygorii Strashko { .compatible = "ti,am654-navss-ringacc", }, 11743277e8aaSGrygorii Strashko {}, 11753277e8aaSGrygorii Strashko }; 11763277e8aaSGrygorii Strashko 11773277e8aaSGrygorii Strashko static struct platform_driver k3_ringacc_driver = { 11783277e8aaSGrygorii Strashko .probe = k3_ringacc_probe, 11793277e8aaSGrygorii Strashko .driver = { 11803277e8aaSGrygorii Strashko .name = "k3-ringacc", 11813277e8aaSGrygorii Strashko .of_match_table = k3_ringacc_of_match, 11823277e8aaSGrygorii Strashko .suppress_bind_attrs = true, 11833277e8aaSGrygorii Strashko }, 11843277e8aaSGrygorii Strashko }; 11853277e8aaSGrygorii Strashko builtin_platform_driver(k3_ringacc_driver); 1186