13277e8aaSGrygorii Strashko // SPDX-License-Identifier: GPL-2.0 23277e8aaSGrygorii Strashko /* 33277e8aaSGrygorii Strashko * TI K3 NAVSS Ring Accelerator subsystem driver 43277e8aaSGrygorii Strashko * 53277e8aaSGrygorii Strashko * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 63277e8aaSGrygorii Strashko */ 73277e8aaSGrygorii Strashko 83277e8aaSGrygorii Strashko #include <linux/dma-mapping.h> 93277e8aaSGrygorii Strashko #include <linux/io.h> 103277e8aaSGrygorii Strashko #include <linux/init.h> 113277e8aaSGrygorii Strashko #include <linux/of.h> 123277e8aaSGrygorii Strashko #include <linux/platform_device.h> 1395e7be06SGrygorii Strashko #include <linux/sys_soc.h> 143277e8aaSGrygorii Strashko #include <linux/soc/ti/k3-ringacc.h> 153277e8aaSGrygorii Strashko #include <linux/soc/ti/ti_sci_protocol.h> 163277e8aaSGrygorii Strashko #include <linux/soc/ti/ti_sci_inta_msi.h> 173277e8aaSGrygorii Strashko #include <linux/of_irq.h> 183277e8aaSGrygorii Strashko #include <linux/irqdomain.h> 193277e8aaSGrygorii Strashko 203277e8aaSGrygorii Strashko static LIST_HEAD(k3_ringacc_list); 213277e8aaSGrygorii Strashko static DEFINE_MUTEX(k3_ringacc_list_lock); 223277e8aaSGrygorii Strashko 233277e8aaSGrygorii Strashko #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) 243277e8aaSGrygorii Strashko 253277e8aaSGrygorii Strashko /** 263277e8aaSGrygorii Strashko * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region 273277e8aaSGrygorii Strashko * 283277e8aaSGrygorii Strashko * @resv_16: Reserved 293277e8aaSGrygorii Strashko * @db: Ring Doorbell Register 303277e8aaSGrygorii Strashko * @resv_4: Reserved 313277e8aaSGrygorii Strashko * @occ: Ring Occupancy Register 323277e8aaSGrygorii Strashko * @indx: Ring Current Index Register 333277e8aaSGrygorii Strashko * @hwocc: Ring Hardware Occupancy Register 343277e8aaSGrygorii Strashko * @hwindx: Ring Hardware Current Index Register 353277e8aaSGrygorii Strashko */ 363277e8aaSGrygorii Strashko struct k3_ring_rt_regs { 373277e8aaSGrygorii Strashko u32 resv_16[4]; 383277e8aaSGrygorii Strashko u32 db; 393277e8aaSGrygorii Strashko u32 resv_4[1]; 403277e8aaSGrygorii Strashko u32 occ; 413277e8aaSGrygorii Strashko u32 indx; 423277e8aaSGrygorii Strashko u32 hwocc; 433277e8aaSGrygorii Strashko u32 hwindx; 443277e8aaSGrygorii Strashko }; 453277e8aaSGrygorii Strashko 463277e8aaSGrygorii Strashko #define K3_RINGACC_RT_REGS_STEP 0x1000 473277e8aaSGrygorii Strashko 483277e8aaSGrygorii Strashko /** 493277e8aaSGrygorii Strashko * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region 503277e8aaSGrygorii Strashko * 513277e8aaSGrygorii Strashko * @head_data: Ring Head Entry Data Registers 523277e8aaSGrygorii Strashko * @tail_data: Ring Tail Entry Data Registers 533277e8aaSGrygorii Strashko * @peek_head_data: Ring Peek Head Entry Data Regs 543277e8aaSGrygorii Strashko * @peek_tail_data: Ring Peek Tail Entry Data Regs 553277e8aaSGrygorii Strashko */ 563277e8aaSGrygorii Strashko struct k3_ring_fifo_regs { 573277e8aaSGrygorii Strashko u32 head_data[128]; 583277e8aaSGrygorii Strashko u32 tail_data[128]; 593277e8aaSGrygorii Strashko u32 peek_head_data[128]; 603277e8aaSGrygorii Strashko u32 peek_tail_data[128]; 613277e8aaSGrygorii Strashko }; 623277e8aaSGrygorii Strashko 633277e8aaSGrygorii Strashko /** 643277e8aaSGrygorii Strashko * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region 653277e8aaSGrygorii Strashko * 663277e8aaSGrygorii Strashko * @revision: Revision Register 673277e8aaSGrygorii Strashko * @config: Config Register 683277e8aaSGrygorii Strashko */ 693277e8aaSGrygorii Strashko struct k3_ringacc_proxy_gcfg_regs { 703277e8aaSGrygorii Strashko u32 revision; 713277e8aaSGrygorii Strashko u32 config; 723277e8aaSGrygorii Strashko }; 733277e8aaSGrygorii Strashko 743277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) 753277e8aaSGrygorii Strashko 763277e8aaSGrygorii Strashko /** 773277e8aaSGrygorii Strashko * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region 783277e8aaSGrygorii Strashko * 793277e8aaSGrygorii Strashko * @control: Proxy Control Register 803277e8aaSGrygorii Strashko * @status: Proxy Status Register 813277e8aaSGrygorii Strashko * @resv_512: Reserved 823277e8aaSGrygorii Strashko * @data: Proxy Data Register 833277e8aaSGrygorii Strashko */ 843277e8aaSGrygorii Strashko struct k3_ringacc_proxy_target_regs { 853277e8aaSGrygorii Strashko u32 control; 863277e8aaSGrygorii Strashko u32 status; 873277e8aaSGrygorii Strashko u8 resv_512[504]; 883277e8aaSGrygorii Strashko u32 data[128]; 893277e8aaSGrygorii Strashko }; 903277e8aaSGrygorii Strashko 913277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_TARGET_STEP 0x1000 923277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_NOT_USED (-1) 933277e8aaSGrygorii Strashko 943277e8aaSGrygorii Strashko enum k3_ringacc_proxy_access_mode { 953277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_HEAD = 0, 963277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_TAIL = 1, 973277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_PEEK_HEAD = 2, 983277e8aaSGrygorii Strashko PROXY_ACCESS_MODE_PEEK_TAIL = 3, 993277e8aaSGrygorii Strashko }; 1003277e8aaSGrygorii Strashko 1013277e8aaSGrygorii Strashko #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) 1023277e8aaSGrygorii Strashko #define K3_RINGACC_FIFO_REGS_STEP 0x1000 1033277e8aaSGrygorii Strashko #define K3_RINGACC_MAX_DB_RING_CNT (127U) 1043277e8aaSGrygorii Strashko 1053277e8aaSGrygorii Strashko struct k3_ring_ops { 1063277e8aaSGrygorii Strashko int (*push_tail)(struct k3_ring *ring, void *elm); 1073277e8aaSGrygorii Strashko int (*push_head)(struct k3_ring *ring, void *elm); 1083277e8aaSGrygorii Strashko int (*pop_tail)(struct k3_ring *ring, void *elm); 1093277e8aaSGrygorii Strashko int (*pop_head)(struct k3_ring *ring, void *elm); 1103277e8aaSGrygorii Strashko }; 1113277e8aaSGrygorii Strashko 1123277e8aaSGrygorii Strashko /** 1136b3da0b4SPeter Ujfalusi * struct k3_ring_state - Internal state tracking structure 1146b3da0b4SPeter Ujfalusi * 1156b3da0b4SPeter Ujfalusi * @free: Number of free entries 1166b3da0b4SPeter Ujfalusi * @occ: Occupancy 1176b3da0b4SPeter Ujfalusi * @windex: Write index 1186b3da0b4SPeter Ujfalusi * @rindex: Read index 1196b3da0b4SPeter Ujfalusi */ 1206b3da0b4SPeter Ujfalusi struct k3_ring_state { 1216b3da0b4SPeter Ujfalusi u32 free; 1226b3da0b4SPeter Ujfalusi u32 occ; 1236b3da0b4SPeter Ujfalusi u32 windex; 1246b3da0b4SPeter Ujfalusi u32 rindex; 1256b3da0b4SPeter Ujfalusi }; 1266b3da0b4SPeter Ujfalusi 1276b3da0b4SPeter Ujfalusi /** 1283277e8aaSGrygorii Strashko * struct k3_ring - RA Ring descriptor 1293277e8aaSGrygorii Strashko * 1303277e8aaSGrygorii Strashko * @rt: Ring control/status registers 1313277e8aaSGrygorii Strashko * @fifos: Ring queues registers 1323277e8aaSGrygorii Strashko * @proxy: Ring Proxy Datapath registers 1333277e8aaSGrygorii Strashko * @ring_mem_dma: Ring buffer dma address 1343277e8aaSGrygorii Strashko * @ring_mem_virt: Ring buffer virt address 1353277e8aaSGrygorii Strashko * @ops: Ring operations 1363277e8aaSGrygorii Strashko * @size: Ring size in elements 1373277e8aaSGrygorii Strashko * @elm_size: Size of the ring element 1383277e8aaSGrygorii Strashko * @mode: Ring mode 1393277e8aaSGrygorii Strashko * @flags: flags 1403277e8aaSGrygorii Strashko * @ring_id: Ring Id 1413277e8aaSGrygorii Strashko * @parent: Pointer on struct @k3_ringacc 1423277e8aaSGrygorii Strashko * @use_count: Use count for shared rings 1433277e8aaSGrygorii Strashko * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) 1443277e8aaSGrygorii Strashko */ 1453277e8aaSGrygorii Strashko struct k3_ring { 1463277e8aaSGrygorii Strashko struct k3_ring_rt_regs __iomem *rt; 1473277e8aaSGrygorii Strashko struct k3_ring_fifo_regs __iomem *fifos; 1483277e8aaSGrygorii Strashko struct k3_ringacc_proxy_target_regs __iomem *proxy; 1493277e8aaSGrygorii Strashko dma_addr_t ring_mem_dma; 1503277e8aaSGrygorii Strashko void *ring_mem_virt; 1513277e8aaSGrygorii Strashko struct k3_ring_ops *ops; 1523277e8aaSGrygorii Strashko u32 size; 1533277e8aaSGrygorii Strashko enum k3_ring_size elm_size; 1543277e8aaSGrygorii Strashko enum k3_ring_mode mode; 1553277e8aaSGrygorii Strashko u32 flags; 1563277e8aaSGrygorii Strashko #define K3_RING_FLAG_BUSY BIT(1) 1573277e8aaSGrygorii Strashko #define K3_RING_FLAG_SHARED BIT(2) 1586b3da0b4SPeter Ujfalusi struct k3_ring_state state; 1593277e8aaSGrygorii Strashko u32 ring_id; 1603277e8aaSGrygorii Strashko struct k3_ringacc *parent; 1613277e8aaSGrygorii Strashko u32 use_count; 1623277e8aaSGrygorii Strashko int proxy_id; 1633277e8aaSGrygorii Strashko }; 1643277e8aaSGrygorii Strashko 16540a2a7c3SGrygorii Strashko struct k3_ringacc_ops { 16640a2a7c3SGrygorii Strashko int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc); 16740a2a7c3SGrygorii Strashko }; 16840a2a7c3SGrygorii Strashko 1693277e8aaSGrygorii Strashko /** 1703277e8aaSGrygorii Strashko * struct k3_ringacc - Rings accelerator descriptor 1713277e8aaSGrygorii Strashko * 1723277e8aaSGrygorii Strashko * @dev: pointer on RA device 1733277e8aaSGrygorii Strashko * @proxy_gcfg: RA proxy global config registers 1743277e8aaSGrygorii Strashko * @proxy_target_base: RA proxy datapath region 1753277e8aaSGrygorii Strashko * @num_rings: number of ring in RA 1763277e8aaSGrygorii Strashko * @rings_inuse: bitfield for ring usage tracking 1773277e8aaSGrygorii Strashko * @rm_gp_range: general purpose rings range from tisci 1783277e8aaSGrygorii Strashko * @dma_ring_reset_quirk: DMA reset w/a enable 1793277e8aaSGrygorii Strashko * @num_proxies: number of RA proxies 1803277e8aaSGrygorii Strashko * @proxy_inuse: bitfield for proxy usage tracking 1813277e8aaSGrygorii Strashko * @rings: array of rings descriptors (struct @k3_ring) 1823277e8aaSGrygorii Strashko * @list: list of RAs in the system 1833277e8aaSGrygorii Strashko * @req_lock: protect rings allocation 1843277e8aaSGrygorii Strashko * @tisci: pointer ti-sci handle 1853277e8aaSGrygorii Strashko * @tisci_ring_ops: ti-sci rings ops 1863277e8aaSGrygorii Strashko * @tisci_dev_id: ti-sci device id 18740a2a7c3SGrygorii Strashko * @ops: SoC specific ringacc operation 1883277e8aaSGrygorii Strashko */ 1893277e8aaSGrygorii Strashko struct k3_ringacc { 1903277e8aaSGrygorii Strashko struct device *dev; 1913277e8aaSGrygorii Strashko struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; 1923277e8aaSGrygorii Strashko void __iomem *proxy_target_base; 1933277e8aaSGrygorii Strashko u32 num_rings; /* number of rings in Ringacc module */ 1943277e8aaSGrygorii Strashko unsigned long *rings_inuse; 1953277e8aaSGrygorii Strashko struct ti_sci_resource *rm_gp_range; 1963277e8aaSGrygorii Strashko 1973277e8aaSGrygorii Strashko bool dma_ring_reset_quirk; 1983277e8aaSGrygorii Strashko u32 num_proxies; 1993277e8aaSGrygorii Strashko unsigned long *proxy_inuse; 2003277e8aaSGrygorii Strashko 2013277e8aaSGrygorii Strashko struct k3_ring *rings; 2023277e8aaSGrygorii Strashko struct list_head list; 2033277e8aaSGrygorii Strashko struct mutex req_lock; /* protect rings allocation */ 2043277e8aaSGrygorii Strashko 2053277e8aaSGrygorii Strashko const struct ti_sci_handle *tisci; 2063277e8aaSGrygorii Strashko const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; 2073277e8aaSGrygorii Strashko u32 tisci_dev_id; 20840a2a7c3SGrygorii Strashko 20940a2a7c3SGrygorii Strashko const struct k3_ringacc_ops *ops; 2103277e8aaSGrygorii Strashko }; 2113277e8aaSGrygorii Strashko 21295e7be06SGrygorii Strashko /** 21395e7be06SGrygorii Strashko * struct k3_ringacc - Rings accelerator SoC data 21495e7be06SGrygorii Strashko * 21595e7be06SGrygorii Strashko * @dma_ring_reset_quirk: DMA reset w/a enable 21695e7be06SGrygorii Strashko */ 21795e7be06SGrygorii Strashko struct k3_ringacc_soc_data { 21895e7be06SGrygorii Strashko unsigned dma_ring_reset_quirk:1; 21995e7be06SGrygorii Strashko }; 22095e7be06SGrygorii Strashko 2213277e8aaSGrygorii Strashko static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) 2223277e8aaSGrygorii Strashko { 2233277e8aaSGrygorii Strashko return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - 2243277e8aaSGrygorii Strashko (4 << ring->elm_size); 2253277e8aaSGrygorii Strashko } 2263277e8aaSGrygorii Strashko 2273277e8aaSGrygorii Strashko static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) 2283277e8aaSGrygorii Strashko { 2293277e8aaSGrygorii Strashko return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); 2303277e8aaSGrygorii Strashko } 2313277e8aaSGrygorii Strashko 2323277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); 2333277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); 2343277e8aaSGrygorii Strashko 2353277e8aaSGrygorii Strashko static struct k3_ring_ops k3_ring_mode_ring_ops = { 2363277e8aaSGrygorii Strashko .push_tail = k3_ringacc_ring_push_mem, 2373277e8aaSGrygorii Strashko .pop_head = k3_ringacc_ring_pop_mem, 2383277e8aaSGrygorii Strashko }; 2393277e8aaSGrygorii Strashko 2403277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); 2413277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); 2423277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); 2433277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); 2443277e8aaSGrygorii Strashko 2453277e8aaSGrygorii Strashko static struct k3_ring_ops k3_ring_mode_msg_ops = { 2463277e8aaSGrygorii Strashko .push_tail = k3_ringacc_ring_push_io, 2473277e8aaSGrygorii Strashko .push_head = k3_ringacc_ring_push_head_io, 2483277e8aaSGrygorii Strashko .pop_tail = k3_ringacc_ring_pop_tail_io, 2493277e8aaSGrygorii Strashko .pop_head = k3_ringacc_ring_pop_io, 2503277e8aaSGrygorii Strashko }; 2513277e8aaSGrygorii Strashko 2523277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); 2533277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); 2543277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); 2553277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); 2563277e8aaSGrygorii Strashko 2573277e8aaSGrygorii Strashko static struct k3_ring_ops k3_ring_mode_proxy_ops = { 2583277e8aaSGrygorii Strashko .push_tail = k3_ringacc_ring_push_tail_proxy, 2593277e8aaSGrygorii Strashko .push_head = k3_ringacc_ring_push_head_proxy, 2603277e8aaSGrygorii Strashko .pop_tail = k3_ringacc_ring_pop_tail_proxy, 2613277e8aaSGrygorii Strashko .pop_head = k3_ringacc_ring_pop_head_proxy, 2623277e8aaSGrygorii Strashko }; 2633277e8aaSGrygorii Strashko 2643277e8aaSGrygorii Strashko static void k3_ringacc_ring_dump(struct k3_ring *ring) 2653277e8aaSGrygorii Strashko { 2663277e8aaSGrygorii Strashko struct device *dev = ring->parent->dev; 2673277e8aaSGrygorii Strashko 2683277e8aaSGrygorii Strashko dev_dbg(dev, "dump ring: %d\n", ring->ring_id); 2693277e8aaSGrygorii Strashko dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, 2703277e8aaSGrygorii Strashko &ring->ring_mem_dma); 2713277e8aaSGrygorii Strashko dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", 2723277e8aaSGrygorii Strashko ring->elm_size, ring->size, ring->mode, ring->proxy_id); 273175e663fSGrygorii Strashko dev_dbg(dev, "dump flags %08X\n", ring->flags); 2743277e8aaSGrygorii Strashko 2753277e8aaSGrygorii Strashko dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); 2763277e8aaSGrygorii Strashko dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); 2773277e8aaSGrygorii Strashko dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); 2783277e8aaSGrygorii Strashko dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); 2793277e8aaSGrygorii Strashko dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); 2803277e8aaSGrygorii Strashko 2813277e8aaSGrygorii Strashko if (ring->ring_mem_virt) 2823277e8aaSGrygorii Strashko print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, 2833277e8aaSGrygorii Strashko 16, 1, ring->ring_mem_virt, 16 * 8, false); 2843277e8aaSGrygorii Strashko } 2853277e8aaSGrygorii Strashko 2863277e8aaSGrygorii Strashko struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, 2873277e8aaSGrygorii Strashko int id, u32 flags) 2883277e8aaSGrygorii Strashko { 2893277e8aaSGrygorii Strashko int proxy_id = K3_RINGACC_PROXY_NOT_USED; 2903277e8aaSGrygorii Strashko 2913277e8aaSGrygorii Strashko mutex_lock(&ringacc->req_lock); 2923277e8aaSGrygorii Strashko 2933277e8aaSGrygorii Strashko if (id == K3_RINGACC_RING_ID_ANY) { 2943277e8aaSGrygorii Strashko /* Request for any general purpose ring */ 2953277e8aaSGrygorii Strashko struct ti_sci_resource_desc *gp_rings = 2963277e8aaSGrygorii Strashko &ringacc->rm_gp_range->desc[0]; 2973277e8aaSGrygorii Strashko unsigned long size; 2983277e8aaSGrygorii Strashko 2993277e8aaSGrygorii Strashko size = gp_rings->start + gp_rings->num; 3003277e8aaSGrygorii Strashko id = find_next_zero_bit(ringacc->rings_inuse, size, 3013277e8aaSGrygorii Strashko gp_rings->start); 3023277e8aaSGrygorii Strashko if (id == size) 3033277e8aaSGrygorii Strashko goto error; 3043277e8aaSGrygorii Strashko } else if (id < 0) { 3053277e8aaSGrygorii Strashko goto error; 3063277e8aaSGrygorii Strashko } 3073277e8aaSGrygorii Strashko 3083277e8aaSGrygorii Strashko if (test_bit(id, ringacc->rings_inuse) && 3093277e8aaSGrygorii Strashko !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) 3103277e8aaSGrygorii Strashko goto error; 3113277e8aaSGrygorii Strashko else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) 3123277e8aaSGrygorii Strashko goto out; 3133277e8aaSGrygorii Strashko 3143277e8aaSGrygorii Strashko if (flags & K3_RINGACC_RING_USE_PROXY) { 3153277e8aaSGrygorii Strashko proxy_id = find_next_zero_bit(ringacc->proxy_inuse, 3163277e8aaSGrygorii Strashko ringacc->num_proxies, 0); 3173277e8aaSGrygorii Strashko if (proxy_id == ringacc->num_proxies) 3183277e8aaSGrygorii Strashko goto error; 3193277e8aaSGrygorii Strashko } 3203277e8aaSGrygorii Strashko 3213277e8aaSGrygorii Strashko if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { 3223277e8aaSGrygorii Strashko set_bit(proxy_id, ringacc->proxy_inuse); 3233277e8aaSGrygorii Strashko ringacc->rings[id].proxy_id = proxy_id; 3243277e8aaSGrygorii Strashko dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, 3253277e8aaSGrygorii Strashko proxy_id); 3263277e8aaSGrygorii Strashko } else { 3273277e8aaSGrygorii Strashko dev_dbg(ringacc->dev, "Giving ring#%d\n", id); 3283277e8aaSGrygorii Strashko } 3293277e8aaSGrygorii Strashko 3303277e8aaSGrygorii Strashko set_bit(id, ringacc->rings_inuse); 3313277e8aaSGrygorii Strashko out: 3323277e8aaSGrygorii Strashko ringacc->rings[id].use_count++; 3333277e8aaSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 3343277e8aaSGrygorii Strashko return &ringacc->rings[id]; 3353277e8aaSGrygorii Strashko 3363277e8aaSGrygorii Strashko error: 3373277e8aaSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 3383277e8aaSGrygorii Strashko return NULL; 3393277e8aaSGrygorii Strashko } 3403277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); 3413277e8aaSGrygorii Strashko 34243148b1cSGrygorii Strashko int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, 34343148b1cSGrygorii Strashko int fwd_id, int compl_id, 34443148b1cSGrygorii Strashko struct k3_ring **fwd_ring, 34543148b1cSGrygorii Strashko struct k3_ring **compl_ring) 34643148b1cSGrygorii Strashko { 34743148b1cSGrygorii Strashko int ret = 0; 34843148b1cSGrygorii Strashko 34943148b1cSGrygorii Strashko if (!fwd_ring || !compl_ring) 35043148b1cSGrygorii Strashko return -EINVAL; 35143148b1cSGrygorii Strashko 35243148b1cSGrygorii Strashko *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); 35343148b1cSGrygorii Strashko if (!(*fwd_ring)) 35443148b1cSGrygorii Strashko return -ENODEV; 35543148b1cSGrygorii Strashko 35643148b1cSGrygorii Strashko *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0); 35743148b1cSGrygorii Strashko if (!(*compl_ring)) { 35843148b1cSGrygorii Strashko k3_ringacc_ring_free(*fwd_ring); 35943148b1cSGrygorii Strashko ret = -ENODEV; 36043148b1cSGrygorii Strashko } 36143148b1cSGrygorii Strashko 36243148b1cSGrygorii Strashko return ret; 36343148b1cSGrygorii Strashko } 36443148b1cSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair); 36543148b1cSGrygorii Strashko 3663277e8aaSGrygorii Strashko static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) 3673277e8aaSGrygorii Strashko { 368*bb49ca00SPeter Ujfalusi struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 3693277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 3703277e8aaSGrygorii Strashko int ret; 3713277e8aaSGrygorii Strashko 372*bb49ca00SPeter Ujfalusi ring_cfg.nav_id = ringacc->tisci_dev_id; 373*bb49ca00SPeter Ujfalusi ring_cfg.index = ring->ring_id; 374*bb49ca00SPeter Ujfalusi ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID; 375*bb49ca00SPeter Ujfalusi ring_cfg.count = ring->size; 376*bb49ca00SPeter Ujfalusi 377*bb49ca00SPeter Ujfalusi ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 3783277e8aaSGrygorii Strashko if (ret) 3793277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", 3803277e8aaSGrygorii Strashko ret, ring->ring_id); 3813277e8aaSGrygorii Strashko } 3823277e8aaSGrygorii Strashko 3833277e8aaSGrygorii Strashko void k3_ringacc_ring_reset(struct k3_ring *ring) 3843277e8aaSGrygorii Strashko { 3853277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 3863277e8aaSGrygorii Strashko return; 3873277e8aaSGrygorii Strashko 3886b3da0b4SPeter Ujfalusi memset(&ring->state, 0, sizeof(ring->state)); 3893277e8aaSGrygorii Strashko 3903277e8aaSGrygorii Strashko k3_ringacc_ring_reset_sci(ring); 3913277e8aaSGrygorii Strashko } 3923277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); 3933277e8aaSGrygorii Strashko 3943277e8aaSGrygorii Strashko static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, 3953277e8aaSGrygorii Strashko enum k3_ring_mode mode) 3963277e8aaSGrygorii Strashko { 397*bb49ca00SPeter Ujfalusi struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 3983277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 3993277e8aaSGrygorii Strashko int ret; 4003277e8aaSGrygorii Strashko 401*bb49ca00SPeter Ujfalusi ring_cfg.nav_id = ringacc->tisci_dev_id; 402*bb49ca00SPeter Ujfalusi ring_cfg.index = ring->ring_id; 403*bb49ca00SPeter Ujfalusi ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID; 404*bb49ca00SPeter Ujfalusi ring_cfg.mode = mode; 405*bb49ca00SPeter Ujfalusi 406*bb49ca00SPeter Ujfalusi ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 4073277e8aaSGrygorii Strashko if (ret) 4083277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", 4093277e8aaSGrygorii Strashko ret, ring->ring_id); 4103277e8aaSGrygorii Strashko } 4113277e8aaSGrygorii Strashko 4123277e8aaSGrygorii Strashko void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) 4133277e8aaSGrygorii Strashko { 4143277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 4153277e8aaSGrygorii Strashko return; 4163277e8aaSGrygorii Strashko 4173277e8aaSGrygorii Strashko if (!ring->parent->dma_ring_reset_quirk) 4183277e8aaSGrygorii Strashko goto reset; 4193277e8aaSGrygorii Strashko 4203277e8aaSGrygorii Strashko if (!occ) 4213277e8aaSGrygorii Strashko occ = readl(&ring->rt->occ); 4223277e8aaSGrygorii Strashko 4233277e8aaSGrygorii Strashko if (occ) { 4243277e8aaSGrygorii Strashko u32 db_ring_cnt, db_ring_cnt_cur; 4253277e8aaSGrygorii Strashko 4263277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, 4273277e8aaSGrygorii Strashko ring->ring_id, occ); 4283277e8aaSGrygorii Strashko /* TI-SCI ring reset */ 4293277e8aaSGrygorii Strashko k3_ringacc_ring_reset_sci(ring); 4303277e8aaSGrygorii Strashko 4313277e8aaSGrygorii Strashko /* 4323277e8aaSGrygorii Strashko * Setup the ring in ring/doorbell mode (if not already in this 4333277e8aaSGrygorii Strashko * mode) 4343277e8aaSGrygorii Strashko */ 4353277e8aaSGrygorii Strashko if (ring->mode != K3_RINGACC_RING_MODE_RING) 4363277e8aaSGrygorii Strashko k3_ringacc_ring_reconfig_qmode_sci( 4373277e8aaSGrygorii Strashko ring, K3_RINGACC_RING_MODE_RING); 4383277e8aaSGrygorii Strashko /* 4393277e8aaSGrygorii Strashko * Ring the doorbell 2**22 – ringOcc times. 4403277e8aaSGrygorii Strashko * This will wrap the internal UDMAP ring state occupancy 4413277e8aaSGrygorii Strashko * counter (which is 21-bits wide) to 0. 4423277e8aaSGrygorii Strashko */ 4433277e8aaSGrygorii Strashko db_ring_cnt = (1U << 22) - occ; 4443277e8aaSGrygorii Strashko 4453277e8aaSGrygorii Strashko while (db_ring_cnt != 0) { 4463277e8aaSGrygorii Strashko /* 4473277e8aaSGrygorii Strashko * Ring the doorbell with the maximum count each 4483277e8aaSGrygorii Strashko * iteration if possible to minimize the total 4493277e8aaSGrygorii Strashko * of writes 4503277e8aaSGrygorii Strashko */ 4513277e8aaSGrygorii Strashko if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) 4523277e8aaSGrygorii Strashko db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; 4533277e8aaSGrygorii Strashko else 4543277e8aaSGrygorii Strashko db_ring_cnt_cur = db_ring_cnt; 4553277e8aaSGrygorii Strashko 4563277e8aaSGrygorii Strashko writel(db_ring_cnt_cur, &ring->rt->db); 4573277e8aaSGrygorii Strashko db_ring_cnt -= db_ring_cnt_cur; 4583277e8aaSGrygorii Strashko } 4593277e8aaSGrygorii Strashko 4603277e8aaSGrygorii Strashko /* Restore the original ring mode (if not ring mode) */ 4613277e8aaSGrygorii Strashko if (ring->mode != K3_RINGACC_RING_MODE_RING) 4623277e8aaSGrygorii Strashko k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); 4633277e8aaSGrygorii Strashko } 4643277e8aaSGrygorii Strashko 4653277e8aaSGrygorii Strashko reset: 4663277e8aaSGrygorii Strashko /* Reset the ring */ 4673277e8aaSGrygorii Strashko k3_ringacc_ring_reset(ring); 4683277e8aaSGrygorii Strashko } 4693277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); 4703277e8aaSGrygorii Strashko 4713277e8aaSGrygorii Strashko static void k3_ringacc_ring_free_sci(struct k3_ring *ring) 4723277e8aaSGrygorii Strashko { 473*bb49ca00SPeter Ujfalusi struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 4743277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 4753277e8aaSGrygorii Strashko int ret; 4763277e8aaSGrygorii Strashko 477*bb49ca00SPeter Ujfalusi ring_cfg.nav_id = ringacc->tisci_dev_id; 478*bb49ca00SPeter Ujfalusi ring_cfg.index = ring->ring_id; 479*bb49ca00SPeter Ujfalusi ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 480*bb49ca00SPeter Ujfalusi 481*bb49ca00SPeter Ujfalusi ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 4823277e8aaSGrygorii Strashko if (ret) 4833277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", 4843277e8aaSGrygorii Strashko ret, ring->ring_id); 4853277e8aaSGrygorii Strashko } 4863277e8aaSGrygorii Strashko 4873277e8aaSGrygorii Strashko int k3_ringacc_ring_free(struct k3_ring *ring) 4883277e8aaSGrygorii Strashko { 4893277e8aaSGrygorii Strashko struct k3_ringacc *ringacc; 4903277e8aaSGrygorii Strashko 4913277e8aaSGrygorii Strashko if (!ring) 4923277e8aaSGrygorii Strashko return -EINVAL; 4933277e8aaSGrygorii Strashko 4943277e8aaSGrygorii Strashko ringacc = ring->parent; 4953277e8aaSGrygorii Strashko 4963277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); 4973277e8aaSGrygorii Strashko 4983277e8aaSGrygorii Strashko if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 4993277e8aaSGrygorii Strashko return -EINVAL; 5003277e8aaSGrygorii Strashko 5013277e8aaSGrygorii Strashko mutex_lock(&ringacc->req_lock); 5023277e8aaSGrygorii Strashko 5033277e8aaSGrygorii Strashko if (--ring->use_count) 5043277e8aaSGrygorii Strashko goto out; 5053277e8aaSGrygorii Strashko 5063277e8aaSGrygorii Strashko if (!(ring->flags & K3_RING_FLAG_BUSY)) 5073277e8aaSGrygorii Strashko goto no_init; 5083277e8aaSGrygorii Strashko 5093277e8aaSGrygorii Strashko k3_ringacc_ring_free_sci(ring); 5103277e8aaSGrygorii Strashko 5113277e8aaSGrygorii Strashko dma_free_coherent(ringacc->dev, 5123277e8aaSGrygorii Strashko ring->size * (4 << ring->elm_size), 5133277e8aaSGrygorii Strashko ring->ring_mem_virt, ring->ring_mem_dma); 5143277e8aaSGrygorii Strashko ring->flags = 0; 5153277e8aaSGrygorii Strashko ring->ops = NULL; 5163277e8aaSGrygorii Strashko if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { 5173277e8aaSGrygorii Strashko clear_bit(ring->proxy_id, ringacc->proxy_inuse); 5183277e8aaSGrygorii Strashko ring->proxy = NULL; 5193277e8aaSGrygorii Strashko ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 5203277e8aaSGrygorii Strashko } 5213277e8aaSGrygorii Strashko 5223277e8aaSGrygorii Strashko no_init: 5233277e8aaSGrygorii Strashko clear_bit(ring->ring_id, ringacc->rings_inuse); 5243277e8aaSGrygorii Strashko 5253277e8aaSGrygorii Strashko out: 5263277e8aaSGrygorii Strashko mutex_unlock(&ringacc->req_lock); 5273277e8aaSGrygorii Strashko return 0; 5283277e8aaSGrygorii Strashko } 5293277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); 5303277e8aaSGrygorii Strashko 5313277e8aaSGrygorii Strashko u32 k3_ringacc_get_ring_id(struct k3_ring *ring) 5323277e8aaSGrygorii Strashko { 5333277e8aaSGrygorii Strashko if (!ring) 5343277e8aaSGrygorii Strashko return -EINVAL; 5353277e8aaSGrygorii Strashko 5363277e8aaSGrygorii Strashko return ring->ring_id; 5373277e8aaSGrygorii Strashko } 5383277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); 5393277e8aaSGrygorii Strashko 5403277e8aaSGrygorii Strashko u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) 5413277e8aaSGrygorii Strashko { 5423277e8aaSGrygorii Strashko if (!ring) 5433277e8aaSGrygorii Strashko return -EINVAL; 5443277e8aaSGrygorii Strashko 5453277e8aaSGrygorii Strashko return ring->parent->tisci_dev_id; 5463277e8aaSGrygorii Strashko } 5473277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); 5483277e8aaSGrygorii Strashko 5493277e8aaSGrygorii Strashko int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) 5503277e8aaSGrygorii Strashko { 5513277e8aaSGrygorii Strashko int irq_num; 5523277e8aaSGrygorii Strashko 5533277e8aaSGrygorii Strashko if (!ring) 5543277e8aaSGrygorii Strashko return -EINVAL; 5553277e8aaSGrygorii Strashko 5563277e8aaSGrygorii Strashko irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); 5573277e8aaSGrygorii Strashko if (irq_num <= 0) 5583277e8aaSGrygorii Strashko irq_num = -EINVAL; 5593277e8aaSGrygorii Strashko return irq_num; 5603277e8aaSGrygorii Strashko } 5613277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); 5623277e8aaSGrygorii Strashko 5633277e8aaSGrygorii Strashko static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) 5643277e8aaSGrygorii Strashko { 565*bb49ca00SPeter Ujfalusi struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 5663277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ring->parent; 5673277e8aaSGrygorii Strashko int ret; 5683277e8aaSGrygorii Strashko 5693277e8aaSGrygorii Strashko if (!ringacc->tisci) 5703277e8aaSGrygorii Strashko return -EINVAL; 5713277e8aaSGrygorii Strashko 572*bb49ca00SPeter Ujfalusi ring_cfg.nav_id = ringacc->tisci_dev_id; 573*bb49ca00SPeter Ujfalusi ring_cfg.index = ring->ring_id; 574*bb49ca00SPeter Ujfalusi ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 575*bb49ca00SPeter Ujfalusi ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma); 576*bb49ca00SPeter Ujfalusi ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma); 577*bb49ca00SPeter Ujfalusi ring_cfg.count = ring->size; 578*bb49ca00SPeter Ujfalusi ring_cfg.mode = ring->mode; 579*bb49ca00SPeter Ujfalusi ring_cfg.size = ring->elm_size; 580*bb49ca00SPeter Ujfalusi 581*bb49ca00SPeter Ujfalusi ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 5823277e8aaSGrygorii Strashko if (ret) 5833277e8aaSGrygorii Strashko dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", 584*bb49ca00SPeter Ujfalusi ret, ring->ring_id); 5853277e8aaSGrygorii Strashko 5863277e8aaSGrygorii Strashko return ret; 5873277e8aaSGrygorii Strashko } 5883277e8aaSGrygorii Strashko 5893277e8aaSGrygorii Strashko int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 5903277e8aaSGrygorii Strashko { 59180ff73f7SGrygorii Strashko struct k3_ringacc *ringacc; 5923277e8aaSGrygorii Strashko int ret = 0; 5933277e8aaSGrygorii Strashko 5943277e8aaSGrygorii Strashko if (!ring || !cfg) 5953277e8aaSGrygorii Strashko return -EINVAL; 59680ff73f7SGrygorii Strashko ringacc = ring->parent; 59780ff73f7SGrygorii Strashko 5983277e8aaSGrygorii Strashko if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || 5993277e8aaSGrygorii Strashko cfg->mode >= K3_RINGACC_RING_MODE_INVALID || 6003277e8aaSGrygorii Strashko cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || 6013277e8aaSGrygorii Strashko !test_bit(ring->ring_id, ringacc->rings_inuse)) 6023277e8aaSGrygorii Strashko return -EINVAL; 6033277e8aaSGrygorii Strashko 6043277e8aaSGrygorii Strashko if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && 6053277e8aaSGrygorii Strashko ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && 6063277e8aaSGrygorii Strashko cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { 6073277e8aaSGrygorii Strashko dev_err(ringacc->dev, 6083277e8aaSGrygorii Strashko "Message mode must use proxy for %u element size\n", 6093277e8aaSGrygorii Strashko 4 << ring->elm_size); 6103277e8aaSGrygorii Strashko return -EINVAL; 6113277e8aaSGrygorii Strashko } 6123277e8aaSGrygorii Strashko 6133277e8aaSGrygorii Strashko /* 6143277e8aaSGrygorii Strashko * In case of shared ring only the first user (master user) can 6153277e8aaSGrygorii Strashko * configure the ring. The sequence should be by the client: 6163277e8aaSGrygorii Strashko * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user 6173277e8aaSGrygorii Strashko * k3_ringacc_ring_cfg(ring, cfg); # master configuration 6183277e8aaSGrygorii Strashko * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 6193277e8aaSGrygorii Strashko * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 6203277e8aaSGrygorii Strashko */ 6213277e8aaSGrygorii Strashko if (ring->use_count != 1) 6223277e8aaSGrygorii Strashko return 0; 6233277e8aaSGrygorii Strashko 6243277e8aaSGrygorii Strashko ring->size = cfg->size; 6253277e8aaSGrygorii Strashko ring->elm_size = cfg->elm_size; 6263277e8aaSGrygorii Strashko ring->mode = cfg->mode; 6276b3da0b4SPeter Ujfalusi memset(&ring->state, 0, sizeof(ring->state)); 6283277e8aaSGrygorii Strashko 6293277e8aaSGrygorii Strashko if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) 6303277e8aaSGrygorii Strashko ring->proxy = ringacc->proxy_target_base + 6313277e8aaSGrygorii Strashko ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; 6323277e8aaSGrygorii Strashko 6333277e8aaSGrygorii Strashko switch (ring->mode) { 6343277e8aaSGrygorii Strashko case K3_RINGACC_RING_MODE_RING: 6353277e8aaSGrygorii Strashko ring->ops = &k3_ring_mode_ring_ops; 6363277e8aaSGrygorii Strashko break; 6373277e8aaSGrygorii Strashko case K3_RINGACC_RING_MODE_MESSAGE: 6383277e8aaSGrygorii Strashko if (ring->proxy) 6393277e8aaSGrygorii Strashko ring->ops = &k3_ring_mode_proxy_ops; 6403277e8aaSGrygorii Strashko else 6413277e8aaSGrygorii Strashko ring->ops = &k3_ring_mode_msg_ops; 6423277e8aaSGrygorii Strashko break; 6433277e8aaSGrygorii Strashko default: 6443277e8aaSGrygorii Strashko ring->ops = NULL; 6453277e8aaSGrygorii Strashko ret = -EINVAL; 6463277e8aaSGrygorii Strashko goto err_free_proxy; 6471d036016Skernel test robot } 6483277e8aaSGrygorii Strashko 6493277e8aaSGrygorii Strashko ring->ring_mem_virt = dma_alloc_coherent(ringacc->dev, 6503277e8aaSGrygorii Strashko ring->size * (4 << ring->elm_size), 6513277e8aaSGrygorii Strashko &ring->ring_mem_dma, GFP_KERNEL); 6523277e8aaSGrygorii Strashko if (!ring->ring_mem_virt) { 6533277e8aaSGrygorii Strashko dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 6543277e8aaSGrygorii Strashko ret = -ENOMEM; 6553277e8aaSGrygorii Strashko goto err_free_ops; 6563277e8aaSGrygorii Strashko } 6573277e8aaSGrygorii Strashko 6583277e8aaSGrygorii Strashko ret = k3_ringacc_ring_cfg_sci(ring); 6593277e8aaSGrygorii Strashko 6603277e8aaSGrygorii Strashko if (ret) 6613277e8aaSGrygorii Strashko goto err_free_mem; 6623277e8aaSGrygorii Strashko 6633277e8aaSGrygorii Strashko ring->flags |= K3_RING_FLAG_BUSY; 6643277e8aaSGrygorii Strashko ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? 6653277e8aaSGrygorii Strashko K3_RING_FLAG_SHARED : 0; 6663277e8aaSGrygorii Strashko 6673277e8aaSGrygorii Strashko k3_ringacc_ring_dump(ring); 6683277e8aaSGrygorii Strashko 6693277e8aaSGrygorii Strashko return 0; 6703277e8aaSGrygorii Strashko 6713277e8aaSGrygorii Strashko err_free_mem: 6723277e8aaSGrygorii Strashko dma_free_coherent(ringacc->dev, 6733277e8aaSGrygorii Strashko ring->size * (4 << ring->elm_size), 6743277e8aaSGrygorii Strashko ring->ring_mem_virt, 6753277e8aaSGrygorii Strashko ring->ring_mem_dma); 6763277e8aaSGrygorii Strashko err_free_ops: 6773277e8aaSGrygorii Strashko ring->ops = NULL; 6783277e8aaSGrygorii Strashko err_free_proxy: 6793277e8aaSGrygorii Strashko ring->proxy = NULL; 6803277e8aaSGrygorii Strashko return ret; 6813277e8aaSGrygorii Strashko } 6823277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); 6833277e8aaSGrygorii Strashko 6843277e8aaSGrygorii Strashko u32 k3_ringacc_ring_get_size(struct k3_ring *ring) 6853277e8aaSGrygorii Strashko { 6863277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 6873277e8aaSGrygorii Strashko return -EINVAL; 6883277e8aaSGrygorii Strashko 6893277e8aaSGrygorii Strashko return ring->size; 6903277e8aaSGrygorii Strashko } 6913277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); 6923277e8aaSGrygorii Strashko 6933277e8aaSGrygorii Strashko u32 k3_ringacc_ring_get_free(struct k3_ring *ring) 6943277e8aaSGrygorii Strashko { 6953277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 6963277e8aaSGrygorii Strashko return -EINVAL; 6973277e8aaSGrygorii Strashko 6986b3da0b4SPeter Ujfalusi if (!ring->state.free) 6996b3da0b4SPeter Ujfalusi ring->state.free = ring->size - readl(&ring->rt->occ); 7003277e8aaSGrygorii Strashko 7016b3da0b4SPeter Ujfalusi return ring->state.free; 7023277e8aaSGrygorii Strashko } 7033277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); 7043277e8aaSGrygorii Strashko 7053277e8aaSGrygorii Strashko u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) 7063277e8aaSGrygorii Strashko { 7073277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 7083277e8aaSGrygorii Strashko return -EINVAL; 7093277e8aaSGrygorii Strashko 7103277e8aaSGrygorii Strashko return readl(&ring->rt->occ); 7113277e8aaSGrygorii Strashko } 7123277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); 7133277e8aaSGrygorii Strashko 7143277e8aaSGrygorii Strashko u32 k3_ringacc_ring_is_full(struct k3_ring *ring) 7153277e8aaSGrygorii Strashko { 7163277e8aaSGrygorii Strashko return !k3_ringacc_ring_get_free(ring); 7173277e8aaSGrygorii Strashko } 7183277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); 7193277e8aaSGrygorii Strashko 7203277e8aaSGrygorii Strashko enum k3_ringacc_access_mode { 7213277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_HEAD, 7223277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD, 7233277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_TAIL, 7243277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_TAIL, 7253277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PEEK_HEAD, 7263277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PEEK_TAIL, 7273277e8aaSGrygorii Strashko }; 7283277e8aaSGrygorii Strashko 7293277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) 7303277e8aaSGrygorii Strashko #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) 7313277e8aaSGrygorii Strashko static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, 7323277e8aaSGrygorii Strashko enum k3_ringacc_proxy_access_mode mode) 7333277e8aaSGrygorii Strashko { 7343277e8aaSGrygorii Strashko u32 val; 7353277e8aaSGrygorii Strashko 7363277e8aaSGrygorii Strashko val = ring->ring_id; 7373277e8aaSGrygorii Strashko val |= K3_RINGACC_PROXY_MODE(mode); 7383277e8aaSGrygorii Strashko val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); 7393277e8aaSGrygorii Strashko writel(val, &ring->proxy->control); 7403277e8aaSGrygorii Strashko return 0; 7413277e8aaSGrygorii Strashko } 7423277e8aaSGrygorii Strashko 7433277e8aaSGrygorii Strashko static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, 7443277e8aaSGrygorii Strashko enum k3_ringacc_access_mode access_mode) 7453277e8aaSGrygorii Strashko { 7463277e8aaSGrygorii Strashko void __iomem *ptr; 7473277e8aaSGrygorii Strashko 7483277e8aaSGrygorii Strashko ptr = (void __iomem *)&ring->proxy->data; 7493277e8aaSGrygorii Strashko 7503277e8aaSGrygorii Strashko switch (access_mode) { 7513277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 7523277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 7533277e8aaSGrygorii Strashko k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); 7543277e8aaSGrygorii Strashko break; 7553277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 7563277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 7573277e8aaSGrygorii Strashko k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); 7583277e8aaSGrygorii Strashko break; 7593277e8aaSGrygorii Strashko default: 7603277e8aaSGrygorii Strashko return -EINVAL; 7613277e8aaSGrygorii Strashko } 7623277e8aaSGrygorii Strashko 7633277e8aaSGrygorii Strashko ptr += k3_ringacc_ring_get_fifo_pos(ring); 7643277e8aaSGrygorii Strashko 7653277e8aaSGrygorii Strashko switch (access_mode) { 7663277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 7673277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 7683277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 7693277e8aaSGrygorii Strashko "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 7703277e8aaSGrygorii Strashko access_mode); 7713277e8aaSGrygorii Strashko memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 7726b3da0b4SPeter Ujfalusi ring->state.occ--; 7733277e8aaSGrygorii Strashko break; 7743277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 7753277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 7763277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 7773277e8aaSGrygorii Strashko "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 7783277e8aaSGrygorii Strashko access_mode); 7793277e8aaSGrygorii Strashko memcpy_toio(ptr, elem, (4 << ring->elm_size)); 7806b3da0b4SPeter Ujfalusi ring->state.free--; 7813277e8aaSGrygorii Strashko break; 7823277e8aaSGrygorii Strashko default: 7833277e8aaSGrygorii Strashko return -EINVAL; 7843277e8aaSGrygorii Strashko } 7853277e8aaSGrygorii Strashko 7866b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, 7876b3da0b4SPeter Ujfalusi ring->state.occ); 7883277e8aaSGrygorii Strashko return 0; 7893277e8aaSGrygorii Strashko } 7903277e8aaSGrygorii Strashko 7913277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) 7923277e8aaSGrygorii Strashko { 7933277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 7943277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 7953277e8aaSGrygorii Strashko } 7963277e8aaSGrygorii Strashko 7973277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) 7983277e8aaSGrygorii Strashko { 7993277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 8003277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 8013277e8aaSGrygorii Strashko } 8023277e8aaSGrygorii Strashko 8033277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) 8043277e8aaSGrygorii Strashko { 8053277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 8063277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 8073277e8aaSGrygorii Strashko } 8083277e8aaSGrygorii Strashko 8093277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) 8103277e8aaSGrygorii Strashko { 8113277e8aaSGrygorii Strashko return k3_ringacc_ring_access_proxy(ring, elem, 8123277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 8133277e8aaSGrygorii Strashko } 8143277e8aaSGrygorii Strashko 8153277e8aaSGrygorii Strashko static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, 8163277e8aaSGrygorii Strashko enum k3_ringacc_access_mode access_mode) 8173277e8aaSGrygorii Strashko { 8183277e8aaSGrygorii Strashko void __iomem *ptr; 8193277e8aaSGrygorii Strashko 8203277e8aaSGrygorii Strashko switch (access_mode) { 8213277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 8223277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 8233277e8aaSGrygorii Strashko ptr = (void __iomem *)&ring->fifos->head_data; 8243277e8aaSGrygorii Strashko break; 8253277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 8263277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 8273277e8aaSGrygorii Strashko ptr = (void __iomem *)&ring->fifos->tail_data; 8283277e8aaSGrygorii Strashko break; 8293277e8aaSGrygorii Strashko default: 8303277e8aaSGrygorii Strashko return -EINVAL; 8313277e8aaSGrygorii Strashko } 8323277e8aaSGrygorii Strashko 8333277e8aaSGrygorii Strashko ptr += k3_ringacc_ring_get_fifo_pos(ring); 8343277e8aaSGrygorii Strashko 8353277e8aaSGrygorii Strashko switch (access_mode) { 8363277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_HEAD: 8373277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_POP_TAIL: 8383277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 8393277e8aaSGrygorii Strashko "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 8403277e8aaSGrygorii Strashko access_mode); 8413277e8aaSGrygorii Strashko memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 8426b3da0b4SPeter Ujfalusi ring->state.occ--; 8433277e8aaSGrygorii Strashko break; 8443277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 8453277e8aaSGrygorii Strashko case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 8463277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, 8473277e8aaSGrygorii Strashko "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 8483277e8aaSGrygorii Strashko access_mode); 8493277e8aaSGrygorii Strashko memcpy_toio(ptr, elem, (4 << ring->elm_size)); 8506b3da0b4SPeter Ujfalusi ring->state.free--; 8513277e8aaSGrygorii Strashko break; 8523277e8aaSGrygorii Strashko default: 8533277e8aaSGrygorii Strashko return -EINVAL; 8543277e8aaSGrygorii Strashko } 8553277e8aaSGrygorii Strashko 8566b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", 8576b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex, ring->state.occ, 8586b3da0b4SPeter Ujfalusi ring->state.rindex); 8593277e8aaSGrygorii Strashko return 0; 8603277e8aaSGrygorii Strashko } 8613277e8aaSGrygorii Strashko 8623277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) 8633277e8aaSGrygorii Strashko { 8643277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 8653277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 8663277e8aaSGrygorii Strashko } 8673277e8aaSGrygorii Strashko 8683277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) 8693277e8aaSGrygorii Strashko { 8703277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 8713277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 8723277e8aaSGrygorii Strashko } 8733277e8aaSGrygorii Strashko 8743277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) 8753277e8aaSGrygorii Strashko { 8763277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 8773277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 8783277e8aaSGrygorii Strashko } 8793277e8aaSGrygorii Strashko 8803277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) 8813277e8aaSGrygorii Strashko { 8823277e8aaSGrygorii Strashko return k3_ringacc_ring_access_io(ring, elem, 8833277e8aaSGrygorii Strashko K3_RINGACC_ACCESS_MODE_POP_HEAD); 8843277e8aaSGrygorii Strashko } 8853277e8aaSGrygorii Strashko 8863277e8aaSGrygorii Strashko static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) 8873277e8aaSGrygorii Strashko { 8883277e8aaSGrygorii Strashko void *elem_ptr; 8893277e8aaSGrygorii Strashko 8906b3da0b4SPeter Ujfalusi elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); 8913277e8aaSGrygorii Strashko 8923277e8aaSGrygorii Strashko memcpy(elem_ptr, elem, (4 << ring->elm_size)); 8933277e8aaSGrygorii Strashko 8946b3da0b4SPeter Ujfalusi ring->state.windex = (ring->state.windex + 1) % ring->size; 8956b3da0b4SPeter Ujfalusi ring->state.free--; 8963277e8aaSGrygorii Strashko writel(1, &ring->rt->db); 8973277e8aaSGrygorii Strashko 8983277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", 8996b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex); 9003277e8aaSGrygorii Strashko 9013277e8aaSGrygorii Strashko return 0; 9023277e8aaSGrygorii Strashko } 9033277e8aaSGrygorii Strashko 9043277e8aaSGrygorii Strashko static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) 9053277e8aaSGrygorii Strashko { 9063277e8aaSGrygorii Strashko void *elem_ptr; 9073277e8aaSGrygorii Strashko 9086b3da0b4SPeter Ujfalusi elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 9093277e8aaSGrygorii Strashko 9103277e8aaSGrygorii Strashko memcpy(elem, elem_ptr, (4 << ring->elm_size)); 9113277e8aaSGrygorii Strashko 9126b3da0b4SPeter Ujfalusi ring->state.rindex = (ring->state.rindex + 1) % ring->size; 9136b3da0b4SPeter Ujfalusi ring->state.occ--; 9143277e8aaSGrygorii Strashko writel(-1, &ring->rt->db); 9153277e8aaSGrygorii Strashko 9163277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", 9176b3da0b4SPeter Ujfalusi ring->state.occ, ring->state.rindex, elem_ptr); 9183277e8aaSGrygorii Strashko return 0; 9193277e8aaSGrygorii Strashko } 9203277e8aaSGrygorii Strashko 9213277e8aaSGrygorii Strashko int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) 9223277e8aaSGrygorii Strashko { 9233277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 9243277e8aaSGrygorii Strashko 9253277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 9263277e8aaSGrygorii Strashko return -EINVAL; 9273277e8aaSGrygorii Strashko 9286b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", 9296b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex); 9303277e8aaSGrygorii Strashko 9313277e8aaSGrygorii Strashko if (k3_ringacc_ring_is_full(ring)) 9323277e8aaSGrygorii Strashko return -ENOMEM; 9333277e8aaSGrygorii Strashko 9343277e8aaSGrygorii Strashko if (ring->ops && ring->ops->push_tail) 9353277e8aaSGrygorii Strashko ret = ring->ops->push_tail(ring, elem); 9363277e8aaSGrygorii Strashko 9373277e8aaSGrygorii Strashko return ret; 9383277e8aaSGrygorii Strashko } 9393277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); 9403277e8aaSGrygorii Strashko 9413277e8aaSGrygorii Strashko int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) 9423277e8aaSGrygorii Strashko { 9433277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 9443277e8aaSGrygorii Strashko 9453277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 9463277e8aaSGrygorii Strashko return -EINVAL; 9473277e8aaSGrygorii Strashko 9483277e8aaSGrygorii Strashko dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", 9496b3da0b4SPeter Ujfalusi ring->state.free, ring->state.windex); 9503277e8aaSGrygorii Strashko 9513277e8aaSGrygorii Strashko if (k3_ringacc_ring_is_full(ring)) 9523277e8aaSGrygorii Strashko return -ENOMEM; 9533277e8aaSGrygorii Strashko 9543277e8aaSGrygorii Strashko if (ring->ops && ring->ops->push_head) 9553277e8aaSGrygorii Strashko ret = ring->ops->push_head(ring, elem); 9563277e8aaSGrygorii Strashko 9573277e8aaSGrygorii Strashko return ret; 9583277e8aaSGrygorii Strashko } 9593277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); 9603277e8aaSGrygorii Strashko 9613277e8aaSGrygorii Strashko int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) 9623277e8aaSGrygorii Strashko { 9633277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 9643277e8aaSGrygorii Strashko 9653277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 9663277e8aaSGrygorii Strashko return -EINVAL; 9673277e8aaSGrygorii Strashko 9686b3da0b4SPeter Ujfalusi if (!ring->state.occ) 9696b3da0b4SPeter Ujfalusi ring->state.occ = k3_ringacc_ring_get_occ(ring); 9703277e8aaSGrygorii Strashko 9716b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, 9726b3da0b4SPeter Ujfalusi ring->state.rindex); 9733277e8aaSGrygorii Strashko 9746b3da0b4SPeter Ujfalusi if (!ring->state.occ) 9753277e8aaSGrygorii Strashko return -ENODATA; 9763277e8aaSGrygorii Strashko 9773277e8aaSGrygorii Strashko if (ring->ops && ring->ops->pop_head) 9783277e8aaSGrygorii Strashko ret = ring->ops->pop_head(ring, elem); 9793277e8aaSGrygorii Strashko 9803277e8aaSGrygorii Strashko return ret; 9813277e8aaSGrygorii Strashko } 9823277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); 9833277e8aaSGrygorii Strashko 9843277e8aaSGrygorii Strashko int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) 9853277e8aaSGrygorii Strashko { 9863277e8aaSGrygorii Strashko int ret = -EOPNOTSUPP; 9873277e8aaSGrygorii Strashko 9883277e8aaSGrygorii Strashko if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 9893277e8aaSGrygorii Strashko return -EINVAL; 9903277e8aaSGrygorii Strashko 9916b3da0b4SPeter Ujfalusi if (!ring->state.occ) 9926b3da0b4SPeter Ujfalusi ring->state.occ = k3_ringacc_ring_get_occ(ring); 9933277e8aaSGrygorii Strashko 9946b3da0b4SPeter Ujfalusi dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", 9956b3da0b4SPeter Ujfalusi ring->state.occ, ring->state.rindex); 9963277e8aaSGrygorii Strashko 9976b3da0b4SPeter Ujfalusi if (!ring->state.occ) 9983277e8aaSGrygorii Strashko return -ENODATA; 9993277e8aaSGrygorii Strashko 10003277e8aaSGrygorii Strashko if (ring->ops && ring->ops->pop_tail) 10013277e8aaSGrygorii Strashko ret = ring->ops->pop_tail(ring, elem); 10023277e8aaSGrygorii Strashko 10033277e8aaSGrygorii Strashko return ret; 10043277e8aaSGrygorii Strashko } 10053277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); 10063277e8aaSGrygorii Strashko 10073277e8aaSGrygorii Strashko struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, 10083277e8aaSGrygorii Strashko const char *property) 10093277e8aaSGrygorii Strashko { 10103277e8aaSGrygorii Strashko struct device_node *ringacc_np; 10113277e8aaSGrygorii Strashko struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); 10123277e8aaSGrygorii Strashko struct k3_ringacc *entry; 10133277e8aaSGrygorii Strashko 10143277e8aaSGrygorii Strashko ringacc_np = of_parse_phandle(np, property, 0); 10153277e8aaSGrygorii Strashko if (!ringacc_np) 10163277e8aaSGrygorii Strashko return ERR_PTR(-ENODEV); 10173277e8aaSGrygorii Strashko 10183277e8aaSGrygorii Strashko mutex_lock(&k3_ringacc_list_lock); 10193277e8aaSGrygorii Strashko list_for_each_entry(entry, &k3_ringacc_list, list) 10203277e8aaSGrygorii Strashko if (entry->dev->of_node == ringacc_np) { 10213277e8aaSGrygorii Strashko ringacc = entry; 10223277e8aaSGrygorii Strashko break; 10233277e8aaSGrygorii Strashko } 10243277e8aaSGrygorii Strashko mutex_unlock(&k3_ringacc_list_lock); 10253277e8aaSGrygorii Strashko of_node_put(ringacc_np); 10263277e8aaSGrygorii Strashko 10273277e8aaSGrygorii Strashko return ringacc; 10283277e8aaSGrygorii Strashko } 10293277e8aaSGrygorii Strashko EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); 10303277e8aaSGrygorii Strashko 10313277e8aaSGrygorii Strashko static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) 10323277e8aaSGrygorii Strashko { 10333277e8aaSGrygorii Strashko struct device_node *node = ringacc->dev->of_node; 10343277e8aaSGrygorii Strashko struct device *dev = ringacc->dev; 10353277e8aaSGrygorii Strashko struct platform_device *pdev = to_platform_device(dev); 10363277e8aaSGrygorii Strashko int ret; 10373277e8aaSGrygorii Strashko 10383277e8aaSGrygorii Strashko if (!node) { 10393277e8aaSGrygorii Strashko dev_err(dev, "device tree info unavailable\n"); 10403277e8aaSGrygorii Strashko return -ENODEV; 10413277e8aaSGrygorii Strashko } 10423277e8aaSGrygorii Strashko 10433277e8aaSGrygorii Strashko ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); 10443277e8aaSGrygorii Strashko if (ret) { 10453277e8aaSGrygorii Strashko dev_err(dev, "ti,num-rings read failure %d\n", ret); 10463277e8aaSGrygorii Strashko return ret; 10473277e8aaSGrygorii Strashko } 10483277e8aaSGrygorii Strashko 10493277e8aaSGrygorii Strashko ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); 10503277e8aaSGrygorii Strashko if (IS_ERR(ringacc->tisci)) { 10513277e8aaSGrygorii Strashko ret = PTR_ERR(ringacc->tisci); 10523277e8aaSGrygorii Strashko if (ret != -EPROBE_DEFER) 10533277e8aaSGrygorii Strashko dev_err(dev, "ti,sci read fail %d\n", ret); 10543277e8aaSGrygorii Strashko ringacc->tisci = NULL; 10553277e8aaSGrygorii Strashko return ret; 10563277e8aaSGrygorii Strashko } 10573277e8aaSGrygorii Strashko 10583277e8aaSGrygorii Strashko ret = of_property_read_u32(node, "ti,sci-dev-id", 10593277e8aaSGrygorii Strashko &ringacc->tisci_dev_id); 10603277e8aaSGrygorii Strashko if (ret) { 10613277e8aaSGrygorii Strashko dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); 10623277e8aaSGrygorii Strashko return ret; 10633277e8aaSGrygorii Strashko } 10643277e8aaSGrygorii Strashko 10653277e8aaSGrygorii Strashko pdev->id = ringacc->tisci_dev_id; 10663277e8aaSGrygorii Strashko 10673277e8aaSGrygorii Strashko ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, 10683277e8aaSGrygorii Strashko ringacc->tisci_dev_id, 10693277e8aaSGrygorii Strashko "ti,sci-rm-range-gp-rings"); 10703277e8aaSGrygorii Strashko if (IS_ERR(ringacc->rm_gp_range)) { 10713277e8aaSGrygorii Strashko dev_err(dev, "Failed to allocate MSI interrupts\n"); 10723277e8aaSGrygorii Strashko return PTR_ERR(ringacc->rm_gp_range); 10733277e8aaSGrygorii Strashko } 10743277e8aaSGrygorii Strashko 10753277e8aaSGrygorii Strashko return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, 10763277e8aaSGrygorii Strashko ringacc->rm_gp_range); 10773277e8aaSGrygorii Strashko } 10783277e8aaSGrygorii Strashko 107995e7be06SGrygorii Strashko static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = { 108095e7be06SGrygorii Strashko .dma_ring_reset_quirk = 1, 108195e7be06SGrygorii Strashko }; 108295e7be06SGrygorii Strashko 108395e7be06SGrygorii Strashko static const struct soc_device_attribute k3_ringacc_socinfo[] = { 108495e7be06SGrygorii Strashko { .family = "AM65X", 108595e7be06SGrygorii Strashko .revision = "SR1.0", 108695e7be06SGrygorii Strashko .data = &k3_ringacc_soc_data_sr1 108795e7be06SGrygorii Strashko }, 108895e7be06SGrygorii Strashko {/* sentinel */} 108995e7be06SGrygorii Strashko }; 109095e7be06SGrygorii Strashko 109140a2a7c3SGrygorii Strashko static int k3_ringacc_init(struct platform_device *pdev, 109240a2a7c3SGrygorii Strashko struct k3_ringacc *ringacc) 10933277e8aaSGrygorii Strashko { 109495e7be06SGrygorii Strashko const struct soc_device_attribute *soc; 10953277e8aaSGrygorii Strashko void __iomem *base_fifo, *base_rt; 10963277e8aaSGrygorii Strashko struct device *dev = &pdev->dev; 10973277e8aaSGrygorii Strashko struct resource *res; 10983277e8aaSGrygorii Strashko int ret, i; 10993277e8aaSGrygorii Strashko 11003277e8aaSGrygorii Strashko dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 11013277e8aaSGrygorii Strashko DOMAIN_BUS_TI_SCI_INTA_MSI); 11023277e8aaSGrygorii Strashko if (!dev->msi_domain) { 11033277e8aaSGrygorii Strashko dev_err(dev, "Failed to get MSI domain\n"); 11043277e8aaSGrygorii Strashko return -EPROBE_DEFER; 11053277e8aaSGrygorii Strashko } 11063277e8aaSGrygorii Strashko 11073277e8aaSGrygorii Strashko ret = k3_ringacc_probe_dt(ringacc); 11083277e8aaSGrygorii Strashko if (ret) 11093277e8aaSGrygorii Strashko return ret; 11103277e8aaSGrygorii Strashko 111195e7be06SGrygorii Strashko soc = soc_device_match(k3_ringacc_socinfo); 111295e7be06SGrygorii Strashko if (soc && soc->data) { 111395e7be06SGrygorii Strashko const struct k3_ringacc_soc_data *soc_data = soc->data; 111495e7be06SGrygorii Strashko 111595e7be06SGrygorii Strashko ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk; 111695e7be06SGrygorii Strashko } 111795e7be06SGrygorii Strashko 11183277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); 11193277e8aaSGrygorii Strashko base_rt = devm_ioremap_resource(dev, res); 11203277e8aaSGrygorii Strashko if (IS_ERR(base_rt)) 11213277e8aaSGrygorii Strashko return PTR_ERR(base_rt); 11223277e8aaSGrygorii Strashko 11233277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos"); 11243277e8aaSGrygorii Strashko base_fifo = devm_ioremap_resource(dev, res); 11253277e8aaSGrygorii Strashko if (IS_ERR(base_fifo)) 11263277e8aaSGrygorii Strashko return PTR_ERR(base_fifo); 11273277e8aaSGrygorii Strashko 11283277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg"); 11293277e8aaSGrygorii Strashko ringacc->proxy_gcfg = devm_ioremap_resource(dev, res); 11303277e8aaSGrygorii Strashko if (IS_ERR(ringacc->proxy_gcfg)) 11313277e8aaSGrygorii Strashko return PTR_ERR(ringacc->proxy_gcfg); 11323277e8aaSGrygorii Strashko 11333277e8aaSGrygorii Strashko res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 11343277e8aaSGrygorii Strashko "proxy_target"); 11353277e8aaSGrygorii Strashko ringacc->proxy_target_base = devm_ioremap_resource(dev, res); 11363277e8aaSGrygorii Strashko if (IS_ERR(ringacc->proxy_target_base)) 11373277e8aaSGrygorii Strashko return PTR_ERR(ringacc->proxy_target_base); 11383277e8aaSGrygorii Strashko 11393277e8aaSGrygorii Strashko ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & 11403277e8aaSGrygorii Strashko K3_RINGACC_PROXY_CFG_THREADS_MASK; 11413277e8aaSGrygorii Strashko 11423277e8aaSGrygorii Strashko ringacc->rings = devm_kzalloc(dev, 11433277e8aaSGrygorii Strashko sizeof(*ringacc->rings) * 11443277e8aaSGrygorii Strashko ringacc->num_rings, 11453277e8aaSGrygorii Strashko GFP_KERNEL); 11463277e8aaSGrygorii Strashko ringacc->rings_inuse = devm_kcalloc(dev, 11473277e8aaSGrygorii Strashko BITS_TO_LONGS(ringacc->num_rings), 11483277e8aaSGrygorii Strashko sizeof(unsigned long), GFP_KERNEL); 11493277e8aaSGrygorii Strashko ringacc->proxy_inuse = devm_kcalloc(dev, 11503277e8aaSGrygorii Strashko BITS_TO_LONGS(ringacc->num_proxies), 11513277e8aaSGrygorii Strashko sizeof(unsigned long), GFP_KERNEL); 11523277e8aaSGrygorii Strashko 11533277e8aaSGrygorii Strashko if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) 11543277e8aaSGrygorii Strashko return -ENOMEM; 11553277e8aaSGrygorii Strashko 11563277e8aaSGrygorii Strashko for (i = 0; i < ringacc->num_rings; i++) { 11573277e8aaSGrygorii Strashko ringacc->rings[i].rt = base_rt + 11583277e8aaSGrygorii Strashko K3_RINGACC_RT_REGS_STEP * i; 11593277e8aaSGrygorii Strashko ringacc->rings[i].fifos = base_fifo + 11603277e8aaSGrygorii Strashko K3_RINGACC_FIFO_REGS_STEP * i; 11613277e8aaSGrygorii Strashko ringacc->rings[i].parent = ringacc; 11623277e8aaSGrygorii Strashko ringacc->rings[i].ring_id = i; 11633277e8aaSGrygorii Strashko ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; 11643277e8aaSGrygorii Strashko } 11653277e8aaSGrygorii Strashko 11663277e8aaSGrygorii Strashko ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 11673277e8aaSGrygorii Strashko 11683277e8aaSGrygorii Strashko dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", 11693277e8aaSGrygorii Strashko ringacc->num_rings, 11703277e8aaSGrygorii Strashko ringacc->rm_gp_range->desc[0].start, 11713277e8aaSGrygorii Strashko ringacc->rm_gp_range->desc[0].num, 11723277e8aaSGrygorii Strashko ringacc->tisci_dev_id); 11733277e8aaSGrygorii Strashko dev_info(dev, "dma-ring-reset-quirk: %s\n", 11743277e8aaSGrygorii Strashko ringacc->dma_ring_reset_quirk ? "enabled" : "disabled"); 11753277e8aaSGrygorii Strashko dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", 11763277e8aaSGrygorii Strashko readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); 117740a2a7c3SGrygorii Strashko 11783277e8aaSGrygorii Strashko return 0; 11793277e8aaSGrygorii Strashko } 11803277e8aaSGrygorii Strashko 118140a2a7c3SGrygorii Strashko struct ringacc_match_data { 118240a2a7c3SGrygorii Strashko struct k3_ringacc_ops ops; 118340a2a7c3SGrygorii Strashko }; 118440a2a7c3SGrygorii Strashko 118540a2a7c3SGrygorii Strashko static struct ringacc_match_data k3_ringacc_data = { 118640a2a7c3SGrygorii Strashko .ops = { 118740a2a7c3SGrygorii Strashko .init = k3_ringacc_init, 118840a2a7c3SGrygorii Strashko }, 118940a2a7c3SGrygorii Strashko }; 119040a2a7c3SGrygorii Strashko 11913277e8aaSGrygorii Strashko /* Match table for of_platform binding */ 11923277e8aaSGrygorii Strashko static const struct of_device_id k3_ringacc_of_match[] = { 119340a2a7c3SGrygorii Strashko { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, }, 11943277e8aaSGrygorii Strashko {}, 11953277e8aaSGrygorii Strashko }; 11963277e8aaSGrygorii Strashko 119740a2a7c3SGrygorii Strashko static int k3_ringacc_probe(struct platform_device *pdev) 119840a2a7c3SGrygorii Strashko { 119940a2a7c3SGrygorii Strashko const struct ringacc_match_data *match_data; 120040a2a7c3SGrygorii Strashko const struct of_device_id *match; 120140a2a7c3SGrygorii Strashko struct device *dev = &pdev->dev; 120240a2a7c3SGrygorii Strashko struct k3_ringacc *ringacc; 120340a2a7c3SGrygorii Strashko int ret; 120440a2a7c3SGrygorii Strashko 120540a2a7c3SGrygorii Strashko match = of_match_node(k3_ringacc_of_match, dev->of_node); 120640a2a7c3SGrygorii Strashko if (!match) 120740a2a7c3SGrygorii Strashko return -ENODEV; 120840a2a7c3SGrygorii Strashko match_data = match->data; 120940a2a7c3SGrygorii Strashko 121040a2a7c3SGrygorii Strashko ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 121140a2a7c3SGrygorii Strashko if (!ringacc) 121240a2a7c3SGrygorii Strashko return -ENOMEM; 121340a2a7c3SGrygorii Strashko 121440a2a7c3SGrygorii Strashko ringacc->dev = dev; 121540a2a7c3SGrygorii Strashko mutex_init(&ringacc->req_lock); 121640a2a7c3SGrygorii Strashko ringacc->ops = &match_data->ops; 121740a2a7c3SGrygorii Strashko 121840a2a7c3SGrygorii Strashko ret = ringacc->ops->init(pdev, ringacc); 121940a2a7c3SGrygorii Strashko if (ret) 122040a2a7c3SGrygorii Strashko return ret; 122140a2a7c3SGrygorii Strashko 122240a2a7c3SGrygorii Strashko dev_set_drvdata(dev, ringacc); 122340a2a7c3SGrygorii Strashko 122440a2a7c3SGrygorii Strashko mutex_lock(&k3_ringacc_list_lock); 122540a2a7c3SGrygorii Strashko list_add_tail(&ringacc->list, &k3_ringacc_list); 122640a2a7c3SGrygorii Strashko mutex_unlock(&k3_ringacc_list_lock); 122740a2a7c3SGrygorii Strashko 122840a2a7c3SGrygorii Strashko return 0; 122940a2a7c3SGrygorii Strashko } 123040a2a7c3SGrygorii Strashko 12313277e8aaSGrygorii Strashko static struct platform_driver k3_ringacc_driver = { 12323277e8aaSGrygorii Strashko .probe = k3_ringacc_probe, 12333277e8aaSGrygorii Strashko .driver = { 12343277e8aaSGrygorii Strashko .name = "k3-ringacc", 12353277e8aaSGrygorii Strashko .of_match_table = k3_ringacc_of_match, 12363277e8aaSGrygorii Strashko .suppress_bind_attrs = true, 12373277e8aaSGrygorii Strashko }, 12383277e8aaSGrygorii Strashko }; 12393277e8aaSGrygorii Strashko builtin_platform_driver(k3_ringacc_driver); 1240