1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * TI K3 NAVSS Ring Accelerator subsystem driver 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 */ 7 8 #include <linux/dma-mapping.h> 9 #include <linux/io.h> 10 #include <linux/init.h> 11 #include <linux/of.h> 12 #include <linux/of_device.h> 13 #include <linux/platform_device.h> 14 #include <linux/sys_soc.h> 15 #include <linux/dma/ti-cppi5.h> 16 #include <linux/soc/ti/k3-ringacc.h> 17 #include <linux/soc/ti/ti_sci_protocol.h> 18 #include <linux/soc/ti/ti_sci_inta_msi.h> 19 #include <linux/of_irq.h> 20 #include <linux/irqdomain.h> 21 22 static LIST_HEAD(k3_ringacc_list); 23 static DEFINE_MUTEX(k3_ringacc_list_lock); 24 25 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) 26 #define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0) 27 28 /** 29 * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region 30 * 31 * @resv_16: Reserved 32 * @db: Ring Doorbell Register 33 * @resv_4: Reserved 34 * @occ: Ring Occupancy Register 35 * @indx: Ring Current Index Register 36 * @hwocc: Ring Hardware Occupancy Register 37 * @hwindx: Ring Hardware Current Index Register 38 */ 39 struct k3_ring_rt_regs { 40 u32 resv_16[4]; 41 u32 db; 42 u32 resv_4[1]; 43 u32 occ; 44 u32 indx; 45 u32 hwocc; 46 u32 hwindx; 47 }; 48 49 #define K3_RINGACC_RT_REGS_STEP 0x1000 50 #define K3_DMARING_RT_REGS_STEP 0x2000 51 #define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000 52 #define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0) 53 #define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31) 54 #define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0) 55 #define K3_DMARING_RT_DB_TDOWN_ACK BIT(31) 56 57 /** 58 * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region 59 * 60 * @head_data: Ring Head Entry Data Registers 61 * @tail_data: Ring Tail Entry Data Registers 62 * @peek_head_data: Ring Peek Head Entry Data Regs 63 * @peek_tail_data: Ring Peek Tail Entry Data Regs 64 */ 65 struct k3_ring_fifo_regs { 66 u32 head_data[128]; 67 u32 tail_data[128]; 68 u32 peek_head_data[128]; 69 u32 peek_tail_data[128]; 70 }; 71 72 /** 73 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region 74 * 75 * @revision: Revision Register 76 * @config: Config Register 77 */ 78 struct k3_ringacc_proxy_gcfg_regs { 79 u32 revision; 80 u32 config; 81 }; 82 83 #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) 84 85 /** 86 * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region 87 * 88 * @control: Proxy Control Register 89 * @status: Proxy Status Register 90 * @resv_512: Reserved 91 * @data: Proxy Data Register 92 */ 93 struct k3_ringacc_proxy_target_regs { 94 u32 control; 95 u32 status; 96 u8 resv_512[504]; 97 u32 data[128]; 98 }; 99 100 #define K3_RINGACC_PROXY_TARGET_STEP 0x1000 101 #define K3_RINGACC_PROXY_NOT_USED (-1) 102 103 enum k3_ringacc_proxy_access_mode { 104 PROXY_ACCESS_MODE_HEAD = 0, 105 PROXY_ACCESS_MODE_TAIL = 1, 106 PROXY_ACCESS_MODE_PEEK_HEAD = 2, 107 PROXY_ACCESS_MODE_PEEK_TAIL = 3, 108 }; 109 110 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) 111 #define K3_RINGACC_FIFO_REGS_STEP 0x1000 112 #define K3_RINGACC_MAX_DB_RING_CNT (127U) 113 114 struct k3_ring_ops { 115 int (*push_tail)(struct k3_ring *ring, void *elm); 116 int (*push_head)(struct k3_ring *ring, void *elm); 117 int (*pop_tail)(struct k3_ring *ring, void *elm); 118 int (*pop_head)(struct k3_ring *ring, void *elm); 119 }; 120 121 /** 122 * struct k3_ring_state - Internal state tracking structure 123 * 124 * @free: Number of free entries 125 * @occ: Occupancy 126 * @windex: Write index 127 * @rindex: Read index 128 */ 129 struct k3_ring_state { 130 u32 free; 131 u32 occ; 132 u32 windex; 133 u32 rindex; 134 u32 tdown_complete:1; 135 }; 136 137 /** 138 * struct k3_ring - RA Ring descriptor 139 * 140 * @rt: Ring control/status registers 141 * @fifos: Ring queues registers 142 * @proxy: Ring Proxy Datapath registers 143 * @ring_mem_dma: Ring buffer dma address 144 * @ring_mem_virt: Ring buffer virt address 145 * @ops: Ring operations 146 * @size: Ring size in elements 147 * @elm_size: Size of the ring element 148 * @mode: Ring mode 149 * @flags: flags 150 * @state: Ring state 151 * @ring_id: Ring Id 152 * @parent: Pointer on struct @k3_ringacc 153 * @use_count: Use count for shared rings 154 * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) 155 * @dma_dev: device to be used for DMA API (allocation, mapping) 156 * @asel: Address Space Select value for physical addresses 157 */ 158 struct k3_ring { 159 struct k3_ring_rt_regs __iomem *rt; 160 struct k3_ring_fifo_regs __iomem *fifos; 161 struct k3_ringacc_proxy_target_regs __iomem *proxy; 162 dma_addr_t ring_mem_dma; 163 void *ring_mem_virt; 164 struct k3_ring_ops *ops; 165 u32 size; 166 enum k3_ring_size elm_size; 167 enum k3_ring_mode mode; 168 u32 flags; 169 #define K3_RING_FLAG_BUSY BIT(1) 170 #define K3_RING_FLAG_SHARED BIT(2) 171 #define K3_RING_FLAG_REVERSE BIT(3) 172 struct k3_ring_state state; 173 u32 ring_id; 174 struct k3_ringacc *parent; 175 u32 use_count; 176 int proxy_id; 177 struct device *dma_dev; 178 u32 asel; 179 #define K3_ADDRESS_ASEL_SHIFT 48 180 }; 181 182 struct k3_ringacc_ops { 183 int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc); 184 }; 185 186 /** 187 * struct k3_ringacc - Rings accelerator descriptor 188 * 189 * @dev: pointer on RA device 190 * @proxy_gcfg: RA proxy global config registers 191 * @proxy_target_base: RA proxy datapath region 192 * @num_rings: number of ring in RA 193 * @rings_inuse: bitfield for ring usage tracking 194 * @rm_gp_range: general purpose rings range from tisci 195 * @dma_ring_reset_quirk: DMA reset w/a enable 196 * @num_proxies: number of RA proxies 197 * @proxy_inuse: bitfield for proxy usage tracking 198 * @rings: array of rings descriptors (struct @k3_ring) 199 * @list: list of RAs in the system 200 * @req_lock: protect rings allocation 201 * @tisci: pointer ti-sci handle 202 * @tisci_ring_ops: ti-sci rings ops 203 * @tisci_dev_id: ti-sci device id 204 * @ops: SoC specific ringacc operation 205 * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA) 206 */ 207 struct k3_ringacc { 208 struct device *dev; 209 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; 210 void __iomem *proxy_target_base; 211 u32 num_rings; /* number of rings in Ringacc module */ 212 unsigned long *rings_inuse; 213 struct ti_sci_resource *rm_gp_range; 214 215 bool dma_ring_reset_quirk; 216 u32 num_proxies; 217 unsigned long *proxy_inuse; 218 219 struct k3_ring *rings; 220 struct list_head list; 221 struct mutex req_lock; /* protect rings allocation */ 222 223 const struct ti_sci_handle *tisci; 224 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; 225 u32 tisci_dev_id; 226 227 const struct k3_ringacc_ops *ops; 228 bool dma_rings; 229 }; 230 231 /** 232 * struct k3_ringacc - Rings accelerator SoC data 233 * 234 * @dma_ring_reset_quirk: DMA reset w/a enable 235 */ 236 struct k3_ringacc_soc_data { 237 unsigned dma_ring_reset_quirk:1; 238 }; 239 240 static int k3_ringacc_ring_read_occ(struct k3_ring *ring) 241 { 242 return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK; 243 } 244 245 static void k3_ringacc_ring_update_occ(struct k3_ring *ring) 246 { 247 u32 val; 248 249 val = readl(&ring->rt->occ); 250 251 ring->state.occ = val & K3_RINGACC_RT_OCC_MASK; 252 ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE); 253 } 254 255 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) 256 { 257 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - 258 (4 << ring->elm_size); 259 } 260 261 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) 262 { 263 return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); 264 } 265 266 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); 267 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); 268 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem); 269 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem); 270 271 static struct k3_ring_ops k3_ring_mode_ring_ops = { 272 .push_tail = k3_ringacc_ring_push_mem, 273 .pop_head = k3_ringacc_ring_pop_mem, 274 }; 275 276 static struct k3_ring_ops k3_dmaring_fwd_ops = { 277 .push_tail = k3_ringacc_ring_push_mem, 278 .pop_head = k3_dmaring_fwd_pop, 279 }; 280 281 static struct k3_ring_ops k3_dmaring_reverse_ops = { 282 /* Reverse side of the DMA ring can only be popped by SW */ 283 .pop_head = k3_dmaring_reverse_pop, 284 }; 285 286 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); 287 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); 288 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); 289 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); 290 291 static struct k3_ring_ops k3_ring_mode_msg_ops = { 292 .push_tail = k3_ringacc_ring_push_io, 293 .push_head = k3_ringacc_ring_push_head_io, 294 .pop_tail = k3_ringacc_ring_pop_tail_io, 295 .pop_head = k3_ringacc_ring_pop_io, 296 }; 297 298 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); 299 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); 300 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); 301 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); 302 303 static struct k3_ring_ops k3_ring_mode_proxy_ops = { 304 .push_tail = k3_ringacc_ring_push_tail_proxy, 305 .push_head = k3_ringacc_ring_push_head_proxy, 306 .pop_tail = k3_ringacc_ring_pop_tail_proxy, 307 .pop_head = k3_ringacc_ring_pop_head_proxy, 308 }; 309 310 static void k3_ringacc_ring_dump(struct k3_ring *ring) 311 { 312 struct device *dev = ring->parent->dev; 313 314 dev_dbg(dev, "dump ring: %d\n", ring->ring_id); 315 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, 316 &ring->ring_mem_dma); 317 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", 318 ring->elm_size, ring->size, ring->mode, ring->proxy_id); 319 dev_dbg(dev, "dump flags %08X\n", ring->flags); 320 321 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); 322 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); 323 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); 324 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); 325 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); 326 327 if (ring->ring_mem_virt) 328 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, 329 16, 1, ring->ring_mem_virt, 16 * 8, false); 330 } 331 332 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, 333 int id, u32 flags) 334 { 335 int proxy_id = K3_RINGACC_PROXY_NOT_USED; 336 337 mutex_lock(&ringacc->req_lock); 338 339 if (id == K3_RINGACC_RING_ID_ANY) { 340 /* Request for any general purpose ring */ 341 struct ti_sci_resource_desc *gp_rings = 342 &ringacc->rm_gp_range->desc[0]; 343 unsigned long size; 344 345 size = gp_rings->start + gp_rings->num; 346 id = find_next_zero_bit(ringacc->rings_inuse, size, 347 gp_rings->start); 348 if (id == size) 349 goto error; 350 } else if (id < 0) { 351 goto error; 352 } 353 354 if (test_bit(id, ringacc->rings_inuse) && 355 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) 356 goto error; 357 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) 358 goto out; 359 360 if (flags & K3_RINGACC_RING_USE_PROXY) { 361 proxy_id = find_first_zero_bit(ringacc->proxy_inuse, 362 ringacc->num_proxies); 363 if (proxy_id == ringacc->num_proxies) 364 goto error; 365 } 366 367 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { 368 set_bit(proxy_id, ringacc->proxy_inuse); 369 ringacc->rings[id].proxy_id = proxy_id; 370 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, 371 proxy_id); 372 } else { 373 dev_dbg(ringacc->dev, "Giving ring#%d\n", id); 374 } 375 376 set_bit(id, ringacc->rings_inuse); 377 out: 378 ringacc->rings[id].use_count++; 379 mutex_unlock(&ringacc->req_lock); 380 return &ringacc->rings[id]; 381 382 error: 383 mutex_unlock(&ringacc->req_lock); 384 return NULL; 385 } 386 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); 387 388 static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id, 389 struct k3_ring **fwd_ring, 390 struct k3_ring **compl_ring) 391 { 392 int ret = 0; 393 394 /* 395 * DMA rings must be requested by ID, completion ring is the reverse 396 * side of the forward ring 397 */ 398 if (fwd_id < 0) 399 return -EINVAL; 400 401 mutex_lock(&ringacc->req_lock); 402 403 if (test_bit(fwd_id, ringacc->rings_inuse)) { 404 ret = -EBUSY; 405 goto error; 406 } 407 408 *fwd_ring = &ringacc->rings[fwd_id]; 409 *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings]; 410 set_bit(fwd_id, ringacc->rings_inuse); 411 ringacc->rings[fwd_id].use_count++; 412 dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id); 413 414 mutex_unlock(&ringacc->req_lock); 415 return 0; 416 417 error: 418 mutex_unlock(&ringacc->req_lock); 419 return ret; 420 } 421 422 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, 423 int fwd_id, int compl_id, 424 struct k3_ring **fwd_ring, 425 struct k3_ring **compl_ring) 426 { 427 int ret = 0; 428 429 if (!fwd_ring || !compl_ring) 430 return -EINVAL; 431 432 if (ringacc->dma_rings) 433 return k3_dmaring_request_dual_ring(ringacc, fwd_id, 434 fwd_ring, compl_ring); 435 436 *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); 437 if (!(*fwd_ring)) 438 return -ENODEV; 439 440 *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0); 441 if (!(*compl_ring)) { 442 k3_ringacc_ring_free(*fwd_ring); 443 ret = -ENODEV; 444 } 445 446 return ret; 447 } 448 EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair); 449 450 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) 451 { 452 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 453 struct k3_ringacc *ringacc = ring->parent; 454 int ret; 455 456 ring_cfg.nav_id = ringacc->tisci_dev_id; 457 ring_cfg.index = ring->ring_id; 458 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID; 459 ring_cfg.count = ring->size; 460 461 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 462 if (ret) 463 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", 464 ret, ring->ring_id); 465 } 466 467 void k3_ringacc_ring_reset(struct k3_ring *ring) 468 { 469 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 470 return; 471 472 memset(&ring->state, 0, sizeof(ring->state)); 473 474 k3_ringacc_ring_reset_sci(ring); 475 } 476 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); 477 478 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, 479 enum k3_ring_mode mode) 480 { 481 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 482 struct k3_ringacc *ringacc = ring->parent; 483 int ret; 484 485 ring_cfg.nav_id = ringacc->tisci_dev_id; 486 ring_cfg.index = ring->ring_id; 487 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID; 488 ring_cfg.mode = mode; 489 490 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 491 if (ret) 492 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", 493 ret, ring->ring_id); 494 } 495 496 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) 497 { 498 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 499 return; 500 501 if (!ring->parent->dma_ring_reset_quirk) 502 goto reset; 503 504 if (!occ) 505 occ = k3_ringacc_ring_read_occ(ring); 506 507 if (occ) { 508 u32 db_ring_cnt, db_ring_cnt_cur; 509 510 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, 511 ring->ring_id, occ); 512 /* TI-SCI ring reset */ 513 k3_ringacc_ring_reset_sci(ring); 514 515 /* 516 * Setup the ring in ring/doorbell mode (if not already in this 517 * mode) 518 */ 519 if (ring->mode != K3_RINGACC_RING_MODE_RING) 520 k3_ringacc_ring_reconfig_qmode_sci( 521 ring, K3_RINGACC_RING_MODE_RING); 522 /* 523 * Ring the doorbell 2**22 – ringOcc times. 524 * This will wrap the internal UDMAP ring state occupancy 525 * counter (which is 21-bits wide) to 0. 526 */ 527 db_ring_cnt = (1U << 22) - occ; 528 529 while (db_ring_cnt != 0) { 530 /* 531 * Ring the doorbell with the maximum count each 532 * iteration if possible to minimize the total 533 * of writes 534 */ 535 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) 536 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; 537 else 538 db_ring_cnt_cur = db_ring_cnt; 539 540 writel(db_ring_cnt_cur, &ring->rt->db); 541 db_ring_cnt -= db_ring_cnt_cur; 542 } 543 544 /* Restore the original ring mode (if not ring mode) */ 545 if (ring->mode != K3_RINGACC_RING_MODE_RING) 546 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); 547 } 548 549 reset: 550 /* Reset the ring */ 551 k3_ringacc_ring_reset(ring); 552 } 553 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); 554 555 static void k3_ringacc_ring_free_sci(struct k3_ring *ring) 556 { 557 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 558 struct k3_ringacc *ringacc = ring->parent; 559 int ret; 560 561 ring_cfg.nav_id = ringacc->tisci_dev_id; 562 ring_cfg.index = ring->ring_id; 563 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 564 565 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 566 if (ret) 567 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", 568 ret, ring->ring_id); 569 } 570 571 int k3_ringacc_ring_free(struct k3_ring *ring) 572 { 573 struct k3_ringacc *ringacc; 574 575 if (!ring) 576 return -EINVAL; 577 578 ringacc = ring->parent; 579 580 /* 581 * DMA rings: rings shared memory and configuration, only forward ring 582 * is configured and reverse ring considered as slave. 583 */ 584 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) 585 return 0; 586 587 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); 588 589 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 590 return -EINVAL; 591 592 mutex_lock(&ringacc->req_lock); 593 594 if (--ring->use_count) 595 goto out; 596 597 if (!(ring->flags & K3_RING_FLAG_BUSY)) 598 goto no_init; 599 600 k3_ringacc_ring_free_sci(ring); 601 602 dma_free_coherent(ring->dma_dev, 603 ring->size * (4 << ring->elm_size), 604 ring->ring_mem_virt, ring->ring_mem_dma); 605 ring->flags = 0; 606 ring->ops = NULL; 607 ring->dma_dev = NULL; 608 ring->asel = 0; 609 610 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { 611 clear_bit(ring->proxy_id, ringacc->proxy_inuse); 612 ring->proxy = NULL; 613 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 614 } 615 616 no_init: 617 clear_bit(ring->ring_id, ringacc->rings_inuse); 618 619 out: 620 mutex_unlock(&ringacc->req_lock); 621 return 0; 622 } 623 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); 624 625 u32 k3_ringacc_get_ring_id(struct k3_ring *ring) 626 { 627 if (!ring) 628 return -EINVAL; 629 630 return ring->ring_id; 631 } 632 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); 633 634 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) 635 { 636 if (!ring) 637 return -EINVAL; 638 639 return ring->parent->tisci_dev_id; 640 } 641 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); 642 643 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) 644 { 645 int irq_num; 646 647 if (!ring) 648 return -EINVAL; 649 650 irq_num = msi_get_virq(ring->parent->dev, ring->ring_id); 651 if (irq_num <= 0) 652 irq_num = -EINVAL; 653 return irq_num; 654 } 655 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); 656 657 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) 658 { 659 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 660 struct k3_ringacc *ringacc = ring->parent; 661 int ret; 662 663 if (!ringacc->tisci) 664 return -EINVAL; 665 666 ring_cfg.nav_id = ringacc->tisci_dev_id; 667 ring_cfg.index = ring->ring_id; 668 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 669 ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma); 670 ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma); 671 ring_cfg.count = ring->size; 672 ring_cfg.mode = ring->mode; 673 ring_cfg.size = ring->elm_size; 674 ring_cfg.asel = ring->asel; 675 676 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 677 if (ret) 678 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", 679 ret, ring->ring_id); 680 681 return ret; 682 } 683 684 static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 685 { 686 struct k3_ringacc *ringacc; 687 struct k3_ring *reverse_ring; 688 int ret = 0; 689 690 if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 || 691 cfg->mode != K3_RINGACC_RING_MODE_RING || 692 cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK) 693 return -EINVAL; 694 695 ringacc = ring->parent; 696 697 /* 698 * DMA rings: rings shared memory and configuration, only forward ring 699 * is configured and reverse ring considered as slave. 700 */ 701 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) 702 return 0; 703 704 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 705 return -EINVAL; 706 707 ring->size = cfg->size; 708 ring->elm_size = cfg->elm_size; 709 ring->mode = cfg->mode; 710 ring->asel = cfg->asel; 711 ring->dma_dev = cfg->dma_dev; 712 if (!ring->dma_dev) { 713 dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n", 714 ring->ring_id); 715 ring->dma_dev = ringacc->dev; 716 } 717 718 memset(&ring->state, 0, sizeof(ring->state)); 719 720 ring->ops = &k3_dmaring_fwd_ops; 721 722 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, 723 ring->size * (4 << ring->elm_size), 724 &ring->ring_mem_dma, GFP_KERNEL); 725 if (!ring->ring_mem_virt) { 726 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 727 ret = -ENOMEM; 728 goto err_free_ops; 729 } 730 731 ret = k3_ringacc_ring_cfg_sci(ring); 732 if (ret) 733 goto err_free_mem; 734 735 ring->flags |= K3_RING_FLAG_BUSY; 736 737 k3_ringacc_ring_dump(ring); 738 739 /* DMA rings: configure reverse ring */ 740 reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings]; 741 reverse_ring->size = cfg->size; 742 reverse_ring->elm_size = cfg->elm_size; 743 reverse_ring->mode = cfg->mode; 744 reverse_ring->asel = cfg->asel; 745 memset(&reverse_ring->state, 0, sizeof(reverse_ring->state)); 746 reverse_ring->ops = &k3_dmaring_reverse_ops; 747 748 reverse_ring->ring_mem_virt = ring->ring_mem_virt; 749 reverse_ring->ring_mem_dma = ring->ring_mem_dma; 750 reverse_ring->flags |= K3_RING_FLAG_BUSY; 751 k3_ringacc_ring_dump(reverse_ring); 752 753 return 0; 754 755 err_free_mem: 756 dma_free_coherent(ring->dma_dev, 757 ring->size * (4 << ring->elm_size), 758 ring->ring_mem_virt, 759 ring->ring_mem_dma); 760 err_free_ops: 761 ring->ops = NULL; 762 ring->proxy = NULL; 763 ring->dma_dev = NULL; 764 ring->asel = 0; 765 return ret; 766 } 767 768 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 769 { 770 struct k3_ringacc *ringacc; 771 int ret = 0; 772 773 if (!ring || !cfg) 774 return -EINVAL; 775 776 ringacc = ring->parent; 777 778 if (ringacc->dma_rings) 779 return k3_dmaring_cfg(ring, cfg); 780 781 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || 782 cfg->mode >= K3_RINGACC_RING_MODE_INVALID || 783 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || 784 !test_bit(ring->ring_id, ringacc->rings_inuse)) 785 return -EINVAL; 786 787 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && 788 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && 789 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { 790 dev_err(ringacc->dev, 791 "Message mode must use proxy for %u element size\n", 792 4 << ring->elm_size); 793 return -EINVAL; 794 } 795 796 /* 797 * In case of shared ring only the first user (master user) can 798 * configure the ring. The sequence should be by the client: 799 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user 800 * k3_ringacc_ring_cfg(ring, cfg); # master configuration 801 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 802 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 803 */ 804 if (ring->use_count != 1) 805 return 0; 806 807 ring->size = cfg->size; 808 ring->elm_size = cfg->elm_size; 809 ring->mode = cfg->mode; 810 memset(&ring->state, 0, sizeof(ring->state)); 811 812 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) 813 ring->proxy = ringacc->proxy_target_base + 814 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; 815 816 switch (ring->mode) { 817 case K3_RINGACC_RING_MODE_RING: 818 ring->ops = &k3_ring_mode_ring_ops; 819 ring->dma_dev = cfg->dma_dev; 820 if (!ring->dma_dev) 821 ring->dma_dev = ringacc->dev; 822 break; 823 case K3_RINGACC_RING_MODE_MESSAGE: 824 ring->dma_dev = ringacc->dev; 825 if (ring->proxy) 826 ring->ops = &k3_ring_mode_proxy_ops; 827 else 828 ring->ops = &k3_ring_mode_msg_ops; 829 break; 830 default: 831 ring->ops = NULL; 832 ret = -EINVAL; 833 goto err_free_proxy; 834 } 835 836 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, 837 ring->size * (4 << ring->elm_size), 838 &ring->ring_mem_dma, GFP_KERNEL); 839 if (!ring->ring_mem_virt) { 840 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 841 ret = -ENOMEM; 842 goto err_free_ops; 843 } 844 845 ret = k3_ringacc_ring_cfg_sci(ring); 846 847 if (ret) 848 goto err_free_mem; 849 850 ring->flags |= K3_RING_FLAG_BUSY; 851 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? 852 K3_RING_FLAG_SHARED : 0; 853 854 k3_ringacc_ring_dump(ring); 855 856 return 0; 857 858 err_free_mem: 859 dma_free_coherent(ring->dma_dev, 860 ring->size * (4 << ring->elm_size), 861 ring->ring_mem_virt, 862 ring->ring_mem_dma); 863 err_free_ops: 864 ring->ops = NULL; 865 ring->dma_dev = NULL; 866 err_free_proxy: 867 ring->proxy = NULL; 868 return ret; 869 } 870 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); 871 872 u32 k3_ringacc_ring_get_size(struct k3_ring *ring) 873 { 874 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 875 return -EINVAL; 876 877 return ring->size; 878 } 879 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); 880 881 u32 k3_ringacc_ring_get_free(struct k3_ring *ring) 882 { 883 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 884 return -EINVAL; 885 886 if (!ring->state.free) 887 ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring); 888 889 return ring->state.free; 890 } 891 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); 892 893 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) 894 { 895 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 896 return -EINVAL; 897 898 return k3_ringacc_ring_read_occ(ring); 899 } 900 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); 901 902 u32 k3_ringacc_ring_is_full(struct k3_ring *ring) 903 { 904 return !k3_ringacc_ring_get_free(ring); 905 } 906 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); 907 908 enum k3_ringacc_access_mode { 909 K3_RINGACC_ACCESS_MODE_PUSH_HEAD, 910 K3_RINGACC_ACCESS_MODE_POP_HEAD, 911 K3_RINGACC_ACCESS_MODE_PUSH_TAIL, 912 K3_RINGACC_ACCESS_MODE_POP_TAIL, 913 K3_RINGACC_ACCESS_MODE_PEEK_HEAD, 914 K3_RINGACC_ACCESS_MODE_PEEK_TAIL, 915 }; 916 917 #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) 918 #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) 919 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, 920 enum k3_ringacc_proxy_access_mode mode) 921 { 922 u32 val; 923 924 val = ring->ring_id; 925 val |= K3_RINGACC_PROXY_MODE(mode); 926 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); 927 writel(val, &ring->proxy->control); 928 return 0; 929 } 930 931 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, 932 enum k3_ringacc_access_mode access_mode) 933 { 934 void __iomem *ptr; 935 936 ptr = (void __iomem *)&ring->proxy->data; 937 938 switch (access_mode) { 939 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 940 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 941 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); 942 break; 943 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 944 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 945 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); 946 break; 947 default: 948 return -EINVAL; 949 } 950 951 ptr += k3_ringacc_ring_get_fifo_pos(ring); 952 953 switch (access_mode) { 954 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 955 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 956 dev_dbg(ring->parent->dev, 957 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 958 access_mode); 959 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 960 ring->state.occ--; 961 break; 962 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 963 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 964 dev_dbg(ring->parent->dev, 965 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 966 access_mode); 967 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 968 ring->state.free--; 969 break; 970 default: 971 return -EINVAL; 972 } 973 974 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, 975 ring->state.occ); 976 return 0; 977 } 978 979 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) 980 { 981 return k3_ringacc_ring_access_proxy(ring, elem, 982 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 983 } 984 985 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) 986 { 987 return k3_ringacc_ring_access_proxy(ring, elem, 988 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 989 } 990 991 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) 992 { 993 return k3_ringacc_ring_access_proxy(ring, elem, 994 K3_RINGACC_ACCESS_MODE_POP_HEAD); 995 } 996 997 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) 998 { 999 return k3_ringacc_ring_access_proxy(ring, elem, 1000 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1001 } 1002 1003 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, 1004 enum k3_ringacc_access_mode access_mode) 1005 { 1006 void __iomem *ptr; 1007 1008 switch (access_mode) { 1009 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 1010 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 1011 ptr = (void __iomem *)&ring->fifos->head_data; 1012 break; 1013 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 1014 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 1015 ptr = (void __iomem *)&ring->fifos->tail_data; 1016 break; 1017 default: 1018 return -EINVAL; 1019 } 1020 1021 ptr += k3_ringacc_ring_get_fifo_pos(ring); 1022 1023 switch (access_mode) { 1024 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 1025 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 1026 dev_dbg(ring->parent->dev, 1027 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 1028 access_mode); 1029 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 1030 ring->state.occ--; 1031 break; 1032 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 1033 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 1034 dev_dbg(ring->parent->dev, 1035 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 1036 access_mode); 1037 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 1038 ring->state.free--; 1039 break; 1040 default: 1041 return -EINVAL; 1042 } 1043 1044 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", 1045 ring->state.free, ring->state.windex, ring->state.occ, 1046 ring->state.rindex); 1047 return 0; 1048 } 1049 1050 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) 1051 { 1052 return k3_ringacc_ring_access_io(ring, elem, 1053 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 1054 } 1055 1056 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) 1057 { 1058 return k3_ringacc_ring_access_io(ring, elem, 1059 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 1060 } 1061 1062 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) 1063 { 1064 return k3_ringacc_ring_access_io(ring, elem, 1065 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1066 } 1067 1068 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) 1069 { 1070 return k3_ringacc_ring_access_io(ring, elem, 1071 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1072 } 1073 1074 /* 1075 * The element is 48 bits of address + ASEL bits in the ring. 1076 * ASEL is used by the DMAs and should be removed for the kernel as it is not 1077 * part of the physical memory address. 1078 */ 1079 static void k3_dmaring_remove_asel_from_elem(u64 *elem) 1080 { 1081 *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0); 1082 } 1083 1084 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem) 1085 { 1086 void *elem_ptr; 1087 u32 elem_idx; 1088 1089 /* 1090 * DMA rings: forward ring is always tied DMA channel and HW does not 1091 * maintain any state data required for POP operation and its unknown 1092 * how much elements were consumed by HW. So, to actually 1093 * do POP, the read pointer has to be recalculated every time. 1094 */ 1095 ring->state.occ = k3_ringacc_ring_read_occ(ring); 1096 if (ring->state.windex >= ring->state.occ) 1097 elem_idx = ring->state.windex - ring->state.occ; 1098 else 1099 elem_idx = ring->size - (ring->state.occ - ring->state.windex); 1100 1101 elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx); 1102 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1103 k3_dmaring_remove_asel_from_elem(elem); 1104 1105 ring->state.occ--; 1106 writel(-1, &ring->rt->db); 1107 1108 dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n", 1109 __func__, ring->state.occ, ring->state.windex, elem_idx, 1110 elem_ptr); 1111 return 0; 1112 } 1113 1114 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem) 1115 { 1116 void *elem_ptr; 1117 1118 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 1119 1120 if (ring->state.occ) { 1121 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1122 k3_dmaring_remove_asel_from_elem(elem); 1123 1124 ring->state.rindex = (ring->state.rindex + 1) % ring->size; 1125 ring->state.occ--; 1126 writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db); 1127 } else if (ring->state.tdown_complete) { 1128 dma_addr_t *value = elem; 1129 1130 *value = CPPI5_TDCM_MARKER; 1131 writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db); 1132 ring->state.tdown_complete = false; 1133 } 1134 1135 dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n", 1136 __func__, ring->state.occ, ring->state.rindex, elem_ptr); 1137 return 0; 1138 } 1139 1140 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) 1141 { 1142 void *elem_ptr; 1143 1144 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); 1145 1146 memcpy(elem_ptr, elem, (4 << ring->elm_size)); 1147 if (ring->parent->dma_rings) { 1148 u64 *addr = elem_ptr; 1149 1150 *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT); 1151 } 1152 1153 ring->state.windex = (ring->state.windex + 1) % ring->size; 1154 ring->state.free--; 1155 writel(1, &ring->rt->db); 1156 1157 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", 1158 ring->state.free, ring->state.windex); 1159 1160 return 0; 1161 } 1162 1163 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) 1164 { 1165 void *elem_ptr; 1166 1167 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 1168 1169 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1170 1171 ring->state.rindex = (ring->state.rindex + 1) % ring->size; 1172 ring->state.occ--; 1173 writel(-1, &ring->rt->db); 1174 1175 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", 1176 ring->state.occ, ring->state.rindex, elem_ptr); 1177 return 0; 1178 } 1179 1180 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) 1181 { 1182 int ret = -EOPNOTSUPP; 1183 1184 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1185 return -EINVAL; 1186 1187 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", 1188 ring->state.free, ring->state.windex); 1189 1190 if (k3_ringacc_ring_is_full(ring)) 1191 return -ENOMEM; 1192 1193 if (ring->ops && ring->ops->push_tail) 1194 ret = ring->ops->push_tail(ring, elem); 1195 1196 return ret; 1197 } 1198 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); 1199 1200 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) 1201 { 1202 int ret = -EOPNOTSUPP; 1203 1204 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1205 return -EINVAL; 1206 1207 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", 1208 ring->state.free, ring->state.windex); 1209 1210 if (k3_ringacc_ring_is_full(ring)) 1211 return -ENOMEM; 1212 1213 if (ring->ops && ring->ops->push_head) 1214 ret = ring->ops->push_head(ring, elem); 1215 1216 return ret; 1217 } 1218 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); 1219 1220 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) 1221 { 1222 int ret = -EOPNOTSUPP; 1223 1224 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1225 return -EINVAL; 1226 1227 if (!ring->state.occ) 1228 k3_ringacc_ring_update_occ(ring); 1229 1230 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, 1231 ring->state.rindex); 1232 1233 if (!ring->state.occ && !ring->state.tdown_complete) 1234 return -ENODATA; 1235 1236 if (ring->ops && ring->ops->pop_head) 1237 ret = ring->ops->pop_head(ring, elem); 1238 1239 return ret; 1240 } 1241 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); 1242 1243 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) 1244 { 1245 int ret = -EOPNOTSUPP; 1246 1247 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1248 return -EINVAL; 1249 1250 if (!ring->state.occ) 1251 k3_ringacc_ring_update_occ(ring); 1252 1253 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", 1254 ring->state.occ, ring->state.rindex); 1255 1256 if (!ring->state.occ) 1257 return -ENODATA; 1258 1259 if (ring->ops && ring->ops->pop_tail) 1260 ret = ring->ops->pop_tail(ring, elem); 1261 1262 return ret; 1263 } 1264 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); 1265 1266 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, 1267 const char *property) 1268 { 1269 struct device_node *ringacc_np; 1270 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); 1271 struct k3_ringacc *entry; 1272 1273 ringacc_np = of_parse_phandle(np, property, 0); 1274 if (!ringacc_np) 1275 return ERR_PTR(-ENODEV); 1276 1277 mutex_lock(&k3_ringacc_list_lock); 1278 list_for_each_entry(entry, &k3_ringacc_list, list) 1279 if (entry->dev->of_node == ringacc_np) { 1280 ringacc = entry; 1281 break; 1282 } 1283 mutex_unlock(&k3_ringacc_list_lock); 1284 of_node_put(ringacc_np); 1285 1286 return ringacc; 1287 } 1288 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); 1289 1290 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) 1291 { 1292 struct device_node *node = ringacc->dev->of_node; 1293 struct device *dev = ringacc->dev; 1294 struct platform_device *pdev = to_platform_device(dev); 1295 int ret; 1296 1297 if (!node) { 1298 dev_err(dev, "device tree info unavailable\n"); 1299 return -ENODEV; 1300 } 1301 1302 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); 1303 if (ret) { 1304 dev_err(dev, "ti,num-rings read failure %d\n", ret); 1305 return ret; 1306 } 1307 1308 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); 1309 if (IS_ERR(ringacc->tisci)) { 1310 ret = PTR_ERR(ringacc->tisci); 1311 if (ret != -EPROBE_DEFER) 1312 dev_err(dev, "ti,sci read fail %d\n", ret); 1313 ringacc->tisci = NULL; 1314 return ret; 1315 } 1316 1317 ret = of_property_read_u32(node, "ti,sci-dev-id", 1318 &ringacc->tisci_dev_id); 1319 if (ret) { 1320 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); 1321 return ret; 1322 } 1323 1324 pdev->id = ringacc->tisci_dev_id; 1325 1326 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, 1327 ringacc->tisci_dev_id, 1328 "ti,sci-rm-range-gp-rings"); 1329 if (IS_ERR(ringacc->rm_gp_range)) { 1330 dev_err(dev, "Failed to allocate MSI interrupts\n"); 1331 return PTR_ERR(ringacc->rm_gp_range); 1332 } 1333 1334 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, 1335 ringacc->rm_gp_range); 1336 } 1337 1338 static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = { 1339 .dma_ring_reset_quirk = 1, 1340 }; 1341 1342 static const struct soc_device_attribute k3_ringacc_socinfo[] = { 1343 { .family = "AM65X", 1344 .revision = "SR1.0", 1345 .data = &k3_ringacc_soc_data_sr1 1346 }, 1347 {/* sentinel */} 1348 }; 1349 1350 static int k3_ringacc_init(struct platform_device *pdev, 1351 struct k3_ringacc *ringacc) 1352 { 1353 const struct soc_device_attribute *soc; 1354 void __iomem *base_fifo, *base_rt; 1355 struct device *dev = &pdev->dev; 1356 struct resource *res; 1357 int ret, i; 1358 1359 dev->msi.domain = of_msi_get_domain(dev, dev->of_node, 1360 DOMAIN_BUS_TI_SCI_INTA_MSI); 1361 if (!dev->msi.domain) { 1362 dev_err(dev, "Failed to get MSI domain\n"); 1363 return -EPROBE_DEFER; 1364 } 1365 1366 ret = k3_ringacc_probe_dt(ringacc); 1367 if (ret) 1368 return ret; 1369 1370 soc = soc_device_match(k3_ringacc_socinfo); 1371 if (soc && soc->data) { 1372 const struct k3_ringacc_soc_data *soc_data = soc->data; 1373 1374 ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk; 1375 } 1376 1377 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); 1378 base_rt = devm_ioremap_resource(dev, res); 1379 if (IS_ERR(base_rt)) 1380 return PTR_ERR(base_rt); 1381 1382 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos"); 1383 base_fifo = devm_ioremap_resource(dev, res); 1384 if (IS_ERR(base_fifo)) 1385 return PTR_ERR(base_fifo); 1386 1387 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg"); 1388 ringacc->proxy_gcfg = devm_ioremap_resource(dev, res); 1389 if (IS_ERR(ringacc->proxy_gcfg)) 1390 return PTR_ERR(ringacc->proxy_gcfg); 1391 1392 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 1393 "proxy_target"); 1394 ringacc->proxy_target_base = devm_ioremap_resource(dev, res); 1395 if (IS_ERR(ringacc->proxy_target_base)) 1396 return PTR_ERR(ringacc->proxy_target_base); 1397 1398 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & 1399 K3_RINGACC_PROXY_CFG_THREADS_MASK; 1400 1401 ringacc->rings = devm_kzalloc(dev, 1402 sizeof(*ringacc->rings) * 1403 ringacc->num_rings, 1404 GFP_KERNEL); 1405 ringacc->rings_inuse = devm_kcalloc(dev, 1406 BITS_TO_LONGS(ringacc->num_rings), 1407 sizeof(unsigned long), GFP_KERNEL); 1408 ringacc->proxy_inuse = devm_kcalloc(dev, 1409 BITS_TO_LONGS(ringacc->num_proxies), 1410 sizeof(unsigned long), GFP_KERNEL); 1411 1412 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) 1413 return -ENOMEM; 1414 1415 for (i = 0; i < ringacc->num_rings; i++) { 1416 ringacc->rings[i].rt = base_rt + 1417 K3_RINGACC_RT_REGS_STEP * i; 1418 ringacc->rings[i].fifos = base_fifo + 1419 K3_RINGACC_FIFO_REGS_STEP * i; 1420 ringacc->rings[i].parent = ringacc; 1421 ringacc->rings[i].ring_id = i; 1422 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; 1423 } 1424 1425 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1426 1427 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", 1428 ringacc->num_rings, 1429 ringacc->rm_gp_range->desc[0].start, 1430 ringacc->rm_gp_range->desc[0].num, 1431 ringacc->tisci_dev_id); 1432 dev_info(dev, "dma-ring-reset-quirk: %s\n", 1433 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled"); 1434 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", 1435 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); 1436 1437 return 0; 1438 } 1439 1440 struct ringacc_match_data { 1441 struct k3_ringacc_ops ops; 1442 }; 1443 1444 static struct ringacc_match_data k3_ringacc_data = { 1445 .ops = { 1446 .init = k3_ringacc_init, 1447 }, 1448 }; 1449 1450 /* Match table for of_platform binding */ 1451 static const struct of_device_id k3_ringacc_of_match[] = { 1452 { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, }, 1453 {}, 1454 }; 1455 1456 struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev, 1457 struct k3_ringacc_init_data *data) 1458 { 1459 struct device *dev = &pdev->dev; 1460 struct k3_ringacc *ringacc; 1461 void __iomem *base_rt; 1462 struct resource *res; 1463 int i; 1464 1465 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1466 if (!ringacc) 1467 return ERR_PTR(-ENOMEM); 1468 1469 ringacc->dev = dev; 1470 ringacc->dma_rings = true; 1471 ringacc->num_rings = data->num_rings; 1472 ringacc->tisci = data->tisci; 1473 ringacc->tisci_dev_id = data->tisci_dev_id; 1474 1475 mutex_init(&ringacc->req_lock); 1476 1477 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt"); 1478 base_rt = devm_ioremap_resource(dev, res); 1479 if (IS_ERR(base_rt)) 1480 return ERR_CAST(base_rt); 1481 1482 ringacc->rings = devm_kzalloc(dev, 1483 sizeof(*ringacc->rings) * 1484 ringacc->num_rings * 2, 1485 GFP_KERNEL); 1486 ringacc->rings_inuse = devm_kcalloc(dev, 1487 BITS_TO_LONGS(ringacc->num_rings), 1488 sizeof(unsigned long), GFP_KERNEL); 1489 1490 if (!ringacc->rings || !ringacc->rings_inuse) 1491 return ERR_PTR(-ENOMEM); 1492 1493 for (i = 0; i < ringacc->num_rings; i++) { 1494 struct k3_ring *ring = &ringacc->rings[i]; 1495 1496 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i; 1497 ring->parent = ringacc; 1498 ring->ring_id = i; 1499 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 1500 1501 ring = &ringacc->rings[ringacc->num_rings + i]; 1502 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i + 1503 K3_DMARING_RT_REGS_REVERSE_OFS; 1504 ring->parent = ringacc; 1505 ring->ring_id = i; 1506 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 1507 ring->flags = K3_RING_FLAG_REVERSE; 1508 } 1509 1510 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1511 1512 dev_info(dev, "Number of rings: %u\n", ringacc->num_rings); 1513 1514 return ringacc; 1515 } 1516 EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init); 1517 1518 static int k3_ringacc_probe(struct platform_device *pdev) 1519 { 1520 const struct ringacc_match_data *match_data; 1521 struct device *dev = &pdev->dev; 1522 struct k3_ringacc *ringacc; 1523 int ret; 1524 1525 match_data = of_device_get_match_data(&pdev->dev); 1526 if (!match_data) 1527 return -ENODEV; 1528 1529 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1530 if (!ringacc) 1531 return -ENOMEM; 1532 1533 ringacc->dev = dev; 1534 mutex_init(&ringacc->req_lock); 1535 ringacc->ops = &match_data->ops; 1536 1537 ret = ringacc->ops->init(pdev, ringacc); 1538 if (ret) 1539 return ret; 1540 1541 dev_set_drvdata(dev, ringacc); 1542 1543 mutex_lock(&k3_ringacc_list_lock); 1544 list_add_tail(&ringacc->list, &k3_ringacc_list); 1545 mutex_unlock(&k3_ringacc_list_lock); 1546 1547 return 0; 1548 } 1549 1550 static struct platform_driver k3_ringacc_driver = { 1551 .probe = k3_ringacc_probe, 1552 .driver = { 1553 .name = "k3-ringacc", 1554 .of_match_table = k3_ringacc_of_match, 1555 .suppress_bind_attrs = true, 1556 }, 1557 }; 1558 builtin_platform_driver(k3_ringacc_driver); 1559