1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * TI K3 NAVSS Ring Accelerator subsystem driver 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 */ 7 8 #include <linux/dma-mapping.h> 9 #include <linux/io.h> 10 #include <linux/module.h> 11 #include <linux/of.h> 12 #include <linux/of_device.h> 13 #include <linux/platform_device.h> 14 #include <linux/sys_soc.h> 15 #include <linux/dma/ti-cppi5.h> 16 #include <linux/soc/ti/k3-ringacc.h> 17 #include <linux/soc/ti/ti_sci_protocol.h> 18 #include <linux/soc/ti/ti_sci_inta_msi.h> 19 #include <linux/of_irq.h> 20 #include <linux/irqdomain.h> 21 22 static LIST_HEAD(k3_ringacc_list); 23 static DEFINE_MUTEX(k3_ringacc_list_lock); 24 25 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) 26 #define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0) 27 28 /** 29 * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region 30 * 31 * @resv_16: Reserved 32 * @db: Ring Doorbell Register 33 * @resv_4: Reserved 34 * @occ: Ring Occupancy Register 35 * @indx: Ring Current Index Register 36 * @hwocc: Ring Hardware Occupancy Register 37 * @hwindx: Ring Hardware Current Index Register 38 */ 39 struct k3_ring_rt_regs { 40 u32 resv_16[4]; 41 u32 db; 42 u32 resv_4[1]; 43 u32 occ; 44 u32 indx; 45 u32 hwocc; 46 u32 hwindx; 47 }; 48 49 #define K3_RINGACC_RT_REGS_STEP 0x1000 50 #define K3_DMARING_RT_REGS_STEP 0x2000 51 #define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000 52 #define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0) 53 #define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31) 54 #define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0) 55 #define K3_DMARING_RT_DB_TDOWN_ACK BIT(31) 56 57 /** 58 * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region 59 * 60 * @head_data: Ring Head Entry Data Registers 61 * @tail_data: Ring Tail Entry Data Registers 62 * @peek_head_data: Ring Peek Head Entry Data Regs 63 * @peek_tail_data: Ring Peek Tail Entry Data Regs 64 */ 65 struct k3_ring_fifo_regs { 66 u32 head_data[128]; 67 u32 tail_data[128]; 68 u32 peek_head_data[128]; 69 u32 peek_tail_data[128]; 70 }; 71 72 /** 73 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region 74 * 75 * @revision: Revision Register 76 * @config: Config Register 77 */ 78 struct k3_ringacc_proxy_gcfg_regs { 79 u32 revision; 80 u32 config; 81 }; 82 83 #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) 84 85 /** 86 * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region 87 * 88 * @control: Proxy Control Register 89 * @status: Proxy Status Register 90 * @resv_512: Reserved 91 * @data: Proxy Data Register 92 */ 93 struct k3_ringacc_proxy_target_regs { 94 u32 control; 95 u32 status; 96 u8 resv_512[504]; 97 u32 data[128]; 98 }; 99 100 #define K3_RINGACC_PROXY_TARGET_STEP 0x1000 101 #define K3_RINGACC_PROXY_NOT_USED (-1) 102 103 enum k3_ringacc_proxy_access_mode { 104 PROXY_ACCESS_MODE_HEAD = 0, 105 PROXY_ACCESS_MODE_TAIL = 1, 106 PROXY_ACCESS_MODE_PEEK_HEAD = 2, 107 PROXY_ACCESS_MODE_PEEK_TAIL = 3, 108 }; 109 110 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) 111 #define K3_RINGACC_FIFO_REGS_STEP 0x1000 112 #define K3_RINGACC_MAX_DB_RING_CNT (127U) 113 114 struct k3_ring_ops { 115 int (*push_tail)(struct k3_ring *ring, void *elm); 116 int (*push_head)(struct k3_ring *ring, void *elm); 117 int (*pop_tail)(struct k3_ring *ring, void *elm); 118 int (*pop_head)(struct k3_ring *ring, void *elm); 119 }; 120 121 /** 122 * struct k3_ring_state - Internal state tracking structure 123 * 124 * @free: Number of free entries 125 * @occ: Occupancy 126 * @windex: Write index 127 * @rindex: Read index 128 */ 129 struct k3_ring_state { 130 u32 free; 131 u32 occ; 132 u32 windex; 133 u32 rindex; 134 u32 tdown_complete:1; 135 }; 136 137 /** 138 * struct k3_ring - RA Ring descriptor 139 * 140 * @rt: Ring control/status registers 141 * @fifos: Ring queues registers 142 * @proxy: Ring Proxy Datapath registers 143 * @ring_mem_dma: Ring buffer dma address 144 * @ring_mem_virt: Ring buffer virt address 145 * @ops: Ring operations 146 * @size: Ring size in elements 147 * @elm_size: Size of the ring element 148 * @mode: Ring mode 149 * @flags: flags 150 * @state: Ring state 151 * @ring_id: Ring Id 152 * @parent: Pointer on struct @k3_ringacc 153 * @use_count: Use count for shared rings 154 * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) 155 * @dma_dev: device to be used for DMA API (allocation, mapping) 156 * @asel: Address Space Select value for physical addresses 157 */ 158 struct k3_ring { 159 struct k3_ring_rt_regs __iomem *rt; 160 struct k3_ring_fifo_regs __iomem *fifos; 161 struct k3_ringacc_proxy_target_regs __iomem *proxy; 162 dma_addr_t ring_mem_dma; 163 void *ring_mem_virt; 164 struct k3_ring_ops *ops; 165 u32 size; 166 enum k3_ring_size elm_size; 167 enum k3_ring_mode mode; 168 u32 flags; 169 #define K3_RING_FLAG_BUSY BIT(1) 170 #define K3_RING_FLAG_SHARED BIT(2) 171 #define K3_RING_FLAG_REVERSE BIT(3) 172 struct k3_ring_state state; 173 u32 ring_id; 174 struct k3_ringacc *parent; 175 u32 use_count; 176 int proxy_id; 177 struct device *dma_dev; 178 u32 asel; 179 #define K3_ADDRESS_ASEL_SHIFT 48 180 }; 181 182 struct k3_ringacc_ops { 183 int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc); 184 }; 185 186 /** 187 * struct k3_ringacc - Rings accelerator descriptor 188 * 189 * @dev: pointer on RA device 190 * @proxy_gcfg: RA proxy global config registers 191 * @proxy_target_base: RA proxy datapath region 192 * @num_rings: number of ring in RA 193 * @rings_inuse: bitfield for ring usage tracking 194 * @rm_gp_range: general purpose rings range from tisci 195 * @dma_ring_reset_quirk: DMA reset w/a enable 196 * @num_proxies: number of RA proxies 197 * @proxy_inuse: bitfield for proxy usage tracking 198 * @rings: array of rings descriptors (struct @k3_ring) 199 * @list: list of RAs in the system 200 * @req_lock: protect rings allocation 201 * @tisci: pointer ti-sci handle 202 * @tisci_ring_ops: ti-sci rings ops 203 * @tisci_dev_id: ti-sci device id 204 * @ops: SoC specific ringacc operation 205 * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA) 206 */ 207 struct k3_ringacc { 208 struct device *dev; 209 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; 210 void __iomem *proxy_target_base; 211 u32 num_rings; /* number of rings in Ringacc module */ 212 unsigned long *rings_inuse; 213 struct ti_sci_resource *rm_gp_range; 214 215 bool dma_ring_reset_quirk; 216 u32 num_proxies; 217 unsigned long *proxy_inuse; 218 219 struct k3_ring *rings; 220 struct list_head list; 221 struct mutex req_lock; /* protect rings allocation */ 222 223 const struct ti_sci_handle *tisci; 224 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; 225 u32 tisci_dev_id; 226 227 const struct k3_ringacc_ops *ops; 228 bool dma_rings; 229 }; 230 231 /** 232 * struct k3_ringacc - Rings accelerator SoC data 233 * 234 * @dma_ring_reset_quirk: DMA reset w/a enable 235 */ 236 struct k3_ringacc_soc_data { 237 unsigned dma_ring_reset_quirk:1; 238 }; 239 240 static int k3_ringacc_ring_read_occ(struct k3_ring *ring) 241 { 242 return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK; 243 } 244 245 static void k3_ringacc_ring_update_occ(struct k3_ring *ring) 246 { 247 u32 val; 248 249 val = readl(&ring->rt->occ); 250 251 ring->state.occ = val & K3_RINGACC_RT_OCC_MASK; 252 ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE); 253 } 254 255 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) 256 { 257 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - 258 (4 << ring->elm_size); 259 } 260 261 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) 262 { 263 return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); 264 } 265 266 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); 267 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); 268 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem); 269 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem); 270 271 static struct k3_ring_ops k3_ring_mode_ring_ops = { 272 .push_tail = k3_ringacc_ring_push_mem, 273 .pop_head = k3_ringacc_ring_pop_mem, 274 }; 275 276 static struct k3_ring_ops k3_dmaring_fwd_ops = { 277 .push_tail = k3_ringacc_ring_push_mem, 278 .pop_head = k3_dmaring_fwd_pop, 279 }; 280 281 static struct k3_ring_ops k3_dmaring_reverse_ops = { 282 /* Reverse side of the DMA ring can only be popped by SW */ 283 .pop_head = k3_dmaring_reverse_pop, 284 }; 285 286 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); 287 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); 288 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); 289 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); 290 291 static struct k3_ring_ops k3_ring_mode_msg_ops = { 292 .push_tail = k3_ringacc_ring_push_io, 293 .push_head = k3_ringacc_ring_push_head_io, 294 .pop_tail = k3_ringacc_ring_pop_tail_io, 295 .pop_head = k3_ringacc_ring_pop_io, 296 }; 297 298 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); 299 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); 300 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); 301 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); 302 303 static struct k3_ring_ops k3_ring_mode_proxy_ops = { 304 .push_tail = k3_ringacc_ring_push_tail_proxy, 305 .push_head = k3_ringacc_ring_push_head_proxy, 306 .pop_tail = k3_ringacc_ring_pop_tail_proxy, 307 .pop_head = k3_ringacc_ring_pop_head_proxy, 308 }; 309 310 static void k3_ringacc_ring_dump(struct k3_ring *ring) 311 { 312 struct device *dev = ring->parent->dev; 313 314 dev_dbg(dev, "dump ring: %d\n", ring->ring_id); 315 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, 316 &ring->ring_mem_dma); 317 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", 318 ring->elm_size, ring->size, ring->mode, ring->proxy_id); 319 dev_dbg(dev, "dump flags %08X\n", ring->flags); 320 321 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); 322 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); 323 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); 324 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); 325 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); 326 327 if (ring->ring_mem_virt) 328 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, 329 16, 1, ring->ring_mem_virt, 16 * 8, false); 330 } 331 332 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, 333 int id, u32 flags) 334 { 335 int proxy_id = K3_RINGACC_PROXY_NOT_USED; 336 337 mutex_lock(&ringacc->req_lock); 338 339 if (!try_module_get(ringacc->dev->driver->owner)) 340 goto err_module_get; 341 342 if (id == K3_RINGACC_RING_ID_ANY) { 343 /* Request for any general purpose ring */ 344 struct ti_sci_resource_desc *gp_rings = 345 &ringacc->rm_gp_range->desc[0]; 346 unsigned long size; 347 348 size = gp_rings->start + gp_rings->num; 349 id = find_next_zero_bit(ringacc->rings_inuse, size, 350 gp_rings->start); 351 if (id == size) 352 goto error; 353 } else if (id < 0) { 354 goto error; 355 } 356 357 if (test_bit(id, ringacc->rings_inuse) && 358 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) 359 goto error; 360 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) 361 goto out; 362 363 if (flags & K3_RINGACC_RING_USE_PROXY) { 364 proxy_id = find_first_zero_bit(ringacc->proxy_inuse, 365 ringacc->num_proxies); 366 if (proxy_id == ringacc->num_proxies) 367 goto error; 368 } 369 370 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { 371 set_bit(proxy_id, ringacc->proxy_inuse); 372 ringacc->rings[id].proxy_id = proxy_id; 373 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, 374 proxy_id); 375 } else { 376 dev_dbg(ringacc->dev, "Giving ring#%d\n", id); 377 } 378 379 set_bit(id, ringacc->rings_inuse); 380 out: 381 ringacc->rings[id].use_count++; 382 mutex_unlock(&ringacc->req_lock); 383 return &ringacc->rings[id]; 384 385 error: 386 module_put(ringacc->dev->driver->owner); 387 388 err_module_get: 389 mutex_unlock(&ringacc->req_lock); 390 return NULL; 391 } 392 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); 393 394 static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id, 395 struct k3_ring **fwd_ring, 396 struct k3_ring **compl_ring) 397 { 398 int ret = 0; 399 400 /* 401 * DMA rings must be requested by ID, completion ring is the reverse 402 * side of the forward ring 403 */ 404 if (fwd_id < 0) 405 return -EINVAL; 406 407 mutex_lock(&ringacc->req_lock); 408 409 if (test_bit(fwd_id, ringacc->rings_inuse)) { 410 ret = -EBUSY; 411 goto error; 412 } 413 414 *fwd_ring = &ringacc->rings[fwd_id]; 415 *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings]; 416 set_bit(fwd_id, ringacc->rings_inuse); 417 ringacc->rings[fwd_id].use_count++; 418 dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id); 419 420 mutex_unlock(&ringacc->req_lock); 421 return 0; 422 423 error: 424 mutex_unlock(&ringacc->req_lock); 425 return ret; 426 } 427 428 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, 429 int fwd_id, int compl_id, 430 struct k3_ring **fwd_ring, 431 struct k3_ring **compl_ring) 432 { 433 int ret = 0; 434 435 if (!fwd_ring || !compl_ring) 436 return -EINVAL; 437 438 if (ringacc->dma_rings) 439 return k3_dmaring_request_dual_ring(ringacc, fwd_id, 440 fwd_ring, compl_ring); 441 442 *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); 443 if (!(*fwd_ring)) 444 return -ENODEV; 445 446 *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0); 447 if (!(*compl_ring)) { 448 k3_ringacc_ring_free(*fwd_ring); 449 ret = -ENODEV; 450 } 451 452 return ret; 453 } 454 EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair); 455 456 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) 457 { 458 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 459 struct k3_ringacc *ringacc = ring->parent; 460 int ret; 461 462 ring_cfg.nav_id = ringacc->tisci_dev_id; 463 ring_cfg.index = ring->ring_id; 464 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID; 465 ring_cfg.count = ring->size; 466 467 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 468 if (ret) 469 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", 470 ret, ring->ring_id); 471 } 472 473 void k3_ringacc_ring_reset(struct k3_ring *ring) 474 { 475 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 476 return; 477 478 memset(&ring->state, 0, sizeof(ring->state)); 479 480 k3_ringacc_ring_reset_sci(ring); 481 } 482 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); 483 484 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, 485 enum k3_ring_mode mode) 486 { 487 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 488 struct k3_ringacc *ringacc = ring->parent; 489 int ret; 490 491 ring_cfg.nav_id = ringacc->tisci_dev_id; 492 ring_cfg.index = ring->ring_id; 493 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID; 494 ring_cfg.mode = mode; 495 496 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 497 if (ret) 498 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", 499 ret, ring->ring_id); 500 } 501 502 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) 503 { 504 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 505 return; 506 507 if (!ring->parent->dma_ring_reset_quirk) 508 goto reset; 509 510 if (!occ) 511 occ = k3_ringacc_ring_read_occ(ring); 512 513 if (occ) { 514 u32 db_ring_cnt, db_ring_cnt_cur; 515 516 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, 517 ring->ring_id, occ); 518 /* TI-SCI ring reset */ 519 k3_ringacc_ring_reset_sci(ring); 520 521 /* 522 * Setup the ring in ring/doorbell mode (if not already in this 523 * mode) 524 */ 525 if (ring->mode != K3_RINGACC_RING_MODE_RING) 526 k3_ringacc_ring_reconfig_qmode_sci( 527 ring, K3_RINGACC_RING_MODE_RING); 528 /* 529 * Ring the doorbell 2**22 – ringOcc times. 530 * This will wrap the internal UDMAP ring state occupancy 531 * counter (which is 21-bits wide) to 0. 532 */ 533 db_ring_cnt = (1U << 22) - occ; 534 535 while (db_ring_cnt != 0) { 536 /* 537 * Ring the doorbell with the maximum count each 538 * iteration if possible to minimize the total 539 * of writes 540 */ 541 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) 542 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; 543 else 544 db_ring_cnt_cur = db_ring_cnt; 545 546 writel(db_ring_cnt_cur, &ring->rt->db); 547 db_ring_cnt -= db_ring_cnt_cur; 548 } 549 550 /* Restore the original ring mode (if not ring mode) */ 551 if (ring->mode != K3_RINGACC_RING_MODE_RING) 552 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); 553 } 554 555 reset: 556 /* Reset the ring */ 557 k3_ringacc_ring_reset(ring); 558 } 559 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); 560 561 static void k3_ringacc_ring_free_sci(struct k3_ring *ring) 562 { 563 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 564 struct k3_ringacc *ringacc = ring->parent; 565 int ret; 566 567 ring_cfg.nav_id = ringacc->tisci_dev_id; 568 ring_cfg.index = ring->ring_id; 569 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 570 571 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 572 if (ret) 573 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", 574 ret, ring->ring_id); 575 } 576 577 int k3_ringacc_ring_free(struct k3_ring *ring) 578 { 579 struct k3_ringacc *ringacc; 580 581 if (!ring) 582 return -EINVAL; 583 584 ringacc = ring->parent; 585 586 /* 587 * DMA rings: rings shared memory and configuration, only forward ring 588 * is configured and reverse ring considered as slave. 589 */ 590 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) 591 return 0; 592 593 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); 594 595 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 596 return -EINVAL; 597 598 mutex_lock(&ringacc->req_lock); 599 600 if (--ring->use_count) 601 goto out; 602 603 if (!(ring->flags & K3_RING_FLAG_BUSY)) 604 goto no_init; 605 606 k3_ringacc_ring_free_sci(ring); 607 608 dma_free_coherent(ring->dma_dev, 609 ring->size * (4 << ring->elm_size), 610 ring->ring_mem_virt, ring->ring_mem_dma); 611 ring->flags = 0; 612 ring->ops = NULL; 613 ring->dma_dev = NULL; 614 ring->asel = 0; 615 616 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { 617 clear_bit(ring->proxy_id, ringacc->proxy_inuse); 618 ring->proxy = NULL; 619 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 620 } 621 622 no_init: 623 clear_bit(ring->ring_id, ringacc->rings_inuse); 624 625 module_put(ringacc->dev->driver->owner); 626 627 out: 628 mutex_unlock(&ringacc->req_lock); 629 return 0; 630 } 631 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); 632 633 u32 k3_ringacc_get_ring_id(struct k3_ring *ring) 634 { 635 if (!ring) 636 return -EINVAL; 637 638 return ring->ring_id; 639 } 640 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); 641 642 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) 643 { 644 if (!ring) 645 return -EINVAL; 646 647 return ring->parent->tisci_dev_id; 648 } 649 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); 650 651 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) 652 { 653 int irq_num; 654 655 if (!ring) 656 return -EINVAL; 657 658 irq_num = msi_get_virq(ring->parent->dev, ring->ring_id); 659 if (irq_num <= 0) 660 irq_num = -EINVAL; 661 return irq_num; 662 } 663 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); 664 665 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) 666 { 667 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 668 struct k3_ringacc *ringacc = ring->parent; 669 int ret; 670 671 if (!ringacc->tisci) 672 return -EINVAL; 673 674 ring_cfg.nav_id = ringacc->tisci_dev_id; 675 ring_cfg.index = ring->ring_id; 676 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 677 ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma); 678 ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma); 679 ring_cfg.count = ring->size; 680 ring_cfg.mode = ring->mode; 681 ring_cfg.size = ring->elm_size; 682 ring_cfg.asel = ring->asel; 683 684 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 685 if (ret) 686 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", 687 ret, ring->ring_id); 688 689 return ret; 690 } 691 692 static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 693 { 694 struct k3_ringacc *ringacc; 695 struct k3_ring *reverse_ring; 696 int ret = 0; 697 698 if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 || 699 cfg->mode != K3_RINGACC_RING_MODE_RING || 700 cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK) 701 return -EINVAL; 702 703 ringacc = ring->parent; 704 705 /* 706 * DMA rings: rings shared memory and configuration, only forward ring 707 * is configured and reverse ring considered as slave. 708 */ 709 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) 710 return 0; 711 712 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 713 return -EINVAL; 714 715 ring->size = cfg->size; 716 ring->elm_size = cfg->elm_size; 717 ring->mode = cfg->mode; 718 ring->asel = cfg->asel; 719 ring->dma_dev = cfg->dma_dev; 720 if (!ring->dma_dev) { 721 dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n", 722 ring->ring_id); 723 ring->dma_dev = ringacc->dev; 724 } 725 726 memset(&ring->state, 0, sizeof(ring->state)); 727 728 ring->ops = &k3_dmaring_fwd_ops; 729 730 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, 731 ring->size * (4 << ring->elm_size), 732 &ring->ring_mem_dma, GFP_KERNEL); 733 if (!ring->ring_mem_virt) { 734 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 735 ret = -ENOMEM; 736 goto err_free_ops; 737 } 738 739 ret = k3_ringacc_ring_cfg_sci(ring); 740 if (ret) 741 goto err_free_mem; 742 743 ring->flags |= K3_RING_FLAG_BUSY; 744 745 k3_ringacc_ring_dump(ring); 746 747 /* DMA rings: configure reverse ring */ 748 reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings]; 749 reverse_ring->size = cfg->size; 750 reverse_ring->elm_size = cfg->elm_size; 751 reverse_ring->mode = cfg->mode; 752 reverse_ring->asel = cfg->asel; 753 memset(&reverse_ring->state, 0, sizeof(reverse_ring->state)); 754 reverse_ring->ops = &k3_dmaring_reverse_ops; 755 756 reverse_ring->ring_mem_virt = ring->ring_mem_virt; 757 reverse_ring->ring_mem_dma = ring->ring_mem_dma; 758 reverse_ring->flags |= K3_RING_FLAG_BUSY; 759 k3_ringacc_ring_dump(reverse_ring); 760 761 return 0; 762 763 err_free_mem: 764 dma_free_coherent(ring->dma_dev, 765 ring->size * (4 << ring->elm_size), 766 ring->ring_mem_virt, 767 ring->ring_mem_dma); 768 err_free_ops: 769 ring->ops = NULL; 770 ring->proxy = NULL; 771 ring->dma_dev = NULL; 772 ring->asel = 0; 773 return ret; 774 } 775 776 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 777 { 778 struct k3_ringacc *ringacc; 779 int ret = 0; 780 781 if (!ring || !cfg) 782 return -EINVAL; 783 784 ringacc = ring->parent; 785 786 if (ringacc->dma_rings) 787 return k3_dmaring_cfg(ring, cfg); 788 789 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || 790 cfg->mode >= K3_RINGACC_RING_MODE_INVALID || 791 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || 792 !test_bit(ring->ring_id, ringacc->rings_inuse)) 793 return -EINVAL; 794 795 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && 796 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && 797 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { 798 dev_err(ringacc->dev, 799 "Message mode must use proxy for %u element size\n", 800 4 << ring->elm_size); 801 return -EINVAL; 802 } 803 804 /* 805 * In case of shared ring only the first user (master user) can 806 * configure the ring. The sequence should be by the client: 807 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user 808 * k3_ringacc_ring_cfg(ring, cfg); # master configuration 809 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 810 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 811 */ 812 if (ring->use_count != 1) 813 return 0; 814 815 ring->size = cfg->size; 816 ring->elm_size = cfg->elm_size; 817 ring->mode = cfg->mode; 818 memset(&ring->state, 0, sizeof(ring->state)); 819 820 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) 821 ring->proxy = ringacc->proxy_target_base + 822 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; 823 824 switch (ring->mode) { 825 case K3_RINGACC_RING_MODE_RING: 826 ring->ops = &k3_ring_mode_ring_ops; 827 ring->dma_dev = cfg->dma_dev; 828 if (!ring->dma_dev) 829 ring->dma_dev = ringacc->dev; 830 break; 831 case K3_RINGACC_RING_MODE_MESSAGE: 832 ring->dma_dev = ringacc->dev; 833 if (ring->proxy) 834 ring->ops = &k3_ring_mode_proxy_ops; 835 else 836 ring->ops = &k3_ring_mode_msg_ops; 837 break; 838 default: 839 ring->ops = NULL; 840 ret = -EINVAL; 841 goto err_free_proxy; 842 } 843 844 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, 845 ring->size * (4 << ring->elm_size), 846 &ring->ring_mem_dma, GFP_KERNEL); 847 if (!ring->ring_mem_virt) { 848 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 849 ret = -ENOMEM; 850 goto err_free_ops; 851 } 852 853 ret = k3_ringacc_ring_cfg_sci(ring); 854 855 if (ret) 856 goto err_free_mem; 857 858 ring->flags |= K3_RING_FLAG_BUSY; 859 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? 860 K3_RING_FLAG_SHARED : 0; 861 862 k3_ringacc_ring_dump(ring); 863 864 return 0; 865 866 err_free_mem: 867 dma_free_coherent(ring->dma_dev, 868 ring->size * (4 << ring->elm_size), 869 ring->ring_mem_virt, 870 ring->ring_mem_dma); 871 err_free_ops: 872 ring->ops = NULL; 873 ring->dma_dev = NULL; 874 err_free_proxy: 875 ring->proxy = NULL; 876 return ret; 877 } 878 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); 879 880 u32 k3_ringacc_ring_get_size(struct k3_ring *ring) 881 { 882 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 883 return -EINVAL; 884 885 return ring->size; 886 } 887 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); 888 889 u32 k3_ringacc_ring_get_free(struct k3_ring *ring) 890 { 891 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 892 return -EINVAL; 893 894 if (!ring->state.free) 895 ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring); 896 897 return ring->state.free; 898 } 899 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); 900 901 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) 902 { 903 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 904 return -EINVAL; 905 906 return k3_ringacc_ring_read_occ(ring); 907 } 908 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); 909 910 u32 k3_ringacc_ring_is_full(struct k3_ring *ring) 911 { 912 return !k3_ringacc_ring_get_free(ring); 913 } 914 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); 915 916 enum k3_ringacc_access_mode { 917 K3_RINGACC_ACCESS_MODE_PUSH_HEAD, 918 K3_RINGACC_ACCESS_MODE_POP_HEAD, 919 K3_RINGACC_ACCESS_MODE_PUSH_TAIL, 920 K3_RINGACC_ACCESS_MODE_POP_TAIL, 921 K3_RINGACC_ACCESS_MODE_PEEK_HEAD, 922 K3_RINGACC_ACCESS_MODE_PEEK_TAIL, 923 }; 924 925 #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) 926 #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) 927 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, 928 enum k3_ringacc_proxy_access_mode mode) 929 { 930 u32 val; 931 932 val = ring->ring_id; 933 val |= K3_RINGACC_PROXY_MODE(mode); 934 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); 935 writel(val, &ring->proxy->control); 936 return 0; 937 } 938 939 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, 940 enum k3_ringacc_access_mode access_mode) 941 { 942 void __iomem *ptr; 943 944 ptr = (void __iomem *)&ring->proxy->data; 945 946 switch (access_mode) { 947 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 948 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 949 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); 950 break; 951 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 952 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 953 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); 954 break; 955 default: 956 return -EINVAL; 957 } 958 959 ptr += k3_ringacc_ring_get_fifo_pos(ring); 960 961 switch (access_mode) { 962 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 963 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 964 dev_dbg(ring->parent->dev, 965 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 966 access_mode); 967 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 968 ring->state.occ--; 969 break; 970 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 971 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 972 dev_dbg(ring->parent->dev, 973 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 974 access_mode); 975 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 976 ring->state.free--; 977 break; 978 default: 979 return -EINVAL; 980 } 981 982 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, 983 ring->state.occ); 984 return 0; 985 } 986 987 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) 988 { 989 return k3_ringacc_ring_access_proxy(ring, elem, 990 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 991 } 992 993 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) 994 { 995 return k3_ringacc_ring_access_proxy(ring, elem, 996 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 997 } 998 999 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) 1000 { 1001 return k3_ringacc_ring_access_proxy(ring, elem, 1002 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1003 } 1004 1005 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) 1006 { 1007 return k3_ringacc_ring_access_proxy(ring, elem, 1008 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1009 } 1010 1011 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, 1012 enum k3_ringacc_access_mode access_mode) 1013 { 1014 void __iomem *ptr; 1015 1016 switch (access_mode) { 1017 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 1018 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 1019 ptr = (void __iomem *)&ring->fifos->head_data; 1020 break; 1021 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 1022 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 1023 ptr = (void __iomem *)&ring->fifos->tail_data; 1024 break; 1025 default: 1026 return -EINVAL; 1027 } 1028 1029 ptr += k3_ringacc_ring_get_fifo_pos(ring); 1030 1031 switch (access_mode) { 1032 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 1033 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 1034 dev_dbg(ring->parent->dev, 1035 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 1036 access_mode); 1037 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 1038 ring->state.occ--; 1039 break; 1040 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 1041 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 1042 dev_dbg(ring->parent->dev, 1043 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 1044 access_mode); 1045 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 1046 ring->state.free--; 1047 break; 1048 default: 1049 return -EINVAL; 1050 } 1051 1052 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", 1053 ring->state.free, ring->state.windex, ring->state.occ, 1054 ring->state.rindex); 1055 return 0; 1056 } 1057 1058 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) 1059 { 1060 return k3_ringacc_ring_access_io(ring, elem, 1061 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 1062 } 1063 1064 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) 1065 { 1066 return k3_ringacc_ring_access_io(ring, elem, 1067 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 1068 } 1069 1070 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) 1071 { 1072 return k3_ringacc_ring_access_io(ring, elem, 1073 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1074 } 1075 1076 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) 1077 { 1078 return k3_ringacc_ring_access_io(ring, elem, 1079 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1080 } 1081 1082 /* 1083 * The element is 48 bits of address + ASEL bits in the ring. 1084 * ASEL is used by the DMAs and should be removed for the kernel as it is not 1085 * part of the physical memory address. 1086 */ 1087 static void k3_dmaring_remove_asel_from_elem(u64 *elem) 1088 { 1089 *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0); 1090 } 1091 1092 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem) 1093 { 1094 void *elem_ptr; 1095 u32 elem_idx; 1096 1097 /* 1098 * DMA rings: forward ring is always tied DMA channel and HW does not 1099 * maintain any state data required for POP operation and its unknown 1100 * how much elements were consumed by HW. So, to actually 1101 * do POP, the read pointer has to be recalculated every time. 1102 */ 1103 ring->state.occ = k3_ringacc_ring_read_occ(ring); 1104 if (ring->state.windex >= ring->state.occ) 1105 elem_idx = ring->state.windex - ring->state.occ; 1106 else 1107 elem_idx = ring->size - (ring->state.occ - ring->state.windex); 1108 1109 elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx); 1110 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1111 k3_dmaring_remove_asel_from_elem(elem); 1112 1113 ring->state.occ--; 1114 writel(-1, &ring->rt->db); 1115 1116 dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n", 1117 __func__, ring->state.occ, ring->state.windex, elem_idx, 1118 elem_ptr); 1119 return 0; 1120 } 1121 1122 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem) 1123 { 1124 void *elem_ptr; 1125 1126 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 1127 1128 if (ring->state.occ) { 1129 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1130 k3_dmaring_remove_asel_from_elem(elem); 1131 1132 ring->state.rindex = (ring->state.rindex + 1) % ring->size; 1133 ring->state.occ--; 1134 writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db); 1135 } else if (ring->state.tdown_complete) { 1136 dma_addr_t *value = elem; 1137 1138 *value = CPPI5_TDCM_MARKER; 1139 writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db); 1140 ring->state.tdown_complete = false; 1141 } 1142 1143 dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n", 1144 __func__, ring->state.occ, ring->state.rindex, elem_ptr); 1145 return 0; 1146 } 1147 1148 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) 1149 { 1150 void *elem_ptr; 1151 1152 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); 1153 1154 memcpy(elem_ptr, elem, (4 << ring->elm_size)); 1155 if (ring->parent->dma_rings) { 1156 u64 *addr = elem_ptr; 1157 1158 *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT); 1159 } 1160 1161 ring->state.windex = (ring->state.windex + 1) % ring->size; 1162 ring->state.free--; 1163 writel(1, &ring->rt->db); 1164 1165 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", 1166 ring->state.free, ring->state.windex); 1167 1168 return 0; 1169 } 1170 1171 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) 1172 { 1173 void *elem_ptr; 1174 1175 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 1176 1177 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1178 1179 ring->state.rindex = (ring->state.rindex + 1) % ring->size; 1180 ring->state.occ--; 1181 writel(-1, &ring->rt->db); 1182 1183 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", 1184 ring->state.occ, ring->state.rindex, elem_ptr); 1185 return 0; 1186 } 1187 1188 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) 1189 { 1190 int ret = -EOPNOTSUPP; 1191 1192 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1193 return -EINVAL; 1194 1195 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", 1196 ring->state.free, ring->state.windex); 1197 1198 if (k3_ringacc_ring_is_full(ring)) 1199 return -ENOMEM; 1200 1201 if (ring->ops && ring->ops->push_tail) 1202 ret = ring->ops->push_tail(ring, elem); 1203 1204 return ret; 1205 } 1206 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); 1207 1208 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) 1209 { 1210 int ret = -EOPNOTSUPP; 1211 1212 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1213 return -EINVAL; 1214 1215 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", 1216 ring->state.free, ring->state.windex); 1217 1218 if (k3_ringacc_ring_is_full(ring)) 1219 return -ENOMEM; 1220 1221 if (ring->ops && ring->ops->push_head) 1222 ret = ring->ops->push_head(ring, elem); 1223 1224 return ret; 1225 } 1226 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); 1227 1228 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) 1229 { 1230 int ret = -EOPNOTSUPP; 1231 1232 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1233 return -EINVAL; 1234 1235 if (!ring->state.occ) 1236 k3_ringacc_ring_update_occ(ring); 1237 1238 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, 1239 ring->state.rindex); 1240 1241 if (!ring->state.occ && !ring->state.tdown_complete) 1242 return -ENODATA; 1243 1244 if (ring->ops && ring->ops->pop_head) 1245 ret = ring->ops->pop_head(ring, elem); 1246 1247 return ret; 1248 } 1249 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); 1250 1251 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) 1252 { 1253 int ret = -EOPNOTSUPP; 1254 1255 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1256 return -EINVAL; 1257 1258 if (!ring->state.occ) 1259 k3_ringacc_ring_update_occ(ring); 1260 1261 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", 1262 ring->state.occ, ring->state.rindex); 1263 1264 if (!ring->state.occ) 1265 return -ENODATA; 1266 1267 if (ring->ops && ring->ops->pop_tail) 1268 ret = ring->ops->pop_tail(ring, elem); 1269 1270 return ret; 1271 } 1272 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); 1273 1274 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, 1275 const char *property) 1276 { 1277 struct device_node *ringacc_np; 1278 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); 1279 struct k3_ringacc *entry; 1280 1281 ringacc_np = of_parse_phandle(np, property, 0); 1282 if (!ringacc_np) 1283 return ERR_PTR(-ENODEV); 1284 1285 mutex_lock(&k3_ringacc_list_lock); 1286 list_for_each_entry(entry, &k3_ringacc_list, list) 1287 if (entry->dev->of_node == ringacc_np) { 1288 ringacc = entry; 1289 break; 1290 } 1291 mutex_unlock(&k3_ringacc_list_lock); 1292 of_node_put(ringacc_np); 1293 1294 return ringacc; 1295 } 1296 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); 1297 1298 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) 1299 { 1300 struct device_node *node = ringacc->dev->of_node; 1301 struct device *dev = ringacc->dev; 1302 struct platform_device *pdev = to_platform_device(dev); 1303 int ret; 1304 1305 if (!node) { 1306 dev_err(dev, "device tree info unavailable\n"); 1307 return -ENODEV; 1308 } 1309 1310 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); 1311 if (ret) { 1312 dev_err(dev, "ti,num-rings read failure %d\n", ret); 1313 return ret; 1314 } 1315 1316 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); 1317 if (IS_ERR(ringacc->tisci)) { 1318 ret = PTR_ERR(ringacc->tisci); 1319 if (ret != -EPROBE_DEFER) 1320 dev_err(dev, "ti,sci read fail %d\n", ret); 1321 ringacc->tisci = NULL; 1322 return ret; 1323 } 1324 1325 ret = of_property_read_u32(node, "ti,sci-dev-id", 1326 &ringacc->tisci_dev_id); 1327 if (ret) { 1328 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); 1329 return ret; 1330 } 1331 1332 pdev->id = ringacc->tisci_dev_id; 1333 1334 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, 1335 ringacc->tisci_dev_id, 1336 "ti,sci-rm-range-gp-rings"); 1337 if (IS_ERR(ringacc->rm_gp_range)) { 1338 dev_err(dev, "Failed to allocate MSI interrupts\n"); 1339 return PTR_ERR(ringacc->rm_gp_range); 1340 } 1341 1342 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, 1343 ringacc->rm_gp_range); 1344 } 1345 1346 static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = { 1347 .dma_ring_reset_quirk = 1, 1348 }; 1349 1350 static const struct soc_device_attribute k3_ringacc_socinfo[] = { 1351 { .family = "AM65X", 1352 .revision = "SR1.0", 1353 .data = &k3_ringacc_soc_data_sr1 1354 }, 1355 {/* sentinel */} 1356 }; 1357 1358 static int k3_ringacc_init(struct platform_device *pdev, 1359 struct k3_ringacc *ringacc) 1360 { 1361 const struct soc_device_attribute *soc; 1362 void __iomem *base_fifo, *base_rt; 1363 struct device *dev = &pdev->dev; 1364 struct resource *res; 1365 int ret, i; 1366 1367 dev->msi.domain = of_msi_get_domain(dev, dev->of_node, 1368 DOMAIN_BUS_TI_SCI_INTA_MSI); 1369 if (!dev->msi.domain) { 1370 dev_err(dev, "Failed to get MSI domain\n"); 1371 return -EPROBE_DEFER; 1372 } 1373 1374 ret = k3_ringacc_probe_dt(ringacc); 1375 if (ret) 1376 return ret; 1377 1378 soc = soc_device_match(k3_ringacc_socinfo); 1379 if (soc && soc->data) { 1380 const struct k3_ringacc_soc_data *soc_data = soc->data; 1381 1382 ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk; 1383 } 1384 1385 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); 1386 base_rt = devm_ioremap_resource(dev, res); 1387 if (IS_ERR(base_rt)) 1388 return PTR_ERR(base_rt); 1389 1390 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos"); 1391 base_fifo = devm_ioremap_resource(dev, res); 1392 if (IS_ERR(base_fifo)) 1393 return PTR_ERR(base_fifo); 1394 1395 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg"); 1396 ringacc->proxy_gcfg = devm_ioremap_resource(dev, res); 1397 if (IS_ERR(ringacc->proxy_gcfg)) 1398 return PTR_ERR(ringacc->proxy_gcfg); 1399 1400 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 1401 "proxy_target"); 1402 ringacc->proxy_target_base = devm_ioremap_resource(dev, res); 1403 if (IS_ERR(ringacc->proxy_target_base)) 1404 return PTR_ERR(ringacc->proxy_target_base); 1405 1406 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & 1407 K3_RINGACC_PROXY_CFG_THREADS_MASK; 1408 1409 ringacc->rings = devm_kzalloc(dev, 1410 sizeof(*ringacc->rings) * 1411 ringacc->num_rings, 1412 GFP_KERNEL); 1413 ringacc->rings_inuse = devm_bitmap_zalloc(dev, ringacc->num_rings, 1414 GFP_KERNEL); 1415 ringacc->proxy_inuse = devm_bitmap_zalloc(dev, ringacc->num_proxies, 1416 GFP_KERNEL); 1417 1418 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) 1419 return -ENOMEM; 1420 1421 for (i = 0; i < ringacc->num_rings; i++) { 1422 ringacc->rings[i].rt = base_rt + 1423 K3_RINGACC_RT_REGS_STEP * i; 1424 ringacc->rings[i].fifos = base_fifo + 1425 K3_RINGACC_FIFO_REGS_STEP * i; 1426 ringacc->rings[i].parent = ringacc; 1427 ringacc->rings[i].ring_id = i; 1428 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; 1429 } 1430 1431 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1432 1433 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", 1434 ringacc->num_rings, 1435 ringacc->rm_gp_range->desc[0].start, 1436 ringacc->rm_gp_range->desc[0].num, 1437 ringacc->tisci_dev_id); 1438 dev_info(dev, "dma-ring-reset-quirk: %s\n", 1439 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled"); 1440 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", 1441 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); 1442 1443 return 0; 1444 } 1445 1446 struct ringacc_match_data { 1447 struct k3_ringacc_ops ops; 1448 }; 1449 1450 static struct ringacc_match_data k3_ringacc_data = { 1451 .ops = { 1452 .init = k3_ringacc_init, 1453 }, 1454 }; 1455 1456 /* Match table for of_platform binding */ 1457 static const struct of_device_id k3_ringacc_of_match[] = { 1458 { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, }, 1459 {}, 1460 }; 1461 MODULE_DEVICE_TABLE(of, k3_ringacc_of_match); 1462 1463 struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev, 1464 struct k3_ringacc_init_data *data) 1465 { 1466 struct device *dev = &pdev->dev; 1467 struct k3_ringacc *ringacc; 1468 void __iomem *base_rt; 1469 struct resource *res; 1470 int i; 1471 1472 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1473 if (!ringacc) 1474 return ERR_PTR(-ENOMEM); 1475 1476 ringacc->dev = dev; 1477 ringacc->dma_rings = true; 1478 ringacc->num_rings = data->num_rings; 1479 ringacc->tisci = data->tisci; 1480 ringacc->tisci_dev_id = data->tisci_dev_id; 1481 1482 mutex_init(&ringacc->req_lock); 1483 1484 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt"); 1485 base_rt = devm_ioremap_resource(dev, res); 1486 if (IS_ERR(base_rt)) 1487 return ERR_CAST(base_rt); 1488 1489 ringacc->rings = devm_kzalloc(dev, 1490 sizeof(*ringacc->rings) * 1491 ringacc->num_rings * 2, 1492 GFP_KERNEL); 1493 ringacc->rings_inuse = devm_bitmap_zalloc(dev, ringacc->num_rings, 1494 GFP_KERNEL); 1495 1496 if (!ringacc->rings || !ringacc->rings_inuse) 1497 return ERR_PTR(-ENOMEM); 1498 1499 for (i = 0; i < ringacc->num_rings; i++) { 1500 struct k3_ring *ring = &ringacc->rings[i]; 1501 1502 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i; 1503 ring->parent = ringacc; 1504 ring->ring_id = i; 1505 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 1506 1507 ring = &ringacc->rings[ringacc->num_rings + i]; 1508 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i + 1509 K3_DMARING_RT_REGS_REVERSE_OFS; 1510 ring->parent = ringacc; 1511 ring->ring_id = i; 1512 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 1513 ring->flags = K3_RING_FLAG_REVERSE; 1514 } 1515 1516 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1517 1518 dev_info(dev, "Number of rings: %u\n", ringacc->num_rings); 1519 1520 return ringacc; 1521 } 1522 EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init); 1523 1524 static int k3_ringacc_probe(struct platform_device *pdev) 1525 { 1526 const struct ringacc_match_data *match_data; 1527 struct device *dev = &pdev->dev; 1528 struct k3_ringacc *ringacc; 1529 int ret; 1530 1531 match_data = of_device_get_match_data(&pdev->dev); 1532 if (!match_data) 1533 return -ENODEV; 1534 1535 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1536 if (!ringacc) 1537 return -ENOMEM; 1538 1539 ringacc->dev = dev; 1540 mutex_init(&ringacc->req_lock); 1541 ringacc->ops = &match_data->ops; 1542 1543 ret = ringacc->ops->init(pdev, ringacc); 1544 if (ret) 1545 return ret; 1546 1547 dev_set_drvdata(dev, ringacc); 1548 1549 mutex_lock(&k3_ringacc_list_lock); 1550 list_add_tail(&ringacc->list, &k3_ringacc_list); 1551 mutex_unlock(&k3_ringacc_list_lock); 1552 1553 return 0; 1554 } 1555 1556 static int k3_ringacc_remove(struct platform_device *pdev) 1557 { 1558 struct k3_ringacc *ringacc = dev_get_drvdata(&pdev->dev); 1559 1560 mutex_lock(&k3_ringacc_list_lock); 1561 list_del(&ringacc->list); 1562 mutex_unlock(&k3_ringacc_list_lock); 1563 return 0; 1564 } 1565 1566 static struct platform_driver k3_ringacc_driver = { 1567 .probe = k3_ringacc_probe, 1568 .remove = k3_ringacc_remove, 1569 .driver = { 1570 .name = "k3-ringacc", 1571 .of_match_table = k3_ringacc_of_match, 1572 .suppress_bind_attrs = true, 1573 }, 1574 }; 1575 module_platform_driver(k3_ringacc_driver); 1576 1577 MODULE_LICENSE("GPL"); 1578 MODULE_DESCRIPTION("TI Ringacc driver for K3 SOCs"); 1579 MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>"); 1580