1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * TI K3 NAVSS Ring Accelerator subsystem driver 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 */ 7 8 #include <linux/dma-mapping.h> 9 #include <linux/io.h> 10 #include <linux/init.h> 11 #include <linux/of.h> 12 #include <linux/platform_device.h> 13 #include <linux/soc/ti/k3-ringacc.h> 14 #include <linux/soc/ti/ti_sci_protocol.h> 15 #include <linux/soc/ti/ti_sci_inta_msi.h> 16 #include <linux/of_irq.h> 17 #include <linux/irqdomain.h> 18 19 static LIST_HEAD(k3_ringacc_list); 20 static DEFINE_MUTEX(k3_ringacc_list_lock); 21 22 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) 23 24 /** 25 * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region 26 * 27 * @resv_16: Reserved 28 * @db: Ring Doorbell Register 29 * @resv_4: Reserved 30 * @occ: Ring Occupancy Register 31 * @indx: Ring Current Index Register 32 * @hwocc: Ring Hardware Occupancy Register 33 * @hwindx: Ring Hardware Current Index Register 34 */ 35 struct k3_ring_rt_regs { 36 u32 resv_16[4]; 37 u32 db; 38 u32 resv_4[1]; 39 u32 occ; 40 u32 indx; 41 u32 hwocc; 42 u32 hwindx; 43 }; 44 45 #define K3_RINGACC_RT_REGS_STEP 0x1000 46 47 /** 48 * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region 49 * 50 * @head_data: Ring Head Entry Data Registers 51 * @tail_data: Ring Tail Entry Data Registers 52 * @peek_head_data: Ring Peek Head Entry Data Regs 53 * @peek_tail_data: Ring Peek Tail Entry Data Regs 54 */ 55 struct k3_ring_fifo_regs { 56 u32 head_data[128]; 57 u32 tail_data[128]; 58 u32 peek_head_data[128]; 59 u32 peek_tail_data[128]; 60 }; 61 62 /** 63 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region 64 * 65 * @revision: Revision Register 66 * @config: Config Register 67 */ 68 struct k3_ringacc_proxy_gcfg_regs { 69 u32 revision; 70 u32 config; 71 }; 72 73 #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) 74 75 /** 76 * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region 77 * 78 * @control: Proxy Control Register 79 * @status: Proxy Status Register 80 * @resv_512: Reserved 81 * @data: Proxy Data Register 82 */ 83 struct k3_ringacc_proxy_target_regs { 84 u32 control; 85 u32 status; 86 u8 resv_512[504]; 87 u32 data[128]; 88 }; 89 90 #define K3_RINGACC_PROXY_TARGET_STEP 0x1000 91 #define K3_RINGACC_PROXY_NOT_USED (-1) 92 93 enum k3_ringacc_proxy_access_mode { 94 PROXY_ACCESS_MODE_HEAD = 0, 95 PROXY_ACCESS_MODE_TAIL = 1, 96 PROXY_ACCESS_MODE_PEEK_HEAD = 2, 97 PROXY_ACCESS_MODE_PEEK_TAIL = 3, 98 }; 99 100 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) 101 #define K3_RINGACC_FIFO_REGS_STEP 0x1000 102 #define K3_RINGACC_MAX_DB_RING_CNT (127U) 103 104 struct k3_ring_ops { 105 int (*push_tail)(struct k3_ring *ring, void *elm); 106 int (*push_head)(struct k3_ring *ring, void *elm); 107 int (*pop_tail)(struct k3_ring *ring, void *elm); 108 int (*pop_head)(struct k3_ring *ring, void *elm); 109 }; 110 111 /** 112 * struct k3_ring_state - Internal state tracking structure 113 * 114 * @free: Number of free entries 115 * @occ: Occupancy 116 * @windex: Write index 117 * @rindex: Read index 118 */ 119 struct k3_ring_state { 120 u32 free; 121 u32 occ; 122 u32 windex; 123 u32 rindex; 124 }; 125 126 /** 127 * struct k3_ring - RA Ring descriptor 128 * 129 * @rt: Ring control/status registers 130 * @fifos: Ring queues registers 131 * @proxy: Ring Proxy Datapath registers 132 * @ring_mem_dma: Ring buffer dma address 133 * @ring_mem_virt: Ring buffer virt address 134 * @ops: Ring operations 135 * @size: Ring size in elements 136 * @elm_size: Size of the ring element 137 * @mode: Ring mode 138 * @flags: flags 139 * @ring_id: Ring Id 140 * @parent: Pointer on struct @k3_ringacc 141 * @use_count: Use count for shared rings 142 * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) 143 */ 144 struct k3_ring { 145 struct k3_ring_rt_regs __iomem *rt; 146 struct k3_ring_fifo_regs __iomem *fifos; 147 struct k3_ringacc_proxy_target_regs __iomem *proxy; 148 dma_addr_t ring_mem_dma; 149 void *ring_mem_virt; 150 struct k3_ring_ops *ops; 151 u32 size; 152 enum k3_ring_size elm_size; 153 enum k3_ring_mode mode; 154 u32 flags; 155 #define K3_RING_FLAG_BUSY BIT(1) 156 #define K3_RING_FLAG_SHARED BIT(2) 157 struct k3_ring_state state; 158 u32 ring_id; 159 struct k3_ringacc *parent; 160 u32 use_count; 161 int proxy_id; 162 }; 163 164 struct k3_ringacc_ops { 165 int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc); 166 }; 167 168 /** 169 * struct k3_ringacc - Rings accelerator descriptor 170 * 171 * @dev: pointer on RA device 172 * @proxy_gcfg: RA proxy global config registers 173 * @proxy_target_base: RA proxy datapath region 174 * @num_rings: number of ring in RA 175 * @rings_inuse: bitfield for ring usage tracking 176 * @rm_gp_range: general purpose rings range from tisci 177 * @dma_ring_reset_quirk: DMA reset w/a enable 178 * @num_proxies: number of RA proxies 179 * @proxy_inuse: bitfield for proxy usage tracking 180 * @rings: array of rings descriptors (struct @k3_ring) 181 * @list: list of RAs in the system 182 * @req_lock: protect rings allocation 183 * @tisci: pointer ti-sci handle 184 * @tisci_ring_ops: ti-sci rings ops 185 * @tisci_dev_id: ti-sci device id 186 * @ops: SoC specific ringacc operation 187 */ 188 struct k3_ringacc { 189 struct device *dev; 190 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; 191 void __iomem *proxy_target_base; 192 u32 num_rings; /* number of rings in Ringacc module */ 193 unsigned long *rings_inuse; 194 struct ti_sci_resource *rm_gp_range; 195 196 bool dma_ring_reset_quirk; 197 u32 num_proxies; 198 unsigned long *proxy_inuse; 199 200 struct k3_ring *rings; 201 struct list_head list; 202 struct mutex req_lock; /* protect rings allocation */ 203 204 const struct ti_sci_handle *tisci; 205 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; 206 u32 tisci_dev_id; 207 208 const struct k3_ringacc_ops *ops; 209 }; 210 211 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) 212 { 213 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - 214 (4 << ring->elm_size); 215 } 216 217 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) 218 { 219 return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); 220 } 221 222 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); 223 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); 224 225 static struct k3_ring_ops k3_ring_mode_ring_ops = { 226 .push_tail = k3_ringacc_ring_push_mem, 227 .pop_head = k3_ringacc_ring_pop_mem, 228 }; 229 230 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); 231 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); 232 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); 233 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); 234 235 static struct k3_ring_ops k3_ring_mode_msg_ops = { 236 .push_tail = k3_ringacc_ring_push_io, 237 .push_head = k3_ringacc_ring_push_head_io, 238 .pop_tail = k3_ringacc_ring_pop_tail_io, 239 .pop_head = k3_ringacc_ring_pop_io, 240 }; 241 242 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); 243 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); 244 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); 245 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); 246 247 static struct k3_ring_ops k3_ring_mode_proxy_ops = { 248 .push_tail = k3_ringacc_ring_push_tail_proxy, 249 .push_head = k3_ringacc_ring_push_head_proxy, 250 .pop_tail = k3_ringacc_ring_pop_tail_proxy, 251 .pop_head = k3_ringacc_ring_pop_head_proxy, 252 }; 253 254 static void k3_ringacc_ring_dump(struct k3_ring *ring) 255 { 256 struct device *dev = ring->parent->dev; 257 258 dev_dbg(dev, "dump ring: %d\n", ring->ring_id); 259 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, 260 &ring->ring_mem_dma); 261 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", 262 ring->elm_size, ring->size, ring->mode, ring->proxy_id); 263 dev_dbg(dev, "dump flags %08X\n", ring->flags); 264 265 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); 266 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); 267 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); 268 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); 269 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); 270 271 if (ring->ring_mem_virt) 272 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, 273 16, 1, ring->ring_mem_virt, 16 * 8, false); 274 } 275 276 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, 277 int id, u32 flags) 278 { 279 int proxy_id = K3_RINGACC_PROXY_NOT_USED; 280 281 mutex_lock(&ringacc->req_lock); 282 283 if (id == K3_RINGACC_RING_ID_ANY) { 284 /* Request for any general purpose ring */ 285 struct ti_sci_resource_desc *gp_rings = 286 &ringacc->rm_gp_range->desc[0]; 287 unsigned long size; 288 289 size = gp_rings->start + gp_rings->num; 290 id = find_next_zero_bit(ringacc->rings_inuse, size, 291 gp_rings->start); 292 if (id == size) 293 goto error; 294 } else if (id < 0) { 295 goto error; 296 } 297 298 if (test_bit(id, ringacc->rings_inuse) && 299 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) 300 goto error; 301 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) 302 goto out; 303 304 if (flags & K3_RINGACC_RING_USE_PROXY) { 305 proxy_id = find_next_zero_bit(ringacc->proxy_inuse, 306 ringacc->num_proxies, 0); 307 if (proxy_id == ringacc->num_proxies) 308 goto error; 309 } 310 311 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { 312 set_bit(proxy_id, ringacc->proxy_inuse); 313 ringacc->rings[id].proxy_id = proxy_id; 314 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, 315 proxy_id); 316 } else { 317 dev_dbg(ringacc->dev, "Giving ring#%d\n", id); 318 } 319 320 set_bit(id, ringacc->rings_inuse); 321 out: 322 ringacc->rings[id].use_count++; 323 mutex_unlock(&ringacc->req_lock); 324 return &ringacc->rings[id]; 325 326 error: 327 mutex_unlock(&ringacc->req_lock); 328 return NULL; 329 } 330 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); 331 332 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, 333 int fwd_id, int compl_id, 334 struct k3_ring **fwd_ring, 335 struct k3_ring **compl_ring) 336 { 337 int ret = 0; 338 339 if (!fwd_ring || !compl_ring) 340 return -EINVAL; 341 342 *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); 343 if (!(*fwd_ring)) 344 return -ENODEV; 345 346 *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0); 347 if (!(*compl_ring)) { 348 k3_ringacc_ring_free(*fwd_ring); 349 ret = -ENODEV; 350 } 351 352 return ret; 353 } 354 EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair); 355 356 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) 357 { 358 struct k3_ringacc *ringacc = ring->parent; 359 int ret; 360 361 ret = ringacc->tisci_ring_ops->config( 362 ringacc->tisci, 363 TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID, 364 ringacc->tisci_dev_id, 365 ring->ring_id, 366 0, 367 0, 368 ring->size, 369 0, 370 0, 371 0); 372 if (ret) 373 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", 374 ret, ring->ring_id); 375 } 376 377 void k3_ringacc_ring_reset(struct k3_ring *ring) 378 { 379 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 380 return; 381 382 memset(&ring->state, 0, sizeof(ring->state)); 383 384 k3_ringacc_ring_reset_sci(ring); 385 } 386 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); 387 388 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, 389 enum k3_ring_mode mode) 390 { 391 struct k3_ringacc *ringacc = ring->parent; 392 int ret; 393 394 ret = ringacc->tisci_ring_ops->config( 395 ringacc->tisci, 396 TI_SCI_MSG_VALUE_RM_RING_MODE_VALID, 397 ringacc->tisci_dev_id, 398 ring->ring_id, 399 0, 400 0, 401 0, 402 mode, 403 0, 404 0); 405 if (ret) 406 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", 407 ret, ring->ring_id); 408 } 409 410 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) 411 { 412 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 413 return; 414 415 if (!ring->parent->dma_ring_reset_quirk) 416 goto reset; 417 418 if (!occ) 419 occ = readl(&ring->rt->occ); 420 421 if (occ) { 422 u32 db_ring_cnt, db_ring_cnt_cur; 423 424 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, 425 ring->ring_id, occ); 426 /* TI-SCI ring reset */ 427 k3_ringacc_ring_reset_sci(ring); 428 429 /* 430 * Setup the ring in ring/doorbell mode (if not already in this 431 * mode) 432 */ 433 if (ring->mode != K3_RINGACC_RING_MODE_RING) 434 k3_ringacc_ring_reconfig_qmode_sci( 435 ring, K3_RINGACC_RING_MODE_RING); 436 /* 437 * Ring the doorbell 2**22 – ringOcc times. 438 * This will wrap the internal UDMAP ring state occupancy 439 * counter (which is 21-bits wide) to 0. 440 */ 441 db_ring_cnt = (1U << 22) - occ; 442 443 while (db_ring_cnt != 0) { 444 /* 445 * Ring the doorbell with the maximum count each 446 * iteration if possible to minimize the total 447 * of writes 448 */ 449 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) 450 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; 451 else 452 db_ring_cnt_cur = db_ring_cnt; 453 454 writel(db_ring_cnt_cur, &ring->rt->db); 455 db_ring_cnt -= db_ring_cnt_cur; 456 } 457 458 /* Restore the original ring mode (if not ring mode) */ 459 if (ring->mode != K3_RINGACC_RING_MODE_RING) 460 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); 461 } 462 463 reset: 464 /* Reset the ring */ 465 k3_ringacc_ring_reset(ring); 466 } 467 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); 468 469 static void k3_ringacc_ring_free_sci(struct k3_ring *ring) 470 { 471 struct k3_ringacc *ringacc = ring->parent; 472 int ret; 473 474 ret = ringacc->tisci_ring_ops->config( 475 ringacc->tisci, 476 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER, 477 ringacc->tisci_dev_id, 478 ring->ring_id, 479 0, 480 0, 481 0, 482 0, 483 0, 484 0); 485 if (ret) 486 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", 487 ret, ring->ring_id); 488 } 489 490 int k3_ringacc_ring_free(struct k3_ring *ring) 491 { 492 struct k3_ringacc *ringacc; 493 494 if (!ring) 495 return -EINVAL; 496 497 ringacc = ring->parent; 498 499 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); 500 501 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 502 return -EINVAL; 503 504 mutex_lock(&ringacc->req_lock); 505 506 if (--ring->use_count) 507 goto out; 508 509 if (!(ring->flags & K3_RING_FLAG_BUSY)) 510 goto no_init; 511 512 k3_ringacc_ring_free_sci(ring); 513 514 dma_free_coherent(ringacc->dev, 515 ring->size * (4 << ring->elm_size), 516 ring->ring_mem_virt, ring->ring_mem_dma); 517 ring->flags = 0; 518 ring->ops = NULL; 519 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { 520 clear_bit(ring->proxy_id, ringacc->proxy_inuse); 521 ring->proxy = NULL; 522 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 523 } 524 525 no_init: 526 clear_bit(ring->ring_id, ringacc->rings_inuse); 527 528 out: 529 mutex_unlock(&ringacc->req_lock); 530 return 0; 531 } 532 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); 533 534 u32 k3_ringacc_get_ring_id(struct k3_ring *ring) 535 { 536 if (!ring) 537 return -EINVAL; 538 539 return ring->ring_id; 540 } 541 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); 542 543 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) 544 { 545 if (!ring) 546 return -EINVAL; 547 548 return ring->parent->tisci_dev_id; 549 } 550 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); 551 552 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) 553 { 554 int irq_num; 555 556 if (!ring) 557 return -EINVAL; 558 559 irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); 560 if (irq_num <= 0) 561 irq_num = -EINVAL; 562 return irq_num; 563 } 564 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); 565 566 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) 567 { 568 struct k3_ringacc *ringacc = ring->parent; 569 u32 ring_idx; 570 int ret; 571 572 if (!ringacc->tisci) 573 return -EINVAL; 574 575 ring_idx = ring->ring_id; 576 ret = ringacc->tisci_ring_ops->config( 577 ringacc->tisci, 578 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER, 579 ringacc->tisci_dev_id, 580 ring_idx, 581 lower_32_bits(ring->ring_mem_dma), 582 upper_32_bits(ring->ring_mem_dma), 583 ring->size, 584 ring->mode, 585 ring->elm_size, 586 0); 587 if (ret) 588 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", 589 ret, ring_idx); 590 591 return ret; 592 } 593 594 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 595 { 596 struct k3_ringacc *ringacc; 597 int ret = 0; 598 599 if (!ring || !cfg) 600 return -EINVAL; 601 ringacc = ring->parent; 602 603 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || 604 cfg->mode >= K3_RINGACC_RING_MODE_INVALID || 605 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || 606 !test_bit(ring->ring_id, ringacc->rings_inuse)) 607 return -EINVAL; 608 609 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && 610 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && 611 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { 612 dev_err(ringacc->dev, 613 "Message mode must use proxy for %u element size\n", 614 4 << ring->elm_size); 615 return -EINVAL; 616 } 617 618 /* 619 * In case of shared ring only the first user (master user) can 620 * configure the ring. The sequence should be by the client: 621 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user 622 * k3_ringacc_ring_cfg(ring, cfg); # master configuration 623 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 624 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 625 */ 626 if (ring->use_count != 1) 627 return 0; 628 629 ring->size = cfg->size; 630 ring->elm_size = cfg->elm_size; 631 ring->mode = cfg->mode; 632 memset(&ring->state, 0, sizeof(ring->state)); 633 634 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) 635 ring->proxy = ringacc->proxy_target_base + 636 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; 637 638 switch (ring->mode) { 639 case K3_RINGACC_RING_MODE_RING: 640 ring->ops = &k3_ring_mode_ring_ops; 641 break; 642 case K3_RINGACC_RING_MODE_MESSAGE: 643 if (ring->proxy) 644 ring->ops = &k3_ring_mode_proxy_ops; 645 else 646 ring->ops = &k3_ring_mode_msg_ops; 647 break; 648 default: 649 ring->ops = NULL; 650 ret = -EINVAL; 651 goto err_free_proxy; 652 } 653 654 ring->ring_mem_virt = dma_alloc_coherent(ringacc->dev, 655 ring->size * (4 << ring->elm_size), 656 &ring->ring_mem_dma, GFP_KERNEL); 657 if (!ring->ring_mem_virt) { 658 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 659 ret = -ENOMEM; 660 goto err_free_ops; 661 } 662 663 ret = k3_ringacc_ring_cfg_sci(ring); 664 665 if (ret) 666 goto err_free_mem; 667 668 ring->flags |= K3_RING_FLAG_BUSY; 669 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? 670 K3_RING_FLAG_SHARED : 0; 671 672 k3_ringacc_ring_dump(ring); 673 674 return 0; 675 676 err_free_mem: 677 dma_free_coherent(ringacc->dev, 678 ring->size * (4 << ring->elm_size), 679 ring->ring_mem_virt, 680 ring->ring_mem_dma); 681 err_free_ops: 682 ring->ops = NULL; 683 err_free_proxy: 684 ring->proxy = NULL; 685 return ret; 686 } 687 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); 688 689 u32 k3_ringacc_ring_get_size(struct k3_ring *ring) 690 { 691 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 692 return -EINVAL; 693 694 return ring->size; 695 } 696 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); 697 698 u32 k3_ringacc_ring_get_free(struct k3_ring *ring) 699 { 700 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 701 return -EINVAL; 702 703 if (!ring->state.free) 704 ring->state.free = ring->size - readl(&ring->rt->occ); 705 706 return ring->state.free; 707 } 708 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); 709 710 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) 711 { 712 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 713 return -EINVAL; 714 715 return readl(&ring->rt->occ); 716 } 717 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); 718 719 u32 k3_ringacc_ring_is_full(struct k3_ring *ring) 720 { 721 return !k3_ringacc_ring_get_free(ring); 722 } 723 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); 724 725 enum k3_ringacc_access_mode { 726 K3_RINGACC_ACCESS_MODE_PUSH_HEAD, 727 K3_RINGACC_ACCESS_MODE_POP_HEAD, 728 K3_RINGACC_ACCESS_MODE_PUSH_TAIL, 729 K3_RINGACC_ACCESS_MODE_POP_TAIL, 730 K3_RINGACC_ACCESS_MODE_PEEK_HEAD, 731 K3_RINGACC_ACCESS_MODE_PEEK_TAIL, 732 }; 733 734 #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) 735 #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) 736 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, 737 enum k3_ringacc_proxy_access_mode mode) 738 { 739 u32 val; 740 741 val = ring->ring_id; 742 val |= K3_RINGACC_PROXY_MODE(mode); 743 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); 744 writel(val, &ring->proxy->control); 745 return 0; 746 } 747 748 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, 749 enum k3_ringacc_access_mode access_mode) 750 { 751 void __iomem *ptr; 752 753 ptr = (void __iomem *)&ring->proxy->data; 754 755 switch (access_mode) { 756 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 757 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 758 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); 759 break; 760 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 761 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 762 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); 763 break; 764 default: 765 return -EINVAL; 766 } 767 768 ptr += k3_ringacc_ring_get_fifo_pos(ring); 769 770 switch (access_mode) { 771 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 772 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 773 dev_dbg(ring->parent->dev, 774 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 775 access_mode); 776 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 777 ring->state.occ--; 778 break; 779 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 780 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 781 dev_dbg(ring->parent->dev, 782 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 783 access_mode); 784 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 785 ring->state.free--; 786 break; 787 default: 788 return -EINVAL; 789 } 790 791 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, 792 ring->state.occ); 793 return 0; 794 } 795 796 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) 797 { 798 return k3_ringacc_ring_access_proxy(ring, elem, 799 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 800 } 801 802 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) 803 { 804 return k3_ringacc_ring_access_proxy(ring, elem, 805 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 806 } 807 808 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) 809 { 810 return k3_ringacc_ring_access_proxy(ring, elem, 811 K3_RINGACC_ACCESS_MODE_POP_HEAD); 812 } 813 814 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) 815 { 816 return k3_ringacc_ring_access_proxy(ring, elem, 817 K3_RINGACC_ACCESS_MODE_POP_HEAD); 818 } 819 820 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, 821 enum k3_ringacc_access_mode access_mode) 822 { 823 void __iomem *ptr; 824 825 switch (access_mode) { 826 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 827 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 828 ptr = (void __iomem *)&ring->fifos->head_data; 829 break; 830 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 831 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 832 ptr = (void __iomem *)&ring->fifos->tail_data; 833 break; 834 default: 835 return -EINVAL; 836 } 837 838 ptr += k3_ringacc_ring_get_fifo_pos(ring); 839 840 switch (access_mode) { 841 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 842 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 843 dev_dbg(ring->parent->dev, 844 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 845 access_mode); 846 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 847 ring->state.occ--; 848 break; 849 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 850 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 851 dev_dbg(ring->parent->dev, 852 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 853 access_mode); 854 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 855 ring->state.free--; 856 break; 857 default: 858 return -EINVAL; 859 } 860 861 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", 862 ring->state.free, ring->state.windex, ring->state.occ, 863 ring->state.rindex); 864 return 0; 865 } 866 867 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) 868 { 869 return k3_ringacc_ring_access_io(ring, elem, 870 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 871 } 872 873 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) 874 { 875 return k3_ringacc_ring_access_io(ring, elem, 876 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 877 } 878 879 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) 880 { 881 return k3_ringacc_ring_access_io(ring, elem, 882 K3_RINGACC_ACCESS_MODE_POP_HEAD); 883 } 884 885 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) 886 { 887 return k3_ringacc_ring_access_io(ring, elem, 888 K3_RINGACC_ACCESS_MODE_POP_HEAD); 889 } 890 891 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) 892 { 893 void *elem_ptr; 894 895 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); 896 897 memcpy(elem_ptr, elem, (4 << ring->elm_size)); 898 899 ring->state.windex = (ring->state.windex + 1) % ring->size; 900 ring->state.free--; 901 writel(1, &ring->rt->db); 902 903 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", 904 ring->state.free, ring->state.windex); 905 906 return 0; 907 } 908 909 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) 910 { 911 void *elem_ptr; 912 913 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 914 915 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 916 917 ring->state.rindex = (ring->state.rindex + 1) % ring->size; 918 ring->state.occ--; 919 writel(-1, &ring->rt->db); 920 921 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", 922 ring->state.occ, ring->state.rindex, elem_ptr); 923 return 0; 924 } 925 926 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) 927 { 928 int ret = -EOPNOTSUPP; 929 930 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 931 return -EINVAL; 932 933 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", 934 ring->state.free, ring->state.windex); 935 936 if (k3_ringacc_ring_is_full(ring)) 937 return -ENOMEM; 938 939 if (ring->ops && ring->ops->push_tail) 940 ret = ring->ops->push_tail(ring, elem); 941 942 return ret; 943 } 944 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); 945 946 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) 947 { 948 int ret = -EOPNOTSUPP; 949 950 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 951 return -EINVAL; 952 953 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", 954 ring->state.free, ring->state.windex); 955 956 if (k3_ringacc_ring_is_full(ring)) 957 return -ENOMEM; 958 959 if (ring->ops && ring->ops->push_head) 960 ret = ring->ops->push_head(ring, elem); 961 962 return ret; 963 } 964 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); 965 966 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) 967 { 968 int ret = -EOPNOTSUPP; 969 970 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 971 return -EINVAL; 972 973 if (!ring->state.occ) 974 ring->state.occ = k3_ringacc_ring_get_occ(ring); 975 976 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, 977 ring->state.rindex); 978 979 if (!ring->state.occ) 980 return -ENODATA; 981 982 if (ring->ops && ring->ops->pop_head) 983 ret = ring->ops->pop_head(ring, elem); 984 985 return ret; 986 } 987 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); 988 989 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) 990 { 991 int ret = -EOPNOTSUPP; 992 993 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 994 return -EINVAL; 995 996 if (!ring->state.occ) 997 ring->state.occ = k3_ringacc_ring_get_occ(ring); 998 999 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", 1000 ring->state.occ, ring->state.rindex); 1001 1002 if (!ring->state.occ) 1003 return -ENODATA; 1004 1005 if (ring->ops && ring->ops->pop_tail) 1006 ret = ring->ops->pop_tail(ring, elem); 1007 1008 return ret; 1009 } 1010 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); 1011 1012 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, 1013 const char *property) 1014 { 1015 struct device_node *ringacc_np; 1016 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); 1017 struct k3_ringacc *entry; 1018 1019 ringacc_np = of_parse_phandle(np, property, 0); 1020 if (!ringacc_np) 1021 return ERR_PTR(-ENODEV); 1022 1023 mutex_lock(&k3_ringacc_list_lock); 1024 list_for_each_entry(entry, &k3_ringacc_list, list) 1025 if (entry->dev->of_node == ringacc_np) { 1026 ringacc = entry; 1027 break; 1028 } 1029 mutex_unlock(&k3_ringacc_list_lock); 1030 of_node_put(ringacc_np); 1031 1032 return ringacc; 1033 } 1034 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); 1035 1036 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) 1037 { 1038 struct device_node *node = ringacc->dev->of_node; 1039 struct device *dev = ringacc->dev; 1040 struct platform_device *pdev = to_platform_device(dev); 1041 int ret; 1042 1043 if (!node) { 1044 dev_err(dev, "device tree info unavailable\n"); 1045 return -ENODEV; 1046 } 1047 1048 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); 1049 if (ret) { 1050 dev_err(dev, "ti,num-rings read failure %d\n", ret); 1051 return ret; 1052 } 1053 1054 ringacc->dma_ring_reset_quirk = 1055 of_property_read_bool(node, "ti,dma-ring-reset-quirk"); 1056 1057 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); 1058 if (IS_ERR(ringacc->tisci)) { 1059 ret = PTR_ERR(ringacc->tisci); 1060 if (ret != -EPROBE_DEFER) 1061 dev_err(dev, "ti,sci read fail %d\n", ret); 1062 ringacc->tisci = NULL; 1063 return ret; 1064 } 1065 1066 ret = of_property_read_u32(node, "ti,sci-dev-id", 1067 &ringacc->tisci_dev_id); 1068 if (ret) { 1069 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); 1070 return ret; 1071 } 1072 1073 pdev->id = ringacc->tisci_dev_id; 1074 1075 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, 1076 ringacc->tisci_dev_id, 1077 "ti,sci-rm-range-gp-rings"); 1078 if (IS_ERR(ringacc->rm_gp_range)) { 1079 dev_err(dev, "Failed to allocate MSI interrupts\n"); 1080 return PTR_ERR(ringacc->rm_gp_range); 1081 } 1082 1083 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, 1084 ringacc->rm_gp_range); 1085 } 1086 1087 static int k3_ringacc_init(struct platform_device *pdev, 1088 struct k3_ringacc *ringacc) 1089 { 1090 void __iomem *base_fifo, *base_rt; 1091 struct device *dev = &pdev->dev; 1092 struct resource *res; 1093 int ret, i; 1094 1095 dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 1096 DOMAIN_BUS_TI_SCI_INTA_MSI); 1097 if (!dev->msi_domain) { 1098 dev_err(dev, "Failed to get MSI domain\n"); 1099 return -EPROBE_DEFER; 1100 } 1101 1102 ret = k3_ringacc_probe_dt(ringacc); 1103 if (ret) 1104 return ret; 1105 1106 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); 1107 base_rt = devm_ioremap_resource(dev, res); 1108 if (IS_ERR(base_rt)) 1109 return PTR_ERR(base_rt); 1110 1111 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos"); 1112 base_fifo = devm_ioremap_resource(dev, res); 1113 if (IS_ERR(base_fifo)) 1114 return PTR_ERR(base_fifo); 1115 1116 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg"); 1117 ringacc->proxy_gcfg = devm_ioremap_resource(dev, res); 1118 if (IS_ERR(ringacc->proxy_gcfg)) 1119 return PTR_ERR(ringacc->proxy_gcfg); 1120 1121 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 1122 "proxy_target"); 1123 ringacc->proxy_target_base = devm_ioremap_resource(dev, res); 1124 if (IS_ERR(ringacc->proxy_target_base)) 1125 return PTR_ERR(ringacc->proxy_target_base); 1126 1127 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & 1128 K3_RINGACC_PROXY_CFG_THREADS_MASK; 1129 1130 ringacc->rings = devm_kzalloc(dev, 1131 sizeof(*ringacc->rings) * 1132 ringacc->num_rings, 1133 GFP_KERNEL); 1134 ringacc->rings_inuse = devm_kcalloc(dev, 1135 BITS_TO_LONGS(ringacc->num_rings), 1136 sizeof(unsigned long), GFP_KERNEL); 1137 ringacc->proxy_inuse = devm_kcalloc(dev, 1138 BITS_TO_LONGS(ringacc->num_proxies), 1139 sizeof(unsigned long), GFP_KERNEL); 1140 1141 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) 1142 return -ENOMEM; 1143 1144 for (i = 0; i < ringacc->num_rings; i++) { 1145 ringacc->rings[i].rt = base_rt + 1146 K3_RINGACC_RT_REGS_STEP * i; 1147 ringacc->rings[i].fifos = base_fifo + 1148 K3_RINGACC_FIFO_REGS_STEP * i; 1149 ringacc->rings[i].parent = ringacc; 1150 ringacc->rings[i].ring_id = i; 1151 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; 1152 } 1153 1154 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1155 1156 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", 1157 ringacc->num_rings, 1158 ringacc->rm_gp_range->desc[0].start, 1159 ringacc->rm_gp_range->desc[0].num, 1160 ringacc->tisci_dev_id); 1161 dev_info(dev, "dma-ring-reset-quirk: %s\n", 1162 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled"); 1163 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", 1164 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); 1165 1166 return 0; 1167 } 1168 1169 struct ringacc_match_data { 1170 struct k3_ringacc_ops ops; 1171 }; 1172 1173 static struct ringacc_match_data k3_ringacc_data = { 1174 .ops = { 1175 .init = k3_ringacc_init, 1176 }, 1177 }; 1178 1179 /* Match table for of_platform binding */ 1180 static const struct of_device_id k3_ringacc_of_match[] = { 1181 { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, }, 1182 {}, 1183 }; 1184 1185 static int k3_ringacc_probe(struct platform_device *pdev) 1186 { 1187 const struct ringacc_match_data *match_data; 1188 const struct of_device_id *match; 1189 struct device *dev = &pdev->dev; 1190 struct k3_ringacc *ringacc; 1191 int ret; 1192 1193 match = of_match_node(k3_ringacc_of_match, dev->of_node); 1194 if (!match) 1195 return -ENODEV; 1196 match_data = match->data; 1197 1198 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1199 if (!ringacc) 1200 return -ENOMEM; 1201 1202 ringacc->dev = dev; 1203 mutex_init(&ringacc->req_lock); 1204 ringacc->ops = &match_data->ops; 1205 1206 ret = ringacc->ops->init(pdev, ringacc); 1207 if (ret) 1208 return ret; 1209 1210 dev_set_drvdata(dev, ringacc); 1211 1212 mutex_lock(&k3_ringacc_list_lock); 1213 list_add_tail(&ringacc->list, &k3_ringacc_list); 1214 mutex_unlock(&k3_ringacc_list_lock); 1215 1216 return 0; 1217 } 1218 1219 static struct platform_driver k3_ringacc_driver = { 1220 .probe = k3_ringacc_probe, 1221 .driver = { 1222 .name = "k3-ringacc", 1223 .of_match_table = k3_ringacc_of_match, 1224 .suppress_bind_attrs = true, 1225 }, 1226 }; 1227 builtin_platform_driver(k3_ringacc_driver); 1228