1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * TI K3 NAVSS Ring Accelerator subsystem driver 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 */ 7 8 #include <linux/dma-mapping.h> 9 #include <linux/io.h> 10 #include <linux/init.h> 11 #include <linux/of.h> 12 #include <linux/platform_device.h> 13 #include <linux/soc/ti/k3-ringacc.h> 14 #include <linux/soc/ti/ti_sci_protocol.h> 15 #include <linux/soc/ti/ti_sci_inta_msi.h> 16 #include <linux/of_irq.h> 17 #include <linux/irqdomain.h> 18 19 static LIST_HEAD(k3_ringacc_list); 20 static DEFINE_MUTEX(k3_ringacc_list_lock); 21 22 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) 23 24 /** 25 * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region 26 * 27 * @resv_16: Reserved 28 * @db: Ring Doorbell Register 29 * @resv_4: Reserved 30 * @occ: Ring Occupancy Register 31 * @indx: Ring Current Index Register 32 * @hwocc: Ring Hardware Occupancy Register 33 * @hwindx: Ring Hardware Current Index Register 34 */ 35 struct k3_ring_rt_regs { 36 u32 resv_16[4]; 37 u32 db; 38 u32 resv_4[1]; 39 u32 occ; 40 u32 indx; 41 u32 hwocc; 42 u32 hwindx; 43 }; 44 45 #define K3_RINGACC_RT_REGS_STEP 0x1000 46 47 /** 48 * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region 49 * 50 * @head_data: Ring Head Entry Data Registers 51 * @tail_data: Ring Tail Entry Data Registers 52 * @peek_head_data: Ring Peek Head Entry Data Regs 53 * @peek_tail_data: Ring Peek Tail Entry Data Regs 54 */ 55 struct k3_ring_fifo_regs { 56 u32 head_data[128]; 57 u32 tail_data[128]; 58 u32 peek_head_data[128]; 59 u32 peek_tail_data[128]; 60 }; 61 62 /** 63 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region 64 * 65 * @revision: Revision Register 66 * @config: Config Register 67 */ 68 struct k3_ringacc_proxy_gcfg_regs { 69 u32 revision; 70 u32 config; 71 }; 72 73 #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) 74 75 /** 76 * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region 77 * 78 * @control: Proxy Control Register 79 * @status: Proxy Status Register 80 * @resv_512: Reserved 81 * @data: Proxy Data Register 82 */ 83 struct k3_ringacc_proxy_target_regs { 84 u32 control; 85 u32 status; 86 u8 resv_512[504]; 87 u32 data[128]; 88 }; 89 90 #define K3_RINGACC_PROXY_TARGET_STEP 0x1000 91 #define K3_RINGACC_PROXY_NOT_USED (-1) 92 93 enum k3_ringacc_proxy_access_mode { 94 PROXY_ACCESS_MODE_HEAD = 0, 95 PROXY_ACCESS_MODE_TAIL = 1, 96 PROXY_ACCESS_MODE_PEEK_HEAD = 2, 97 PROXY_ACCESS_MODE_PEEK_TAIL = 3, 98 }; 99 100 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) 101 #define K3_RINGACC_FIFO_REGS_STEP 0x1000 102 #define K3_RINGACC_MAX_DB_RING_CNT (127U) 103 104 struct k3_ring_ops { 105 int (*push_tail)(struct k3_ring *ring, void *elm); 106 int (*push_head)(struct k3_ring *ring, void *elm); 107 int (*pop_tail)(struct k3_ring *ring, void *elm); 108 int (*pop_head)(struct k3_ring *ring, void *elm); 109 }; 110 111 /** 112 * struct k3_ring - RA Ring descriptor 113 * 114 * @rt: Ring control/status registers 115 * @fifos: Ring queues registers 116 * @proxy: Ring Proxy Datapath registers 117 * @ring_mem_dma: Ring buffer dma address 118 * @ring_mem_virt: Ring buffer virt address 119 * @ops: Ring operations 120 * @size: Ring size in elements 121 * @elm_size: Size of the ring element 122 * @mode: Ring mode 123 * @flags: flags 124 * @free: Number of free elements 125 * @occ: Ring occupancy 126 * @windex: Write index (only for @K3_RINGACC_RING_MODE_RING) 127 * @rindex: Read index (only for @K3_RINGACC_RING_MODE_RING) 128 * @ring_id: Ring Id 129 * @parent: Pointer on struct @k3_ringacc 130 * @use_count: Use count for shared rings 131 * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) 132 */ 133 struct k3_ring { 134 struct k3_ring_rt_regs __iomem *rt; 135 struct k3_ring_fifo_regs __iomem *fifos; 136 struct k3_ringacc_proxy_target_regs __iomem *proxy; 137 dma_addr_t ring_mem_dma; 138 void *ring_mem_virt; 139 struct k3_ring_ops *ops; 140 u32 size; 141 enum k3_ring_size elm_size; 142 enum k3_ring_mode mode; 143 u32 flags; 144 #define K3_RING_FLAG_BUSY BIT(1) 145 #define K3_RING_FLAG_SHARED BIT(2) 146 u32 free; 147 u32 occ; 148 u32 windex; 149 u32 rindex; 150 u32 ring_id; 151 struct k3_ringacc *parent; 152 u32 use_count; 153 int proxy_id; 154 }; 155 156 /** 157 * struct k3_ringacc - Rings accelerator descriptor 158 * 159 * @dev: pointer on RA device 160 * @proxy_gcfg: RA proxy global config registers 161 * @proxy_target_base: RA proxy datapath region 162 * @num_rings: number of ring in RA 163 * @rings_inuse: bitfield for ring usage tracking 164 * @rm_gp_range: general purpose rings range from tisci 165 * @dma_ring_reset_quirk: DMA reset w/a enable 166 * @num_proxies: number of RA proxies 167 * @proxy_inuse: bitfield for proxy usage tracking 168 * @rings: array of rings descriptors (struct @k3_ring) 169 * @list: list of RAs in the system 170 * @req_lock: protect rings allocation 171 * @tisci: pointer ti-sci handle 172 * @tisci_ring_ops: ti-sci rings ops 173 * @tisci_dev_id: ti-sci device id 174 */ 175 struct k3_ringacc { 176 struct device *dev; 177 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; 178 void __iomem *proxy_target_base; 179 u32 num_rings; /* number of rings in Ringacc module */ 180 unsigned long *rings_inuse; 181 struct ti_sci_resource *rm_gp_range; 182 183 bool dma_ring_reset_quirk; 184 u32 num_proxies; 185 unsigned long *proxy_inuse; 186 187 struct k3_ring *rings; 188 struct list_head list; 189 struct mutex req_lock; /* protect rings allocation */ 190 191 const struct ti_sci_handle *tisci; 192 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; 193 u32 tisci_dev_id; 194 }; 195 196 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) 197 { 198 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - 199 (4 << ring->elm_size); 200 } 201 202 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) 203 { 204 return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); 205 } 206 207 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); 208 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); 209 210 static struct k3_ring_ops k3_ring_mode_ring_ops = { 211 .push_tail = k3_ringacc_ring_push_mem, 212 .pop_head = k3_ringacc_ring_pop_mem, 213 }; 214 215 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); 216 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); 217 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); 218 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); 219 220 static struct k3_ring_ops k3_ring_mode_msg_ops = { 221 .push_tail = k3_ringacc_ring_push_io, 222 .push_head = k3_ringacc_ring_push_head_io, 223 .pop_tail = k3_ringacc_ring_pop_tail_io, 224 .pop_head = k3_ringacc_ring_pop_io, 225 }; 226 227 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); 228 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); 229 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); 230 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); 231 232 static struct k3_ring_ops k3_ring_mode_proxy_ops = { 233 .push_tail = k3_ringacc_ring_push_tail_proxy, 234 .push_head = k3_ringacc_ring_push_head_proxy, 235 .pop_tail = k3_ringacc_ring_pop_tail_proxy, 236 .pop_head = k3_ringacc_ring_pop_head_proxy, 237 }; 238 239 static void k3_ringacc_ring_dump(struct k3_ring *ring) 240 { 241 struct device *dev = ring->parent->dev; 242 243 dev_dbg(dev, "dump ring: %d\n", ring->ring_id); 244 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, 245 &ring->ring_mem_dma); 246 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", 247 ring->elm_size, ring->size, ring->mode, ring->proxy_id); 248 249 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); 250 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); 251 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); 252 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); 253 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); 254 255 if (ring->ring_mem_virt) 256 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, 257 16, 1, ring->ring_mem_virt, 16 * 8, false); 258 } 259 260 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, 261 int id, u32 flags) 262 { 263 int proxy_id = K3_RINGACC_PROXY_NOT_USED; 264 265 mutex_lock(&ringacc->req_lock); 266 267 if (id == K3_RINGACC_RING_ID_ANY) { 268 /* Request for any general purpose ring */ 269 struct ti_sci_resource_desc *gp_rings = 270 &ringacc->rm_gp_range->desc[0]; 271 unsigned long size; 272 273 size = gp_rings->start + gp_rings->num; 274 id = find_next_zero_bit(ringacc->rings_inuse, size, 275 gp_rings->start); 276 if (id == size) 277 goto error; 278 } else if (id < 0) { 279 goto error; 280 } 281 282 if (test_bit(id, ringacc->rings_inuse) && 283 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) 284 goto error; 285 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) 286 goto out; 287 288 if (flags & K3_RINGACC_RING_USE_PROXY) { 289 proxy_id = find_next_zero_bit(ringacc->proxy_inuse, 290 ringacc->num_proxies, 0); 291 if (proxy_id == ringacc->num_proxies) 292 goto error; 293 } 294 295 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { 296 set_bit(proxy_id, ringacc->proxy_inuse); 297 ringacc->rings[id].proxy_id = proxy_id; 298 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, 299 proxy_id); 300 } else { 301 dev_dbg(ringacc->dev, "Giving ring#%d\n", id); 302 } 303 304 set_bit(id, ringacc->rings_inuse); 305 out: 306 ringacc->rings[id].use_count++; 307 mutex_unlock(&ringacc->req_lock); 308 return &ringacc->rings[id]; 309 310 error: 311 mutex_unlock(&ringacc->req_lock); 312 return NULL; 313 } 314 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); 315 316 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) 317 { 318 struct k3_ringacc *ringacc = ring->parent; 319 int ret; 320 321 ret = ringacc->tisci_ring_ops->config( 322 ringacc->tisci, 323 TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID, 324 ringacc->tisci_dev_id, 325 ring->ring_id, 326 0, 327 0, 328 ring->size, 329 0, 330 0, 331 0); 332 if (ret) 333 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", 334 ret, ring->ring_id); 335 } 336 337 void k3_ringacc_ring_reset(struct k3_ring *ring) 338 { 339 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 340 return; 341 342 ring->occ = 0; 343 ring->free = 0; 344 ring->rindex = 0; 345 ring->windex = 0; 346 347 k3_ringacc_ring_reset_sci(ring); 348 } 349 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); 350 351 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, 352 enum k3_ring_mode mode) 353 { 354 struct k3_ringacc *ringacc = ring->parent; 355 int ret; 356 357 ret = ringacc->tisci_ring_ops->config( 358 ringacc->tisci, 359 TI_SCI_MSG_VALUE_RM_RING_MODE_VALID, 360 ringacc->tisci_dev_id, 361 ring->ring_id, 362 0, 363 0, 364 0, 365 mode, 366 0, 367 0); 368 if (ret) 369 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", 370 ret, ring->ring_id); 371 } 372 373 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) 374 { 375 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 376 return; 377 378 if (!ring->parent->dma_ring_reset_quirk) 379 goto reset; 380 381 if (!occ) 382 occ = readl(&ring->rt->occ); 383 384 if (occ) { 385 u32 db_ring_cnt, db_ring_cnt_cur; 386 387 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, 388 ring->ring_id, occ); 389 /* TI-SCI ring reset */ 390 k3_ringacc_ring_reset_sci(ring); 391 392 /* 393 * Setup the ring in ring/doorbell mode (if not already in this 394 * mode) 395 */ 396 if (ring->mode != K3_RINGACC_RING_MODE_RING) 397 k3_ringacc_ring_reconfig_qmode_sci( 398 ring, K3_RINGACC_RING_MODE_RING); 399 /* 400 * Ring the doorbell 2**22 – ringOcc times. 401 * This will wrap the internal UDMAP ring state occupancy 402 * counter (which is 21-bits wide) to 0. 403 */ 404 db_ring_cnt = (1U << 22) - occ; 405 406 while (db_ring_cnt != 0) { 407 /* 408 * Ring the doorbell with the maximum count each 409 * iteration if possible to minimize the total 410 * of writes 411 */ 412 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) 413 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; 414 else 415 db_ring_cnt_cur = db_ring_cnt; 416 417 writel(db_ring_cnt_cur, &ring->rt->db); 418 db_ring_cnt -= db_ring_cnt_cur; 419 } 420 421 /* Restore the original ring mode (if not ring mode) */ 422 if (ring->mode != K3_RINGACC_RING_MODE_RING) 423 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); 424 } 425 426 reset: 427 /* Reset the ring */ 428 k3_ringacc_ring_reset(ring); 429 } 430 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); 431 432 static void k3_ringacc_ring_free_sci(struct k3_ring *ring) 433 { 434 struct k3_ringacc *ringacc = ring->parent; 435 int ret; 436 437 ret = ringacc->tisci_ring_ops->config( 438 ringacc->tisci, 439 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER, 440 ringacc->tisci_dev_id, 441 ring->ring_id, 442 0, 443 0, 444 0, 445 0, 446 0, 447 0); 448 if (ret) 449 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", 450 ret, ring->ring_id); 451 } 452 453 int k3_ringacc_ring_free(struct k3_ring *ring) 454 { 455 struct k3_ringacc *ringacc; 456 457 if (!ring) 458 return -EINVAL; 459 460 ringacc = ring->parent; 461 462 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); 463 464 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 465 return -EINVAL; 466 467 mutex_lock(&ringacc->req_lock); 468 469 if (--ring->use_count) 470 goto out; 471 472 if (!(ring->flags & K3_RING_FLAG_BUSY)) 473 goto no_init; 474 475 k3_ringacc_ring_free_sci(ring); 476 477 dma_free_coherent(ringacc->dev, 478 ring->size * (4 << ring->elm_size), 479 ring->ring_mem_virt, ring->ring_mem_dma); 480 ring->flags = 0; 481 ring->ops = NULL; 482 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { 483 clear_bit(ring->proxy_id, ringacc->proxy_inuse); 484 ring->proxy = NULL; 485 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 486 } 487 488 no_init: 489 clear_bit(ring->ring_id, ringacc->rings_inuse); 490 491 out: 492 mutex_unlock(&ringacc->req_lock); 493 return 0; 494 } 495 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); 496 497 u32 k3_ringacc_get_ring_id(struct k3_ring *ring) 498 { 499 if (!ring) 500 return -EINVAL; 501 502 return ring->ring_id; 503 } 504 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); 505 506 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) 507 { 508 if (!ring) 509 return -EINVAL; 510 511 return ring->parent->tisci_dev_id; 512 } 513 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); 514 515 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) 516 { 517 int irq_num; 518 519 if (!ring) 520 return -EINVAL; 521 522 irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); 523 if (irq_num <= 0) 524 irq_num = -EINVAL; 525 return irq_num; 526 } 527 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); 528 529 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) 530 { 531 struct k3_ringacc *ringacc = ring->parent; 532 u32 ring_idx; 533 int ret; 534 535 if (!ringacc->tisci) 536 return -EINVAL; 537 538 ring_idx = ring->ring_id; 539 ret = ringacc->tisci_ring_ops->config( 540 ringacc->tisci, 541 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER, 542 ringacc->tisci_dev_id, 543 ring_idx, 544 lower_32_bits(ring->ring_mem_dma), 545 upper_32_bits(ring->ring_mem_dma), 546 ring->size, 547 ring->mode, 548 ring->elm_size, 549 0); 550 if (ret) 551 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", 552 ret, ring_idx); 553 554 return ret; 555 } 556 557 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 558 { 559 struct k3_ringacc *ringacc = ring->parent; 560 int ret = 0; 561 562 if (!ring || !cfg) 563 return -EINVAL; 564 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || 565 cfg->mode >= K3_RINGACC_RING_MODE_INVALID || 566 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || 567 !test_bit(ring->ring_id, ringacc->rings_inuse)) 568 return -EINVAL; 569 570 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && 571 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && 572 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { 573 dev_err(ringacc->dev, 574 "Message mode must use proxy for %u element size\n", 575 4 << ring->elm_size); 576 return -EINVAL; 577 } 578 579 /* 580 * In case of shared ring only the first user (master user) can 581 * configure the ring. The sequence should be by the client: 582 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user 583 * k3_ringacc_ring_cfg(ring, cfg); # master configuration 584 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 585 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 586 */ 587 if (ring->use_count != 1) 588 return 0; 589 590 ring->size = cfg->size; 591 ring->elm_size = cfg->elm_size; 592 ring->mode = cfg->mode; 593 ring->occ = 0; 594 ring->free = 0; 595 ring->rindex = 0; 596 ring->windex = 0; 597 598 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) 599 ring->proxy = ringacc->proxy_target_base + 600 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; 601 602 switch (ring->mode) { 603 case K3_RINGACC_RING_MODE_RING: 604 ring->ops = &k3_ring_mode_ring_ops; 605 break; 606 case K3_RINGACC_RING_MODE_MESSAGE: 607 if (ring->proxy) 608 ring->ops = &k3_ring_mode_proxy_ops; 609 else 610 ring->ops = &k3_ring_mode_msg_ops; 611 break; 612 default: 613 ring->ops = NULL; 614 ret = -EINVAL; 615 goto err_free_proxy; 616 }; 617 618 ring->ring_mem_virt = dma_alloc_coherent(ringacc->dev, 619 ring->size * (4 << ring->elm_size), 620 &ring->ring_mem_dma, GFP_KERNEL); 621 if (!ring->ring_mem_virt) { 622 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 623 ret = -ENOMEM; 624 goto err_free_ops; 625 } 626 627 ret = k3_ringacc_ring_cfg_sci(ring); 628 629 if (ret) 630 goto err_free_mem; 631 632 ring->flags |= K3_RING_FLAG_BUSY; 633 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? 634 K3_RING_FLAG_SHARED : 0; 635 636 k3_ringacc_ring_dump(ring); 637 638 return 0; 639 640 err_free_mem: 641 dma_free_coherent(ringacc->dev, 642 ring->size * (4 << ring->elm_size), 643 ring->ring_mem_virt, 644 ring->ring_mem_dma); 645 err_free_ops: 646 ring->ops = NULL; 647 err_free_proxy: 648 ring->proxy = NULL; 649 return ret; 650 } 651 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); 652 653 u32 k3_ringacc_ring_get_size(struct k3_ring *ring) 654 { 655 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 656 return -EINVAL; 657 658 return ring->size; 659 } 660 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); 661 662 u32 k3_ringacc_ring_get_free(struct k3_ring *ring) 663 { 664 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 665 return -EINVAL; 666 667 if (!ring->free) 668 ring->free = ring->size - readl(&ring->rt->occ); 669 670 return ring->free; 671 } 672 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); 673 674 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) 675 { 676 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 677 return -EINVAL; 678 679 return readl(&ring->rt->occ); 680 } 681 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); 682 683 u32 k3_ringacc_ring_is_full(struct k3_ring *ring) 684 { 685 return !k3_ringacc_ring_get_free(ring); 686 } 687 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); 688 689 enum k3_ringacc_access_mode { 690 K3_RINGACC_ACCESS_MODE_PUSH_HEAD, 691 K3_RINGACC_ACCESS_MODE_POP_HEAD, 692 K3_RINGACC_ACCESS_MODE_PUSH_TAIL, 693 K3_RINGACC_ACCESS_MODE_POP_TAIL, 694 K3_RINGACC_ACCESS_MODE_PEEK_HEAD, 695 K3_RINGACC_ACCESS_MODE_PEEK_TAIL, 696 }; 697 698 #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) 699 #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) 700 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, 701 enum k3_ringacc_proxy_access_mode mode) 702 { 703 u32 val; 704 705 val = ring->ring_id; 706 val |= K3_RINGACC_PROXY_MODE(mode); 707 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); 708 writel(val, &ring->proxy->control); 709 return 0; 710 } 711 712 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, 713 enum k3_ringacc_access_mode access_mode) 714 { 715 void __iomem *ptr; 716 717 ptr = (void __iomem *)&ring->proxy->data; 718 719 switch (access_mode) { 720 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 721 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 722 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); 723 break; 724 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 725 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 726 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); 727 break; 728 default: 729 return -EINVAL; 730 } 731 732 ptr += k3_ringacc_ring_get_fifo_pos(ring); 733 734 switch (access_mode) { 735 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 736 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 737 dev_dbg(ring->parent->dev, 738 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 739 access_mode); 740 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 741 ring->occ--; 742 break; 743 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 744 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 745 dev_dbg(ring->parent->dev, 746 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 747 access_mode); 748 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 749 ring->free--; 750 break; 751 default: 752 return -EINVAL; 753 } 754 755 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->free, 756 ring->occ); 757 return 0; 758 } 759 760 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) 761 { 762 return k3_ringacc_ring_access_proxy(ring, elem, 763 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 764 } 765 766 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) 767 { 768 return k3_ringacc_ring_access_proxy(ring, elem, 769 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 770 } 771 772 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) 773 { 774 return k3_ringacc_ring_access_proxy(ring, elem, 775 K3_RINGACC_ACCESS_MODE_POP_HEAD); 776 } 777 778 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) 779 { 780 return k3_ringacc_ring_access_proxy(ring, elem, 781 K3_RINGACC_ACCESS_MODE_POP_HEAD); 782 } 783 784 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, 785 enum k3_ringacc_access_mode access_mode) 786 { 787 void __iomem *ptr; 788 789 switch (access_mode) { 790 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 791 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 792 ptr = (void __iomem *)&ring->fifos->head_data; 793 break; 794 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 795 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 796 ptr = (void __iomem *)&ring->fifos->tail_data; 797 break; 798 default: 799 return -EINVAL; 800 } 801 802 ptr += k3_ringacc_ring_get_fifo_pos(ring); 803 804 switch (access_mode) { 805 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 806 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 807 dev_dbg(ring->parent->dev, 808 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 809 access_mode); 810 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 811 ring->occ--; 812 break; 813 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 814 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 815 dev_dbg(ring->parent->dev, 816 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 817 access_mode); 818 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 819 ring->free--; 820 break; 821 default: 822 return -EINVAL; 823 } 824 825 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", ring->free, 826 ring->windex, ring->occ, ring->rindex); 827 return 0; 828 } 829 830 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) 831 { 832 return k3_ringacc_ring_access_io(ring, elem, 833 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 834 } 835 836 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) 837 { 838 return k3_ringacc_ring_access_io(ring, elem, 839 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 840 } 841 842 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) 843 { 844 return k3_ringacc_ring_access_io(ring, elem, 845 K3_RINGACC_ACCESS_MODE_POP_HEAD); 846 } 847 848 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) 849 { 850 return k3_ringacc_ring_access_io(ring, elem, 851 K3_RINGACC_ACCESS_MODE_POP_HEAD); 852 } 853 854 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) 855 { 856 void *elem_ptr; 857 858 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->windex); 859 860 memcpy(elem_ptr, elem, (4 << ring->elm_size)); 861 862 ring->windex = (ring->windex + 1) % ring->size; 863 ring->free--; 864 writel(1, &ring->rt->db); 865 866 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", 867 ring->free, ring->windex); 868 869 return 0; 870 } 871 872 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) 873 { 874 void *elem_ptr; 875 876 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->rindex); 877 878 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 879 880 ring->rindex = (ring->rindex + 1) % ring->size; 881 ring->occ--; 882 writel(-1, &ring->rt->db); 883 884 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", 885 ring->occ, ring->rindex, elem_ptr); 886 return 0; 887 } 888 889 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) 890 { 891 int ret = -EOPNOTSUPP; 892 893 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 894 return -EINVAL; 895 896 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", ring->free, 897 ring->windex); 898 899 if (k3_ringacc_ring_is_full(ring)) 900 return -ENOMEM; 901 902 if (ring->ops && ring->ops->push_tail) 903 ret = ring->ops->push_tail(ring, elem); 904 905 return ret; 906 } 907 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); 908 909 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) 910 { 911 int ret = -EOPNOTSUPP; 912 913 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 914 return -EINVAL; 915 916 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", 917 ring->free, ring->windex); 918 919 if (k3_ringacc_ring_is_full(ring)) 920 return -ENOMEM; 921 922 if (ring->ops && ring->ops->push_head) 923 ret = ring->ops->push_head(ring, elem); 924 925 return ret; 926 } 927 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); 928 929 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) 930 { 931 int ret = -EOPNOTSUPP; 932 933 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 934 return -EINVAL; 935 936 if (!ring->occ) 937 ring->occ = k3_ringacc_ring_get_occ(ring); 938 939 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->occ, 940 ring->rindex); 941 942 if (!ring->occ) 943 return -ENODATA; 944 945 if (ring->ops && ring->ops->pop_head) 946 ret = ring->ops->pop_head(ring, elem); 947 948 return ret; 949 } 950 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); 951 952 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) 953 { 954 int ret = -EOPNOTSUPP; 955 956 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 957 return -EINVAL; 958 959 if (!ring->occ) 960 ring->occ = k3_ringacc_ring_get_occ(ring); 961 962 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", ring->occ, 963 ring->rindex); 964 965 if (!ring->occ) 966 return -ENODATA; 967 968 if (ring->ops && ring->ops->pop_tail) 969 ret = ring->ops->pop_tail(ring, elem); 970 971 return ret; 972 } 973 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); 974 975 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, 976 const char *property) 977 { 978 struct device_node *ringacc_np; 979 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); 980 struct k3_ringacc *entry; 981 982 ringacc_np = of_parse_phandle(np, property, 0); 983 if (!ringacc_np) 984 return ERR_PTR(-ENODEV); 985 986 mutex_lock(&k3_ringacc_list_lock); 987 list_for_each_entry(entry, &k3_ringacc_list, list) 988 if (entry->dev->of_node == ringacc_np) { 989 ringacc = entry; 990 break; 991 } 992 mutex_unlock(&k3_ringacc_list_lock); 993 of_node_put(ringacc_np); 994 995 return ringacc; 996 } 997 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); 998 999 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) 1000 { 1001 struct device_node *node = ringacc->dev->of_node; 1002 struct device *dev = ringacc->dev; 1003 struct platform_device *pdev = to_platform_device(dev); 1004 int ret; 1005 1006 if (!node) { 1007 dev_err(dev, "device tree info unavailable\n"); 1008 return -ENODEV; 1009 } 1010 1011 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); 1012 if (ret) { 1013 dev_err(dev, "ti,num-rings read failure %d\n", ret); 1014 return ret; 1015 } 1016 1017 ringacc->dma_ring_reset_quirk = 1018 of_property_read_bool(node, "ti,dma-ring-reset-quirk"); 1019 1020 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); 1021 if (IS_ERR(ringacc->tisci)) { 1022 ret = PTR_ERR(ringacc->tisci); 1023 if (ret != -EPROBE_DEFER) 1024 dev_err(dev, "ti,sci read fail %d\n", ret); 1025 ringacc->tisci = NULL; 1026 return ret; 1027 } 1028 1029 ret = of_property_read_u32(node, "ti,sci-dev-id", 1030 &ringacc->tisci_dev_id); 1031 if (ret) { 1032 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); 1033 return ret; 1034 } 1035 1036 pdev->id = ringacc->tisci_dev_id; 1037 1038 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, 1039 ringacc->tisci_dev_id, 1040 "ti,sci-rm-range-gp-rings"); 1041 if (IS_ERR(ringacc->rm_gp_range)) { 1042 dev_err(dev, "Failed to allocate MSI interrupts\n"); 1043 return PTR_ERR(ringacc->rm_gp_range); 1044 } 1045 1046 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, 1047 ringacc->rm_gp_range); 1048 } 1049 1050 static int k3_ringacc_probe(struct platform_device *pdev) 1051 { 1052 struct k3_ringacc *ringacc; 1053 void __iomem *base_fifo, *base_rt; 1054 struct device *dev = &pdev->dev; 1055 struct resource *res; 1056 int ret, i; 1057 1058 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1059 if (!ringacc) 1060 return -ENOMEM; 1061 1062 ringacc->dev = dev; 1063 mutex_init(&ringacc->req_lock); 1064 1065 dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 1066 DOMAIN_BUS_TI_SCI_INTA_MSI); 1067 if (!dev->msi_domain) { 1068 dev_err(dev, "Failed to get MSI domain\n"); 1069 return -EPROBE_DEFER; 1070 } 1071 1072 ret = k3_ringacc_probe_dt(ringacc); 1073 if (ret) 1074 return ret; 1075 1076 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); 1077 base_rt = devm_ioremap_resource(dev, res); 1078 if (IS_ERR(base_rt)) 1079 return PTR_ERR(base_rt); 1080 1081 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos"); 1082 base_fifo = devm_ioremap_resource(dev, res); 1083 if (IS_ERR(base_fifo)) 1084 return PTR_ERR(base_fifo); 1085 1086 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg"); 1087 ringacc->proxy_gcfg = devm_ioremap_resource(dev, res); 1088 if (IS_ERR(ringacc->proxy_gcfg)) 1089 return PTR_ERR(ringacc->proxy_gcfg); 1090 1091 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 1092 "proxy_target"); 1093 ringacc->proxy_target_base = devm_ioremap_resource(dev, res); 1094 if (IS_ERR(ringacc->proxy_target_base)) 1095 return PTR_ERR(ringacc->proxy_target_base); 1096 1097 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & 1098 K3_RINGACC_PROXY_CFG_THREADS_MASK; 1099 1100 ringacc->rings = devm_kzalloc(dev, 1101 sizeof(*ringacc->rings) * 1102 ringacc->num_rings, 1103 GFP_KERNEL); 1104 ringacc->rings_inuse = devm_kcalloc(dev, 1105 BITS_TO_LONGS(ringacc->num_rings), 1106 sizeof(unsigned long), GFP_KERNEL); 1107 ringacc->proxy_inuse = devm_kcalloc(dev, 1108 BITS_TO_LONGS(ringacc->num_proxies), 1109 sizeof(unsigned long), GFP_KERNEL); 1110 1111 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) 1112 return -ENOMEM; 1113 1114 for (i = 0; i < ringacc->num_rings; i++) { 1115 ringacc->rings[i].rt = base_rt + 1116 K3_RINGACC_RT_REGS_STEP * i; 1117 ringacc->rings[i].fifos = base_fifo + 1118 K3_RINGACC_FIFO_REGS_STEP * i; 1119 ringacc->rings[i].parent = ringacc; 1120 ringacc->rings[i].ring_id = i; 1121 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; 1122 } 1123 dev_set_drvdata(dev, ringacc); 1124 1125 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1126 1127 mutex_lock(&k3_ringacc_list_lock); 1128 list_add_tail(&ringacc->list, &k3_ringacc_list); 1129 mutex_unlock(&k3_ringacc_list_lock); 1130 1131 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", 1132 ringacc->num_rings, 1133 ringacc->rm_gp_range->desc[0].start, 1134 ringacc->rm_gp_range->desc[0].num, 1135 ringacc->tisci_dev_id); 1136 dev_info(dev, "dma-ring-reset-quirk: %s\n", 1137 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled"); 1138 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", 1139 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); 1140 return 0; 1141 } 1142 1143 /* Match table for of_platform binding */ 1144 static const struct of_device_id k3_ringacc_of_match[] = { 1145 { .compatible = "ti,am654-navss-ringacc", }, 1146 {}, 1147 }; 1148 1149 static struct platform_driver k3_ringacc_driver = { 1150 .probe = k3_ringacc_probe, 1151 .driver = { 1152 .name = "k3-ringacc", 1153 .of_match_table = k3_ringacc_of_match, 1154 .suppress_bind_attrs = true, 1155 }, 1156 }; 1157 builtin_platform_driver(k3_ringacc_driver); 1158