1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * K3 NAVSS DMA glue interface 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 * 7 */ 8 9 #include <linux/module.h> 10 #include <linux/atomic.h> 11 #include <linux/delay.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/io.h> 14 #include <linux/init.h> 15 #include <linux/of.h> 16 #include <linux/platform_device.h> 17 #include <linux/soc/ti/k3-ringacc.h> 18 #include <linux/dma/ti-cppi5.h> 19 #include <linux/dma/k3-udma-glue.h> 20 21 #include "k3-udma.h" 22 #include "k3-psil-priv.h" 23 24 struct k3_udma_glue_common { 25 struct device *dev; 26 struct device chan_dev; 27 struct udma_dev *udmax; 28 const struct udma_tisci_rm *tisci_rm; 29 struct k3_ringacc *ringacc; 30 u32 src_thread; 31 u32 dst_thread; 32 33 u32 hdesc_size; 34 bool epib; 35 u32 psdata_size; 36 u32 swdata_size; 37 u32 atype_asel; 38 struct psil_endpoint_config *ep_config; 39 }; 40 41 struct k3_udma_glue_tx_channel { 42 struct k3_udma_glue_common common; 43 44 struct udma_tchan *udma_tchanx; 45 int udma_tchan_id; 46 47 struct k3_ring *ringtx; 48 struct k3_ring *ringtxcq; 49 50 bool psil_paired; 51 52 int virq; 53 54 atomic_t free_pkts; 55 bool tx_pause_on_err; 56 bool tx_filt_einfo; 57 bool tx_filt_pswords; 58 bool tx_supr_tdpkt; 59 60 int udma_tflow_id; 61 }; 62 63 struct k3_udma_glue_rx_flow { 64 struct udma_rflow *udma_rflow; 65 int udma_rflow_id; 66 struct k3_ring *ringrx; 67 struct k3_ring *ringrxfdq; 68 69 int virq; 70 }; 71 72 struct k3_udma_glue_rx_channel { 73 struct k3_udma_glue_common common; 74 75 struct udma_rchan *udma_rchanx; 76 int udma_rchan_id; 77 bool remote; 78 79 bool psil_paired; 80 81 u32 swdata_size; 82 int flow_id_base; 83 84 struct k3_udma_glue_rx_flow *flows; 85 u32 flow_num; 86 u32 flows_ready; 87 }; 88 89 static void k3_udma_chan_dev_release(struct device *dev) 90 { 91 /* The struct containing the device is devm managed */ 92 } 93 94 static struct class k3_udma_glue_devclass = { 95 .name = "k3_udma_glue_chan", 96 .dev_release = k3_udma_chan_dev_release, 97 }; 98 99 #define K3_UDMAX_TDOWN_TIMEOUT_US 1000 100 101 static int of_k3_udma_glue_parse(struct device_node *udmax_np, 102 struct k3_udma_glue_common *common) 103 { 104 common->udmax = of_xudma_dev_get(udmax_np, NULL); 105 if (IS_ERR(common->udmax)) 106 return PTR_ERR(common->udmax); 107 108 common->ringacc = xudma_get_ringacc(common->udmax); 109 common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax); 110 111 return 0; 112 } 113 114 static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, 115 const char *name, struct k3_udma_glue_common *common, 116 bool tx_chn) 117 { 118 struct of_phandle_args dma_spec; 119 u32 thread_id; 120 int ret = 0; 121 int index; 122 123 if (unlikely(!name)) 124 return -EINVAL; 125 126 index = of_property_match_string(chn_np, "dma-names", name); 127 if (index < 0) 128 return index; 129 130 if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index, 131 &dma_spec)) 132 return -ENOENT; 133 134 ret = of_k3_udma_glue_parse(dma_spec.np, common); 135 if (ret) 136 goto out_put_spec; 137 138 thread_id = dma_spec.args[0]; 139 if (dma_spec.args_count == 2) { 140 if (dma_spec.args[1] > 2 && !xudma_is_pktdma(common->udmax)) { 141 dev_err(common->dev, "Invalid channel atype: %u\n", 142 dma_spec.args[1]); 143 ret = -EINVAL; 144 goto out_put_spec; 145 } 146 if (dma_spec.args[1] > 15 && xudma_is_pktdma(common->udmax)) { 147 dev_err(common->dev, "Invalid channel asel: %u\n", 148 dma_spec.args[1]); 149 ret = -EINVAL; 150 goto out_put_spec; 151 } 152 153 common->atype_asel = dma_spec.args[1]; 154 } 155 156 if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 157 ret = -EINVAL; 158 goto out_put_spec; 159 } 160 161 if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 162 ret = -EINVAL; 163 goto out_put_spec; 164 } 165 166 /* get psil endpoint config */ 167 common->ep_config = psil_get_ep_config(thread_id); 168 if (IS_ERR(common->ep_config)) { 169 dev_err(common->dev, 170 "No configuration for psi-l thread 0x%04x\n", 171 thread_id); 172 ret = PTR_ERR(common->ep_config); 173 goto out_put_spec; 174 } 175 176 common->epib = common->ep_config->needs_epib; 177 common->psdata_size = common->ep_config->psd_size; 178 179 if (tx_chn) 180 common->dst_thread = thread_id; 181 else 182 common->src_thread = thread_id; 183 184 out_put_spec: 185 of_node_put(dma_spec.np); 186 return ret; 187 }; 188 189 static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 190 { 191 struct device *dev = tx_chn->common.dev; 192 193 dev_dbg(dev, "dump_tx_chn:\n" 194 "udma_tchan_id: %d\n" 195 "src_thread: %08x\n" 196 "dst_thread: %08x\n", 197 tx_chn->udma_tchan_id, 198 tx_chn->common.src_thread, 199 tx_chn->common.dst_thread); 200 } 201 202 static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn, 203 char *mark) 204 { 205 struct device *dev = chn->common.dev; 206 207 dev_dbg(dev, "=== dump ===> %s\n", mark); 208 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG, 209 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG)); 210 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG, 211 xudma_tchanrt_read(chn->udma_tchanx, 212 UDMA_CHAN_RT_PEER_RT_EN_REG)); 213 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG, 214 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_PCNT_REG)); 215 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG, 216 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_BCNT_REG)); 217 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG, 218 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_SBCNT_REG)); 219 } 220 221 static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 222 { 223 const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm; 224 struct ti_sci_msg_rm_udmap_tx_ch_cfg req; 225 226 memset(&req, 0, sizeof(req)); 227 228 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | 229 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | 230 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | 231 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 232 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | 233 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 234 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 235 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; 236 req.nav_id = tisci_rm->tisci_dev_id; 237 req.index = tx_chn->udma_tchan_id; 238 if (tx_chn->tx_pause_on_err) 239 req.tx_pause_on_err = 1; 240 if (tx_chn->tx_filt_einfo) 241 req.tx_filt_einfo = 1; 242 if (tx_chn->tx_filt_pswords) 243 req.tx_filt_pswords = 1; 244 req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 245 if (tx_chn->tx_supr_tdpkt) 246 req.tx_supr_tdpkt = 1; 247 req.tx_fetch_size = tx_chn->common.hdesc_size >> 2; 248 req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 249 req.tx_atype = tx_chn->common.atype_asel; 250 251 return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); 252 } 253 254 struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, 255 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg) 256 { 257 struct k3_udma_glue_tx_channel *tx_chn; 258 int ret; 259 260 tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); 261 if (!tx_chn) 262 return ERR_PTR(-ENOMEM); 263 264 tx_chn->common.dev = dev; 265 tx_chn->common.swdata_size = cfg->swdata_size; 266 tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; 267 tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; 268 tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; 269 tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; 270 271 /* parse of udmap channel */ 272 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 273 &tx_chn->common, true); 274 if (ret) 275 goto err; 276 277 tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib, 278 tx_chn->common.psdata_size, 279 tx_chn->common.swdata_size); 280 281 if (xudma_is_pktdma(tx_chn->common.udmax)) 282 tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id; 283 else 284 tx_chn->udma_tchan_id = -1; 285 286 /* request and cfg UDMAP TX channel */ 287 tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, 288 tx_chn->udma_tchan_id); 289 if (IS_ERR(tx_chn->udma_tchanx)) { 290 ret = PTR_ERR(tx_chn->udma_tchanx); 291 dev_err(dev, "UDMAX tchanx get err %d\n", ret); 292 goto err; 293 } 294 tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx); 295 296 tx_chn->common.chan_dev.class = &k3_udma_glue_devclass; 297 tx_chn->common.chan_dev.parent = xudma_get_device(tx_chn->common.udmax); 298 dev_set_name(&tx_chn->common.chan_dev, "tchan%d-0x%04x", 299 tx_chn->udma_tchan_id, tx_chn->common.dst_thread); 300 ret = device_register(&tx_chn->common.chan_dev); 301 if (ret) { 302 dev_err(dev, "Channel Device registration failed %d\n", ret); 303 put_device(&tx_chn->common.chan_dev); 304 tx_chn->common.chan_dev.parent = NULL; 305 goto err; 306 } 307 308 if (xudma_is_pktdma(tx_chn->common.udmax)) { 309 /* prepare the channel device as coherent */ 310 tx_chn->common.chan_dev.dma_coherent = true; 311 dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev, 312 DMA_BIT_MASK(48)); 313 } 314 315 atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size); 316 317 if (xudma_is_pktdma(tx_chn->common.udmax)) 318 tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id; 319 else 320 tx_chn->udma_tflow_id = tx_chn->udma_tchan_id; 321 322 /* request and cfg rings */ 323 ret = k3_ringacc_request_rings_pair(tx_chn->common.ringacc, 324 tx_chn->udma_tflow_id, -1, 325 &tx_chn->ringtx, 326 &tx_chn->ringtxcq); 327 if (ret) { 328 dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret); 329 goto err; 330 } 331 332 /* Set the dma_dev for the rings to be configured */ 333 cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn); 334 cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev; 335 336 /* Set the ASEL value for DMA rings of PKTDMA */ 337 if (xudma_is_pktdma(tx_chn->common.udmax)) { 338 cfg->tx_cfg.asel = tx_chn->common.atype_asel; 339 cfg->txcq_cfg.asel = tx_chn->common.atype_asel; 340 } 341 342 ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg); 343 if (ret) { 344 dev_err(dev, "Failed to cfg ringtx %d\n", ret); 345 goto err; 346 } 347 348 ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg); 349 if (ret) { 350 dev_err(dev, "Failed to cfg ringtx %d\n", ret); 351 goto err; 352 } 353 354 /* request and cfg psi-l */ 355 tx_chn->common.src_thread = 356 xudma_dev_get_psil_base(tx_chn->common.udmax) + 357 tx_chn->udma_tchan_id; 358 359 ret = k3_udma_glue_cfg_tx_chn(tx_chn); 360 if (ret) { 361 dev_err(dev, "Failed to cfg tchan %d\n", ret); 362 goto err; 363 } 364 365 k3_udma_glue_dump_tx_chn(tx_chn); 366 367 return tx_chn; 368 369 err: 370 k3_udma_glue_release_tx_chn(tx_chn); 371 return ERR_PTR(ret); 372 } 373 EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn); 374 375 void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 376 { 377 if (tx_chn->psil_paired) { 378 xudma_navss_psil_unpair(tx_chn->common.udmax, 379 tx_chn->common.src_thread, 380 tx_chn->common.dst_thread); 381 tx_chn->psil_paired = false; 382 } 383 384 if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx)) 385 xudma_tchan_put(tx_chn->common.udmax, 386 tx_chn->udma_tchanx); 387 388 if (tx_chn->ringtxcq) 389 k3_ringacc_ring_free(tx_chn->ringtxcq); 390 391 if (tx_chn->ringtx) 392 k3_ringacc_ring_free(tx_chn->ringtx); 393 394 if (tx_chn->common.chan_dev.parent) { 395 device_unregister(&tx_chn->common.chan_dev); 396 tx_chn->common.chan_dev.parent = NULL; 397 } 398 } 399 EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn); 400 401 int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 402 struct cppi5_host_desc_t *desc_tx, 403 dma_addr_t desc_dma) 404 { 405 u32 ringtxcq_id; 406 407 if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0)) 408 return -ENOMEM; 409 410 ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 411 cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id); 412 413 return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma); 414 } 415 EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn); 416 417 int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 418 dma_addr_t *desc_dma) 419 { 420 int ret; 421 422 ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma); 423 if (!ret) 424 atomic_inc(&tx_chn->free_pkts); 425 426 return ret; 427 } 428 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn); 429 430 int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 431 { 432 int ret; 433 434 ret = xudma_navss_psil_pair(tx_chn->common.udmax, 435 tx_chn->common.src_thread, 436 tx_chn->common.dst_thread); 437 if (ret) { 438 dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret); 439 return ret; 440 } 441 442 tx_chn->psil_paired = true; 443 444 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, 445 UDMA_PEER_RT_EN_ENABLE); 446 447 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 448 UDMA_CHAN_RT_CTL_EN); 449 450 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en"); 451 return 0; 452 } 453 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn); 454 455 void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 456 { 457 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1"); 458 459 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 0); 460 461 xudma_tchanrt_write(tx_chn->udma_tchanx, 462 UDMA_CHAN_RT_PEER_RT_EN_REG, 0); 463 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2"); 464 465 if (tx_chn->psil_paired) { 466 xudma_navss_psil_unpair(tx_chn->common.udmax, 467 tx_chn->common.src_thread, 468 tx_chn->common.dst_thread); 469 tx_chn->psil_paired = false; 470 } 471 } 472 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn); 473 474 void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 475 bool sync) 476 { 477 int i = 0; 478 u32 val; 479 480 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1"); 481 482 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 483 UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN); 484 485 val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG); 486 487 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { 488 val = xudma_tchanrt_read(tx_chn->udma_tchanx, 489 UDMA_CHAN_RT_CTL_REG); 490 udelay(1); 491 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { 492 dev_err(tx_chn->common.dev, "TX tdown timeout\n"); 493 break; 494 } 495 i++; 496 } 497 498 val = xudma_tchanrt_read(tx_chn->udma_tchanx, 499 UDMA_CHAN_RT_PEER_RT_EN_REG); 500 if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) 501 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n"); 502 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2"); 503 } 504 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn); 505 506 void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 507 void *data, 508 void (*cleanup)(void *data, dma_addr_t desc_dma)) 509 { 510 struct device *dev = tx_chn->common.dev; 511 dma_addr_t desc_dma; 512 int occ_tx, i, ret; 513 514 /* 515 * TXQ reset need to be special way as it is input for udma and its 516 * state cached by udma, so: 517 * 1) save TXQ occ 518 * 2) clean up TXQ and call callback .cleanup() for each desc 519 * 3) reset TXQ in a special way 520 */ 521 occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx); 522 dev_dbg(dev, "TX reset occ_tx %u\n", occ_tx); 523 524 for (i = 0; i < occ_tx; i++) { 525 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma); 526 if (ret) { 527 if (ret != -ENODATA) 528 dev_err(dev, "TX reset pop %d\n", ret); 529 break; 530 } 531 cleanup(data, desc_dma); 532 } 533 534 /* reset TXCQ as it is not input for udma - expected to be empty */ 535 k3_ringacc_ring_reset(tx_chn->ringtxcq); 536 k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx); 537 } 538 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn); 539 540 u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn) 541 { 542 return tx_chn->common.hdesc_size; 543 } 544 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size); 545 546 u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn) 547 { 548 return k3_ringacc_get_ring_id(tx_chn->ringtxcq); 549 } 550 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id); 551 552 int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn) 553 { 554 if (xudma_is_pktdma(tx_chn->common.udmax)) { 555 tx_chn->virq = xudma_pktdma_tflow_get_irq(tx_chn->common.udmax, 556 tx_chn->udma_tflow_id); 557 } else { 558 tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq); 559 } 560 561 if (!tx_chn->virq) 562 return -ENXIO; 563 564 return tx_chn->virq; 565 } 566 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq); 567 568 struct device * 569 k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn) 570 { 571 if (xudma_is_pktdma(tx_chn->common.udmax) && 572 (tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15)) 573 return &tx_chn->common.chan_dev; 574 575 return xudma_get_device(tx_chn->common.udmax); 576 } 577 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device); 578 579 void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn, 580 dma_addr_t *addr) 581 { 582 if (!xudma_is_pktdma(tx_chn->common.udmax) || 583 !tx_chn->common.atype_asel) 584 return; 585 586 *addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT; 587 } 588 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr); 589 590 void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn, 591 dma_addr_t *addr) 592 { 593 if (!xudma_is_pktdma(tx_chn->common.udmax) || 594 !tx_chn->common.atype_asel) 595 return; 596 597 *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0); 598 } 599 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr); 600 601 static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 602 { 603 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 604 struct ti_sci_msg_rm_udmap_rx_ch_cfg req; 605 int ret; 606 607 memset(&req, 0, sizeof(req)); 608 609 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 610 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 611 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 612 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; 613 614 req.nav_id = tisci_rm->tisci_dev_id; 615 req.index = rx_chn->udma_rchan_id; 616 req.rx_fetch_size = rx_chn->common.hdesc_size >> 2; 617 /* 618 * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw 619 * and udmax impl, so just configure it to invalid value. 620 * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx); 621 */ 622 req.rxcq_qnum = 0xFFFF; 623 if (!xudma_is_pktdma(rx_chn->common.udmax) && rx_chn->flow_num && 624 rx_chn->flow_id_base != rx_chn->udma_rchan_id) { 625 /* Default flow + extra ones */ 626 req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | 627 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID; 628 req.flowid_start = rx_chn->flow_id_base; 629 req.flowid_cnt = rx_chn->flow_num; 630 } 631 req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 632 req.rx_atype = rx_chn->common.atype_asel; 633 634 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req); 635 if (ret) 636 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n", 637 rx_chn->udma_rchan_id, ret); 638 639 return ret; 640 } 641 642 static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, 643 u32 flow_num) 644 { 645 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 646 647 if (IS_ERR_OR_NULL(flow->udma_rflow)) 648 return; 649 650 if (flow->ringrxfdq) 651 k3_ringacc_ring_free(flow->ringrxfdq); 652 653 if (flow->ringrx) 654 k3_ringacc_ring_free(flow->ringrx); 655 656 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); 657 flow->udma_rflow = NULL; 658 rx_chn->flows_ready--; 659 } 660 661 static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, 662 u32 flow_idx, 663 struct k3_udma_glue_rx_flow_cfg *flow_cfg) 664 { 665 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 666 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 667 struct device *dev = rx_chn->common.dev; 668 struct ti_sci_msg_rm_udmap_flow_cfg req; 669 int rx_ring_id; 670 int rx_ringfdq_id; 671 int ret = 0; 672 673 flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax, 674 flow->udma_rflow_id); 675 if (IS_ERR(flow->udma_rflow)) { 676 ret = PTR_ERR(flow->udma_rflow); 677 dev_err(dev, "UDMAX rflow get err %d\n", ret); 678 return ret; 679 } 680 681 if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) { 682 ret = -ENODEV; 683 goto err_rflow_put; 684 } 685 686 if (xudma_is_pktdma(rx_chn->common.udmax)) { 687 rx_ringfdq_id = flow->udma_rflow_id + 688 xudma_get_rflow_ring_offset(rx_chn->common.udmax); 689 rx_ring_id = 0; 690 } else { 691 rx_ring_id = flow_cfg->ring_rxq_id; 692 rx_ringfdq_id = flow_cfg->ring_rxfdq0_id; 693 } 694 695 /* request and cfg rings */ 696 ret = k3_ringacc_request_rings_pair(rx_chn->common.ringacc, 697 rx_ringfdq_id, rx_ring_id, 698 &flow->ringrxfdq, 699 &flow->ringrx); 700 if (ret) { 701 dev_err(dev, "Failed to get RX/RXFDQ rings %d\n", ret); 702 goto err_rflow_put; 703 } 704 705 /* Set the dma_dev for the rings to be configured */ 706 flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn); 707 flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev; 708 709 /* Set the ASEL value for DMA rings of PKTDMA */ 710 if (xudma_is_pktdma(rx_chn->common.udmax)) { 711 flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel; 712 flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel; 713 } 714 715 ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg); 716 if (ret) { 717 dev_err(dev, "Failed to cfg ringrx %d\n", ret); 718 goto err_ringrxfdq_free; 719 } 720 721 ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg); 722 if (ret) { 723 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret); 724 goto err_ringrxfdq_free; 725 } 726 727 if (rx_chn->remote) { 728 rx_ring_id = TI_SCI_RESOURCE_NULL; 729 rx_ringfdq_id = TI_SCI_RESOURCE_NULL; 730 } else { 731 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx); 732 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq); 733 } 734 735 memset(&req, 0, sizeof(req)); 736 737 req.valid_params = 738 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 739 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 740 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 741 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 742 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 743 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 744 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 745 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 746 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 747 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 748 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 749 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 750 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 751 req.nav_id = tisci_rm->tisci_dev_id; 752 req.flow_index = flow->udma_rflow_id; 753 if (rx_chn->common.epib) 754 req.rx_einfo_present = 1; 755 if (rx_chn->common.psdata_size) 756 req.rx_psinfo_present = 1; 757 if (flow_cfg->rx_error_handling) 758 req.rx_error_handling = 1; 759 req.rx_desc_type = 0; 760 req.rx_dest_qnum = rx_ring_id; 761 req.rx_src_tag_hi_sel = 0; 762 req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel; 763 req.rx_dest_tag_hi_sel = 0; 764 req.rx_dest_tag_lo_sel = 0; 765 req.rx_fdq0_sz0_qnum = rx_ringfdq_id; 766 req.rx_fdq1_qnum = rx_ringfdq_id; 767 req.rx_fdq2_qnum = rx_ringfdq_id; 768 req.rx_fdq3_qnum = rx_ringfdq_id; 769 770 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 771 if (ret) { 772 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id, 773 ret); 774 goto err_ringrxfdq_free; 775 } 776 777 rx_chn->flows_ready++; 778 dev_dbg(dev, "flow%d config done. ready:%d\n", 779 flow->udma_rflow_id, rx_chn->flows_ready); 780 781 return 0; 782 783 err_ringrxfdq_free: 784 k3_ringacc_ring_free(flow->ringrxfdq); 785 k3_ringacc_ring_free(flow->ringrx); 786 787 err_rflow_put: 788 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); 789 flow->udma_rflow = NULL; 790 791 return ret; 792 } 793 794 static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn) 795 { 796 struct device *dev = chn->common.dev; 797 798 dev_dbg(dev, "dump_rx_chn:\n" 799 "udma_rchan_id: %d\n" 800 "src_thread: %08x\n" 801 "dst_thread: %08x\n" 802 "epib: %d\n" 803 "hdesc_size: %u\n" 804 "psdata_size: %u\n" 805 "swdata_size: %u\n" 806 "flow_id_base: %d\n" 807 "flow_num: %d\n", 808 chn->udma_rchan_id, 809 chn->common.src_thread, 810 chn->common.dst_thread, 811 chn->common.epib, 812 chn->common.hdesc_size, 813 chn->common.psdata_size, 814 chn->common.swdata_size, 815 chn->flow_id_base, 816 chn->flow_num); 817 } 818 819 static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn, 820 char *mark) 821 { 822 struct device *dev = chn->common.dev; 823 824 dev_dbg(dev, "=== dump ===> %s\n", mark); 825 826 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG, 827 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG)); 828 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG, 829 xudma_rchanrt_read(chn->udma_rchanx, 830 UDMA_CHAN_RT_PEER_RT_EN_REG)); 831 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG, 832 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_PCNT_REG)); 833 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG, 834 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_BCNT_REG)); 835 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG, 836 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_SBCNT_REG)); 837 } 838 839 static int 840 k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn, 841 struct k3_udma_glue_rx_channel_cfg *cfg) 842 { 843 int ret; 844 845 /* default rflow */ 846 if (cfg->flow_id_use_rxchan_id) 847 return 0; 848 849 /* not a GP rflows */ 850 if (rx_chn->flow_id_base != -1 && 851 !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) 852 return 0; 853 854 /* Allocate range of GP rflows */ 855 ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax, 856 rx_chn->flow_id_base, 857 rx_chn->flow_num); 858 if (ret < 0) { 859 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n", 860 rx_chn->flow_id_base, rx_chn->flow_num, ret); 861 return ret; 862 } 863 rx_chn->flow_id_base = ret; 864 865 return 0; 866 } 867 868 static struct k3_udma_glue_rx_channel * 869 k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, 870 struct k3_udma_glue_rx_channel_cfg *cfg) 871 { 872 struct k3_udma_glue_rx_channel *rx_chn; 873 struct psil_endpoint_config *ep_cfg; 874 int ret, i; 875 876 if (cfg->flow_id_num <= 0) 877 return ERR_PTR(-EINVAL); 878 879 if (cfg->flow_id_num != 1 && 880 (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id)) 881 return ERR_PTR(-EINVAL); 882 883 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); 884 if (!rx_chn) 885 return ERR_PTR(-ENOMEM); 886 887 rx_chn->common.dev = dev; 888 rx_chn->common.swdata_size = cfg->swdata_size; 889 rx_chn->remote = false; 890 891 /* parse of udmap channel */ 892 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 893 &rx_chn->common, false); 894 if (ret) 895 goto err; 896 897 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, 898 rx_chn->common.psdata_size, 899 rx_chn->common.swdata_size); 900 901 ep_cfg = rx_chn->common.ep_config; 902 903 if (xudma_is_pktdma(rx_chn->common.udmax)) 904 rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id; 905 else 906 rx_chn->udma_rchan_id = -1; 907 908 /* request and cfg UDMAP RX channel */ 909 rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, 910 rx_chn->udma_rchan_id); 911 if (IS_ERR(rx_chn->udma_rchanx)) { 912 ret = PTR_ERR(rx_chn->udma_rchanx); 913 dev_err(dev, "UDMAX rchanx get err %d\n", ret); 914 goto err; 915 } 916 rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx); 917 918 rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; 919 rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax); 920 dev_set_name(&rx_chn->common.chan_dev, "rchan%d-0x%04x", 921 rx_chn->udma_rchan_id, rx_chn->common.src_thread); 922 ret = device_register(&rx_chn->common.chan_dev); 923 if (ret) { 924 dev_err(dev, "Channel Device registration failed %d\n", ret); 925 put_device(&rx_chn->common.chan_dev); 926 rx_chn->common.chan_dev.parent = NULL; 927 goto err; 928 } 929 930 if (xudma_is_pktdma(rx_chn->common.udmax)) { 931 /* prepare the channel device as coherent */ 932 rx_chn->common.chan_dev.dma_coherent = true; 933 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, 934 DMA_BIT_MASK(48)); 935 } 936 937 if (xudma_is_pktdma(rx_chn->common.udmax)) { 938 int flow_start = cfg->flow_id_base; 939 int flow_end; 940 941 if (flow_start == -1) 942 flow_start = ep_cfg->flow_start; 943 944 flow_end = flow_start + cfg->flow_id_num - 1; 945 if (flow_start < ep_cfg->flow_start || 946 flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) { 947 dev_err(dev, "Invalid flow range requested\n"); 948 ret = -EINVAL; 949 goto err; 950 } 951 rx_chn->flow_id_base = flow_start; 952 } else { 953 rx_chn->flow_id_base = cfg->flow_id_base; 954 955 /* Use RX channel id as flow id: target dev can't generate flow_id */ 956 if (cfg->flow_id_use_rxchan_id) 957 rx_chn->flow_id_base = rx_chn->udma_rchan_id; 958 } 959 960 rx_chn->flow_num = cfg->flow_id_num; 961 962 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, 963 sizeof(*rx_chn->flows), GFP_KERNEL); 964 if (!rx_chn->flows) { 965 ret = -ENOMEM; 966 goto err; 967 } 968 969 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); 970 if (ret) 971 goto err; 972 973 for (i = 0; i < rx_chn->flow_num; i++) 974 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; 975 976 /* request and cfg psi-l */ 977 rx_chn->common.dst_thread = 978 xudma_dev_get_psil_base(rx_chn->common.udmax) + 979 rx_chn->udma_rchan_id; 980 981 ret = k3_udma_glue_cfg_rx_chn(rx_chn); 982 if (ret) { 983 dev_err(dev, "Failed to cfg rchan %d\n", ret); 984 goto err; 985 } 986 987 /* init default RX flow only if flow_num = 1 */ 988 if (cfg->def_flow_cfg) { 989 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg); 990 if (ret) 991 goto err; 992 } 993 994 k3_udma_glue_dump_rx_chn(rx_chn); 995 996 return rx_chn; 997 998 err: 999 k3_udma_glue_release_rx_chn(rx_chn); 1000 return ERR_PTR(ret); 1001 } 1002 1003 static struct k3_udma_glue_rx_channel * 1004 k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, 1005 struct k3_udma_glue_rx_channel_cfg *cfg) 1006 { 1007 struct k3_udma_glue_rx_channel *rx_chn; 1008 int ret, i; 1009 1010 if (cfg->flow_id_num <= 0 || 1011 cfg->flow_id_use_rxchan_id || 1012 cfg->def_flow_cfg || 1013 cfg->flow_id_base < 0) 1014 return ERR_PTR(-EINVAL); 1015 1016 /* 1017 * Remote RX channel is under control of Remote CPU core, so 1018 * Linux can only request and manipulate by dedicated RX flows 1019 */ 1020 1021 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); 1022 if (!rx_chn) 1023 return ERR_PTR(-ENOMEM); 1024 1025 rx_chn->common.dev = dev; 1026 rx_chn->common.swdata_size = cfg->swdata_size; 1027 rx_chn->remote = true; 1028 rx_chn->udma_rchan_id = -1; 1029 rx_chn->flow_num = cfg->flow_id_num; 1030 rx_chn->flow_id_base = cfg->flow_id_base; 1031 rx_chn->psil_paired = false; 1032 1033 /* parse of udmap channel */ 1034 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 1035 &rx_chn->common, false); 1036 if (ret) 1037 goto err; 1038 1039 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, 1040 rx_chn->common.psdata_size, 1041 rx_chn->common.swdata_size); 1042 1043 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, 1044 sizeof(*rx_chn->flows), GFP_KERNEL); 1045 if (!rx_chn->flows) { 1046 ret = -ENOMEM; 1047 goto err; 1048 } 1049 1050 rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; 1051 rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax); 1052 dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x", 1053 rx_chn->common.src_thread); 1054 ret = device_register(&rx_chn->common.chan_dev); 1055 if (ret) { 1056 dev_err(dev, "Channel Device registration failed %d\n", ret); 1057 put_device(&rx_chn->common.chan_dev); 1058 rx_chn->common.chan_dev.parent = NULL; 1059 goto err; 1060 } 1061 1062 if (xudma_is_pktdma(rx_chn->common.udmax)) { 1063 /* prepare the channel device as coherent */ 1064 rx_chn->common.chan_dev.dma_coherent = true; 1065 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, 1066 DMA_BIT_MASK(48)); 1067 } 1068 1069 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); 1070 if (ret) 1071 goto err; 1072 1073 for (i = 0; i < rx_chn->flow_num; i++) 1074 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; 1075 1076 k3_udma_glue_dump_rx_chn(rx_chn); 1077 1078 return rx_chn; 1079 1080 err: 1081 k3_udma_glue_release_rx_chn(rx_chn); 1082 return ERR_PTR(ret); 1083 } 1084 1085 struct k3_udma_glue_rx_channel * 1086 k3_udma_glue_request_rx_chn(struct device *dev, const char *name, 1087 struct k3_udma_glue_rx_channel_cfg *cfg) 1088 { 1089 if (cfg->remote) 1090 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg); 1091 else 1092 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg); 1093 } 1094 EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn); 1095 1096 void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1097 { 1098 int i; 1099 1100 if (IS_ERR_OR_NULL(rx_chn->common.udmax)) 1101 return; 1102 1103 if (rx_chn->psil_paired) { 1104 xudma_navss_psil_unpair(rx_chn->common.udmax, 1105 rx_chn->common.src_thread, 1106 rx_chn->common.dst_thread); 1107 rx_chn->psil_paired = false; 1108 } 1109 1110 for (i = 0; i < rx_chn->flow_num; i++) 1111 k3_udma_glue_release_rx_flow(rx_chn, i); 1112 1113 if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) 1114 xudma_free_gp_rflow_range(rx_chn->common.udmax, 1115 rx_chn->flow_id_base, 1116 rx_chn->flow_num); 1117 1118 if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx)) 1119 xudma_rchan_put(rx_chn->common.udmax, 1120 rx_chn->udma_rchanx); 1121 1122 if (rx_chn->common.chan_dev.parent) { 1123 device_unregister(&rx_chn->common.chan_dev); 1124 rx_chn->common.chan_dev.parent = NULL; 1125 } 1126 } 1127 EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn); 1128 1129 int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn, 1130 u32 flow_idx, 1131 struct k3_udma_glue_rx_flow_cfg *flow_cfg) 1132 { 1133 if (flow_idx >= rx_chn->flow_num) 1134 return -EINVAL; 1135 1136 return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg); 1137 } 1138 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init); 1139 1140 u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn, 1141 u32 flow_idx) 1142 { 1143 struct k3_udma_glue_rx_flow *flow; 1144 1145 if (flow_idx >= rx_chn->flow_num) 1146 return -EINVAL; 1147 1148 flow = &rx_chn->flows[flow_idx]; 1149 1150 return k3_ringacc_get_ring_id(flow->ringrxfdq); 1151 } 1152 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id); 1153 1154 u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn) 1155 { 1156 return rx_chn->flow_id_base; 1157 } 1158 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base); 1159 1160 int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn, 1161 u32 flow_idx) 1162 { 1163 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 1164 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 1165 struct device *dev = rx_chn->common.dev; 1166 struct ti_sci_msg_rm_udmap_flow_cfg req; 1167 int rx_ring_id; 1168 int rx_ringfdq_id; 1169 int ret = 0; 1170 1171 if (!rx_chn->remote) 1172 return -EINVAL; 1173 1174 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx); 1175 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq); 1176 1177 memset(&req, 0, sizeof(req)); 1178 1179 req.valid_params = 1180 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1181 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1182 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1183 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1184 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1185 req.nav_id = tisci_rm->tisci_dev_id; 1186 req.flow_index = flow->udma_rflow_id; 1187 req.rx_dest_qnum = rx_ring_id; 1188 req.rx_fdq0_sz0_qnum = rx_ringfdq_id; 1189 req.rx_fdq1_qnum = rx_ringfdq_id; 1190 req.rx_fdq2_qnum = rx_ringfdq_id; 1191 req.rx_fdq3_qnum = rx_ringfdq_id; 1192 1193 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 1194 if (ret) { 1195 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id, 1196 ret); 1197 } 1198 1199 return ret; 1200 } 1201 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable); 1202 1203 int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn, 1204 u32 flow_idx) 1205 { 1206 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 1207 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 1208 struct device *dev = rx_chn->common.dev; 1209 struct ti_sci_msg_rm_udmap_flow_cfg req; 1210 int ret = 0; 1211 1212 if (!rx_chn->remote) 1213 return -EINVAL; 1214 1215 memset(&req, 0, sizeof(req)); 1216 req.valid_params = 1217 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1218 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1219 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1220 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1221 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1222 req.nav_id = tisci_rm->tisci_dev_id; 1223 req.flow_index = flow->udma_rflow_id; 1224 req.rx_dest_qnum = TI_SCI_RESOURCE_NULL; 1225 req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL; 1226 req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL; 1227 req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL; 1228 req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL; 1229 1230 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 1231 if (ret) { 1232 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id, 1233 ret); 1234 } 1235 1236 return ret; 1237 } 1238 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable); 1239 1240 int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1241 { 1242 int ret; 1243 1244 if (rx_chn->remote) 1245 return -EINVAL; 1246 1247 if (rx_chn->flows_ready < rx_chn->flow_num) 1248 return -EINVAL; 1249 1250 ret = xudma_navss_psil_pair(rx_chn->common.udmax, 1251 rx_chn->common.src_thread, 1252 rx_chn->common.dst_thread); 1253 if (ret) { 1254 dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret); 1255 return ret; 1256 } 1257 1258 rx_chn->psil_paired = true; 1259 1260 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 1261 UDMA_CHAN_RT_CTL_EN); 1262 1263 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, 1264 UDMA_PEER_RT_EN_ENABLE); 1265 1266 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en"); 1267 return 0; 1268 } 1269 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn); 1270 1271 void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1272 { 1273 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1"); 1274 1275 xudma_rchanrt_write(rx_chn->udma_rchanx, 1276 UDMA_CHAN_RT_PEER_RT_EN_REG, 0); 1277 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0); 1278 1279 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2"); 1280 1281 if (rx_chn->psil_paired) { 1282 xudma_navss_psil_unpair(rx_chn->common.udmax, 1283 rx_chn->common.src_thread, 1284 rx_chn->common.dst_thread); 1285 rx_chn->psil_paired = false; 1286 } 1287 } 1288 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn); 1289 1290 void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1291 bool sync) 1292 { 1293 int i = 0; 1294 u32 val; 1295 1296 if (rx_chn->remote) 1297 return; 1298 1299 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1"); 1300 1301 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, 1302 UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN); 1303 1304 val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG); 1305 1306 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { 1307 val = xudma_rchanrt_read(rx_chn->udma_rchanx, 1308 UDMA_CHAN_RT_CTL_REG); 1309 udelay(1); 1310 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { 1311 dev_err(rx_chn->common.dev, "RX tdown timeout\n"); 1312 break; 1313 } 1314 i++; 1315 } 1316 1317 val = xudma_rchanrt_read(rx_chn->udma_rchanx, 1318 UDMA_CHAN_RT_PEER_RT_EN_REG); 1319 if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) 1320 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n"); 1321 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2"); 1322 } 1323 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn); 1324 1325 void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1326 u32 flow_num, void *data, 1327 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq) 1328 { 1329 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1330 struct device *dev = rx_chn->common.dev; 1331 dma_addr_t desc_dma; 1332 int occ_rx, i, ret; 1333 1334 /* reset RXCQ as it is not input for udma - expected to be empty */ 1335 occ_rx = k3_ringacc_ring_get_occ(flow->ringrx); 1336 dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx); 1337 1338 /* Skip RX FDQ in case one FDQ is used for the set of flows */ 1339 if (skip_fdq) 1340 goto do_reset; 1341 1342 /* 1343 * RX FDQ reset need to be special way as it is input for udma and its 1344 * state cached by udma, so: 1345 * 1) save RX FDQ occ 1346 * 2) clean up RX FDQ and call callback .cleanup() for each desc 1347 * 3) reset RX FDQ in a special way 1348 */ 1349 occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq); 1350 dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx); 1351 1352 for (i = 0; i < occ_rx; i++) { 1353 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma); 1354 if (ret) { 1355 if (ret != -ENODATA) 1356 dev_err(dev, "RX reset pop %d\n", ret); 1357 break; 1358 } 1359 cleanup(data, desc_dma); 1360 } 1361 1362 k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx); 1363 1364 do_reset: 1365 k3_ringacc_ring_reset(flow->ringrx); 1366 } 1367 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn); 1368 1369 int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1370 u32 flow_num, struct cppi5_host_desc_t *desc_rx, 1371 dma_addr_t desc_dma) 1372 { 1373 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1374 1375 return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma); 1376 } 1377 EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn); 1378 1379 int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1380 u32 flow_num, dma_addr_t *desc_dma) 1381 { 1382 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1383 1384 return k3_ringacc_ring_pop(flow->ringrx, desc_dma); 1385 } 1386 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn); 1387 1388 int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn, 1389 u32 flow_num) 1390 { 1391 struct k3_udma_glue_rx_flow *flow; 1392 1393 flow = &rx_chn->flows[flow_num]; 1394 1395 if (xudma_is_pktdma(rx_chn->common.udmax)) { 1396 flow->virq = xudma_pktdma_rflow_get_irq(rx_chn->common.udmax, 1397 flow->udma_rflow_id); 1398 } else { 1399 flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx); 1400 } 1401 1402 return flow->virq; 1403 } 1404 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq); 1405 1406 struct device * 1407 k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn) 1408 { 1409 if (xudma_is_pktdma(rx_chn->common.udmax) && 1410 (rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15)) 1411 return &rx_chn->common.chan_dev; 1412 1413 return xudma_get_device(rx_chn->common.udmax); 1414 } 1415 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device); 1416 1417 void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn, 1418 dma_addr_t *addr) 1419 { 1420 if (!xudma_is_pktdma(rx_chn->common.udmax) || 1421 !rx_chn->common.atype_asel) 1422 return; 1423 1424 *addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT; 1425 } 1426 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr); 1427 1428 void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn, 1429 dma_addr_t *addr) 1430 { 1431 if (!xudma_is_pktdma(rx_chn->common.udmax) || 1432 !rx_chn->common.atype_asel) 1433 return; 1434 1435 *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0); 1436 } 1437 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr); 1438 1439 static int __init k3_udma_glue_class_init(void) 1440 { 1441 return class_register(&k3_udma_glue_devclass); 1442 } 1443 1444 module_init(k3_udma_glue_class_init); 1445 MODULE_LICENSE("GPL v2"); 1446