1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * K3 NAVSS DMA glue interface 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 * 7 */ 8 9 #include <linux/atomic.h> 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/io.h> 13 #include <linux/init.h> 14 #include <linux/of.h> 15 #include <linux/platform_device.h> 16 #include <linux/soc/ti/k3-ringacc.h> 17 #include <linux/dma/ti-cppi5.h> 18 #include <linux/dma/k3-udma-glue.h> 19 20 #include "k3-udma.h" 21 #include "k3-psil-priv.h" 22 23 struct k3_udma_glue_common { 24 struct device *dev; 25 struct udma_dev *udmax; 26 const struct udma_tisci_rm *tisci_rm; 27 struct k3_ringacc *ringacc; 28 u32 src_thread; 29 u32 dst_thread; 30 31 u32 hdesc_size; 32 bool epib; 33 u32 psdata_size; 34 u32 swdata_size; 35 u32 atype; 36 }; 37 38 struct k3_udma_glue_tx_channel { 39 struct k3_udma_glue_common common; 40 41 struct udma_tchan *udma_tchanx; 42 int udma_tchan_id; 43 44 struct k3_ring *ringtx; 45 struct k3_ring *ringtxcq; 46 47 bool psil_paired; 48 49 int virq; 50 51 atomic_t free_pkts; 52 bool tx_pause_on_err; 53 bool tx_filt_einfo; 54 bool tx_filt_pswords; 55 bool tx_supr_tdpkt; 56 }; 57 58 struct k3_udma_glue_rx_flow { 59 struct udma_rflow *udma_rflow; 60 int udma_rflow_id; 61 struct k3_ring *ringrx; 62 struct k3_ring *ringrxfdq; 63 64 int virq; 65 }; 66 67 struct k3_udma_glue_rx_channel { 68 struct k3_udma_glue_common common; 69 70 struct udma_rchan *udma_rchanx; 71 int udma_rchan_id; 72 bool remote; 73 74 bool psil_paired; 75 76 u32 swdata_size; 77 int flow_id_base; 78 79 struct k3_udma_glue_rx_flow *flows; 80 u32 flow_num; 81 u32 flows_ready; 82 }; 83 84 #define K3_UDMAX_TDOWN_TIMEOUT_US 1000 85 86 static int of_k3_udma_glue_parse(struct device_node *udmax_np, 87 struct k3_udma_glue_common *common) 88 { 89 common->ringacc = of_k3_ringacc_get_by_phandle(udmax_np, 90 "ti,ringacc"); 91 if (IS_ERR(common->ringacc)) 92 return PTR_ERR(common->ringacc); 93 94 common->udmax = of_xudma_dev_get(udmax_np, NULL); 95 if (IS_ERR(common->udmax)) 96 return PTR_ERR(common->udmax); 97 98 common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax); 99 100 return 0; 101 } 102 103 static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, 104 const char *name, struct k3_udma_glue_common *common, 105 bool tx_chn) 106 { 107 struct psil_endpoint_config *ep_config; 108 struct of_phandle_args dma_spec; 109 u32 thread_id; 110 int ret = 0; 111 int index; 112 113 if (unlikely(!name)) 114 return -EINVAL; 115 116 index = of_property_match_string(chn_np, "dma-names", name); 117 if (index < 0) 118 return index; 119 120 if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index, 121 &dma_spec)) 122 return -ENOENT; 123 124 thread_id = dma_spec.args[0]; 125 if (dma_spec.args_count == 2) { 126 if (dma_spec.args[1] > 2) { 127 dev_err(common->dev, "Invalid channel atype: %u\n", 128 dma_spec.args[1]); 129 ret = -EINVAL; 130 goto out_put_spec; 131 } 132 common->atype = dma_spec.args[1]; 133 } 134 135 if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 136 ret = -EINVAL; 137 goto out_put_spec; 138 } 139 140 if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 141 ret = -EINVAL; 142 goto out_put_spec; 143 } 144 145 /* get psil endpoint config */ 146 ep_config = psil_get_ep_config(thread_id); 147 if (IS_ERR(ep_config)) { 148 dev_err(common->dev, 149 "No configuration for psi-l thread 0x%04x\n", 150 thread_id); 151 ret = PTR_ERR(ep_config); 152 goto out_put_spec; 153 } 154 155 common->epib = ep_config->needs_epib; 156 common->psdata_size = ep_config->psd_size; 157 158 if (tx_chn) 159 common->dst_thread = thread_id; 160 else 161 common->src_thread = thread_id; 162 163 ret = of_k3_udma_glue_parse(dma_spec.np, common); 164 165 out_put_spec: 166 of_node_put(dma_spec.np); 167 return ret; 168 }; 169 170 static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 171 { 172 struct device *dev = tx_chn->common.dev; 173 174 dev_dbg(dev, "dump_tx_chn:\n" 175 "udma_tchan_id: %d\n" 176 "src_thread: %08x\n" 177 "dst_thread: %08x\n", 178 tx_chn->udma_tchan_id, 179 tx_chn->common.src_thread, 180 tx_chn->common.dst_thread); 181 } 182 183 static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn, 184 char *mark) 185 { 186 struct device *dev = chn->common.dev; 187 188 dev_dbg(dev, "=== dump ===> %s\n", mark); 189 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_CTL_REG, 190 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG)); 191 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_PEER_RT_EN_REG, 192 xudma_tchanrt_read(chn->udma_tchanx, 193 UDMA_TCHAN_RT_PEER_RT_EN_REG)); 194 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_PCNT_REG, 195 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_PCNT_REG)); 196 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_BCNT_REG, 197 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_BCNT_REG)); 198 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_SBCNT_REG, 199 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_SBCNT_REG)); 200 } 201 202 static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 203 { 204 const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm; 205 struct ti_sci_msg_rm_udmap_tx_ch_cfg req; 206 207 memset(&req, 0, sizeof(req)); 208 209 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | 210 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | 211 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | 212 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 213 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | 214 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 215 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 216 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; 217 req.nav_id = tisci_rm->tisci_dev_id; 218 req.index = tx_chn->udma_tchan_id; 219 if (tx_chn->tx_pause_on_err) 220 req.tx_pause_on_err = 1; 221 if (tx_chn->tx_filt_einfo) 222 req.tx_filt_einfo = 1; 223 if (tx_chn->tx_filt_pswords) 224 req.tx_filt_pswords = 1; 225 req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 226 if (tx_chn->tx_supr_tdpkt) 227 req.tx_supr_tdpkt = 1; 228 req.tx_fetch_size = tx_chn->common.hdesc_size >> 2; 229 req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 230 req.tx_atype = tx_chn->common.atype; 231 232 return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); 233 } 234 235 struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, 236 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg) 237 { 238 struct k3_udma_glue_tx_channel *tx_chn; 239 int ret; 240 241 tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); 242 if (!tx_chn) 243 return ERR_PTR(-ENOMEM); 244 245 tx_chn->common.dev = dev; 246 tx_chn->common.swdata_size = cfg->swdata_size; 247 tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; 248 tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; 249 tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; 250 tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; 251 252 /* parse of udmap channel */ 253 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 254 &tx_chn->common, true); 255 if (ret) 256 goto err; 257 258 tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib, 259 tx_chn->common.psdata_size, 260 tx_chn->common.swdata_size); 261 262 /* request and cfg UDMAP TX channel */ 263 tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, -1); 264 if (IS_ERR(tx_chn->udma_tchanx)) { 265 ret = PTR_ERR(tx_chn->udma_tchanx); 266 dev_err(dev, "UDMAX tchanx get err %d\n", ret); 267 goto err; 268 } 269 tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx); 270 271 atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size); 272 273 /* request and cfg rings */ 274 tx_chn->ringtx = k3_ringacc_request_ring(tx_chn->common.ringacc, 275 tx_chn->udma_tchan_id, 0); 276 if (!tx_chn->ringtx) { 277 ret = -ENODEV; 278 dev_err(dev, "Failed to get TX ring %u\n", 279 tx_chn->udma_tchan_id); 280 goto err; 281 } 282 283 tx_chn->ringtxcq = k3_ringacc_request_ring(tx_chn->common.ringacc, 284 -1, 0); 285 if (!tx_chn->ringtxcq) { 286 ret = -ENODEV; 287 dev_err(dev, "Failed to get TXCQ ring\n"); 288 goto err; 289 } 290 291 ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg); 292 if (ret) { 293 dev_err(dev, "Failed to cfg ringtx %d\n", ret); 294 goto err; 295 } 296 297 ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg); 298 if (ret) { 299 dev_err(dev, "Failed to cfg ringtx %d\n", ret); 300 goto err; 301 } 302 303 /* request and cfg psi-l */ 304 tx_chn->common.src_thread = 305 xudma_dev_get_psil_base(tx_chn->common.udmax) + 306 tx_chn->udma_tchan_id; 307 308 ret = k3_udma_glue_cfg_tx_chn(tx_chn); 309 if (ret) { 310 dev_err(dev, "Failed to cfg tchan %d\n", ret); 311 goto err; 312 } 313 314 ret = xudma_navss_psil_pair(tx_chn->common.udmax, 315 tx_chn->common.src_thread, 316 tx_chn->common.dst_thread); 317 if (ret) { 318 dev_err(dev, "PSI-L request err %d\n", ret); 319 goto err; 320 } 321 322 tx_chn->psil_paired = true; 323 324 /* reset TX RT registers */ 325 k3_udma_glue_disable_tx_chn(tx_chn); 326 327 k3_udma_glue_dump_tx_chn(tx_chn); 328 329 return tx_chn; 330 331 err: 332 k3_udma_glue_release_tx_chn(tx_chn); 333 return ERR_PTR(ret); 334 } 335 EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn); 336 337 void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 338 { 339 if (tx_chn->psil_paired) { 340 xudma_navss_psil_unpair(tx_chn->common.udmax, 341 tx_chn->common.src_thread, 342 tx_chn->common.dst_thread); 343 tx_chn->psil_paired = false; 344 } 345 346 if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx)) 347 xudma_tchan_put(tx_chn->common.udmax, 348 tx_chn->udma_tchanx); 349 350 if (tx_chn->ringtxcq) 351 k3_ringacc_ring_free(tx_chn->ringtxcq); 352 353 if (tx_chn->ringtx) 354 k3_ringacc_ring_free(tx_chn->ringtx); 355 } 356 EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn); 357 358 int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 359 struct cppi5_host_desc_t *desc_tx, 360 dma_addr_t desc_dma) 361 { 362 u32 ringtxcq_id; 363 364 if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0)) 365 return -ENOMEM; 366 367 ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 368 cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id); 369 370 return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma); 371 } 372 EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn); 373 374 int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 375 dma_addr_t *desc_dma) 376 { 377 int ret; 378 379 ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma); 380 if (!ret) 381 atomic_inc(&tx_chn->free_pkts); 382 383 return ret; 384 } 385 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn); 386 387 int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 388 { 389 u32 txrt_ctl; 390 391 txrt_ctl = UDMA_PEER_RT_EN_ENABLE; 392 xudma_tchanrt_write(tx_chn->udma_tchanx, 393 UDMA_TCHAN_RT_PEER_RT_EN_REG, 394 txrt_ctl); 395 396 txrt_ctl = xudma_tchanrt_read(tx_chn->udma_tchanx, 397 UDMA_TCHAN_RT_CTL_REG); 398 txrt_ctl |= UDMA_CHAN_RT_CTL_EN; 399 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 400 txrt_ctl); 401 402 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en"); 403 return 0; 404 } 405 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn); 406 407 void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 408 { 409 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1"); 410 411 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 0); 412 413 xudma_tchanrt_write(tx_chn->udma_tchanx, 414 UDMA_TCHAN_RT_PEER_RT_EN_REG, 0); 415 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2"); 416 } 417 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn); 418 419 void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 420 bool sync) 421 { 422 int i = 0; 423 u32 val; 424 425 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1"); 426 427 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 428 UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN); 429 430 val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG); 431 432 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { 433 val = xudma_tchanrt_read(tx_chn->udma_tchanx, 434 UDMA_TCHAN_RT_CTL_REG); 435 udelay(1); 436 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { 437 dev_err(tx_chn->common.dev, "TX tdown timeout\n"); 438 break; 439 } 440 i++; 441 } 442 443 val = xudma_tchanrt_read(tx_chn->udma_tchanx, 444 UDMA_TCHAN_RT_PEER_RT_EN_REG); 445 if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) 446 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n"); 447 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2"); 448 } 449 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn); 450 451 void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 452 void *data, 453 void (*cleanup)(void *data, dma_addr_t desc_dma)) 454 { 455 dma_addr_t desc_dma; 456 int occ_tx, i, ret; 457 458 /* reset TXCQ as it is not input for udma - expected to be empty */ 459 if (tx_chn->ringtxcq) 460 k3_ringacc_ring_reset(tx_chn->ringtxcq); 461 462 /* 463 * TXQ reset need to be special way as it is input for udma and its 464 * state cached by udma, so: 465 * 1) save TXQ occ 466 * 2) clean up TXQ and call callback .cleanup() for each desc 467 * 3) reset TXQ in a special way 468 */ 469 occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx); 470 dev_dbg(tx_chn->common.dev, "TX reset occ_tx %u\n", occ_tx); 471 472 for (i = 0; i < occ_tx; i++) { 473 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma); 474 if (ret) { 475 dev_err(tx_chn->common.dev, "TX reset pop %d\n", ret); 476 break; 477 } 478 cleanup(data, desc_dma); 479 } 480 481 k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx); 482 } 483 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn); 484 485 u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn) 486 { 487 return tx_chn->common.hdesc_size; 488 } 489 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size); 490 491 u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn) 492 { 493 return k3_ringacc_get_ring_id(tx_chn->ringtxcq); 494 } 495 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id); 496 497 int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn) 498 { 499 tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq); 500 501 return tx_chn->virq; 502 } 503 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq); 504 505 static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 506 { 507 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 508 struct ti_sci_msg_rm_udmap_rx_ch_cfg req; 509 int ret; 510 511 memset(&req, 0, sizeof(req)); 512 513 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 514 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 515 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 516 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | 517 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | 518 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; 519 520 req.nav_id = tisci_rm->tisci_dev_id; 521 req.index = rx_chn->udma_rchan_id; 522 req.rx_fetch_size = rx_chn->common.hdesc_size >> 2; 523 /* 524 * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw 525 * and udmax impl, so just configure it to invalid value. 526 * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx); 527 */ 528 req.rxcq_qnum = 0xFFFF; 529 if (rx_chn->flow_num && rx_chn->flow_id_base != rx_chn->udma_rchan_id) { 530 /* Default flow + extra ones */ 531 req.flowid_start = rx_chn->flow_id_base; 532 req.flowid_cnt = rx_chn->flow_num; 533 } 534 req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 535 req.rx_atype = rx_chn->common.atype; 536 537 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req); 538 if (ret) 539 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n", 540 rx_chn->udma_rchan_id, ret); 541 542 return ret; 543 } 544 545 static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, 546 u32 flow_num) 547 { 548 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 549 550 if (IS_ERR_OR_NULL(flow->udma_rflow)) 551 return; 552 553 if (flow->ringrxfdq) 554 k3_ringacc_ring_free(flow->ringrxfdq); 555 556 if (flow->ringrx) 557 k3_ringacc_ring_free(flow->ringrx); 558 559 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); 560 flow->udma_rflow = NULL; 561 rx_chn->flows_ready--; 562 } 563 564 static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, 565 u32 flow_idx, 566 struct k3_udma_glue_rx_flow_cfg *flow_cfg) 567 { 568 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 569 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 570 struct device *dev = rx_chn->common.dev; 571 struct ti_sci_msg_rm_udmap_flow_cfg req; 572 int rx_ring_id; 573 int rx_ringfdq_id; 574 int ret = 0; 575 576 flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax, 577 flow->udma_rflow_id); 578 if (IS_ERR(flow->udma_rflow)) { 579 ret = PTR_ERR(flow->udma_rflow); 580 dev_err(dev, "UDMAX rflow get err %d\n", ret); 581 return ret; 582 } 583 584 if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) { 585 ret = -ENODEV; 586 goto err_rflow_put; 587 } 588 589 /* request and cfg rings */ 590 flow->ringrx = k3_ringacc_request_ring(rx_chn->common.ringacc, 591 flow_cfg->ring_rxq_id, 0); 592 if (!flow->ringrx) { 593 ret = -ENODEV; 594 dev_err(dev, "Failed to get RX ring\n"); 595 goto err_rflow_put; 596 } 597 598 flow->ringrxfdq = k3_ringacc_request_ring(rx_chn->common.ringacc, 599 flow_cfg->ring_rxfdq0_id, 0); 600 if (!flow->ringrxfdq) { 601 ret = -ENODEV; 602 dev_err(dev, "Failed to get RXFDQ ring\n"); 603 goto err_ringrx_free; 604 } 605 606 ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg); 607 if (ret) { 608 dev_err(dev, "Failed to cfg ringrx %d\n", ret); 609 goto err_ringrxfdq_free; 610 } 611 612 ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg); 613 if (ret) { 614 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret); 615 goto err_ringrxfdq_free; 616 } 617 618 if (rx_chn->remote) { 619 rx_ring_id = TI_SCI_RESOURCE_NULL; 620 rx_ringfdq_id = TI_SCI_RESOURCE_NULL; 621 } else { 622 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx); 623 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq); 624 } 625 626 memset(&req, 0, sizeof(req)); 627 628 req.valid_params = 629 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 630 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 631 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 632 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 633 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 634 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 635 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 636 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 637 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 638 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 639 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 640 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 641 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 642 req.nav_id = tisci_rm->tisci_dev_id; 643 req.flow_index = flow->udma_rflow_id; 644 if (rx_chn->common.epib) 645 req.rx_einfo_present = 1; 646 if (rx_chn->common.psdata_size) 647 req.rx_psinfo_present = 1; 648 if (flow_cfg->rx_error_handling) 649 req.rx_error_handling = 1; 650 req.rx_desc_type = 0; 651 req.rx_dest_qnum = rx_ring_id; 652 req.rx_src_tag_hi_sel = 0; 653 req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel; 654 req.rx_dest_tag_hi_sel = 0; 655 req.rx_dest_tag_lo_sel = 0; 656 req.rx_fdq0_sz0_qnum = rx_ringfdq_id; 657 req.rx_fdq1_qnum = rx_ringfdq_id; 658 req.rx_fdq2_qnum = rx_ringfdq_id; 659 req.rx_fdq3_qnum = rx_ringfdq_id; 660 661 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 662 if (ret) { 663 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id, 664 ret); 665 goto err_ringrxfdq_free; 666 } 667 668 rx_chn->flows_ready++; 669 dev_dbg(dev, "flow%d config done. ready:%d\n", 670 flow->udma_rflow_id, rx_chn->flows_ready); 671 672 return 0; 673 674 err_ringrxfdq_free: 675 k3_ringacc_ring_free(flow->ringrxfdq); 676 677 err_ringrx_free: 678 k3_ringacc_ring_free(flow->ringrx); 679 680 err_rflow_put: 681 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); 682 flow->udma_rflow = NULL; 683 684 return ret; 685 } 686 687 static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn) 688 { 689 struct device *dev = chn->common.dev; 690 691 dev_dbg(dev, "dump_rx_chn:\n" 692 "udma_rchan_id: %d\n" 693 "src_thread: %08x\n" 694 "dst_thread: %08x\n" 695 "epib: %d\n" 696 "hdesc_size: %u\n" 697 "psdata_size: %u\n" 698 "swdata_size: %u\n" 699 "flow_id_base: %d\n" 700 "flow_num: %d\n", 701 chn->udma_rchan_id, 702 chn->common.src_thread, 703 chn->common.dst_thread, 704 chn->common.epib, 705 chn->common.hdesc_size, 706 chn->common.psdata_size, 707 chn->common.swdata_size, 708 chn->flow_id_base, 709 chn->flow_num); 710 } 711 712 static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn, 713 char *mark) 714 { 715 struct device *dev = chn->common.dev; 716 717 dev_dbg(dev, "=== dump ===> %s\n", mark); 718 719 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_CTL_REG, 720 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG)); 721 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_PEER_RT_EN_REG, 722 xudma_rchanrt_read(chn->udma_rchanx, 723 UDMA_RCHAN_RT_PEER_RT_EN_REG)); 724 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_PCNT_REG, 725 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_PCNT_REG)); 726 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_BCNT_REG, 727 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_BCNT_REG)); 728 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_SBCNT_REG, 729 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_SBCNT_REG)); 730 } 731 732 static int 733 k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn, 734 struct k3_udma_glue_rx_channel_cfg *cfg) 735 { 736 int ret; 737 738 /* default rflow */ 739 if (cfg->flow_id_use_rxchan_id) 740 return 0; 741 742 /* not a GP rflows */ 743 if (rx_chn->flow_id_base != -1 && 744 !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) 745 return 0; 746 747 /* Allocate range of GP rflows */ 748 ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax, 749 rx_chn->flow_id_base, 750 rx_chn->flow_num); 751 if (ret < 0) { 752 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n", 753 rx_chn->flow_id_base, rx_chn->flow_num, ret); 754 return ret; 755 } 756 rx_chn->flow_id_base = ret; 757 758 return 0; 759 } 760 761 static struct k3_udma_glue_rx_channel * 762 k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, 763 struct k3_udma_glue_rx_channel_cfg *cfg) 764 { 765 struct k3_udma_glue_rx_channel *rx_chn; 766 int ret, i; 767 768 if (cfg->flow_id_num <= 0) 769 return ERR_PTR(-EINVAL); 770 771 if (cfg->flow_id_num != 1 && 772 (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id)) 773 return ERR_PTR(-EINVAL); 774 775 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); 776 if (!rx_chn) 777 return ERR_PTR(-ENOMEM); 778 779 rx_chn->common.dev = dev; 780 rx_chn->common.swdata_size = cfg->swdata_size; 781 rx_chn->remote = false; 782 783 /* parse of udmap channel */ 784 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 785 &rx_chn->common, false); 786 if (ret) 787 goto err; 788 789 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, 790 rx_chn->common.psdata_size, 791 rx_chn->common.swdata_size); 792 793 /* request and cfg UDMAP RX channel */ 794 rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, -1); 795 if (IS_ERR(rx_chn->udma_rchanx)) { 796 ret = PTR_ERR(rx_chn->udma_rchanx); 797 dev_err(dev, "UDMAX rchanx get err %d\n", ret); 798 goto err; 799 } 800 rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx); 801 802 rx_chn->flow_num = cfg->flow_id_num; 803 rx_chn->flow_id_base = cfg->flow_id_base; 804 805 /* Use RX channel id as flow id: target dev can't generate flow_id */ 806 if (cfg->flow_id_use_rxchan_id) 807 rx_chn->flow_id_base = rx_chn->udma_rchan_id; 808 809 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, 810 sizeof(*rx_chn->flows), GFP_KERNEL); 811 if (!rx_chn->flows) { 812 ret = -ENOMEM; 813 goto err; 814 } 815 816 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); 817 if (ret) 818 goto err; 819 820 for (i = 0; i < rx_chn->flow_num; i++) 821 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; 822 823 /* request and cfg psi-l */ 824 rx_chn->common.dst_thread = 825 xudma_dev_get_psil_base(rx_chn->common.udmax) + 826 rx_chn->udma_rchan_id; 827 828 ret = k3_udma_glue_cfg_rx_chn(rx_chn); 829 if (ret) { 830 dev_err(dev, "Failed to cfg rchan %d\n", ret); 831 goto err; 832 } 833 834 /* init default RX flow only if flow_num = 1 */ 835 if (cfg->def_flow_cfg) { 836 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg); 837 if (ret) 838 goto err; 839 } 840 841 ret = xudma_navss_psil_pair(rx_chn->common.udmax, 842 rx_chn->common.src_thread, 843 rx_chn->common.dst_thread); 844 if (ret) { 845 dev_err(dev, "PSI-L request err %d\n", ret); 846 goto err; 847 } 848 849 rx_chn->psil_paired = true; 850 851 /* reset RX RT registers */ 852 k3_udma_glue_disable_rx_chn(rx_chn); 853 854 k3_udma_glue_dump_rx_chn(rx_chn); 855 856 return rx_chn; 857 858 err: 859 k3_udma_glue_release_rx_chn(rx_chn); 860 return ERR_PTR(ret); 861 } 862 863 static struct k3_udma_glue_rx_channel * 864 k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, 865 struct k3_udma_glue_rx_channel_cfg *cfg) 866 { 867 struct k3_udma_glue_rx_channel *rx_chn; 868 int ret, i; 869 870 if (cfg->flow_id_num <= 0 || 871 cfg->flow_id_use_rxchan_id || 872 cfg->def_flow_cfg || 873 cfg->flow_id_base < 0) 874 return ERR_PTR(-EINVAL); 875 876 /* 877 * Remote RX channel is under control of Remote CPU core, so 878 * Linux can only request and manipulate by dedicated RX flows 879 */ 880 881 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); 882 if (!rx_chn) 883 return ERR_PTR(-ENOMEM); 884 885 rx_chn->common.dev = dev; 886 rx_chn->common.swdata_size = cfg->swdata_size; 887 rx_chn->remote = true; 888 rx_chn->udma_rchan_id = -1; 889 rx_chn->flow_num = cfg->flow_id_num; 890 rx_chn->flow_id_base = cfg->flow_id_base; 891 rx_chn->psil_paired = false; 892 893 /* parse of udmap channel */ 894 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 895 &rx_chn->common, false); 896 if (ret) 897 goto err; 898 899 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, 900 rx_chn->common.psdata_size, 901 rx_chn->common.swdata_size); 902 903 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, 904 sizeof(*rx_chn->flows), GFP_KERNEL); 905 if (!rx_chn->flows) { 906 ret = -ENOMEM; 907 goto err; 908 } 909 910 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); 911 if (ret) 912 goto err; 913 914 for (i = 0; i < rx_chn->flow_num; i++) 915 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; 916 917 k3_udma_glue_dump_rx_chn(rx_chn); 918 919 return rx_chn; 920 921 err: 922 k3_udma_glue_release_rx_chn(rx_chn); 923 return ERR_PTR(ret); 924 } 925 926 struct k3_udma_glue_rx_channel * 927 k3_udma_glue_request_rx_chn(struct device *dev, const char *name, 928 struct k3_udma_glue_rx_channel_cfg *cfg) 929 { 930 if (cfg->remote) 931 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg); 932 else 933 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg); 934 } 935 EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn); 936 937 void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 938 { 939 int i; 940 941 if (IS_ERR_OR_NULL(rx_chn->common.udmax)) 942 return; 943 944 if (rx_chn->psil_paired) { 945 xudma_navss_psil_unpair(rx_chn->common.udmax, 946 rx_chn->common.src_thread, 947 rx_chn->common.dst_thread); 948 rx_chn->psil_paired = false; 949 } 950 951 for (i = 0; i < rx_chn->flow_num; i++) 952 k3_udma_glue_release_rx_flow(rx_chn, i); 953 954 if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) 955 xudma_free_gp_rflow_range(rx_chn->common.udmax, 956 rx_chn->flow_id_base, 957 rx_chn->flow_num); 958 959 if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx)) 960 xudma_rchan_put(rx_chn->common.udmax, 961 rx_chn->udma_rchanx); 962 } 963 EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn); 964 965 int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn, 966 u32 flow_idx, 967 struct k3_udma_glue_rx_flow_cfg *flow_cfg) 968 { 969 if (flow_idx >= rx_chn->flow_num) 970 return -EINVAL; 971 972 return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg); 973 } 974 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init); 975 976 u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn, 977 u32 flow_idx) 978 { 979 struct k3_udma_glue_rx_flow *flow; 980 981 if (flow_idx >= rx_chn->flow_num) 982 return -EINVAL; 983 984 flow = &rx_chn->flows[flow_idx]; 985 986 return k3_ringacc_get_ring_id(flow->ringrxfdq); 987 } 988 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id); 989 990 u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn) 991 { 992 return rx_chn->flow_id_base; 993 } 994 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base); 995 996 int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn, 997 u32 flow_idx) 998 { 999 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 1000 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 1001 struct device *dev = rx_chn->common.dev; 1002 struct ti_sci_msg_rm_udmap_flow_cfg req; 1003 int rx_ring_id; 1004 int rx_ringfdq_id; 1005 int ret = 0; 1006 1007 if (!rx_chn->remote) 1008 return -EINVAL; 1009 1010 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx); 1011 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq); 1012 1013 memset(&req, 0, sizeof(req)); 1014 1015 req.valid_params = 1016 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1017 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1018 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1019 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1020 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1021 req.nav_id = tisci_rm->tisci_dev_id; 1022 req.flow_index = flow->udma_rflow_id; 1023 req.rx_dest_qnum = rx_ring_id; 1024 req.rx_fdq0_sz0_qnum = rx_ringfdq_id; 1025 req.rx_fdq1_qnum = rx_ringfdq_id; 1026 req.rx_fdq2_qnum = rx_ringfdq_id; 1027 req.rx_fdq3_qnum = rx_ringfdq_id; 1028 1029 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 1030 if (ret) { 1031 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id, 1032 ret); 1033 } 1034 1035 return ret; 1036 } 1037 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable); 1038 1039 int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn, 1040 u32 flow_idx) 1041 { 1042 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 1043 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 1044 struct device *dev = rx_chn->common.dev; 1045 struct ti_sci_msg_rm_udmap_flow_cfg req; 1046 int ret = 0; 1047 1048 if (!rx_chn->remote) 1049 return -EINVAL; 1050 1051 memset(&req, 0, sizeof(req)); 1052 req.valid_params = 1053 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1054 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1055 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1056 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1057 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1058 req.nav_id = tisci_rm->tisci_dev_id; 1059 req.flow_index = flow->udma_rflow_id; 1060 req.rx_dest_qnum = TI_SCI_RESOURCE_NULL; 1061 req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL; 1062 req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL; 1063 req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL; 1064 req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL; 1065 1066 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 1067 if (ret) { 1068 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id, 1069 ret); 1070 } 1071 1072 return ret; 1073 } 1074 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable); 1075 1076 int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1077 { 1078 u32 rxrt_ctl; 1079 1080 if (rx_chn->remote) 1081 return -EINVAL; 1082 1083 if (rx_chn->flows_ready < rx_chn->flow_num) 1084 return -EINVAL; 1085 1086 rxrt_ctl = xudma_rchanrt_read(rx_chn->udma_rchanx, 1087 UDMA_RCHAN_RT_CTL_REG); 1088 rxrt_ctl |= UDMA_CHAN_RT_CTL_EN; 1089 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG, 1090 rxrt_ctl); 1091 1092 xudma_rchanrt_write(rx_chn->udma_rchanx, 1093 UDMA_RCHAN_RT_PEER_RT_EN_REG, 1094 UDMA_PEER_RT_EN_ENABLE); 1095 1096 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en"); 1097 return 0; 1098 } 1099 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn); 1100 1101 void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1102 { 1103 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1"); 1104 1105 xudma_rchanrt_write(rx_chn->udma_rchanx, 1106 UDMA_RCHAN_RT_PEER_RT_EN_REG, 1107 0); 1108 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG, 0); 1109 1110 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2"); 1111 } 1112 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn); 1113 1114 void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1115 bool sync) 1116 { 1117 int i = 0; 1118 u32 val; 1119 1120 if (rx_chn->remote) 1121 return; 1122 1123 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1"); 1124 1125 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_PEER_RT_EN_REG, 1126 UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN); 1127 1128 val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG); 1129 1130 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { 1131 val = xudma_rchanrt_read(rx_chn->udma_rchanx, 1132 UDMA_RCHAN_RT_CTL_REG); 1133 udelay(1); 1134 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { 1135 dev_err(rx_chn->common.dev, "RX tdown timeout\n"); 1136 break; 1137 } 1138 i++; 1139 } 1140 1141 val = xudma_rchanrt_read(rx_chn->udma_rchanx, 1142 UDMA_RCHAN_RT_PEER_RT_EN_REG); 1143 if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) 1144 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n"); 1145 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2"); 1146 } 1147 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn); 1148 1149 void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1150 u32 flow_num, void *data, 1151 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq) 1152 { 1153 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1154 struct device *dev = rx_chn->common.dev; 1155 dma_addr_t desc_dma; 1156 int occ_rx, i, ret; 1157 1158 /* reset RXCQ as it is not input for udma - expected to be empty */ 1159 occ_rx = k3_ringacc_ring_get_occ(flow->ringrx); 1160 dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx); 1161 if (flow->ringrx) 1162 k3_ringacc_ring_reset(flow->ringrx); 1163 1164 /* Skip RX FDQ in case one FDQ is used for the set of flows */ 1165 if (skip_fdq) 1166 return; 1167 1168 /* 1169 * RX FDQ reset need to be special way as it is input for udma and its 1170 * state cached by udma, so: 1171 * 1) save RX FDQ occ 1172 * 2) clean up RX FDQ and call callback .cleanup() for each desc 1173 * 3) reset RX FDQ in a special way 1174 */ 1175 occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq); 1176 dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx); 1177 1178 for (i = 0; i < occ_rx; i++) { 1179 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma); 1180 if (ret) { 1181 dev_err(dev, "RX reset pop %d\n", ret); 1182 break; 1183 } 1184 cleanup(data, desc_dma); 1185 } 1186 1187 k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx); 1188 } 1189 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn); 1190 1191 int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1192 u32 flow_num, struct cppi5_host_desc_t *desc_rx, 1193 dma_addr_t desc_dma) 1194 { 1195 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1196 1197 return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma); 1198 } 1199 EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn); 1200 1201 int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1202 u32 flow_num, dma_addr_t *desc_dma) 1203 { 1204 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1205 1206 return k3_ringacc_ring_pop(flow->ringrx, desc_dma); 1207 } 1208 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn); 1209 1210 int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn, 1211 u32 flow_num) 1212 { 1213 struct k3_udma_glue_rx_flow *flow; 1214 1215 flow = &rx_chn->flows[flow_num]; 1216 1217 flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx); 1218 1219 return flow->virq; 1220 } 1221 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq); 1222