1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * K3 NAVSS DMA glue interface 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 * 7 */ 8 9 #include <linux/atomic.h> 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/io.h> 13 #include <linux/init.h> 14 #include <linux/of.h> 15 #include <linux/platform_device.h> 16 #include <linux/soc/ti/k3-ringacc.h> 17 #include <linux/dma/ti-cppi5.h> 18 #include <linux/dma/k3-udma-glue.h> 19 20 #include "k3-udma.h" 21 #include "k3-psil-priv.h" 22 23 struct k3_udma_glue_common { 24 struct device *dev; 25 struct udma_dev *udmax; 26 const struct udma_tisci_rm *tisci_rm; 27 struct k3_ringacc *ringacc; 28 u32 src_thread; 29 u32 dst_thread; 30 31 u32 hdesc_size; 32 bool epib; 33 u32 psdata_size; 34 u32 swdata_size; 35 u32 atype; 36 }; 37 38 struct k3_udma_glue_tx_channel { 39 struct k3_udma_glue_common common; 40 41 struct udma_tchan *udma_tchanx; 42 int udma_tchan_id; 43 44 struct k3_ring *ringtx; 45 struct k3_ring *ringtxcq; 46 47 bool psil_paired; 48 49 int virq; 50 51 atomic_t free_pkts; 52 bool tx_pause_on_err; 53 bool tx_filt_einfo; 54 bool tx_filt_pswords; 55 bool tx_supr_tdpkt; 56 }; 57 58 struct k3_udma_glue_rx_flow { 59 struct udma_rflow *udma_rflow; 60 int udma_rflow_id; 61 struct k3_ring *ringrx; 62 struct k3_ring *ringrxfdq; 63 64 int virq; 65 }; 66 67 struct k3_udma_glue_rx_channel { 68 struct k3_udma_glue_common common; 69 70 struct udma_rchan *udma_rchanx; 71 int udma_rchan_id; 72 bool remote; 73 74 bool psil_paired; 75 76 u32 swdata_size; 77 int flow_id_base; 78 79 struct k3_udma_glue_rx_flow *flows; 80 u32 flow_num; 81 u32 flows_ready; 82 }; 83 84 #define K3_UDMAX_TDOWN_TIMEOUT_US 1000 85 86 static int of_k3_udma_glue_parse(struct device_node *udmax_np, 87 struct k3_udma_glue_common *common) 88 { 89 common->ringacc = of_k3_ringacc_get_by_phandle(udmax_np, 90 "ti,ringacc"); 91 if (IS_ERR(common->ringacc)) 92 return PTR_ERR(common->ringacc); 93 94 common->udmax = of_xudma_dev_get(udmax_np, NULL); 95 if (IS_ERR(common->udmax)) 96 return PTR_ERR(common->udmax); 97 98 common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax); 99 100 return 0; 101 } 102 103 static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, 104 const char *name, struct k3_udma_glue_common *common, 105 bool tx_chn) 106 { 107 struct psil_endpoint_config *ep_config; 108 struct of_phandle_args dma_spec; 109 u32 thread_id; 110 int ret = 0; 111 int index; 112 113 if (unlikely(!name)) 114 return -EINVAL; 115 116 index = of_property_match_string(chn_np, "dma-names", name); 117 if (index < 0) 118 return index; 119 120 if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index, 121 &dma_spec)) 122 return -ENOENT; 123 124 thread_id = dma_spec.args[0]; 125 if (dma_spec.args_count == 2) { 126 if (dma_spec.args[1] > 2) { 127 dev_err(common->dev, "Invalid channel atype: %u\n", 128 dma_spec.args[1]); 129 ret = -EINVAL; 130 goto out_put_spec; 131 } 132 common->atype = dma_spec.args[1]; 133 } 134 135 if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 136 ret = -EINVAL; 137 goto out_put_spec; 138 } 139 140 if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 141 ret = -EINVAL; 142 goto out_put_spec; 143 } 144 145 /* get psil endpoint config */ 146 ep_config = psil_get_ep_config(thread_id); 147 if (IS_ERR(ep_config)) { 148 dev_err(common->dev, 149 "No configuration for psi-l thread 0x%04x\n", 150 thread_id); 151 ret = PTR_ERR(ep_config); 152 goto out_put_spec; 153 } 154 155 common->epib = ep_config->needs_epib; 156 common->psdata_size = ep_config->psd_size; 157 158 if (tx_chn) 159 common->dst_thread = thread_id; 160 else 161 common->src_thread = thread_id; 162 163 ret = of_k3_udma_glue_parse(dma_spec.np, common); 164 165 out_put_spec: 166 of_node_put(dma_spec.np); 167 return ret; 168 }; 169 170 static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 171 { 172 struct device *dev = tx_chn->common.dev; 173 174 dev_dbg(dev, "dump_tx_chn:\n" 175 "udma_tchan_id: %d\n" 176 "src_thread: %08x\n" 177 "dst_thread: %08x\n", 178 tx_chn->udma_tchan_id, 179 tx_chn->common.src_thread, 180 tx_chn->common.dst_thread); 181 } 182 183 static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn, 184 char *mark) 185 { 186 struct device *dev = chn->common.dev; 187 188 dev_dbg(dev, "=== dump ===> %s\n", mark); 189 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_CTL_REG, 190 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG)); 191 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_PEER_RT_EN_REG, 192 xudma_tchanrt_read(chn->udma_tchanx, 193 UDMA_TCHAN_RT_PEER_RT_EN_REG)); 194 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_PCNT_REG, 195 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_PCNT_REG)); 196 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_BCNT_REG, 197 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_BCNT_REG)); 198 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_SBCNT_REG, 199 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_SBCNT_REG)); 200 } 201 202 static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 203 { 204 const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm; 205 struct ti_sci_msg_rm_udmap_tx_ch_cfg req; 206 207 memset(&req, 0, sizeof(req)); 208 209 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | 210 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | 211 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | 212 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 213 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | 214 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 215 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 216 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; 217 req.nav_id = tisci_rm->tisci_dev_id; 218 req.index = tx_chn->udma_tchan_id; 219 if (tx_chn->tx_pause_on_err) 220 req.tx_pause_on_err = 1; 221 if (tx_chn->tx_filt_einfo) 222 req.tx_filt_einfo = 1; 223 if (tx_chn->tx_filt_pswords) 224 req.tx_filt_pswords = 1; 225 req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 226 if (tx_chn->tx_supr_tdpkt) 227 req.tx_supr_tdpkt = 1; 228 req.tx_fetch_size = tx_chn->common.hdesc_size >> 2; 229 req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 230 req.tx_atype = tx_chn->common.atype; 231 232 return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); 233 } 234 235 struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, 236 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg) 237 { 238 struct k3_udma_glue_tx_channel *tx_chn; 239 int ret; 240 241 tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); 242 if (!tx_chn) 243 return ERR_PTR(-ENOMEM); 244 245 tx_chn->common.dev = dev; 246 tx_chn->common.swdata_size = cfg->swdata_size; 247 tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; 248 tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; 249 tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; 250 tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; 251 252 /* parse of udmap channel */ 253 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 254 &tx_chn->common, true); 255 if (ret) 256 goto err; 257 258 tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib, 259 tx_chn->common.psdata_size, 260 tx_chn->common.swdata_size); 261 262 /* request and cfg UDMAP TX channel */ 263 tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, -1); 264 if (IS_ERR(tx_chn->udma_tchanx)) { 265 ret = PTR_ERR(tx_chn->udma_tchanx); 266 dev_err(dev, "UDMAX tchanx get err %d\n", ret); 267 goto err; 268 } 269 tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx); 270 271 atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size); 272 273 /* request and cfg rings */ 274 ret = k3_ringacc_request_rings_pair(tx_chn->common.ringacc, 275 tx_chn->udma_tchan_id, -1, 276 &tx_chn->ringtx, 277 &tx_chn->ringtxcq); 278 if (ret) { 279 dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret); 280 goto err; 281 } 282 283 ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg); 284 if (ret) { 285 dev_err(dev, "Failed to cfg ringtx %d\n", ret); 286 goto err; 287 } 288 289 ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg); 290 if (ret) { 291 dev_err(dev, "Failed to cfg ringtx %d\n", ret); 292 goto err; 293 } 294 295 /* request and cfg psi-l */ 296 tx_chn->common.src_thread = 297 xudma_dev_get_psil_base(tx_chn->common.udmax) + 298 tx_chn->udma_tchan_id; 299 300 ret = k3_udma_glue_cfg_tx_chn(tx_chn); 301 if (ret) { 302 dev_err(dev, "Failed to cfg tchan %d\n", ret); 303 goto err; 304 } 305 306 ret = xudma_navss_psil_pair(tx_chn->common.udmax, 307 tx_chn->common.src_thread, 308 tx_chn->common.dst_thread); 309 if (ret) { 310 dev_err(dev, "PSI-L request err %d\n", ret); 311 goto err; 312 } 313 314 tx_chn->psil_paired = true; 315 316 /* reset TX RT registers */ 317 k3_udma_glue_disable_tx_chn(tx_chn); 318 319 k3_udma_glue_dump_tx_chn(tx_chn); 320 321 return tx_chn; 322 323 err: 324 k3_udma_glue_release_tx_chn(tx_chn); 325 return ERR_PTR(ret); 326 } 327 EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn); 328 329 void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 330 { 331 if (tx_chn->psil_paired) { 332 xudma_navss_psil_unpair(tx_chn->common.udmax, 333 tx_chn->common.src_thread, 334 tx_chn->common.dst_thread); 335 tx_chn->psil_paired = false; 336 } 337 338 if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx)) 339 xudma_tchan_put(tx_chn->common.udmax, 340 tx_chn->udma_tchanx); 341 342 if (tx_chn->ringtxcq) 343 k3_ringacc_ring_free(tx_chn->ringtxcq); 344 345 if (tx_chn->ringtx) 346 k3_ringacc_ring_free(tx_chn->ringtx); 347 } 348 EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn); 349 350 int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 351 struct cppi5_host_desc_t *desc_tx, 352 dma_addr_t desc_dma) 353 { 354 u32 ringtxcq_id; 355 356 if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0)) 357 return -ENOMEM; 358 359 ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 360 cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id); 361 362 return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma); 363 } 364 EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn); 365 366 int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 367 dma_addr_t *desc_dma) 368 { 369 int ret; 370 371 ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma); 372 if (!ret) 373 atomic_inc(&tx_chn->free_pkts); 374 375 return ret; 376 } 377 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn); 378 379 int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 380 { 381 u32 txrt_ctl; 382 383 txrt_ctl = UDMA_PEER_RT_EN_ENABLE; 384 xudma_tchanrt_write(tx_chn->udma_tchanx, 385 UDMA_TCHAN_RT_PEER_RT_EN_REG, 386 txrt_ctl); 387 388 txrt_ctl = xudma_tchanrt_read(tx_chn->udma_tchanx, 389 UDMA_TCHAN_RT_CTL_REG); 390 txrt_ctl |= UDMA_CHAN_RT_CTL_EN; 391 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 392 txrt_ctl); 393 394 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en"); 395 return 0; 396 } 397 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn); 398 399 void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 400 { 401 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1"); 402 403 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 0); 404 405 xudma_tchanrt_write(tx_chn->udma_tchanx, 406 UDMA_TCHAN_RT_PEER_RT_EN_REG, 0); 407 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2"); 408 } 409 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn); 410 411 void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 412 bool sync) 413 { 414 int i = 0; 415 u32 val; 416 417 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1"); 418 419 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 420 UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN); 421 422 val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG); 423 424 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { 425 val = xudma_tchanrt_read(tx_chn->udma_tchanx, 426 UDMA_TCHAN_RT_CTL_REG); 427 udelay(1); 428 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { 429 dev_err(tx_chn->common.dev, "TX tdown timeout\n"); 430 break; 431 } 432 i++; 433 } 434 435 val = xudma_tchanrt_read(tx_chn->udma_tchanx, 436 UDMA_TCHAN_RT_PEER_RT_EN_REG); 437 if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) 438 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n"); 439 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2"); 440 } 441 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn); 442 443 void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 444 void *data, 445 void (*cleanup)(void *data, dma_addr_t desc_dma)) 446 { 447 dma_addr_t desc_dma; 448 int occ_tx, i, ret; 449 450 /* reset TXCQ as it is not input for udma - expected to be empty */ 451 if (tx_chn->ringtxcq) 452 k3_ringacc_ring_reset(tx_chn->ringtxcq); 453 454 /* 455 * TXQ reset need to be special way as it is input for udma and its 456 * state cached by udma, so: 457 * 1) save TXQ occ 458 * 2) clean up TXQ and call callback .cleanup() for each desc 459 * 3) reset TXQ in a special way 460 */ 461 occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx); 462 dev_dbg(tx_chn->common.dev, "TX reset occ_tx %u\n", occ_tx); 463 464 for (i = 0; i < occ_tx; i++) { 465 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma); 466 if (ret) { 467 dev_err(tx_chn->common.dev, "TX reset pop %d\n", ret); 468 break; 469 } 470 cleanup(data, desc_dma); 471 } 472 473 k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx); 474 } 475 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn); 476 477 u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn) 478 { 479 return tx_chn->common.hdesc_size; 480 } 481 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size); 482 483 u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn) 484 { 485 return k3_ringacc_get_ring_id(tx_chn->ringtxcq); 486 } 487 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id); 488 489 int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn) 490 { 491 tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq); 492 493 return tx_chn->virq; 494 } 495 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq); 496 497 static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 498 { 499 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 500 struct ti_sci_msg_rm_udmap_rx_ch_cfg req; 501 int ret; 502 503 memset(&req, 0, sizeof(req)); 504 505 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 506 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 507 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 508 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | 509 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | 510 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; 511 512 req.nav_id = tisci_rm->tisci_dev_id; 513 req.index = rx_chn->udma_rchan_id; 514 req.rx_fetch_size = rx_chn->common.hdesc_size >> 2; 515 /* 516 * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw 517 * and udmax impl, so just configure it to invalid value. 518 * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx); 519 */ 520 req.rxcq_qnum = 0xFFFF; 521 if (rx_chn->flow_num && rx_chn->flow_id_base != rx_chn->udma_rchan_id) { 522 /* Default flow + extra ones */ 523 req.flowid_start = rx_chn->flow_id_base; 524 req.flowid_cnt = rx_chn->flow_num; 525 } 526 req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 527 req.rx_atype = rx_chn->common.atype; 528 529 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req); 530 if (ret) 531 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n", 532 rx_chn->udma_rchan_id, ret); 533 534 return ret; 535 } 536 537 static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, 538 u32 flow_num) 539 { 540 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 541 542 if (IS_ERR_OR_NULL(flow->udma_rflow)) 543 return; 544 545 if (flow->ringrxfdq) 546 k3_ringacc_ring_free(flow->ringrxfdq); 547 548 if (flow->ringrx) 549 k3_ringacc_ring_free(flow->ringrx); 550 551 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); 552 flow->udma_rflow = NULL; 553 rx_chn->flows_ready--; 554 } 555 556 static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, 557 u32 flow_idx, 558 struct k3_udma_glue_rx_flow_cfg *flow_cfg) 559 { 560 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 561 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 562 struct device *dev = rx_chn->common.dev; 563 struct ti_sci_msg_rm_udmap_flow_cfg req; 564 int rx_ring_id; 565 int rx_ringfdq_id; 566 int ret = 0; 567 568 flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax, 569 flow->udma_rflow_id); 570 if (IS_ERR(flow->udma_rflow)) { 571 ret = PTR_ERR(flow->udma_rflow); 572 dev_err(dev, "UDMAX rflow get err %d\n", ret); 573 return ret; 574 } 575 576 if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) { 577 ret = -ENODEV; 578 goto err_rflow_put; 579 } 580 581 /* request and cfg rings */ 582 ret = k3_ringacc_request_rings_pair(rx_chn->common.ringacc, 583 flow_cfg->ring_rxq_id, 584 flow_cfg->ring_rxfdq0_id, 585 &flow->ringrxfdq, 586 &flow->ringrx); 587 if (ret) { 588 dev_err(dev, "Failed to get RX/RXFDQ rings %d\n", ret); 589 goto err_rflow_put; 590 } 591 592 ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg); 593 if (ret) { 594 dev_err(dev, "Failed to cfg ringrx %d\n", ret); 595 goto err_ringrxfdq_free; 596 } 597 598 ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg); 599 if (ret) { 600 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret); 601 goto err_ringrxfdq_free; 602 } 603 604 if (rx_chn->remote) { 605 rx_ring_id = TI_SCI_RESOURCE_NULL; 606 rx_ringfdq_id = TI_SCI_RESOURCE_NULL; 607 } else { 608 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx); 609 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq); 610 } 611 612 memset(&req, 0, sizeof(req)); 613 614 req.valid_params = 615 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 616 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 617 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 618 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 619 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 620 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 621 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 622 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 623 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 624 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 625 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 626 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 627 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 628 req.nav_id = tisci_rm->tisci_dev_id; 629 req.flow_index = flow->udma_rflow_id; 630 if (rx_chn->common.epib) 631 req.rx_einfo_present = 1; 632 if (rx_chn->common.psdata_size) 633 req.rx_psinfo_present = 1; 634 if (flow_cfg->rx_error_handling) 635 req.rx_error_handling = 1; 636 req.rx_desc_type = 0; 637 req.rx_dest_qnum = rx_ring_id; 638 req.rx_src_tag_hi_sel = 0; 639 req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel; 640 req.rx_dest_tag_hi_sel = 0; 641 req.rx_dest_tag_lo_sel = 0; 642 req.rx_fdq0_sz0_qnum = rx_ringfdq_id; 643 req.rx_fdq1_qnum = rx_ringfdq_id; 644 req.rx_fdq2_qnum = rx_ringfdq_id; 645 req.rx_fdq3_qnum = rx_ringfdq_id; 646 647 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 648 if (ret) { 649 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id, 650 ret); 651 goto err_ringrxfdq_free; 652 } 653 654 rx_chn->flows_ready++; 655 dev_dbg(dev, "flow%d config done. ready:%d\n", 656 flow->udma_rflow_id, rx_chn->flows_ready); 657 658 return 0; 659 660 err_ringrxfdq_free: 661 k3_ringacc_ring_free(flow->ringrxfdq); 662 k3_ringacc_ring_free(flow->ringrx); 663 664 err_rflow_put: 665 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); 666 flow->udma_rflow = NULL; 667 668 return ret; 669 } 670 671 static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn) 672 { 673 struct device *dev = chn->common.dev; 674 675 dev_dbg(dev, "dump_rx_chn:\n" 676 "udma_rchan_id: %d\n" 677 "src_thread: %08x\n" 678 "dst_thread: %08x\n" 679 "epib: %d\n" 680 "hdesc_size: %u\n" 681 "psdata_size: %u\n" 682 "swdata_size: %u\n" 683 "flow_id_base: %d\n" 684 "flow_num: %d\n", 685 chn->udma_rchan_id, 686 chn->common.src_thread, 687 chn->common.dst_thread, 688 chn->common.epib, 689 chn->common.hdesc_size, 690 chn->common.psdata_size, 691 chn->common.swdata_size, 692 chn->flow_id_base, 693 chn->flow_num); 694 } 695 696 static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn, 697 char *mark) 698 { 699 struct device *dev = chn->common.dev; 700 701 dev_dbg(dev, "=== dump ===> %s\n", mark); 702 703 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_CTL_REG, 704 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG)); 705 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_PEER_RT_EN_REG, 706 xudma_rchanrt_read(chn->udma_rchanx, 707 UDMA_RCHAN_RT_PEER_RT_EN_REG)); 708 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_PCNT_REG, 709 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_PCNT_REG)); 710 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_BCNT_REG, 711 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_BCNT_REG)); 712 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_SBCNT_REG, 713 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_SBCNT_REG)); 714 } 715 716 static int 717 k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn, 718 struct k3_udma_glue_rx_channel_cfg *cfg) 719 { 720 int ret; 721 722 /* default rflow */ 723 if (cfg->flow_id_use_rxchan_id) 724 return 0; 725 726 /* not a GP rflows */ 727 if (rx_chn->flow_id_base != -1 && 728 !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) 729 return 0; 730 731 /* Allocate range of GP rflows */ 732 ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax, 733 rx_chn->flow_id_base, 734 rx_chn->flow_num); 735 if (ret < 0) { 736 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n", 737 rx_chn->flow_id_base, rx_chn->flow_num, ret); 738 return ret; 739 } 740 rx_chn->flow_id_base = ret; 741 742 return 0; 743 } 744 745 static struct k3_udma_glue_rx_channel * 746 k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, 747 struct k3_udma_glue_rx_channel_cfg *cfg) 748 { 749 struct k3_udma_glue_rx_channel *rx_chn; 750 int ret, i; 751 752 if (cfg->flow_id_num <= 0) 753 return ERR_PTR(-EINVAL); 754 755 if (cfg->flow_id_num != 1 && 756 (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id)) 757 return ERR_PTR(-EINVAL); 758 759 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); 760 if (!rx_chn) 761 return ERR_PTR(-ENOMEM); 762 763 rx_chn->common.dev = dev; 764 rx_chn->common.swdata_size = cfg->swdata_size; 765 rx_chn->remote = false; 766 767 /* parse of udmap channel */ 768 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 769 &rx_chn->common, false); 770 if (ret) 771 goto err; 772 773 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, 774 rx_chn->common.psdata_size, 775 rx_chn->common.swdata_size); 776 777 /* request and cfg UDMAP RX channel */ 778 rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, -1); 779 if (IS_ERR(rx_chn->udma_rchanx)) { 780 ret = PTR_ERR(rx_chn->udma_rchanx); 781 dev_err(dev, "UDMAX rchanx get err %d\n", ret); 782 goto err; 783 } 784 rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx); 785 786 rx_chn->flow_num = cfg->flow_id_num; 787 rx_chn->flow_id_base = cfg->flow_id_base; 788 789 /* Use RX channel id as flow id: target dev can't generate flow_id */ 790 if (cfg->flow_id_use_rxchan_id) 791 rx_chn->flow_id_base = rx_chn->udma_rchan_id; 792 793 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, 794 sizeof(*rx_chn->flows), GFP_KERNEL); 795 if (!rx_chn->flows) { 796 ret = -ENOMEM; 797 goto err; 798 } 799 800 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); 801 if (ret) 802 goto err; 803 804 for (i = 0; i < rx_chn->flow_num; i++) 805 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; 806 807 /* request and cfg psi-l */ 808 rx_chn->common.dst_thread = 809 xudma_dev_get_psil_base(rx_chn->common.udmax) + 810 rx_chn->udma_rchan_id; 811 812 ret = k3_udma_glue_cfg_rx_chn(rx_chn); 813 if (ret) { 814 dev_err(dev, "Failed to cfg rchan %d\n", ret); 815 goto err; 816 } 817 818 /* init default RX flow only if flow_num = 1 */ 819 if (cfg->def_flow_cfg) { 820 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg); 821 if (ret) 822 goto err; 823 } 824 825 ret = xudma_navss_psil_pair(rx_chn->common.udmax, 826 rx_chn->common.src_thread, 827 rx_chn->common.dst_thread); 828 if (ret) { 829 dev_err(dev, "PSI-L request err %d\n", ret); 830 goto err; 831 } 832 833 rx_chn->psil_paired = true; 834 835 /* reset RX RT registers */ 836 k3_udma_glue_disable_rx_chn(rx_chn); 837 838 k3_udma_glue_dump_rx_chn(rx_chn); 839 840 return rx_chn; 841 842 err: 843 k3_udma_glue_release_rx_chn(rx_chn); 844 return ERR_PTR(ret); 845 } 846 847 static struct k3_udma_glue_rx_channel * 848 k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, 849 struct k3_udma_glue_rx_channel_cfg *cfg) 850 { 851 struct k3_udma_glue_rx_channel *rx_chn; 852 int ret, i; 853 854 if (cfg->flow_id_num <= 0 || 855 cfg->flow_id_use_rxchan_id || 856 cfg->def_flow_cfg || 857 cfg->flow_id_base < 0) 858 return ERR_PTR(-EINVAL); 859 860 /* 861 * Remote RX channel is under control of Remote CPU core, so 862 * Linux can only request and manipulate by dedicated RX flows 863 */ 864 865 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); 866 if (!rx_chn) 867 return ERR_PTR(-ENOMEM); 868 869 rx_chn->common.dev = dev; 870 rx_chn->common.swdata_size = cfg->swdata_size; 871 rx_chn->remote = true; 872 rx_chn->udma_rchan_id = -1; 873 rx_chn->flow_num = cfg->flow_id_num; 874 rx_chn->flow_id_base = cfg->flow_id_base; 875 rx_chn->psil_paired = false; 876 877 /* parse of udmap channel */ 878 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 879 &rx_chn->common, false); 880 if (ret) 881 goto err; 882 883 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, 884 rx_chn->common.psdata_size, 885 rx_chn->common.swdata_size); 886 887 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, 888 sizeof(*rx_chn->flows), GFP_KERNEL); 889 if (!rx_chn->flows) { 890 ret = -ENOMEM; 891 goto err; 892 } 893 894 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); 895 if (ret) 896 goto err; 897 898 for (i = 0; i < rx_chn->flow_num; i++) 899 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; 900 901 k3_udma_glue_dump_rx_chn(rx_chn); 902 903 return rx_chn; 904 905 err: 906 k3_udma_glue_release_rx_chn(rx_chn); 907 return ERR_PTR(ret); 908 } 909 910 struct k3_udma_glue_rx_channel * 911 k3_udma_glue_request_rx_chn(struct device *dev, const char *name, 912 struct k3_udma_glue_rx_channel_cfg *cfg) 913 { 914 if (cfg->remote) 915 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg); 916 else 917 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg); 918 } 919 EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn); 920 921 void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 922 { 923 int i; 924 925 if (IS_ERR_OR_NULL(rx_chn->common.udmax)) 926 return; 927 928 if (rx_chn->psil_paired) { 929 xudma_navss_psil_unpair(rx_chn->common.udmax, 930 rx_chn->common.src_thread, 931 rx_chn->common.dst_thread); 932 rx_chn->psil_paired = false; 933 } 934 935 for (i = 0; i < rx_chn->flow_num; i++) 936 k3_udma_glue_release_rx_flow(rx_chn, i); 937 938 if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) 939 xudma_free_gp_rflow_range(rx_chn->common.udmax, 940 rx_chn->flow_id_base, 941 rx_chn->flow_num); 942 943 if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx)) 944 xudma_rchan_put(rx_chn->common.udmax, 945 rx_chn->udma_rchanx); 946 } 947 EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn); 948 949 int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn, 950 u32 flow_idx, 951 struct k3_udma_glue_rx_flow_cfg *flow_cfg) 952 { 953 if (flow_idx >= rx_chn->flow_num) 954 return -EINVAL; 955 956 return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg); 957 } 958 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init); 959 960 u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn, 961 u32 flow_idx) 962 { 963 struct k3_udma_glue_rx_flow *flow; 964 965 if (flow_idx >= rx_chn->flow_num) 966 return -EINVAL; 967 968 flow = &rx_chn->flows[flow_idx]; 969 970 return k3_ringacc_get_ring_id(flow->ringrxfdq); 971 } 972 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id); 973 974 u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn) 975 { 976 return rx_chn->flow_id_base; 977 } 978 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base); 979 980 int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn, 981 u32 flow_idx) 982 { 983 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 984 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 985 struct device *dev = rx_chn->common.dev; 986 struct ti_sci_msg_rm_udmap_flow_cfg req; 987 int rx_ring_id; 988 int rx_ringfdq_id; 989 int ret = 0; 990 991 if (!rx_chn->remote) 992 return -EINVAL; 993 994 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx); 995 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq); 996 997 memset(&req, 0, sizeof(req)); 998 999 req.valid_params = 1000 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1001 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1002 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1003 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1004 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1005 req.nav_id = tisci_rm->tisci_dev_id; 1006 req.flow_index = flow->udma_rflow_id; 1007 req.rx_dest_qnum = rx_ring_id; 1008 req.rx_fdq0_sz0_qnum = rx_ringfdq_id; 1009 req.rx_fdq1_qnum = rx_ringfdq_id; 1010 req.rx_fdq2_qnum = rx_ringfdq_id; 1011 req.rx_fdq3_qnum = rx_ringfdq_id; 1012 1013 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 1014 if (ret) { 1015 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id, 1016 ret); 1017 } 1018 1019 return ret; 1020 } 1021 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable); 1022 1023 int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn, 1024 u32 flow_idx) 1025 { 1026 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 1027 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 1028 struct device *dev = rx_chn->common.dev; 1029 struct ti_sci_msg_rm_udmap_flow_cfg req; 1030 int ret = 0; 1031 1032 if (!rx_chn->remote) 1033 return -EINVAL; 1034 1035 memset(&req, 0, sizeof(req)); 1036 req.valid_params = 1037 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1038 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1039 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1040 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1041 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1042 req.nav_id = tisci_rm->tisci_dev_id; 1043 req.flow_index = flow->udma_rflow_id; 1044 req.rx_dest_qnum = TI_SCI_RESOURCE_NULL; 1045 req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL; 1046 req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL; 1047 req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL; 1048 req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL; 1049 1050 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 1051 if (ret) { 1052 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id, 1053 ret); 1054 } 1055 1056 return ret; 1057 } 1058 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable); 1059 1060 int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1061 { 1062 u32 rxrt_ctl; 1063 1064 if (rx_chn->remote) 1065 return -EINVAL; 1066 1067 if (rx_chn->flows_ready < rx_chn->flow_num) 1068 return -EINVAL; 1069 1070 rxrt_ctl = xudma_rchanrt_read(rx_chn->udma_rchanx, 1071 UDMA_RCHAN_RT_CTL_REG); 1072 rxrt_ctl |= UDMA_CHAN_RT_CTL_EN; 1073 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG, 1074 rxrt_ctl); 1075 1076 xudma_rchanrt_write(rx_chn->udma_rchanx, 1077 UDMA_RCHAN_RT_PEER_RT_EN_REG, 1078 UDMA_PEER_RT_EN_ENABLE); 1079 1080 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en"); 1081 return 0; 1082 } 1083 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn); 1084 1085 void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1086 { 1087 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1"); 1088 1089 xudma_rchanrt_write(rx_chn->udma_rchanx, 1090 UDMA_RCHAN_RT_PEER_RT_EN_REG, 1091 0); 1092 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG, 0); 1093 1094 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2"); 1095 } 1096 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn); 1097 1098 void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1099 bool sync) 1100 { 1101 int i = 0; 1102 u32 val; 1103 1104 if (rx_chn->remote) 1105 return; 1106 1107 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1"); 1108 1109 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_PEER_RT_EN_REG, 1110 UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN); 1111 1112 val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG); 1113 1114 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { 1115 val = xudma_rchanrt_read(rx_chn->udma_rchanx, 1116 UDMA_RCHAN_RT_CTL_REG); 1117 udelay(1); 1118 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { 1119 dev_err(rx_chn->common.dev, "RX tdown timeout\n"); 1120 break; 1121 } 1122 i++; 1123 } 1124 1125 val = xudma_rchanrt_read(rx_chn->udma_rchanx, 1126 UDMA_RCHAN_RT_PEER_RT_EN_REG); 1127 if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) 1128 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n"); 1129 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2"); 1130 } 1131 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn); 1132 1133 void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1134 u32 flow_num, void *data, 1135 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq) 1136 { 1137 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1138 struct device *dev = rx_chn->common.dev; 1139 dma_addr_t desc_dma; 1140 int occ_rx, i, ret; 1141 1142 /* reset RXCQ as it is not input for udma - expected to be empty */ 1143 occ_rx = k3_ringacc_ring_get_occ(flow->ringrx); 1144 dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx); 1145 if (flow->ringrx) 1146 k3_ringacc_ring_reset(flow->ringrx); 1147 1148 /* Skip RX FDQ in case one FDQ is used for the set of flows */ 1149 if (skip_fdq) 1150 return; 1151 1152 /* 1153 * RX FDQ reset need to be special way as it is input for udma and its 1154 * state cached by udma, so: 1155 * 1) save RX FDQ occ 1156 * 2) clean up RX FDQ and call callback .cleanup() for each desc 1157 * 3) reset RX FDQ in a special way 1158 */ 1159 occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq); 1160 dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx); 1161 1162 for (i = 0; i < occ_rx; i++) { 1163 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma); 1164 if (ret) { 1165 dev_err(dev, "RX reset pop %d\n", ret); 1166 break; 1167 } 1168 cleanup(data, desc_dma); 1169 } 1170 1171 k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx); 1172 } 1173 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn); 1174 1175 int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1176 u32 flow_num, struct cppi5_host_desc_t *desc_rx, 1177 dma_addr_t desc_dma) 1178 { 1179 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1180 1181 return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma); 1182 } 1183 EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn); 1184 1185 int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1186 u32 flow_num, dma_addr_t *desc_dma) 1187 { 1188 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1189 1190 return k3_ringacc_ring_pop(flow->ringrx, desc_dma); 1191 } 1192 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn); 1193 1194 int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn, 1195 u32 flow_num) 1196 { 1197 struct k3_udma_glue_rx_flow *flow; 1198 1199 flow = &rx_chn->flows[flow_num]; 1200 1201 flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx); 1202 1203 return flow->virq; 1204 } 1205 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq); 1206