1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2021, MediaTek Inc. 4 * Copyright (c) 2021-2022, Intel Corporation. 5 * 6 * Authors: 7 * Amir Hanania <amir.hanania@intel.com> 8 * Haijun Liu <haijun.liu@mediatek.com> 9 * Eliot Lee <eliot.lee@intel.com> 10 * Moises Veleta <moises.veleta@intel.com> 11 * Ricardo Martinez <ricardo.martinez@linux.intel.com> 12 * 13 * Contributors: 14 * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> 15 * Sreehari Kancharla <sreehari.kancharla@intel.com> 16 */ 17 18 #include <linux/atomic.h> 19 #include <linux/bitfield.h> 20 #include <linux/delay.h> 21 #include <linux/device.h> 22 #include <linux/dma-direction.h> 23 #include <linux/dma-mapping.h> 24 #include <linux/err.h> 25 #include <linux/gfp.h> 26 #include <linux/kernel.h> 27 #include <linux/kthread.h> 28 #include <linux/list.h> 29 #include <linux/minmax.h> 30 #include <linux/netdevice.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/sched.h> 33 #include <linux/spinlock.h> 34 #include <linux/skbuff.h> 35 #include <linux/types.h> 36 #include <linux/wait.h> 37 #include <linux/workqueue.h> 38 39 #include "t7xx_dpmaif.h" 40 #include "t7xx_hif_dpmaif.h" 41 #include "t7xx_hif_dpmaif_tx.h" 42 #include "t7xx_pci.h" 43 44 #define DPMAIF_SKB_TX_BURST_CNT 5 45 #define DPMAIF_DRB_LIST_LEN 6144 46 47 /* DRB dtype */ 48 #define DES_DTYP_PD 0 49 #define DES_DTYP_MSG 1 50 51 static unsigned int t7xx_dpmaif_update_drb_rd_idx(struct dpmaif_ctrl *dpmaif_ctrl, 52 unsigned int q_num) 53 { 54 struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; 55 unsigned int old_sw_rd_idx, new_hw_rd_idx, drb_cnt; 56 unsigned long flags; 57 58 if (!txq->que_started) 59 return 0; 60 61 old_sw_rd_idx = txq->drb_rd_idx; 62 new_hw_rd_idx = t7xx_dpmaif_ul_get_rd_idx(&dpmaif_ctrl->hw_info, q_num); 63 if (new_hw_rd_idx >= DPMAIF_DRB_LIST_LEN) { 64 dev_err(dpmaif_ctrl->dev, "Out of range read index: %u\n", new_hw_rd_idx); 65 return 0; 66 } 67 68 if (old_sw_rd_idx <= new_hw_rd_idx) 69 drb_cnt = new_hw_rd_idx - old_sw_rd_idx; 70 else 71 drb_cnt = txq->drb_size_cnt - old_sw_rd_idx + new_hw_rd_idx; 72 73 spin_lock_irqsave(&txq->tx_lock, flags); 74 txq->drb_rd_idx = new_hw_rd_idx; 75 spin_unlock_irqrestore(&txq->tx_lock, flags); 76 77 return drb_cnt; 78 } 79 80 static unsigned int t7xx_dpmaif_release_tx_buffer(struct dpmaif_ctrl *dpmaif_ctrl, 81 unsigned int q_num, unsigned int release_cnt) 82 { 83 struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; 84 struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; 85 struct dpmaif_drb_skb *cur_drb_skb, *drb_skb_base; 86 struct dpmaif_drb *cur_drb, *drb_base; 87 unsigned int drb_cnt, i, cur_idx; 88 unsigned long flags; 89 90 drb_skb_base = txq->drb_skb_base; 91 drb_base = txq->drb_base; 92 93 spin_lock_irqsave(&txq->tx_lock, flags); 94 drb_cnt = txq->drb_size_cnt; 95 cur_idx = txq->drb_release_rd_idx; 96 spin_unlock_irqrestore(&txq->tx_lock, flags); 97 98 for (i = 0; i < release_cnt; i++) { 99 cur_drb = drb_base + cur_idx; 100 if (FIELD_GET(DRB_HDR_DTYP, le32_to_cpu(cur_drb->header)) == DES_DTYP_PD) { 101 cur_drb_skb = drb_skb_base + cur_idx; 102 if (!cur_drb_skb->is_msg) 103 dma_unmap_single(dpmaif_ctrl->dev, cur_drb_skb->bus_addr, 104 cur_drb_skb->data_len, DMA_TO_DEVICE); 105 106 if (!FIELD_GET(DRB_HDR_CONT, le32_to_cpu(cur_drb->header))) { 107 if (!cur_drb_skb->skb) { 108 dev_err(dpmaif_ctrl->dev, 109 "txq%u: DRB check fail, invalid skb\n", q_num); 110 continue; 111 } 112 113 dev_kfree_skb_any(cur_drb_skb->skb); 114 } 115 116 cur_drb_skb->skb = NULL; 117 } 118 119 spin_lock_irqsave(&txq->tx_lock, flags); 120 cur_idx = t7xx_ring_buf_get_next_wr_idx(drb_cnt, cur_idx); 121 txq->drb_release_rd_idx = cur_idx; 122 spin_unlock_irqrestore(&txq->tx_lock, flags); 123 124 if (atomic_inc_return(&txq->tx_budget) > txq->drb_size_cnt / 8) 125 cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_IRQ, txq->index); 126 } 127 128 if (FIELD_GET(DRB_HDR_CONT, le32_to_cpu(cur_drb->header))) 129 dev_err(dpmaif_ctrl->dev, "txq%u: DRB not marked as the last one\n", q_num); 130 131 return i; 132 } 133 134 static int t7xx_dpmaif_tx_release(struct dpmaif_ctrl *dpmaif_ctrl, 135 unsigned int q_num, unsigned int budget) 136 { 137 struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; 138 unsigned int rel_cnt, real_rel_cnt; 139 140 /* Update read index from HW */ 141 t7xx_dpmaif_update_drb_rd_idx(dpmaif_ctrl, q_num); 142 143 rel_cnt = t7xx_ring_buf_rd_wr_count(txq->drb_size_cnt, txq->drb_release_rd_idx, 144 txq->drb_rd_idx, DPMAIF_READ); 145 146 real_rel_cnt = min_not_zero(budget, rel_cnt); 147 if (real_rel_cnt) 148 real_rel_cnt = t7xx_dpmaif_release_tx_buffer(dpmaif_ctrl, q_num, real_rel_cnt); 149 150 return real_rel_cnt < rel_cnt ? -EAGAIN : 0; 151 } 152 153 static bool t7xx_dpmaif_drb_ring_not_empty(struct dpmaif_tx_queue *txq) 154 { 155 return !!t7xx_dpmaif_update_drb_rd_idx(txq->dpmaif_ctrl, txq->index); 156 } 157 158 static void t7xx_dpmaif_tx_done(struct work_struct *work) 159 { 160 struct dpmaif_tx_queue *txq = container_of(work, struct dpmaif_tx_queue, dpmaif_tx_work); 161 struct dpmaif_ctrl *dpmaif_ctrl = txq->dpmaif_ctrl; 162 struct dpmaif_hw_info *hw_info; 163 int ret; 164 165 ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev); 166 if (ret < 0 && ret != -EACCES) 167 return; 168 169 /* The device may be in low power state. Disable sleep if needed */ 170 t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev); 171 if (t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev)) { 172 hw_info = &dpmaif_ctrl->hw_info; 173 ret = t7xx_dpmaif_tx_release(dpmaif_ctrl, txq->index, txq->drb_size_cnt); 174 if (ret == -EAGAIN || 175 (t7xx_dpmaif_ul_clr_done(hw_info, txq->index) && 176 t7xx_dpmaif_drb_ring_not_empty(txq))) { 177 queue_work(dpmaif_ctrl->txq[txq->index].worker, 178 &dpmaif_ctrl->txq[txq->index].dpmaif_tx_work); 179 /* Give the device time to enter the low power state */ 180 t7xx_dpmaif_clr_ip_busy_sts(hw_info); 181 } else { 182 t7xx_dpmaif_clr_ip_busy_sts(hw_info); 183 t7xx_dpmaif_unmask_ulq_intr(hw_info, txq->index); 184 } 185 } 186 187 t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev); 188 pm_runtime_mark_last_busy(dpmaif_ctrl->dev); 189 pm_runtime_put_autosuspend(dpmaif_ctrl->dev); 190 } 191 192 static void t7xx_setup_msg_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, 193 unsigned int cur_idx, unsigned int pkt_len, unsigned int count_l, 194 unsigned int channel_id) 195 { 196 struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base; 197 struct dpmaif_drb *drb = drb_base + cur_idx; 198 199 drb->header = cpu_to_le32(FIELD_PREP(DRB_HDR_DTYP, DES_DTYP_MSG) | 200 FIELD_PREP(DRB_HDR_CONT, 1) | 201 FIELD_PREP(DRB_HDR_DATA_LEN, pkt_len)); 202 203 drb->msg.msg_hdr = cpu_to_le32(FIELD_PREP(DRB_MSG_COUNT_L, count_l) | 204 FIELD_PREP(DRB_MSG_CHANNEL_ID, channel_id) | 205 FIELD_PREP(DRB_MSG_L4_CHK, 1)); 206 } 207 208 static void t7xx_setup_payload_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, 209 unsigned int cur_idx, dma_addr_t data_addr, 210 unsigned int pkt_size, bool last_one) 211 { 212 struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base; 213 struct dpmaif_drb *drb = drb_base + cur_idx; 214 u32 header; 215 216 header = FIELD_PREP(DRB_HDR_DTYP, DES_DTYP_PD) | FIELD_PREP(DRB_HDR_DATA_LEN, pkt_size); 217 if (!last_one) 218 header |= FIELD_PREP(DRB_HDR_CONT, 1); 219 220 drb->header = cpu_to_le32(header); 221 drb->pd.data_addr_l = cpu_to_le32(lower_32_bits(data_addr)); 222 drb->pd.data_addr_h = cpu_to_le32(upper_32_bits(data_addr)); 223 } 224 225 static void t7xx_record_drb_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, 226 unsigned int cur_idx, struct sk_buff *skb, bool is_msg, 227 bool is_frag, bool is_last_one, dma_addr_t bus_addr, 228 unsigned int data_len) 229 { 230 struct dpmaif_drb_skb *drb_skb_base = dpmaif_ctrl->txq[q_num].drb_skb_base; 231 struct dpmaif_drb_skb *drb_skb = drb_skb_base + cur_idx; 232 233 drb_skb->skb = skb; 234 drb_skb->bus_addr = bus_addr; 235 drb_skb->data_len = data_len; 236 drb_skb->index = cur_idx; 237 drb_skb->is_msg = is_msg; 238 drb_skb->is_frag = is_frag; 239 drb_skb->is_last = is_last_one; 240 } 241 242 static int t7xx_dpmaif_add_skb_to_ring(struct dpmaif_ctrl *dpmaif_ctrl, struct sk_buff *skb) 243 { 244 struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; 245 unsigned int wr_cnt, send_cnt, payload_cnt; 246 unsigned int cur_idx, drb_wr_idx_backup; 247 struct skb_shared_info *shinfo; 248 struct dpmaif_tx_queue *txq; 249 struct t7xx_skb_cb *skb_cb; 250 unsigned long flags; 251 252 skb_cb = T7XX_SKB_CB(skb); 253 txq = &dpmaif_ctrl->txq[skb_cb->txq_number]; 254 if (!txq->que_started || dpmaif_ctrl->state != DPMAIF_STATE_PWRON) 255 return -ENODEV; 256 257 atomic_set(&txq->tx_processing, 1); 258 /* Ensure tx_processing is changed to 1 before actually begin TX flow */ 259 smp_mb(); 260 261 shinfo = skb_shinfo(skb); 262 if (shinfo->frag_list) 263 dev_warn_ratelimited(dpmaif_ctrl->dev, "frag_list not supported\n"); 264 265 payload_cnt = shinfo->nr_frags + 1; 266 /* nr_frags: frag cnt, 1: skb->data, 1: msg DRB */ 267 send_cnt = payload_cnt + 1; 268 269 spin_lock_irqsave(&txq->tx_lock, flags); 270 cur_idx = txq->drb_wr_idx; 271 drb_wr_idx_backup = cur_idx; 272 txq->drb_wr_idx += send_cnt; 273 if (txq->drb_wr_idx >= txq->drb_size_cnt) 274 txq->drb_wr_idx -= txq->drb_size_cnt; 275 t7xx_setup_msg_drb(dpmaif_ctrl, txq->index, cur_idx, skb->len, 0, skb_cb->netif_idx); 276 t7xx_record_drb_skb(dpmaif_ctrl, txq->index, cur_idx, skb, true, 0, 0, 0, 0); 277 spin_unlock_irqrestore(&txq->tx_lock, flags); 278 279 for (wr_cnt = 0; wr_cnt < payload_cnt; wr_cnt++) { 280 bool is_frag, is_last_one = wr_cnt == payload_cnt - 1; 281 unsigned int data_len; 282 dma_addr_t bus_addr; 283 void *data_addr; 284 285 if (!wr_cnt) { 286 data_len = skb_headlen(skb); 287 data_addr = skb->data; 288 is_frag = false; 289 } else { 290 skb_frag_t *frag = shinfo->frags + wr_cnt - 1; 291 292 data_len = skb_frag_size(frag); 293 data_addr = skb_frag_address(frag); 294 is_frag = true; 295 } 296 297 bus_addr = dma_map_single(dpmaif_ctrl->dev, data_addr, data_len, DMA_TO_DEVICE); 298 if (dma_mapping_error(dpmaif_ctrl->dev, bus_addr)) 299 goto unmap_buffers; 300 301 cur_idx = t7xx_ring_buf_get_next_wr_idx(txq->drb_size_cnt, cur_idx); 302 303 spin_lock_irqsave(&txq->tx_lock, flags); 304 t7xx_setup_payload_drb(dpmaif_ctrl, txq->index, cur_idx, bus_addr, data_len, 305 is_last_one); 306 t7xx_record_drb_skb(dpmaif_ctrl, txq->index, cur_idx, skb, false, is_frag, 307 is_last_one, bus_addr, data_len); 308 spin_unlock_irqrestore(&txq->tx_lock, flags); 309 } 310 311 if (atomic_sub_return(send_cnt, &txq->tx_budget) <= (MAX_SKB_FRAGS + 2)) 312 cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq->index); 313 314 atomic_set(&txq->tx_processing, 0); 315 316 return 0; 317 318 unmap_buffers: 319 while (wr_cnt--) { 320 struct dpmaif_drb_skb *drb_skb = txq->drb_skb_base; 321 322 cur_idx = cur_idx ? cur_idx - 1 : txq->drb_size_cnt - 1; 323 drb_skb += cur_idx; 324 dma_unmap_single(dpmaif_ctrl->dev, drb_skb->bus_addr, 325 drb_skb->data_len, DMA_TO_DEVICE); 326 } 327 328 txq->drb_wr_idx = drb_wr_idx_backup; 329 atomic_set(&txq->tx_processing, 0); 330 331 return -ENOMEM; 332 } 333 334 static bool t7xx_tx_lists_are_all_empty(const struct dpmaif_ctrl *dpmaif_ctrl) 335 { 336 int i; 337 338 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 339 if (!skb_queue_empty(&dpmaif_ctrl->txq[i].tx_skb_head)) 340 return false; 341 } 342 343 return true; 344 } 345 346 /* Currently, only the default TX queue is used */ 347 static struct dpmaif_tx_queue *t7xx_select_tx_queue(struct dpmaif_ctrl *dpmaif_ctrl) 348 { 349 struct dpmaif_tx_queue *txq; 350 351 txq = &dpmaif_ctrl->txq[DPMAIF_TX_DEFAULT_QUEUE]; 352 if (!txq->que_started) 353 return NULL; 354 355 return txq; 356 } 357 358 static unsigned int t7xx_txq_drb_wr_available(struct dpmaif_tx_queue *txq) 359 { 360 return t7xx_ring_buf_rd_wr_count(txq->drb_size_cnt, txq->drb_release_rd_idx, 361 txq->drb_wr_idx, DPMAIF_WRITE); 362 } 363 364 static unsigned int t7xx_skb_drb_cnt(struct sk_buff *skb) 365 { 366 /* Normal DRB (frags data + skb linear data) + msg DRB */ 367 return skb_shinfo(skb)->nr_frags + 2; 368 } 369 370 static int t7xx_txq_burst_send_skb(struct dpmaif_tx_queue *txq) 371 { 372 unsigned int drb_remain_cnt, i; 373 unsigned int send_drb_cnt; 374 int drb_cnt = 0; 375 int ret = 0; 376 377 drb_remain_cnt = t7xx_txq_drb_wr_available(txq); 378 379 for (i = 0; i < DPMAIF_SKB_TX_BURST_CNT; i++) { 380 struct sk_buff *skb; 381 382 skb = skb_peek(&txq->tx_skb_head); 383 if (!skb) 384 break; 385 386 send_drb_cnt = t7xx_skb_drb_cnt(skb); 387 if (drb_remain_cnt < send_drb_cnt) { 388 drb_remain_cnt = t7xx_txq_drb_wr_available(txq); 389 continue; 390 } 391 392 drb_remain_cnt -= send_drb_cnt; 393 394 ret = t7xx_dpmaif_add_skb_to_ring(txq->dpmaif_ctrl, skb); 395 if (ret < 0) { 396 dev_err(txq->dpmaif_ctrl->dev, 397 "Failed to add skb to device's ring: %d\n", ret); 398 break; 399 } 400 401 drb_cnt += send_drb_cnt; 402 skb_unlink(skb, &txq->tx_skb_head); 403 } 404 405 if (drb_cnt > 0) 406 return drb_cnt; 407 408 return ret; 409 } 410 411 static void t7xx_do_tx_hw_push(struct dpmaif_ctrl *dpmaif_ctrl) 412 { 413 bool wait_disable_sleep = true; 414 415 do { 416 struct dpmaif_tx_queue *txq; 417 int drb_send_cnt; 418 419 txq = t7xx_select_tx_queue(dpmaif_ctrl); 420 if (!txq) 421 return; 422 423 drb_send_cnt = t7xx_txq_burst_send_skb(txq); 424 if (drb_send_cnt <= 0) { 425 usleep_range(10, 20); 426 cond_resched(); 427 continue; 428 } 429 430 /* Wait for the PCIe resource to unlock */ 431 if (wait_disable_sleep) { 432 if (!t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev)) 433 return; 434 435 wait_disable_sleep = false; 436 } 437 438 t7xx_dpmaif_ul_update_hw_drb_cnt(&dpmaif_ctrl->hw_info, txq->index, 439 drb_send_cnt * DPMAIF_UL_DRB_SIZE_WORD); 440 441 cond_resched(); 442 } while (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) && !kthread_should_stop() && 443 (dpmaif_ctrl->state == DPMAIF_STATE_PWRON)); 444 } 445 446 static int t7xx_dpmaif_tx_hw_push_thread(void *arg) 447 { 448 struct dpmaif_ctrl *dpmaif_ctrl = arg; 449 int ret; 450 451 while (!kthread_should_stop()) { 452 if (t7xx_tx_lists_are_all_empty(dpmaif_ctrl) || 453 dpmaif_ctrl->state != DPMAIF_STATE_PWRON) { 454 if (wait_event_interruptible(dpmaif_ctrl->tx_wq, 455 (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) && 456 dpmaif_ctrl->state == DPMAIF_STATE_PWRON) || 457 kthread_should_stop())) 458 continue; 459 460 if (kthread_should_stop()) 461 break; 462 } 463 464 ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev); 465 if (ret < 0 && ret != -EACCES) 466 return ret; 467 468 t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev); 469 t7xx_do_tx_hw_push(dpmaif_ctrl); 470 t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev); 471 pm_runtime_mark_last_busy(dpmaif_ctrl->dev); 472 pm_runtime_put_autosuspend(dpmaif_ctrl->dev); 473 } 474 475 return 0; 476 } 477 478 int t7xx_dpmaif_tx_thread_init(struct dpmaif_ctrl *dpmaif_ctrl) 479 { 480 init_waitqueue_head(&dpmaif_ctrl->tx_wq); 481 dpmaif_ctrl->tx_thread = kthread_run(t7xx_dpmaif_tx_hw_push_thread, 482 dpmaif_ctrl, "dpmaif_tx_hw_push"); 483 return PTR_ERR_OR_ZERO(dpmaif_ctrl->tx_thread); 484 } 485 486 void t7xx_dpmaif_tx_thread_rel(struct dpmaif_ctrl *dpmaif_ctrl) 487 { 488 if (dpmaif_ctrl->tx_thread) 489 kthread_stop(dpmaif_ctrl->tx_thread); 490 } 491 492 /** 493 * t7xx_dpmaif_tx_send_skb() - Add skb to the transmit queue. 494 * @dpmaif_ctrl: Pointer to struct dpmaif_ctrl. 495 * @txq_number: Queue number to xmit on. 496 * @skb: Pointer to the skb to transmit. 497 * 498 * Add the skb to the queue of the skbs to be transmit. 499 * Wake up the thread that push the skbs from the queue to the HW. 500 * 501 * Return: 502 * * 0 - Success. 503 * * -EBUSY - Tx budget exhausted. 504 * In normal circumstances t7xx_dpmaif_add_skb_to_ring() must report the txq full 505 * state to prevent this error condition. 506 */ 507 int t7xx_dpmaif_tx_send_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int txq_number, 508 struct sk_buff *skb) 509 { 510 struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[txq_number]; 511 struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; 512 struct t7xx_skb_cb *skb_cb; 513 514 if (atomic_read(&txq->tx_budget) <= t7xx_skb_drb_cnt(skb)) { 515 cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq_number); 516 return -EBUSY; 517 } 518 519 skb_cb = T7XX_SKB_CB(skb); 520 skb_cb->txq_number = txq_number; 521 skb_queue_tail(&txq->tx_skb_head, skb); 522 wake_up(&dpmaif_ctrl->tx_wq); 523 524 return 0; 525 } 526 527 void t7xx_dpmaif_irq_tx_done(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int que_mask) 528 { 529 int i; 530 531 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 532 if (que_mask & BIT(i)) 533 queue_work(dpmaif_ctrl->txq[i].worker, &dpmaif_ctrl->txq[i].dpmaif_tx_work); 534 } 535 } 536 537 static int t7xx_dpmaif_tx_drb_buf_init(struct dpmaif_tx_queue *txq) 538 { 539 size_t brb_skb_size, brb_pd_size; 540 541 brb_pd_size = DPMAIF_DRB_LIST_LEN * sizeof(struct dpmaif_drb); 542 brb_skb_size = DPMAIF_DRB_LIST_LEN * sizeof(struct dpmaif_drb_skb); 543 544 txq->drb_size_cnt = DPMAIF_DRB_LIST_LEN; 545 546 /* For HW && AP SW */ 547 txq->drb_base = dma_alloc_coherent(txq->dpmaif_ctrl->dev, brb_pd_size, 548 &txq->drb_bus_addr, GFP_KERNEL | __GFP_ZERO); 549 if (!txq->drb_base) 550 return -ENOMEM; 551 552 /* For AP SW to record the skb information */ 553 txq->drb_skb_base = devm_kzalloc(txq->dpmaif_ctrl->dev, brb_skb_size, GFP_KERNEL); 554 if (!txq->drb_skb_base) { 555 dma_free_coherent(txq->dpmaif_ctrl->dev, brb_pd_size, 556 txq->drb_base, txq->drb_bus_addr); 557 return -ENOMEM; 558 } 559 560 return 0; 561 } 562 563 static void t7xx_dpmaif_tx_free_drb_skb(struct dpmaif_tx_queue *txq) 564 { 565 struct dpmaif_drb_skb *drb_skb, *drb_skb_base = txq->drb_skb_base; 566 unsigned int i; 567 568 if (!drb_skb_base) 569 return; 570 571 for (i = 0; i < txq->drb_size_cnt; i++) { 572 drb_skb = drb_skb_base + i; 573 if (!drb_skb->skb) 574 continue; 575 576 if (!drb_skb->is_msg) 577 dma_unmap_single(txq->dpmaif_ctrl->dev, drb_skb->bus_addr, 578 drb_skb->data_len, DMA_TO_DEVICE); 579 580 if (drb_skb->is_last) { 581 dev_kfree_skb(drb_skb->skb); 582 drb_skb->skb = NULL; 583 } 584 } 585 } 586 587 static void t7xx_dpmaif_tx_drb_buf_rel(struct dpmaif_tx_queue *txq) 588 { 589 if (txq->drb_base) 590 dma_free_coherent(txq->dpmaif_ctrl->dev, 591 txq->drb_size_cnt * sizeof(struct dpmaif_drb), 592 txq->drb_base, txq->drb_bus_addr); 593 594 t7xx_dpmaif_tx_free_drb_skb(txq); 595 } 596 597 /** 598 * t7xx_dpmaif_txq_init() - Initialize TX queue. 599 * @txq: Pointer to struct dpmaif_tx_queue. 600 * 601 * Initialize the TX queue data structure and allocate memory for it to use. 602 * 603 * Return: 604 * * 0 - Success. 605 * * -ERROR - Error code from failure sub-initializations. 606 */ 607 int t7xx_dpmaif_txq_init(struct dpmaif_tx_queue *txq) 608 { 609 int ret; 610 611 skb_queue_head_init(&txq->tx_skb_head); 612 init_waitqueue_head(&txq->req_wq); 613 atomic_set(&txq->tx_budget, DPMAIF_DRB_LIST_LEN); 614 615 ret = t7xx_dpmaif_tx_drb_buf_init(txq); 616 if (ret) { 617 dev_err(txq->dpmaif_ctrl->dev, "Failed to initialize DRB buffers: %d\n", ret); 618 return ret; 619 } 620 621 txq->worker = alloc_ordered_workqueue("md_dpmaif_tx%d_worker", 622 WQ_MEM_RECLAIM | (txq->index ? 0 : WQ_HIGHPRI), 623 txq->index); 624 if (!txq->worker) 625 return -ENOMEM; 626 627 INIT_WORK(&txq->dpmaif_tx_work, t7xx_dpmaif_tx_done); 628 spin_lock_init(&txq->tx_lock); 629 630 return 0; 631 } 632 633 void t7xx_dpmaif_txq_free(struct dpmaif_tx_queue *txq) 634 { 635 if (txq->worker) 636 destroy_workqueue(txq->worker); 637 638 skb_queue_purge(&txq->tx_skb_head); 639 t7xx_dpmaif_tx_drb_buf_rel(txq); 640 } 641 642 void t7xx_dpmaif_tx_stop(struct dpmaif_ctrl *dpmaif_ctrl) 643 { 644 int i; 645 646 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 647 struct dpmaif_tx_queue *txq; 648 int count = 0; 649 650 txq = &dpmaif_ctrl->txq[i]; 651 txq->que_started = false; 652 /* Make sure TXQ is disabled */ 653 smp_mb(); 654 655 /* Wait for active Tx to be done */ 656 while (atomic_read(&txq->tx_processing)) { 657 if (++count >= DPMAIF_MAX_CHECK_COUNT) { 658 dev_err(dpmaif_ctrl->dev, "TX queue stop failed\n"); 659 break; 660 } 661 } 662 } 663 } 664 665 static void t7xx_dpmaif_txq_flush_rel(struct dpmaif_tx_queue *txq) 666 { 667 txq->que_started = false; 668 669 cancel_work_sync(&txq->dpmaif_tx_work); 670 flush_work(&txq->dpmaif_tx_work); 671 t7xx_dpmaif_tx_free_drb_skb(txq); 672 673 txq->drb_rd_idx = 0; 674 txq->drb_wr_idx = 0; 675 txq->drb_release_rd_idx = 0; 676 } 677 678 void t7xx_dpmaif_tx_clear(struct dpmaif_ctrl *dpmaif_ctrl) 679 { 680 int i; 681 682 for (i = 0; i < DPMAIF_TXQ_NUM; i++) 683 t7xx_dpmaif_txq_flush_rel(&dpmaif_ctrl->txq[i]); 684 } 685