1 // SPDX-License-Identifier: ISC 2 /* 3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> 4 */ 5 6 #include <linux/dma-mapping.h> 7 #include "mt76.h" 8 #include "dma.h" 9 10 static int 11 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, 12 int idx, int n_desc, int bufsize, 13 u32 ring_base) 14 { 15 int size; 16 int i; 17 18 spin_lock_init(&q->lock); 19 20 q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE; 21 q->ndesc = n_desc; 22 q->buf_size = bufsize; 23 q->hw_idx = idx; 24 25 size = q->ndesc * sizeof(struct mt76_desc); 26 q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL); 27 if (!q->desc) 28 return -ENOMEM; 29 30 size = q->ndesc * sizeof(*q->entry); 31 q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL); 32 if (!q->entry) 33 return -ENOMEM; 34 35 /* clear descriptors */ 36 for (i = 0; i < q->ndesc; i++) 37 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE); 38 39 writel(q->desc_dma, &q->regs->desc_base); 40 writel(0, &q->regs->cpu_idx); 41 writel(0, &q->regs->dma_idx); 42 writel(q->ndesc, &q->regs->ring_size); 43 44 return 0; 45 } 46 47 static int 48 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q, 49 struct mt76_queue_buf *buf, int nbufs, u32 info, 50 struct sk_buff *skb, void *txwi) 51 { 52 struct mt76_desc *desc; 53 u32 ctrl; 54 int i, idx = -1; 55 56 if (txwi) 57 q->entry[q->head].txwi = DMA_DUMMY_DATA; 58 59 for (i = 0; i < nbufs; i += 2, buf += 2) { 60 u32 buf0 = buf[0].addr, buf1 = 0; 61 62 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len); 63 if (i < nbufs - 1) { 64 buf1 = buf[1].addr; 65 ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len); 66 } 67 68 if (i == nbufs - 1) 69 ctrl |= MT_DMA_CTL_LAST_SEC0; 70 else if (i == nbufs - 2) 71 ctrl |= MT_DMA_CTL_LAST_SEC1; 72 73 idx = q->head; 74 q->head = (q->head + 1) % q->ndesc; 75 76 desc = &q->desc[idx]; 77 78 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0)); 79 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1)); 80 WRITE_ONCE(desc->info, cpu_to_le32(info)); 81 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl)); 82 83 q->queued++; 84 } 85 86 q->entry[idx].txwi = txwi; 87 q->entry[idx].skb = skb; 88 89 return idx; 90 } 91 92 static void 93 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx, 94 struct mt76_queue_entry *prev_e) 95 { 96 struct mt76_queue_entry *e = &q->entry[idx]; 97 __le32 __ctrl = READ_ONCE(q->desc[idx].ctrl); 98 u32 ctrl = le32_to_cpu(__ctrl); 99 100 if (!e->txwi || !e->skb) { 101 __le32 addr = READ_ONCE(q->desc[idx].buf0); 102 u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl); 103 104 dma_unmap_single(dev->dev, le32_to_cpu(addr), len, 105 DMA_TO_DEVICE); 106 } 107 108 if (!(ctrl & MT_DMA_CTL_LAST_SEC0)) { 109 __le32 addr = READ_ONCE(q->desc[idx].buf1); 110 u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN1, ctrl); 111 112 dma_unmap_single(dev->dev, le32_to_cpu(addr), len, 113 DMA_TO_DEVICE); 114 } 115 116 if (e->txwi == DMA_DUMMY_DATA) 117 e->txwi = NULL; 118 119 if (e->skb == DMA_DUMMY_DATA) 120 e->skb = NULL; 121 122 *prev_e = *e; 123 memset(e, 0, sizeof(*e)); 124 } 125 126 static void 127 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) 128 { 129 writel(q->desc_dma, &q->regs->desc_base); 130 writel(q->ndesc, &q->regs->ring_size); 131 q->head = readl(&q->regs->dma_idx); 132 q->tail = q->head; 133 writel(q->head, &q->regs->cpu_idx); 134 } 135 136 static void 137 mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush) 138 { 139 struct mt76_sw_queue *sq = &dev->q_tx[qid]; 140 struct mt76_queue *q = sq->q; 141 struct mt76_queue_entry entry; 142 unsigned int n_swq_queued[4] = {}; 143 unsigned int n_queued = 0; 144 bool wake = false; 145 int i, last; 146 147 if (!q) 148 return; 149 150 if (flush) 151 last = -1; 152 else 153 last = readl(&q->regs->dma_idx); 154 155 while ((q->queued > n_queued) && q->tail != last) { 156 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry); 157 if (entry.schedule) 158 n_swq_queued[entry.qid]++; 159 160 q->tail = (q->tail + 1) % q->ndesc; 161 n_queued++; 162 163 if (entry.skb) 164 dev->drv->tx_complete_skb(dev, qid, &entry); 165 166 if (entry.txwi) { 167 if (!(dev->drv->txwi_flags & MT_TXWI_NO_FREE)) 168 mt76_put_txwi(dev, entry.txwi); 169 wake = !flush; 170 } 171 172 if (!flush && q->tail == last) 173 last = readl(&q->regs->dma_idx); 174 } 175 176 spin_lock_bh(&q->lock); 177 178 q->queued -= n_queued; 179 for (i = 0; i < ARRAY_SIZE(n_swq_queued); i++) { 180 if (!n_swq_queued[i]) 181 continue; 182 183 dev->q_tx[i].swq_queued -= n_swq_queued[i]; 184 } 185 186 if (flush) 187 mt76_dma_sync_idx(dev, q); 188 189 wake = wake && q->stopped && 190 qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8; 191 if (wake) 192 q->stopped = false; 193 194 if (!q->queued) 195 wake_up(&dev->tx_wait); 196 197 spin_unlock_bh(&q->lock); 198 199 if (wake) 200 ieee80211_wake_queue(dev->hw, qid); 201 } 202 203 static void * 204 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, 205 int *len, u32 *info, bool *more) 206 { 207 struct mt76_queue_entry *e = &q->entry[idx]; 208 struct mt76_desc *desc = &q->desc[idx]; 209 dma_addr_t buf_addr; 210 void *buf = e->buf; 211 int buf_len = SKB_WITH_OVERHEAD(q->buf_size); 212 213 buf_addr = le32_to_cpu(READ_ONCE(desc->buf0)); 214 if (len) { 215 u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl)); 216 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl); 217 *more = !(ctl & MT_DMA_CTL_LAST_SEC0); 218 } 219 220 if (info) 221 *info = le32_to_cpu(desc->info); 222 223 dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE); 224 e->buf = NULL; 225 226 return buf; 227 } 228 229 static void * 230 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, 231 int *len, u32 *info, bool *more) 232 { 233 int idx = q->tail; 234 235 *more = false; 236 if (!q->queued) 237 return NULL; 238 239 if (!flush && !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) 240 return NULL; 241 242 q->tail = (q->tail + 1) % q->ndesc; 243 q->queued--; 244 245 return mt76_dma_get_buf(dev, q, idx, len, info, more); 246 } 247 248 static void 249 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q) 250 { 251 writel(q->head, &q->regs->cpu_idx); 252 } 253 254 static int 255 mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, enum mt76_txq_id qid, 256 struct sk_buff *skb, u32 tx_info) 257 { 258 struct mt76_queue *q = dev->q_tx[qid].q; 259 struct mt76_queue_buf buf; 260 dma_addr_t addr; 261 262 addr = dma_map_single(dev->dev, skb->data, skb->len, 263 DMA_TO_DEVICE); 264 if (unlikely(dma_mapping_error(dev->dev, addr))) 265 return -ENOMEM; 266 267 buf.addr = addr; 268 buf.len = skb->len; 269 270 spin_lock_bh(&q->lock); 271 mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL); 272 mt76_dma_kick_queue(dev, q); 273 spin_unlock_bh(&q->lock); 274 275 return 0; 276 } 277 278 static int 279 mt76_dma_tx_queue_skb(struct mt76_dev *dev, enum mt76_txq_id qid, 280 struct sk_buff *skb, struct mt76_wcid *wcid, 281 struct ieee80211_sta *sta) 282 { 283 struct mt76_queue *q = dev->q_tx[qid].q; 284 struct mt76_tx_info tx_info = { 285 .skb = skb, 286 }; 287 int len, n = 0, ret = -ENOMEM; 288 struct mt76_queue_entry e; 289 struct mt76_txwi_cache *t; 290 struct sk_buff *iter; 291 dma_addr_t addr; 292 u8 *txwi; 293 294 t = mt76_get_txwi(dev); 295 if (!t) { 296 ieee80211_free_txskb(dev->hw, skb); 297 return -ENOMEM; 298 } 299 txwi = mt76_get_txwi_ptr(dev, t); 300 301 skb->prev = skb->next = NULL; 302 if (dev->drv->tx_aligned4_skbs) 303 mt76_insert_hdr_pad(skb); 304 305 len = skb_headlen(skb); 306 addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE); 307 if (unlikely(dma_mapping_error(dev->dev, addr))) 308 goto free; 309 310 tx_info.buf[n].addr = t->dma_addr; 311 tx_info.buf[n++].len = dev->drv->txwi_size; 312 tx_info.buf[n].addr = addr; 313 tx_info.buf[n++].len = len; 314 315 skb_walk_frags(skb, iter) { 316 if (n == ARRAY_SIZE(tx_info.buf)) 317 goto unmap; 318 319 addr = dma_map_single(dev->dev, iter->data, iter->len, 320 DMA_TO_DEVICE); 321 if (unlikely(dma_mapping_error(dev->dev, addr))) 322 goto unmap; 323 324 tx_info.buf[n].addr = addr; 325 tx_info.buf[n++].len = iter->len; 326 } 327 tx_info.nbuf = n; 328 329 dma_sync_single_for_cpu(dev->dev, t->dma_addr, dev->drv->txwi_size, 330 DMA_TO_DEVICE); 331 ret = dev->drv->tx_prepare_skb(dev, txwi, qid, wcid, sta, &tx_info); 332 dma_sync_single_for_device(dev->dev, t->dma_addr, dev->drv->txwi_size, 333 DMA_TO_DEVICE); 334 if (ret < 0) 335 goto unmap; 336 337 if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) { 338 ret = -ENOMEM; 339 goto unmap; 340 } 341 342 return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf, 343 tx_info.info, tx_info.skb, t); 344 345 unmap: 346 for (n--; n > 0; n--) 347 dma_unmap_single(dev->dev, tx_info.buf[n].addr, 348 tx_info.buf[n].len, DMA_TO_DEVICE); 349 350 free: 351 e.skb = tx_info.skb; 352 e.txwi = t; 353 dev->drv->tx_complete_skb(dev, qid, &e); 354 mt76_put_txwi(dev, t); 355 return ret; 356 } 357 358 static int 359 mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q) 360 { 361 dma_addr_t addr; 362 void *buf; 363 int frames = 0; 364 int len = SKB_WITH_OVERHEAD(q->buf_size); 365 int offset = q->buf_offset; 366 int idx; 367 368 spin_lock_bh(&q->lock); 369 370 while (q->queued < q->ndesc - 1) { 371 struct mt76_queue_buf qbuf; 372 373 buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC); 374 if (!buf) 375 break; 376 377 addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE); 378 if (unlikely(dma_mapping_error(dev->dev, addr))) { 379 skb_free_frag(buf); 380 break; 381 } 382 383 qbuf.addr = addr + offset; 384 qbuf.len = len - offset; 385 idx = mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL); 386 frames++; 387 } 388 389 if (frames) 390 mt76_dma_kick_queue(dev, q); 391 392 spin_unlock_bh(&q->lock); 393 394 return frames; 395 } 396 397 static void 398 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q) 399 { 400 struct page *page; 401 void *buf; 402 bool more; 403 404 spin_lock_bh(&q->lock); 405 do { 406 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more); 407 if (!buf) 408 break; 409 410 skb_free_frag(buf); 411 } while (1); 412 spin_unlock_bh(&q->lock); 413 414 if (!q->rx_page.va) 415 return; 416 417 page = virt_to_page(q->rx_page.va); 418 __page_frag_cache_drain(page, q->rx_page.pagecnt_bias); 419 memset(&q->rx_page, 0, sizeof(q->rx_page)); 420 } 421 422 static void 423 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid) 424 { 425 struct mt76_queue *q = &dev->q_rx[qid]; 426 int i; 427 428 for (i = 0; i < q->ndesc; i++) 429 q->desc[i].ctrl &= ~cpu_to_le32(MT_DMA_CTL_DMA_DONE); 430 431 mt76_dma_rx_cleanup(dev, q); 432 mt76_dma_sync_idx(dev, q); 433 mt76_dma_rx_fill(dev, q); 434 435 if (!q->rx_head) 436 return; 437 438 dev_kfree_skb(q->rx_head); 439 q->rx_head = NULL; 440 } 441 442 static void 443 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data, 444 int len, bool more) 445 { 446 struct page *page = virt_to_head_page(data); 447 int offset = data - page_address(page); 448 struct sk_buff *skb = q->rx_head; 449 450 offset += q->buf_offset; 451 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, len, 452 q->buf_size); 453 454 if (more) 455 return; 456 457 q->rx_head = NULL; 458 dev->drv->rx_skb(dev, q - dev->q_rx, skb); 459 } 460 461 static int 462 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) 463 { 464 int len, data_len, done = 0; 465 struct sk_buff *skb; 466 unsigned char *data; 467 bool more; 468 469 while (done < budget) { 470 u32 info; 471 472 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more); 473 if (!data) 474 break; 475 476 if (q->rx_head) 477 data_len = q->buf_size; 478 else 479 data_len = SKB_WITH_OVERHEAD(q->buf_size); 480 481 if (data_len < len + q->buf_offset) { 482 dev_kfree_skb(q->rx_head); 483 q->rx_head = NULL; 484 485 skb_free_frag(data); 486 continue; 487 } 488 489 if (q->rx_head) { 490 mt76_add_fragment(dev, q, data, len, more); 491 continue; 492 } 493 494 skb = build_skb(data, q->buf_size); 495 if (!skb) { 496 skb_free_frag(data); 497 continue; 498 } 499 skb_reserve(skb, q->buf_offset); 500 501 if (q == &dev->q_rx[MT_RXQ_MCU]) { 502 u32 *rxfce = (u32 *)skb->cb; 503 *rxfce = info; 504 } 505 506 __skb_put(skb, len); 507 done++; 508 509 if (more) { 510 q->rx_head = skb; 511 continue; 512 } 513 514 dev->drv->rx_skb(dev, q - dev->q_rx, skb); 515 } 516 517 mt76_dma_rx_fill(dev, q); 518 return done; 519 } 520 521 static int 522 mt76_dma_rx_poll(struct napi_struct *napi, int budget) 523 { 524 struct mt76_dev *dev; 525 int qid, done = 0, cur; 526 527 dev = container_of(napi->dev, struct mt76_dev, napi_dev); 528 qid = napi - dev->napi; 529 530 rcu_read_lock(); 531 532 do { 533 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done); 534 mt76_rx_poll_complete(dev, qid, napi); 535 done += cur; 536 } while (cur && done < budget); 537 538 rcu_read_unlock(); 539 540 if (done < budget) { 541 napi_complete(napi); 542 dev->drv->rx_poll_complete(dev, qid); 543 } 544 545 return done; 546 } 547 548 static int 549 mt76_dma_init(struct mt76_dev *dev) 550 { 551 int i; 552 553 init_dummy_netdev(&dev->napi_dev); 554 555 for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) { 556 netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll, 557 64); 558 mt76_dma_rx_fill(dev, &dev->q_rx[i]); 559 skb_queue_head_init(&dev->rx_skb[i]); 560 napi_enable(&dev->napi[i]); 561 } 562 563 return 0; 564 } 565 566 static const struct mt76_queue_ops mt76_dma_ops = { 567 .init = mt76_dma_init, 568 .alloc = mt76_dma_alloc_queue, 569 .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw, 570 .tx_queue_skb = mt76_dma_tx_queue_skb, 571 .tx_cleanup = mt76_dma_tx_cleanup, 572 .rx_reset = mt76_dma_rx_reset, 573 .kick = mt76_dma_kick_queue, 574 }; 575 576 void mt76_dma_attach(struct mt76_dev *dev) 577 { 578 dev->queue_ops = &mt76_dma_ops; 579 } 580 EXPORT_SYMBOL_GPL(mt76_dma_attach); 581 582 void mt76_dma_cleanup(struct mt76_dev *dev) 583 { 584 int i; 585 586 netif_napi_del(&dev->tx_napi); 587 for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++) 588 mt76_dma_tx_cleanup(dev, i, true); 589 590 for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) { 591 netif_napi_del(&dev->napi[i]); 592 mt76_dma_rx_cleanup(dev, &dev->q_rx[i]); 593 } 594 } 595 EXPORT_SYMBOL_GPL(mt76_dma_cleanup); 596