1 // SPDX-License-Identifier: ISC 2 /* 3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> 4 */ 5 6 #include "mt76.h" 7 8 static struct mt76_txwi_cache * 9 mt76_alloc_txwi(struct mt76_dev *dev) 10 { 11 struct mt76_txwi_cache *t; 12 dma_addr_t addr; 13 u8 *txwi; 14 int size; 15 16 size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t)); 17 txwi = devm_kzalloc(dev->dev, size, GFP_ATOMIC); 18 if (!txwi) 19 return NULL; 20 21 addr = dma_map_single(dev->dev, txwi, dev->drv->txwi_size, 22 DMA_TO_DEVICE); 23 t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size); 24 t->dma_addr = addr; 25 26 return t; 27 } 28 29 static struct mt76_txwi_cache * 30 __mt76_get_txwi(struct mt76_dev *dev) 31 { 32 struct mt76_txwi_cache *t = NULL; 33 34 spin_lock_bh(&dev->lock); 35 if (!list_empty(&dev->txwi_cache)) { 36 t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache, 37 list); 38 list_del(&t->list); 39 } 40 spin_unlock_bh(&dev->lock); 41 42 return t; 43 } 44 45 struct mt76_txwi_cache * 46 mt76_get_txwi(struct mt76_dev *dev) 47 { 48 struct mt76_txwi_cache *t = __mt76_get_txwi(dev); 49 50 if (t) 51 return t; 52 53 return mt76_alloc_txwi(dev); 54 } 55 56 void 57 mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t) 58 { 59 if (!t) 60 return; 61 62 spin_lock_bh(&dev->lock); 63 list_add(&t->list, &dev->txwi_cache); 64 spin_unlock_bh(&dev->lock); 65 } 66 EXPORT_SYMBOL_GPL(mt76_put_txwi); 67 68 void mt76_tx_free(struct mt76_dev *dev) 69 { 70 struct mt76_txwi_cache *t; 71 72 while ((t = __mt76_get_txwi(dev)) != NULL) 73 dma_unmap_single(dev->dev, t->dma_addr, dev->drv->txwi_size, 74 DMA_TO_DEVICE); 75 } 76 77 static int 78 mt76_txq_get_qid(struct ieee80211_txq *txq) 79 { 80 if (!txq->sta) 81 return MT_TXQ_BE; 82 83 return txq->ac; 84 } 85 86 static void 87 mt76_check_agg_ssn(struct mt76_txq *mtxq, struct sk_buff *skb) 88 { 89 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 90 91 if (!ieee80211_is_data_qos(hdr->frame_control) || 92 !ieee80211_is_data_present(hdr->frame_control)) 93 return; 94 95 mtxq->agg_ssn = le16_to_cpu(hdr->seq_ctrl) + 0x10; 96 } 97 98 void 99 mt76_tx_status_lock(struct mt76_dev *dev, struct sk_buff_head *list) 100 __acquires(&dev->status_list.lock) 101 { 102 __skb_queue_head_init(list); 103 spin_lock_bh(&dev->status_list.lock); 104 __acquire(&dev->status_list.lock); 105 } 106 EXPORT_SYMBOL_GPL(mt76_tx_status_lock); 107 108 void 109 mt76_tx_status_unlock(struct mt76_dev *dev, struct sk_buff_head *list) 110 __releases(&dev->status_list.unlock) 111 { 112 struct ieee80211_hw *hw; 113 struct sk_buff *skb; 114 115 spin_unlock_bh(&dev->status_list.lock); 116 __release(&dev->status_list.unlock); 117 118 while ((skb = __skb_dequeue(list)) != NULL) { 119 hw = mt76_tx_status_get_hw(dev, skb); 120 ieee80211_tx_status(hw, skb); 121 } 122 123 } 124 EXPORT_SYMBOL_GPL(mt76_tx_status_unlock); 125 126 static void 127 __mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, u8 flags, 128 struct sk_buff_head *list) 129 { 130 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 131 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 132 u8 done = MT_TX_CB_DMA_DONE | MT_TX_CB_TXS_DONE; 133 134 flags |= cb->flags; 135 cb->flags = flags; 136 137 if ((flags & done) != done) 138 return; 139 140 __skb_unlink(skb, &dev->status_list); 141 142 /* Tx status can be unreliable. if it fails, mark the frame as ACKed */ 143 if (flags & MT_TX_CB_TXS_FAILED) { 144 ieee80211_tx_info_clear_status(info); 145 info->status.rates[0].idx = -1; 146 info->flags |= IEEE80211_TX_STAT_ACK; 147 } 148 149 __skb_queue_tail(list, skb); 150 } 151 152 void 153 mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, 154 struct sk_buff_head *list) 155 { 156 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_DONE, list); 157 } 158 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_done); 159 160 int 161 mt76_tx_status_skb_add(struct mt76_dev *dev, struct mt76_wcid *wcid, 162 struct sk_buff *skb) 163 { 164 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 165 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 166 int pid; 167 168 if (!wcid) 169 return MT_PACKET_ID_NO_ACK; 170 171 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 172 return MT_PACKET_ID_NO_ACK; 173 174 if (!(info->flags & (IEEE80211_TX_CTL_REQ_TX_STATUS | 175 IEEE80211_TX_CTL_RATE_CTRL_PROBE))) 176 return MT_PACKET_ID_NO_SKB; 177 178 spin_lock_bh(&dev->status_list.lock); 179 180 memset(cb, 0, sizeof(*cb)); 181 wcid->packet_id = (wcid->packet_id + 1) & MT_PACKET_ID_MASK; 182 if (wcid->packet_id == MT_PACKET_ID_NO_ACK || 183 wcid->packet_id == MT_PACKET_ID_NO_SKB) 184 wcid->packet_id = MT_PACKET_ID_FIRST; 185 186 pid = wcid->packet_id; 187 cb->wcid = wcid->idx; 188 cb->pktid = pid; 189 cb->jiffies = jiffies; 190 191 __skb_queue_tail(&dev->status_list, skb); 192 spin_unlock_bh(&dev->status_list.lock); 193 194 return pid; 195 } 196 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_add); 197 198 struct sk_buff * 199 mt76_tx_status_skb_get(struct mt76_dev *dev, struct mt76_wcid *wcid, int pktid, 200 struct sk_buff_head *list) 201 { 202 struct sk_buff *skb, *tmp; 203 204 skb_queue_walk_safe(&dev->status_list, skb, tmp) { 205 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 206 207 if (wcid && cb->wcid != wcid->idx) 208 continue; 209 210 if (cb->pktid == pktid) 211 return skb; 212 213 if (pktid >= 0 && !time_after(jiffies, cb->jiffies + 214 MT_TX_STATUS_SKB_TIMEOUT)) 215 continue; 216 217 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_FAILED | 218 MT_TX_CB_TXS_DONE, list); 219 } 220 221 return NULL; 222 } 223 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_get); 224 225 void 226 mt76_tx_status_check(struct mt76_dev *dev, struct mt76_wcid *wcid, bool flush) 227 { 228 struct sk_buff_head list; 229 230 mt76_tx_status_lock(dev, &list); 231 mt76_tx_status_skb_get(dev, wcid, flush ? -1 : 0, &list); 232 mt76_tx_status_unlock(dev, &list); 233 } 234 EXPORT_SYMBOL_GPL(mt76_tx_status_check); 235 236 void mt76_tx_complete_skb(struct mt76_dev *dev, struct sk_buff *skb) 237 { 238 struct ieee80211_hw *hw; 239 struct sk_buff_head list; 240 241 if (!skb->prev) { 242 hw = mt76_tx_status_get_hw(dev, skb); 243 ieee80211_free_txskb(hw, skb); 244 return; 245 } 246 247 mt76_tx_status_lock(dev, &list); 248 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_DMA_DONE, &list); 249 mt76_tx_status_unlock(dev, &list); 250 } 251 EXPORT_SYMBOL_GPL(mt76_tx_complete_skb); 252 253 void 254 mt76_tx(struct mt76_phy *phy, struct ieee80211_sta *sta, 255 struct mt76_wcid *wcid, struct sk_buff *skb) 256 { 257 struct mt76_dev *dev = phy->dev; 258 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 259 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 260 struct mt76_queue *q; 261 int qid = skb_get_queue_mapping(skb); 262 bool ext_phy = phy != &dev->phy; 263 264 if (WARN_ON(qid >= MT_TXQ_PSD)) { 265 qid = MT_TXQ_BE; 266 skb_set_queue_mapping(skb, qid); 267 } 268 269 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 270 ieee80211_get_tx_rates(info->control.vif, sta, skb, 271 info->control.rates, 1); 272 273 if (sta && ieee80211_is_data_qos(hdr->frame_control)) { 274 struct ieee80211_txq *txq; 275 struct mt76_txq *mtxq; 276 u8 tid; 277 278 tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK; 279 txq = sta->txq[tid]; 280 mtxq = (struct mt76_txq *)txq->drv_priv; 281 282 if (mtxq->aggr) 283 mt76_check_agg_ssn(mtxq, skb); 284 } 285 286 if (ext_phy) 287 info->hw_queue |= MT_TX_HW_QUEUE_EXT_PHY; 288 289 q = dev->q_tx[qid].q; 290 291 spin_lock_bh(&q->lock); 292 dev->queue_ops->tx_queue_skb(dev, qid, skb, wcid, sta); 293 dev->queue_ops->kick(dev, q); 294 295 if (q->queued > q->ndesc - 8 && !q->stopped) { 296 ieee80211_stop_queue(phy->hw, skb_get_queue_mapping(skb)); 297 q->stopped = true; 298 } 299 300 spin_unlock_bh(&q->lock); 301 } 302 EXPORT_SYMBOL_GPL(mt76_tx); 303 304 static struct sk_buff * 305 mt76_txq_dequeue(struct mt76_phy *phy, struct mt76_txq *mtxq, bool ps) 306 { 307 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 308 struct ieee80211_tx_info *info; 309 bool ext_phy = phy != &phy->dev->phy; 310 struct sk_buff *skb; 311 312 skb = skb_dequeue(&mtxq->retry_q); 313 if (skb) { 314 u8 tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK; 315 316 if (ps && skb_queue_empty(&mtxq->retry_q)) 317 ieee80211_sta_set_buffered(txq->sta, tid, false); 318 319 return skb; 320 } 321 322 skb = ieee80211_tx_dequeue(phy->hw, txq); 323 if (!skb) 324 return NULL; 325 326 info = IEEE80211_SKB_CB(skb); 327 if (ext_phy) 328 info->hw_queue |= MT_TX_HW_QUEUE_EXT_PHY; 329 330 return skb; 331 } 332 333 static void 334 mt76_queue_ps_skb(struct mt76_dev *dev, struct ieee80211_sta *sta, 335 struct sk_buff *skb, bool last) 336 { 337 struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; 338 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 339 340 info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE; 341 if (last) 342 info->flags |= IEEE80211_TX_STATUS_EOSP | 343 IEEE80211_TX_CTL_REQ_TX_STATUS; 344 345 mt76_skb_set_moredata(skb, !last); 346 dev->queue_ops->tx_queue_skb(dev, MT_TXQ_PSD, skb, wcid, sta); 347 } 348 349 void 350 mt76_release_buffered_frames(struct ieee80211_hw *hw, struct ieee80211_sta *sta, 351 u16 tids, int nframes, 352 enum ieee80211_frame_release_type reason, 353 bool more_data) 354 { 355 struct mt76_phy *phy = hw->priv; 356 struct mt76_dev *dev = phy->dev; 357 struct sk_buff *last_skb = NULL; 358 struct mt76_queue *hwq = dev->q_tx[MT_TXQ_PSD].q; 359 int i; 360 361 spin_lock_bh(&hwq->lock); 362 for (i = 0; tids && nframes; i++, tids >>= 1) { 363 struct ieee80211_txq *txq = sta->txq[i]; 364 struct mt76_txq *mtxq = (struct mt76_txq *)txq->drv_priv; 365 struct sk_buff *skb; 366 367 if (!(tids & 1)) 368 continue; 369 370 do { 371 skb = mt76_txq_dequeue(phy, mtxq, true); 372 if (!skb) 373 break; 374 375 if (mtxq->aggr) 376 mt76_check_agg_ssn(mtxq, skb); 377 378 nframes--; 379 if (last_skb) 380 mt76_queue_ps_skb(dev, sta, last_skb, false); 381 382 last_skb = skb; 383 } while (nframes); 384 } 385 386 if (last_skb) { 387 mt76_queue_ps_skb(dev, sta, last_skb, true); 388 dev->queue_ops->kick(dev, hwq); 389 } else { 390 ieee80211_sta_eosp(sta); 391 } 392 393 spin_unlock_bh(&hwq->lock); 394 } 395 EXPORT_SYMBOL_GPL(mt76_release_buffered_frames); 396 397 static int 398 mt76_txq_send_burst(struct mt76_phy *phy, struct mt76_sw_queue *sq, 399 struct mt76_txq *mtxq) 400 { 401 struct mt76_dev *dev = phy->dev; 402 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 403 enum mt76_txq_id qid = mt76_txq_get_qid(txq); 404 struct mt76_wcid *wcid = mtxq->wcid; 405 struct mt76_queue *hwq = sq->q; 406 struct ieee80211_tx_info *info; 407 struct sk_buff *skb; 408 int n_frames = 1, limit; 409 struct ieee80211_tx_rate tx_rate; 410 bool ampdu; 411 bool probe; 412 int idx; 413 414 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) 415 return 0; 416 417 skb = mt76_txq_dequeue(phy, mtxq, false); 418 if (!skb) 419 return 0; 420 421 info = IEEE80211_SKB_CB(skb); 422 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 423 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 424 info->control.rates, 1); 425 tx_rate = info->control.rates[0]; 426 427 probe = (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE); 428 ampdu = IEEE80211_SKB_CB(skb)->flags & IEEE80211_TX_CTL_AMPDU; 429 limit = ampdu ? 16 : 3; 430 431 if (ampdu) 432 mt76_check_agg_ssn(mtxq, skb); 433 434 idx = dev->queue_ops->tx_queue_skb(dev, qid, skb, wcid, txq->sta); 435 436 if (idx < 0) 437 return idx; 438 439 do { 440 bool cur_ampdu; 441 442 if (probe) 443 break; 444 445 if (test_bit(MT76_RESET, &phy->state)) 446 return -EBUSY; 447 448 skb = mt76_txq_dequeue(phy, mtxq, false); 449 if (!skb) 450 break; 451 452 info = IEEE80211_SKB_CB(skb); 453 cur_ampdu = info->flags & IEEE80211_TX_CTL_AMPDU; 454 455 if (ampdu != cur_ampdu || 456 (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE)) { 457 skb_queue_tail(&mtxq->retry_q, skb); 458 break; 459 } 460 461 info->control.rates[0] = tx_rate; 462 463 if (cur_ampdu) 464 mt76_check_agg_ssn(mtxq, skb); 465 466 idx = dev->queue_ops->tx_queue_skb(dev, qid, skb, wcid, 467 txq->sta); 468 if (idx < 0) 469 return idx; 470 471 n_frames++; 472 } while (n_frames < limit); 473 474 if (!probe) { 475 hwq->entry[idx].qid = sq - dev->q_tx; 476 hwq->entry[idx].schedule = true; 477 sq->swq_queued++; 478 } 479 480 dev->queue_ops->kick(dev, hwq); 481 482 return n_frames; 483 } 484 485 static int 486 mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid) 487 { 488 struct mt76_dev *dev = phy->dev; 489 struct mt76_sw_queue *sq = &dev->q_tx[qid]; 490 struct mt76_queue *hwq = sq->q; 491 struct ieee80211_txq *txq; 492 struct mt76_txq *mtxq; 493 struct mt76_wcid *wcid; 494 int ret = 0; 495 496 spin_lock_bh(&hwq->lock); 497 while (1) { 498 if (sq->swq_queued >= 4) 499 break; 500 501 if (test_bit(MT76_RESET, &phy->state)) { 502 ret = -EBUSY; 503 break; 504 } 505 506 txq = ieee80211_next_txq(phy->hw, qid); 507 if (!txq) 508 break; 509 510 mtxq = (struct mt76_txq *)txq->drv_priv; 511 wcid = mtxq->wcid; 512 if (wcid && test_bit(MT_WCID_FLAG_PS, &wcid->flags)) 513 continue; 514 515 if (mtxq->send_bar && mtxq->aggr) { 516 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 517 struct ieee80211_sta *sta = txq->sta; 518 struct ieee80211_vif *vif = txq->vif; 519 u16 agg_ssn = mtxq->agg_ssn; 520 u8 tid = txq->tid; 521 522 mtxq->send_bar = false; 523 spin_unlock_bh(&hwq->lock); 524 ieee80211_send_bar(vif, sta->addr, tid, agg_ssn); 525 spin_lock_bh(&hwq->lock); 526 } 527 528 ret += mt76_txq_send_burst(phy, sq, mtxq); 529 ieee80211_return_txq(phy->hw, txq, 530 !skb_queue_empty(&mtxq->retry_q)); 531 } 532 spin_unlock_bh(&hwq->lock); 533 534 return ret; 535 } 536 537 void mt76_txq_schedule(struct mt76_phy *phy, enum mt76_txq_id qid) 538 { 539 struct mt76_dev *dev = phy->dev; 540 struct mt76_sw_queue *sq = &dev->q_tx[qid]; 541 int len; 542 543 if (qid >= 4) 544 return; 545 546 if (sq->swq_queued >= 4) 547 return; 548 549 rcu_read_lock(); 550 551 do { 552 ieee80211_txq_schedule_start(phy->hw, qid); 553 len = mt76_txq_schedule_list(phy, qid); 554 ieee80211_txq_schedule_end(phy->hw, qid); 555 } while (len > 0); 556 557 rcu_read_unlock(); 558 } 559 EXPORT_SYMBOL_GPL(mt76_txq_schedule); 560 561 void mt76_txq_schedule_all(struct mt76_phy *phy) 562 { 563 int i; 564 565 for (i = 0; i <= MT_TXQ_BK; i++) 566 mt76_txq_schedule(phy, i); 567 } 568 EXPORT_SYMBOL_GPL(mt76_txq_schedule_all); 569 570 void mt76_tx_tasklet(unsigned long data) 571 { 572 struct mt76_dev *dev = (struct mt76_dev *)data; 573 574 mt76_txq_schedule_all(&dev->phy); 575 if (dev->phy2) 576 mt76_txq_schedule_all(dev->phy2); 577 } 578 579 void mt76_stop_tx_queues(struct mt76_dev *dev, struct ieee80211_sta *sta, 580 bool send_bar) 581 { 582 int i; 583 584 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { 585 struct ieee80211_txq *txq = sta->txq[i]; 586 struct mt76_queue *hwq; 587 struct mt76_txq *mtxq; 588 589 if (!txq) 590 continue; 591 592 mtxq = (struct mt76_txq *)txq->drv_priv; 593 hwq = mtxq->swq->q; 594 595 spin_lock_bh(&hwq->lock); 596 mtxq->send_bar = mtxq->aggr && send_bar; 597 spin_unlock_bh(&hwq->lock); 598 } 599 } 600 EXPORT_SYMBOL_GPL(mt76_stop_tx_queues); 601 602 void mt76_wake_tx_queue(struct ieee80211_hw *hw, struct ieee80211_txq *txq) 603 { 604 struct mt76_phy *phy = hw->priv; 605 struct mt76_dev *dev = phy->dev; 606 607 if (!test_bit(MT76_STATE_RUNNING, &phy->state)) 608 return; 609 610 tasklet_schedule(&dev->tx_tasklet); 611 } 612 EXPORT_SYMBOL_GPL(mt76_wake_tx_queue); 613 614 void mt76_txq_remove(struct mt76_dev *dev, struct ieee80211_txq *txq) 615 { 616 struct ieee80211_hw *hw; 617 struct mt76_txq *mtxq; 618 struct sk_buff *skb; 619 620 if (!txq) 621 return; 622 623 mtxq = (struct mt76_txq *)txq->drv_priv; 624 625 while ((skb = skb_dequeue(&mtxq->retry_q)) != NULL) { 626 hw = mt76_tx_status_get_hw(dev, skb); 627 ieee80211_free_txskb(hw, skb); 628 } 629 } 630 EXPORT_SYMBOL_GPL(mt76_txq_remove); 631 632 void mt76_txq_init(struct mt76_dev *dev, struct ieee80211_txq *txq) 633 { 634 struct mt76_txq *mtxq = (struct mt76_txq *)txq->drv_priv; 635 636 skb_queue_head_init(&mtxq->retry_q); 637 638 mtxq->swq = &dev->q_tx[mt76_txq_get_qid(txq)]; 639 } 640 EXPORT_SYMBOL_GPL(mt76_txq_init); 641 642 u8 mt76_ac_to_hwq(u8 ac) 643 { 644 static const u8 wmm_queue_map[] = { 645 [IEEE80211_AC_BE] = 0, 646 [IEEE80211_AC_BK] = 1, 647 [IEEE80211_AC_VI] = 2, 648 [IEEE80211_AC_VO] = 3, 649 }; 650 651 if (WARN_ON(ac >= IEEE80211_NUM_ACS)) 652 return 0; 653 654 return wmm_queue_map[ac]; 655 } 656 EXPORT_SYMBOL_GPL(mt76_ac_to_hwq); 657