1 // SPDX-License-Identifier: ISC 2 /* Copyright (C) 2020 MediaTek Inc. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/platform_device.h> 6 #include <linux/pci.h> 7 #include <linux/module.h> 8 #include "mt7915.h" 9 #include "mcu.h" 10 11 static bool mt7915_dev_running(struct mt7915_dev *dev) 12 { 13 struct mt7915_phy *phy; 14 15 if (test_bit(MT76_STATE_RUNNING, &dev->mphy.state)) 16 return true; 17 18 phy = mt7915_ext_phy(dev); 19 20 return phy && test_bit(MT76_STATE_RUNNING, &phy->mt76->state); 21 } 22 23 int mt7915_run(struct ieee80211_hw *hw) 24 { 25 struct mt7915_dev *dev = mt7915_hw_dev(hw); 26 struct mt7915_phy *phy = mt7915_hw_phy(hw); 27 bool running; 28 int ret; 29 30 running = mt7915_dev_running(dev); 31 32 if (!running) { 33 ret = mt76_connac_mcu_set_pm(&dev->mt76, 34 dev->phy.mt76->band_idx, 0); 35 if (ret) 36 goto out; 37 38 ret = mt7915_mcu_set_mac(dev, dev->phy.mt76->band_idx, 39 true, true); 40 if (ret) 41 goto out; 42 43 mt7915_mac_enable_nf(dev, dev->phy.mt76->band_idx); 44 } 45 46 if (phy != &dev->phy) { 47 ret = mt76_connac_mcu_set_pm(&dev->mt76, 48 phy->mt76->band_idx, 0); 49 if (ret) 50 goto out; 51 52 ret = mt7915_mcu_set_mac(dev, phy->mt76->band_idx, 53 true, true); 54 if (ret) 55 goto out; 56 57 mt7915_mac_enable_nf(dev, phy->mt76->band_idx); 58 } 59 60 ret = mt7915_mcu_set_thermal_throttling(phy, 61 MT7915_THERMAL_THROTTLE_MAX); 62 63 if (ret) 64 goto out; 65 66 ret = mt7915_mcu_set_thermal_protect(phy); 67 68 if (ret) 69 goto out; 70 71 ret = mt76_connac_mcu_set_rts_thresh(&dev->mt76, 0x92b, 72 phy->mt76->band_idx); 73 if (ret) 74 goto out; 75 76 ret = mt7915_mcu_set_sku_en(phy, true); 77 if (ret) 78 goto out; 79 80 ret = mt7915_mcu_set_chan_info(phy, MCU_EXT_CMD(SET_RX_PATH)); 81 if (ret) 82 goto out; 83 84 set_bit(MT76_STATE_RUNNING, &phy->mt76->state); 85 86 if (!mt76_testmode_enabled(phy->mt76)) 87 ieee80211_queue_delayed_work(hw, &phy->mt76->mac_work, 88 MT7915_WATCHDOG_TIME); 89 90 if (!running) 91 mt7915_mac_reset_counters(phy); 92 93 out: 94 return ret; 95 } 96 97 static int mt7915_start(struct ieee80211_hw *hw) 98 { 99 struct mt7915_dev *dev = mt7915_hw_dev(hw); 100 int ret; 101 102 flush_work(&dev->init_work); 103 104 mutex_lock(&dev->mt76.mutex); 105 ret = mt7915_run(hw); 106 mutex_unlock(&dev->mt76.mutex); 107 108 return ret; 109 } 110 111 static void mt7915_stop(struct ieee80211_hw *hw) 112 { 113 struct mt7915_dev *dev = mt7915_hw_dev(hw); 114 struct mt7915_phy *phy = mt7915_hw_phy(hw); 115 116 cancel_delayed_work_sync(&phy->mt76->mac_work); 117 118 mutex_lock(&dev->mt76.mutex); 119 120 mt76_testmode_reset(phy->mt76, true); 121 122 clear_bit(MT76_STATE_RUNNING, &phy->mt76->state); 123 124 if (phy != &dev->phy) { 125 mt76_connac_mcu_set_pm(&dev->mt76, phy->mt76->band_idx, 1); 126 mt7915_mcu_set_mac(dev, phy->mt76->band_idx, false, false); 127 } 128 129 if (!mt7915_dev_running(dev)) { 130 mt76_connac_mcu_set_pm(&dev->mt76, dev->phy.mt76->band_idx, 1); 131 mt7915_mcu_set_mac(dev, dev->phy.mt76->band_idx, false, false); 132 } 133 134 mutex_unlock(&dev->mt76.mutex); 135 } 136 137 static inline int get_free_idx(u32 mask, u8 start, u8 end) 138 { 139 return ffs(~mask & GENMASK(end, start)); 140 } 141 142 static int get_omac_idx(enum nl80211_iftype type, u64 mask) 143 { 144 int i; 145 146 switch (type) { 147 case NL80211_IFTYPE_MESH_POINT: 148 case NL80211_IFTYPE_ADHOC: 149 case NL80211_IFTYPE_STATION: 150 /* prefer hw bssid slot 1-3 */ 151 i = get_free_idx(mask, HW_BSSID_1, HW_BSSID_3); 152 if (i) 153 return i - 1; 154 155 if (type != NL80211_IFTYPE_STATION) 156 break; 157 158 i = get_free_idx(mask, EXT_BSSID_1, EXT_BSSID_MAX); 159 if (i) 160 return i - 1; 161 162 if (~mask & BIT(HW_BSSID_0)) 163 return HW_BSSID_0; 164 165 break; 166 case NL80211_IFTYPE_MONITOR: 167 case NL80211_IFTYPE_AP: 168 /* ap uses hw bssid 0 and ext bssid */ 169 if (~mask & BIT(HW_BSSID_0)) 170 return HW_BSSID_0; 171 172 i = get_free_idx(mask, EXT_BSSID_1, EXT_BSSID_MAX); 173 if (i) 174 return i - 1; 175 176 break; 177 default: 178 WARN_ON(1); 179 break; 180 } 181 182 return -1; 183 } 184 185 static void mt7915_init_bitrate_mask(struct ieee80211_vif *vif) 186 { 187 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 188 int i; 189 190 for (i = 0; i < ARRAY_SIZE(mvif->bitrate_mask.control); i++) { 191 mvif->bitrate_mask.control[i].gi = NL80211_TXRATE_DEFAULT_GI; 192 mvif->bitrate_mask.control[i].he_gi = 0xff; 193 mvif->bitrate_mask.control[i].he_ltf = 0xff; 194 mvif->bitrate_mask.control[i].legacy = GENMASK(31, 0); 195 memset(mvif->bitrate_mask.control[i].ht_mcs, 0xff, 196 sizeof(mvif->bitrate_mask.control[i].ht_mcs)); 197 memset(mvif->bitrate_mask.control[i].vht_mcs, 0xff, 198 sizeof(mvif->bitrate_mask.control[i].vht_mcs)); 199 memset(mvif->bitrate_mask.control[i].he_mcs, 0xff, 200 sizeof(mvif->bitrate_mask.control[i].he_mcs)); 201 } 202 } 203 204 static int mt7915_add_interface(struct ieee80211_hw *hw, 205 struct ieee80211_vif *vif) 206 { 207 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 208 struct mt7915_dev *dev = mt7915_hw_dev(hw); 209 struct mt7915_phy *phy = mt7915_hw_phy(hw); 210 struct mt76_txq *mtxq; 211 bool ext_phy = phy != &dev->phy; 212 int idx, ret = 0; 213 214 mutex_lock(&dev->mt76.mutex); 215 216 mt76_testmode_reset(phy->mt76, true); 217 218 if (vif->type == NL80211_IFTYPE_MONITOR && 219 is_zero_ether_addr(vif->addr)) 220 phy->monitor_vif = vif; 221 222 mvif->mt76.idx = __ffs64(~dev->mt76.vif_mask); 223 if (mvif->mt76.idx >= (MT7915_MAX_INTERFACES << dev->dbdc_support)) { 224 ret = -ENOSPC; 225 goto out; 226 } 227 228 idx = get_omac_idx(vif->type, phy->omac_mask); 229 if (idx < 0) { 230 ret = -ENOSPC; 231 goto out; 232 } 233 mvif->mt76.omac_idx = idx; 234 mvif->phy = phy; 235 mvif->mt76.band_idx = phy->mt76->band_idx; 236 237 mvif->mt76.wmm_idx = vif->type != NL80211_IFTYPE_AP; 238 if (ext_phy) 239 mvif->mt76.wmm_idx += 2; 240 241 ret = mt7915_mcu_add_dev_info(phy, vif, true); 242 if (ret) 243 goto out; 244 245 dev->mt76.vif_mask |= BIT_ULL(mvif->mt76.idx); 246 phy->omac_mask |= BIT_ULL(mvif->mt76.omac_idx); 247 248 idx = MT7915_WTBL_RESERVED - mvif->mt76.idx; 249 250 INIT_LIST_HEAD(&mvif->sta.rc_list); 251 INIT_LIST_HEAD(&mvif->sta.wcid.poll_list); 252 mvif->sta.wcid.idx = idx; 253 mvif->sta.wcid.phy_idx = ext_phy; 254 mvif->sta.wcid.hw_key_idx = -1; 255 mvif->sta.wcid.tx_info |= MT_WCID_TX_INFO_SET; 256 mt76_wcid_init(&mvif->sta.wcid); 257 258 mt7915_mac_wtbl_update(dev, idx, 259 MT_WTBL_UPDATE_ADM_COUNT_CLEAR); 260 261 if (vif->txq) { 262 mtxq = (struct mt76_txq *)vif->txq->drv_priv; 263 mtxq->wcid = idx; 264 } 265 266 if (vif->type != NL80211_IFTYPE_AP && 267 (!mvif->mt76.omac_idx || mvif->mt76.omac_idx > 3)) 268 vif->offload_flags = 0; 269 vif->offload_flags |= IEEE80211_OFFLOAD_ENCAP_4ADDR; 270 271 mt7915_init_bitrate_mask(vif); 272 memset(&mvif->cap, -1, sizeof(mvif->cap)); 273 274 mt7915_mcu_add_bss_info(phy, vif, true); 275 mt7915_mcu_add_sta(dev, vif, NULL, true); 276 rcu_assign_pointer(dev->mt76.wcid[idx], &mvif->sta.wcid); 277 278 out: 279 mutex_unlock(&dev->mt76.mutex); 280 281 return ret; 282 } 283 284 static void mt7915_remove_interface(struct ieee80211_hw *hw, 285 struct ieee80211_vif *vif) 286 { 287 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 288 struct mt7915_sta *msta = &mvif->sta; 289 struct mt7915_dev *dev = mt7915_hw_dev(hw); 290 struct mt7915_phy *phy = mt7915_hw_phy(hw); 291 int idx = msta->wcid.idx; 292 293 mt7915_mcu_add_bss_info(phy, vif, false); 294 mt7915_mcu_add_sta(dev, vif, NULL, false); 295 296 mutex_lock(&dev->mt76.mutex); 297 mt76_testmode_reset(phy->mt76, true); 298 mutex_unlock(&dev->mt76.mutex); 299 300 if (vif == phy->monitor_vif) 301 phy->monitor_vif = NULL; 302 303 mt7915_mcu_add_dev_info(phy, vif, false); 304 305 rcu_assign_pointer(dev->mt76.wcid[idx], NULL); 306 307 mutex_lock(&dev->mt76.mutex); 308 dev->mt76.vif_mask &= ~BIT_ULL(mvif->mt76.idx); 309 phy->omac_mask &= ~BIT_ULL(mvif->mt76.omac_idx); 310 mutex_unlock(&dev->mt76.mutex); 311 312 spin_lock_bh(&dev->mt76.sta_poll_lock); 313 if (!list_empty(&msta->wcid.poll_list)) 314 list_del_init(&msta->wcid.poll_list); 315 spin_unlock_bh(&dev->mt76.sta_poll_lock); 316 317 mt76_wcid_cleanup(&dev->mt76, &msta->wcid); 318 } 319 320 int mt7915_set_channel(struct mt7915_phy *phy) 321 { 322 struct mt7915_dev *dev = phy->dev; 323 int ret; 324 325 cancel_delayed_work_sync(&phy->mt76->mac_work); 326 327 mutex_lock(&dev->mt76.mutex); 328 set_bit(MT76_RESET, &phy->mt76->state); 329 330 mt76_set_channel(phy->mt76); 331 332 if (dev->flash_mode) { 333 ret = mt7915_mcu_apply_tx_dpd(phy); 334 if (ret) 335 goto out; 336 } 337 338 ret = mt7915_mcu_set_chan_info(phy, MCU_EXT_CMD(CHANNEL_SWITCH)); 339 if (ret) 340 goto out; 341 342 mt7915_mac_set_timing(phy); 343 ret = mt7915_dfs_init_radar_detector(phy); 344 mt7915_mac_cca_stats_reset(phy); 345 346 mt7915_mac_reset_counters(phy); 347 phy->noise = 0; 348 349 out: 350 clear_bit(MT76_RESET, &phy->mt76->state); 351 mutex_unlock(&dev->mt76.mutex); 352 353 mt76_txq_schedule_all(phy->mt76); 354 355 if (!mt76_testmode_enabled(phy->mt76)) 356 ieee80211_queue_delayed_work(phy->mt76->hw, 357 &phy->mt76->mac_work, 358 MT7915_WATCHDOG_TIME); 359 360 return ret; 361 } 362 363 static int mt7915_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, 364 struct ieee80211_vif *vif, struct ieee80211_sta *sta, 365 struct ieee80211_key_conf *key) 366 { 367 struct mt7915_dev *dev = mt7915_hw_dev(hw); 368 struct mt7915_phy *phy = mt7915_hw_phy(hw); 369 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 370 struct mt7915_sta *msta = sta ? (struct mt7915_sta *)sta->drv_priv : 371 &mvif->sta; 372 struct mt76_wcid *wcid = &msta->wcid; 373 u8 *wcid_keyidx = &wcid->hw_key_idx; 374 int idx = key->keyidx; 375 int err = 0; 376 377 /* The hardware does not support per-STA RX GTK, fallback 378 * to software mode for these. 379 */ 380 if ((vif->type == NL80211_IFTYPE_ADHOC || 381 vif->type == NL80211_IFTYPE_MESH_POINT) && 382 (key->cipher == WLAN_CIPHER_SUITE_TKIP || 383 key->cipher == WLAN_CIPHER_SUITE_CCMP) && 384 !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) 385 return -EOPNOTSUPP; 386 387 /* fall back to sw encryption for unsupported ciphers */ 388 switch (key->cipher) { 389 case WLAN_CIPHER_SUITE_AES_CMAC: 390 wcid_keyidx = &wcid->hw_key_idx2; 391 key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIE; 392 break; 393 case WLAN_CIPHER_SUITE_TKIP: 394 case WLAN_CIPHER_SUITE_CCMP: 395 case WLAN_CIPHER_SUITE_CCMP_256: 396 case WLAN_CIPHER_SUITE_GCMP: 397 case WLAN_CIPHER_SUITE_GCMP_256: 398 case WLAN_CIPHER_SUITE_SMS4: 399 break; 400 case WLAN_CIPHER_SUITE_WEP40: 401 case WLAN_CIPHER_SUITE_WEP104: 402 default: 403 return -EOPNOTSUPP; 404 } 405 406 mutex_lock(&dev->mt76.mutex); 407 408 if (cmd == SET_KEY && !sta && !mvif->mt76.cipher) { 409 mvif->mt76.cipher = mt76_connac_mcu_get_cipher(key->cipher); 410 mt7915_mcu_add_bss_info(phy, vif, true); 411 } 412 413 if (cmd == SET_KEY) { 414 *wcid_keyidx = idx; 415 } else { 416 if (idx == *wcid_keyidx) 417 *wcid_keyidx = -1; 418 goto out; 419 } 420 421 mt76_wcid_key_setup(&dev->mt76, wcid, key); 422 err = mt76_connac_mcu_add_key(&dev->mt76, vif, &msta->bip, 423 key, MCU_EXT_CMD(STA_REC_UPDATE), 424 &msta->wcid, cmd); 425 out: 426 mutex_unlock(&dev->mt76.mutex); 427 428 return err; 429 } 430 431 static int mt7915_set_sar_specs(struct ieee80211_hw *hw, 432 const struct cfg80211_sar_specs *sar) 433 { 434 struct mt7915_phy *phy = mt7915_hw_phy(hw); 435 struct mt7915_dev *dev = mt7915_hw_dev(hw); 436 int err = -EINVAL; 437 438 mutex_lock(&dev->mt76.mutex); 439 if (!cfg80211_chandef_valid(&phy->mt76->chandef)) 440 goto out; 441 442 err = mt76_init_sar_power(hw, sar); 443 if (err) 444 goto out; 445 446 err = mt7915_mcu_set_txpower_sku(phy); 447 out: 448 mutex_unlock(&dev->mt76.mutex); 449 450 return err; 451 } 452 453 static int mt7915_config(struct ieee80211_hw *hw, u32 changed) 454 { 455 struct mt7915_dev *dev = mt7915_hw_dev(hw); 456 struct mt7915_phy *phy = mt7915_hw_phy(hw); 457 int ret; 458 459 if (changed & IEEE80211_CONF_CHANGE_CHANNEL) { 460 #ifdef CONFIG_NL80211_TESTMODE 461 if (phy->mt76->test.state != MT76_TM_STATE_OFF) { 462 mutex_lock(&dev->mt76.mutex); 463 mt76_testmode_reset(phy->mt76, false); 464 mutex_unlock(&dev->mt76.mutex); 465 } 466 #endif 467 ieee80211_stop_queues(hw); 468 ret = mt7915_set_channel(phy); 469 if (ret) 470 return ret; 471 ieee80211_wake_queues(hw); 472 } 473 474 if (changed & (IEEE80211_CONF_CHANGE_POWER | 475 IEEE80211_CONF_CHANGE_CHANNEL)) { 476 ret = mt7915_mcu_set_txpower_sku(phy); 477 if (ret) 478 return ret; 479 } 480 481 mutex_lock(&dev->mt76.mutex); 482 483 if (changed & IEEE80211_CONF_CHANGE_MONITOR) { 484 bool enabled = !!(hw->conf.flags & IEEE80211_CONF_MONITOR); 485 bool band = phy->mt76->band_idx; 486 487 if (!enabled) 488 phy->rxfilter |= MT_WF_RFCR_DROP_OTHER_UC; 489 else 490 phy->rxfilter &= ~MT_WF_RFCR_DROP_OTHER_UC; 491 492 mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_DMA_DCR0_RXD_G5_EN, 493 enabled); 494 mt76_testmode_reset(phy->mt76, true); 495 mt76_wr(dev, MT_WF_RFCR(band), phy->rxfilter); 496 } 497 498 mutex_unlock(&dev->mt76.mutex); 499 500 return 0; 501 } 502 503 static int 504 mt7915_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 505 unsigned int link_id, u16 queue, 506 const struct ieee80211_tx_queue_params *params) 507 { 508 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 509 510 /* no need to update right away, we'll get BSS_CHANGED_QOS */ 511 queue = mt76_connac_lmac_mapping(queue); 512 mvif->queue_params[queue] = *params; 513 514 return 0; 515 } 516 517 static void mt7915_configure_filter(struct ieee80211_hw *hw, 518 unsigned int changed_flags, 519 unsigned int *total_flags, 520 u64 multicast) 521 { 522 struct mt7915_dev *dev = mt7915_hw_dev(hw); 523 struct mt7915_phy *phy = mt7915_hw_phy(hw); 524 bool band = phy->mt76->band_idx; 525 u32 ctl_flags = MT_WF_RFCR1_DROP_ACK | 526 MT_WF_RFCR1_DROP_BF_POLL | 527 MT_WF_RFCR1_DROP_BA | 528 MT_WF_RFCR1_DROP_CFEND | 529 MT_WF_RFCR1_DROP_CFACK; 530 u32 flags = 0; 531 532 #define MT76_FILTER(_flag, _hw) do { \ 533 flags |= *total_flags & FIF_##_flag; \ 534 phy->rxfilter &= ~(_hw); \ 535 phy->rxfilter |= !(flags & FIF_##_flag) * (_hw); \ 536 } while (0) 537 538 mutex_lock(&dev->mt76.mutex); 539 540 phy->rxfilter &= ~(MT_WF_RFCR_DROP_OTHER_BSS | 541 MT_WF_RFCR_DROP_OTHER_BEACON | 542 MT_WF_RFCR_DROP_FRAME_REPORT | 543 MT_WF_RFCR_DROP_PROBEREQ | 544 MT_WF_RFCR_DROP_MCAST_FILTERED | 545 MT_WF_RFCR_DROP_MCAST | 546 MT_WF_RFCR_DROP_BCAST | 547 MT_WF_RFCR_DROP_DUPLICATE | 548 MT_WF_RFCR_DROP_A2_BSSID | 549 MT_WF_RFCR_DROP_UNWANTED_CTL | 550 MT_WF_RFCR_DROP_STBC_MULTI); 551 552 MT76_FILTER(OTHER_BSS, MT_WF_RFCR_DROP_OTHER_TIM | 553 MT_WF_RFCR_DROP_A3_MAC | 554 MT_WF_RFCR_DROP_A3_BSSID); 555 556 MT76_FILTER(FCSFAIL, MT_WF_RFCR_DROP_FCSFAIL); 557 558 MT76_FILTER(CONTROL, MT_WF_RFCR_DROP_CTS | 559 MT_WF_RFCR_DROP_RTS | 560 MT_WF_RFCR_DROP_CTL_RSV); 561 562 *total_flags = flags; 563 mt76_wr(dev, MT_WF_RFCR(band), phy->rxfilter); 564 565 if (*total_flags & FIF_CONTROL) 566 mt76_clear(dev, MT_WF_RFCR1(band), ctl_flags); 567 else 568 mt76_set(dev, MT_WF_RFCR1(band), ctl_flags); 569 570 mutex_unlock(&dev->mt76.mutex); 571 } 572 573 static void 574 mt7915_update_bss_color(struct ieee80211_hw *hw, 575 struct ieee80211_vif *vif, 576 struct cfg80211_he_bss_color *bss_color) 577 { 578 struct mt7915_dev *dev = mt7915_hw_dev(hw); 579 580 switch (vif->type) { 581 case NL80211_IFTYPE_AP: { 582 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 583 584 if (mvif->mt76.omac_idx > HW_BSSID_MAX) 585 return; 586 fallthrough; 587 } 588 case NL80211_IFTYPE_STATION: 589 mt7915_mcu_update_bss_color(dev, vif, bss_color); 590 break; 591 default: 592 break; 593 } 594 } 595 596 static void mt7915_bss_info_changed(struct ieee80211_hw *hw, 597 struct ieee80211_vif *vif, 598 struct ieee80211_bss_conf *info, 599 u64 changed) 600 { 601 struct mt7915_phy *phy = mt7915_hw_phy(hw); 602 struct mt7915_dev *dev = mt7915_hw_dev(hw); 603 int set_bss_info = -1, set_sta = -1; 604 605 mutex_lock(&dev->mt76.mutex); 606 607 /* 608 * station mode uses BSSID to map the wlan entry to a peer, 609 * and then peer references bss_info_rfch to set bandwidth cap. 610 */ 611 if (changed & BSS_CHANGED_BSSID && 612 vif->type == NL80211_IFTYPE_STATION) 613 set_bss_info = set_sta = !is_zero_ether_addr(info->bssid); 614 if (changed & BSS_CHANGED_ASSOC) 615 set_bss_info = vif->cfg.assoc; 616 if (changed & BSS_CHANGED_BEACON_ENABLED && 617 vif->type != NL80211_IFTYPE_AP) 618 set_bss_info = set_sta = info->enable_beacon; 619 620 if (set_bss_info == 1) 621 mt7915_mcu_add_bss_info(phy, vif, true); 622 if (set_sta == 1) 623 mt7915_mcu_add_sta(dev, vif, NULL, true); 624 625 if (changed & BSS_CHANGED_ERP_CTS_PROT) 626 mt7915_mac_enable_rtscts(dev, vif, info->use_cts_prot); 627 628 if (changed & BSS_CHANGED_ERP_SLOT) { 629 int slottime = info->use_short_slot ? 9 : 20; 630 631 if (slottime != phy->slottime) { 632 phy->slottime = slottime; 633 mt7915_mac_set_timing(phy); 634 } 635 } 636 637 /* ensure that enable txcmd_mode after bss_info */ 638 if (changed & (BSS_CHANGED_QOS | BSS_CHANGED_BEACON_ENABLED)) 639 mt7915_mcu_set_tx(dev, vif); 640 641 if (changed & BSS_CHANGED_HE_OBSS_PD) 642 mt7915_mcu_add_obss_spr(phy, vif, &info->he_obss_pd); 643 644 if (changed & BSS_CHANGED_HE_BSS_COLOR) 645 mt7915_update_bss_color(hw, vif, &info->he_bss_color); 646 647 if (changed & (BSS_CHANGED_BEACON | 648 BSS_CHANGED_BEACON_ENABLED)) 649 mt7915_mcu_add_beacon(hw, vif, info->enable_beacon, changed); 650 651 if (changed & (BSS_CHANGED_UNSOL_BCAST_PROBE_RESP | 652 BSS_CHANGED_FILS_DISCOVERY)) 653 mt7915_mcu_add_inband_discov(dev, vif, changed); 654 655 if (set_bss_info == 0) 656 mt7915_mcu_add_bss_info(phy, vif, false); 657 if (set_sta == 0) 658 mt7915_mcu_add_sta(dev, vif, NULL, false); 659 660 mutex_unlock(&dev->mt76.mutex); 661 } 662 663 static void 664 mt7915_vif_check_caps(struct mt7915_phy *phy, struct ieee80211_vif *vif) 665 { 666 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 667 struct mt7915_vif_cap *vc = &mvif->cap; 668 669 vc->ht_ldpc = vif->bss_conf.ht_ldpc; 670 vc->vht_ldpc = vif->bss_conf.vht_ldpc; 671 vc->vht_su_ebfer = vif->bss_conf.vht_su_beamformer; 672 vc->vht_su_ebfee = vif->bss_conf.vht_su_beamformee; 673 vc->vht_mu_ebfer = vif->bss_conf.vht_mu_beamformer; 674 vc->vht_mu_ebfee = vif->bss_conf.vht_mu_beamformee; 675 vc->he_ldpc = vif->bss_conf.he_ldpc; 676 vc->he_su_ebfer = vif->bss_conf.he_su_beamformer; 677 vc->he_su_ebfee = vif->bss_conf.he_su_beamformee; 678 vc->he_mu_ebfer = vif->bss_conf.he_mu_beamformer; 679 } 680 681 static int 682 mt7915_start_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 683 struct ieee80211_bss_conf *link_conf) 684 { 685 struct mt7915_phy *phy = mt7915_hw_phy(hw); 686 struct mt7915_dev *dev = mt7915_hw_dev(hw); 687 int err; 688 689 mutex_lock(&dev->mt76.mutex); 690 691 mt7915_vif_check_caps(phy, vif); 692 693 err = mt7915_mcu_add_bss_info(phy, vif, true); 694 if (err) 695 goto out; 696 err = mt7915_mcu_add_sta(dev, vif, NULL, true); 697 out: 698 mutex_unlock(&dev->mt76.mutex); 699 700 return err; 701 } 702 703 static void 704 mt7915_stop_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 705 struct ieee80211_bss_conf *link_conf) 706 { 707 struct mt7915_dev *dev = mt7915_hw_dev(hw); 708 709 mutex_lock(&dev->mt76.mutex); 710 mt7915_mcu_add_sta(dev, vif, NULL, false); 711 mutex_unlock(&dev->mt76.mutex); 712 } 713 714 static void 715 mt7915_channel_switch_beacon(struct ieee80211_hw *hw, 716 struct ieee80211_vif *vif, 717 struct cfg80211_chan_def *chandef) 718 { 719 struct mt7915_dev *dev = mt7915_hw_dev(hw); 720 721 mutex_lock(&dev->mt76.mutex); 722 mt7915_mcu_add_beacon(hw, vif, true, BSS_CHANGED_BEACON); 723 mutex_unlock(&dev->mt76.mutex); 724 } 725 726 int mt7915_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, 727 struct ieee80211_sta *sta) 728 { 729 struct mt7915_dev *dev = container_of(mdev, struct mt7915_dev, mt76); 730 struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; 731 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 732 bool ext_phy = mvif->phy != &dev->phy; 733 int ret, idx; 734 735 idx = mt76_wcid_alloc(dev->mt76.wcid_mask, MT7915_WTBL_STA); 736 if (idx < 0) 737 return -ENOSPC; 738 739 INIT_LIST_HEAD(&msta->rc_list); 740 INIT_LIST_HEAD(&msta->wcid.poll_list); 741 msta->vif = mvif; 742 msta->wcid.sta = 1; 743 msta->wcid.idx = idx; 744 msta->wcid.phy_idx = ext_phy; 745 msta->wcid.tx_info |= MT_WCID_TX_INFO_SET; 746 msta->jiffies = jiffies; 747 748 ewma_avg_signal_init(&msta->avg_ack_signal); 749 750 mt7915_mac_wtbl_update(dev, idx, 751 MT_WTBL_UPDATE_ADM_COUNT_CLEAR); 752 753 ret = mt7915_mcu_add_sta(dev, vif, sta, true); 754 if (ret) 755 return ret; 756 757 return mt7915_mcu_add_rate_ctrl(dev, vif, sta, false); 758 } 759 760 void mt7915_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif, 761 struct ieee80211_sta *sta) 762 { 763 struct mt7915_dev *dev = container_of(mdev, struct mt7915_dev, mt76); 764 struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; 765 int i; 766 767 mt7915_mcu_add_sta(dev, vif, sta, false); 768 769 mt7915_mac_wtbl_update(dev, msta->wcid.idx, 770 MT_WTBL_UPDATE_ADM_COUNT_CLEAR); 771 772 for (i = 0; i < ARRAY_SIZE(msta->twt.flow); i++) 773 mt7915_mac_twt_teardown_flow(dev, msta, i); 774 775 spin_lock_bh(&mdev->sta_poll_lock); 776 if (!list_empty(&msta->wcid.poll_list)) 777 list_del_init(&msta->wcid.poll_list); 778 if (!list_empty(&msta->rc_list)) 779 list_del_init(&msta->rc_list); 780 spin_unlock_bh(&mdev->sta_poll_lock); 781 } 782 783 static void mt7915_tx(struct ieee80211_hw *hw, 784 struct ieee80211_tx_control *control, 785 struct sk_buff *skb) 786 { 787 struct mt7915_dev *dev = mt7915_hw_dev(hw); 788 struct mt76_phy *mphy = hw->priv; 789 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 790 struct ieee80211_vif *vif = info->control.vif; 791 struct mt76_wcid *wcid = &dev->mt76.global_wcid; 792 793 if (control->sta) { 794 struct mt7915_sta *sta; 795 796 sta = (struct mt7915_sta *)control->sta->drv_priv; 797 wcid = &sta->wcid; 798 } 799 800 if (vif && !control->sta) { 801 struct mt7915_vif *mvif; 802 803 mvif = (struct mt7915_vif *)vif->drv_priv; 804 wcid = &mvif->sta.wcid; 805 } 806 807 mt76_tx(mphy, control->sta, wcid, skb); 808 } 809 810 static int mt7915_set_rts_threshold(struct ieee80211_hw *hw, u32 val) 811 { 812 struct mt7915_dev *dev = mt7915_hw_dev(hw); 813 struct mt7915_phy *phy = mt7915_hw_phy(hw); 814 int ret; 815 816 mutex_lock(&dev->mt76.mutex); 817 ret = mt76_connac_mcu_set_rts_thresh(&dev->mt76, val, 818 phy->mt76->band_idx); 819 mutex_unlock(&dev->mt76.mutex); 820 821 return ret; 822 } 823 824 static int 825 mt7915_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 826 struct ieee80211_ampdu_params *params) 827 { 828 enum ieee80211_ampdu_mlme_action action = params->action; 829 struct mt7915_dev *dev = mt7915_hw_dev(hw); 830 struct ieee80211_sta *sta = params->sta; 831 struct ieee80211_txq *txq = sta->txq[params->tid]; 832 struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; 833 u16 tid = params->tid; 834 u16 ssn = params->ssn; 835 struct mt76_txq *mtxq; 836 int ret = 0; 837 838 if (!txq) 839 return -EINVAL; 840 841 mtxq = (struct mt76_txq *)txq->drv_priv; 842 843 mutex_lock(&dev->mt76.mutex); 844 switch (action) { 845 case IEEE80211_AMPDU_RX_START: 846 mt76_rx_aggr_start(&dev->mt76, &msta->wcid, tid, ssn, 847 params->buf_size); 848 ret = mt7915_mcu_add_rx_ba(dev, params, true); 849 break; 850 case IEEE80211_AMPDU_RX_STOP: 851 mt76_rx_aggr_stop(&dev->mt76, &msta->wcid, tid); 852 ret = mt7915_mcu_add_rx_ba(dev, params, false); 853 break; 854 case IEEE80211_AMPDU_TX_OPERATIONAL: 855 mtxq->aggr = true; 856 mtxq->send_bar = false; 857 ret = mt7915_mcu_add_tx_ba(dev, params, true); 858 break; 859 case IEEE80211_AMPDU_TX_STOP_FLUSH: 860 case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT: 861 mtxq->aggr = false; 862 clear_bit(tid, &msta->wcid.ampdu_state); 863 ret = mt7915_mcu_add_tx_ba(dev, params, false); 864 break; 865 case IEEE80211_AMPDU_TX_START: 866 set_bit(tid, &msta->wcid.ampdu_state); 867 ret = IEEE80211_AMPDU_TX_START_IMMEDIATE; 868 break; 869 case IEEE80211_AMPDU_TX_STOP_CONT: 870 mtxq->aggr = false; 871 clear_bit(tid, &msta->wcid.ampdu_state); 872 ret = mt7915_mcu_add_tx_ba(dev, params, false); 873 ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid); 874 break; 875 } 876 mutex_unlock(&dev->mt76.mutex); 877 878 return ret; 879 } 880 881 static int 882 mt7915_sta_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 883 struct ieee80211_sta *sta) 884 { 885 return mt76_sta_state(hw, vif, sta, IEEE80211_STA_NOTEXIST, 886 IEEE80211_STA_NONE); 887 } 888 889 static int 890 mt7915_sta_remove(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 891 struct ieee80211_sta *sta) 892 { 893 return mt76_sta_state(hw, vif, sta, IEEE80211_STA_NONE, 894 IEEE80211_STA_NOTEXIST); 895 } 896 897 static int 898 mt7915_get_stats(struct ieee80211_hw *hw, 899 struct ieee80211_low_level_stats *stats) 900 { 901 struct mt7915_phy *phy = mt7915_hw_phy(hw); 902 struct mt7915_dev *dev = mt7915_hw_dev(hw); 903 struct mt76_mib_stats *mib = &phy->mib; 904 905 mutex_lock(&dev->mt76.mutex); 906 907 stats->dot11RTSSuccessCount = mib->rts_cnt; 908 stats->dot11RTSFailureCount = mib->rts_retries_cnt; 909 stats->dot11FCSErrorCount = mib->fcs_err_cnt; 910 stats->dot11ACKFailureCount = mib->ack_fail_cnt; 911 912 mutex_unlock(&dev->mt76.mutex); 913 914 return 0; 915 } 916 917 u64 __mt7915_get_tsf(struct ieee80211_hw *hw, struct mt7915_vif *mvif) 918 { 919 struct mt7915_dev *dev = mt7915_hw_dev(hw); 920 struct mt7915_phy *phy = mt7915_hw_phy(hw); 921 bool band = phy->mt76->band_idx; 922 union { 923 u64 t64; 924 u32 t32[2]; 925 } tsf; 926 u16 n; 927 928 lockdep_assert_held(&dev->mt76.mutex); 929 930 n = mvif->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0 931 : mvif->mt76.omac_idx; 932 /* TSF software read */ 933 if (is_mt7915(&dev->mt76)) 934 mt76_rmw(dev, MT_LPON_TCR(band, n), MT_LPON_TCR_SW_MODE, 935 MT_LPON_TCR_SW_READ); 936 else 937 mt76_rmw(dev, MT_LPON_TCR_MT7916(band, n), MT_LPON_TCR_SW_MODE, 938 MT_LPON_TCR_SW_READ); 939 tsf.t32[0] = mt76_rr(dev, MT_LPON_UTTR0(band)); 940 tsf.t32[1] = mt76_rr(dev, MT_LPON_UTTR1(band)); 941 942 return tsf.t64; 943 } 944 945 static u64 946 mt7915_get_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif) 947 { 948 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 949 struct mt7915_dev *dev = mt7915_hw_dev(hw); 950 u64 ret; 951 952 mutex_lock(&dev->mt76.mutex); 953 ret = __mt7915_get_tsf(hw, mvif); 954 mutex_unlock(&dev->mt76.mutex); 955 956 return ret; 957 } 958 959 static void 960 mt7915_set_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 961 u64 timestamp) 962 { 963 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 964 struct mt7915_dev *dev = mt7915_hw_dev(hw); 965 struct mt7915_phy *phy = mt7915_hw_phy(hw); 966 bool band = phy->mt76->band_idx; 967 union { 968 u64 t64; 969 u32 t32[2]; 970 } tsf = { .t64 = timestamp, }; 971 u16 n; 972 973 mutex_lock(&dev->mt76.mutex); 974 975 n = mvif->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0 976 : mvif->mt76.omac_idx; 977 mt76_wr(dev, MT_LPON_UTTR0(band), tsf.t32[0]); 978 mt76_wr(dev, MT_LPON_UTTR1(band), tsf.t32[1]); 979 /* TSF software overwrite */ 980 if (is_mt7915(&dev->mt76)) 981 mt76_rmw(dev, MT_LPON_TCR(band, n), MT_LPON_TCR_SW_MODE, 982 MT_LPON_TCR_SW_WRITE); 983 else 984 mt76_rmw(dev, MT_LPON_TCR_MT7916(band, n), MT_LPON_TCR_SW_MODE, 985 MT_LPON_TCR_SW_WRITE); 986 987 mutex_unlock(&dev->mt76.mutex); 988 } 989 990 static void 991 mt7915_offset_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 992 s64 timestamp) 993 { 994 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 995 struct mt7915_dev *dev = mt7915_hw_dev(hw); 996 struct mt7915_phy *phy = mt7915_hw_phy(hw); 997 bool band = phy->mt76->band_idx; 998 union { 999 u64 t64; 1000 u32 t32[2]; 1001 } tsf = { .t64 = timestamp, }; 1002 u16 n; 1003 1004 mutex_lock(&dev->mt76.mutex); 1005 1006 n = mvif->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0 1007 : mvif->mt76.omac_idx; 1008 mt76_wr(dev, MT_LPON_UTTR0(band), tsf.t32[0]); 1009 mt76_wr(dev, MT_LPON_UTTR1(band), tsf.t32[1]); 1010 /* TSF software adjust*/ 1011 if (is_mt7915(&dev->mt76)) 1012 mt76_rmw(dev, MT_LPON_TCR(band, n), MT_LPON_TCR_SW_MODE, 1013 MT_LPON_TCR_SW_ADJUST); 1014 else 1015 mt76_rmw(dev, MT_LPON_TCR_MT7916(band, n), MT_LPON_TCR_SW_MODE, 1016 MT_LPON_TCR_SW_ADJUST); 1017 1018 mutex_unlock(&dev->mt76.mutex); 1019 } 1020 1021 static void 1022 mt7915_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class) 1023 { 1024 struct mt7915_phy *phy = mt7915_hw_phy(hw); 1025 struct mt7915_dev *dev = phy->dev; 1026 1027 mutex_lock(&dev->mt76.mutex); 1028 phy->coverage_class = max_t(s16, coverage_class, 0); 1029 mt7915_mac_set_timing(phy); 1030 mutex_unlock(&dev->mt76.mutex); 1031 } 1032 1033 static int 1034 mt7915_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant) 1035 { 1036 struct mt7915_dev *dev = mt7915_hw_dev(hw); 1037 struct mt7915_phy *phy = mt7915_hw_phy(hw); 1038 int max_nss = hweight8(hw->wiphy->available_antennas_tx); 1039 u8 chainshift = dev->chainshift; 1040 u8 band = phy->mt76->band_idx; 1041 1042 if (!tx_ant || tx_ant != rx_ant || ffs(tx_ant) > max_nss) 1043 return -EINVAL; 1044 1045 mutex_lock(&dev->mt76.mutex); 1046 1047 phy->mt76->antenna_mask = tx_ant; 1048 1049 /* handle a variant of mt7916/mt7981 which has 3T3R but nss2 on 5 GHz band */ 1050 if ((is_mt7916(&dev->mt76) || is_mt7981(&dev->mt76)) && 1051 band && hweight8(tx_ant) == max_nss) 1052 phy->mt76->chainmask = (dev->chainmask >> chainshift) << chainshift; 1053 else 1054 phy->mt76->chainmask = tx_ant << (chainshift * band); 1055 1056 mt76_set_stream_caps(phy->mt76, true); 1057 mt7915_set_stream_vht_txbf_caps(phy); 1058 mt7915_set_stream_he_caps(phy); 1059 1060 mutex_unlock(&dev->mt76.mutex); 1061 1062 return 0; 1063 } 1064 1065 static void mt7915_sta_statistics(struct ieee80211_hw *hw, 1066 struct ieee80211_vif *vif, 1067 struct ieee80211_sta *sta, 1068 struct station_info *sinfo) 1069 { 1070 struct mt7915_phy *phy = mt7915_hw_phy(hw); 1071 struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; 1072 struct rate_info *txrate = &msta->wcid.rate; 1073 struct rate_info rxrate = {}; 1074 1075 if (is_mt7915(&phy->dev->mt76) && 1076 !mt7915_mcu_get_rx_rate(phy, vif, sta, &rxrate)) { 1077 sinfo->rxrate = rxrate; 1078 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BITRATE); 1079 } 1080 1081 if (txrate->legacy || txrate->flags) { 1082 if (txrate->legacy) { 1083 sinfo->txrate.legacy = txrate->legacy; 1084 } else { 1085 sinfo->txrate.mcs = txrate->mcs; 1086 sinfo->txrate.nss = txrate->nss; 1087 sinfo->txrate.bw = txrate->bw; 1088 sinfo->txrate.he_gi = txrate->he_gi; 1089 sinfo->txrate.he_dcm = txrate->he_dcm; 1090 sinfo->txrate.he_ru_alloc = txrate->he_ru_alloc; 1091 } 1092 sinfo->txrate.flags = txrate->flags; 1093 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); 1094 } 1095 1096 /* offloading flows bypass networking stack, so driver counts and 1097 * reports sta statistics via NL80211_STA_INFO when WED is active. 1098 */ 1099 if (mtk_wed_device_active(&phy->dev->mt76.mmio.wed)) { 1100 sinfo->tx_bytes = msta->wcid.stats.tx_bytes; 1101 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES64); 1102 1103 if (!mt7915_mcu_wed_wa_tx_stats(phy->dev, msta->wcid.idx)) { 1104 sinfo->tx_packets = msta->wcid.stats.tx_packets; 1105 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_PACKETS); 1106 } 1107 1108 if (mtk_wed_get_rx_capa(&phy->dev->mt76.mmio.wed)) { 1109 sinfo->rx_bytes = msta->wcid.stats.rx_bytes; 1110 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES64); 1111 1112 sinfo->rx_packets = msta->wcid.stats.rx_packets; 1113 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS); 1114 } 1115 } 1116 1117 sinfo->tx_failed = msta->wcid.stats.tx_failed; 1118 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED); 1119 1120 sinfo->tx_retries = msta->wcid.stats.tx_retries; 1121 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES); 1122 1123 sinfo->ack_signal = (s8)msta->ack_signal; 1124 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL); 1125 1126 sinfo->avg_ack_signal = -(s8)ewma_avg_signal_read(&msta->avg_ack_signal); 1127 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL_AVG); 1128 } 1129 1130 static void mt7915_sta_rc_work(void *data, struct ieee80211_sta *sta) 1131 { 1132 struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; 1133 struct mt7915_dev *dev = msta->vif->phy->dev; 1134 u32 *changed = data; 1135 1136 spin_lock_bh(&dev->mt76.sta_poll_lock); 1137 msta->changed |= *changed; 1138 if (list_empty(&msta->rc_list)) 1139 list_add_tail(&msta->rc_list, &dev->sta_rc_list); 1140 spin_unlock_bh(&dev->mt76.sta_poll_lock); 1141 } 1142 1143 static void mt7915_sta_rc_update(struct ieee80211_hw *hw, 1144 struct ieee80211_vif *vif, 1145 struct ieee80211_sta *sta, 1146 u32 changed) 1147 { 1148 struct mt7915_phy *phy = mt7915_hw_phy(hw); 1149 struct mt7915_dev *dev = phy->dev; 1150 1151 mt7915_sta_rc_work(&changed, sta); 1152 ieee80211_queue_work(hw, &dev->rc_work); 1153 } 1154 1155 static int 1156 mt7915_set_bitrate_mask(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 1157 const struct cfg80211_bitrate_mask *mask) 1158 { 1159 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 1160 struct mt7915_phy *phy = mt7915_hw_phy(hw); 1161 struct mt7915_dev *dev = phy->dev; 1162 u32 changed = IEEE80211_RC_SUPP_RATES_CHANGED; 1163 1164 mvif->bitrate_mask = *mask; 1165 1166 /* if multiple rates across different preambles are given we can 1167 * reconfigure this info with all peers using sta_rec command with 1168 * the below exception cases. 1169 * - single rate : if a rate is passed along with different preambles, 1170 * we select the highest one as fixed rate. i.e VHT MCS for VHT peers. 1171 * - multiple rates: if it's not in range format i.e 0-{7,8,9} for VHT 1172 * then multiple MCS setting (MCS 4,5,6) is not supported. 1173 */ 1174 ieee80211_iterate_stations_atomic(hw, mt7915_sta_rc_work, &changed); 1175 ieee80211_queue_work(hw, &dev->rc_work); 1176 1177 return 0; 1178 } 1179 1180 static void mt7915_sta_set_4addr(struct ieee80211_hw *hw, 1181 struct ieee80211_vif *vif, 1182 struct ieee80211_sta *sta, 1183 bool enabled) 1184 { 1185 struct mt7915_dev *dev = mt7915_hw_dev(hw); 1186 struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; 1187 1188 if (enabled) 1189 set_bit(MT_WCID_FLAG_4ADDR, &msta->wcid.flags); 1190 else 1191 clear_bit(MT_WCID_FLAG_4ADDR, &msta->wcid.flags); 1192 1193 mt76_connac_mcu_wtbl_update_hdr_trans(&dev->mt76, vif, sta); 1194 } 1195 1196 static void mt7915_sta_set_decap_offload(struct ieee80211_hw *hw, 1197 struct ieee80211_vif *vif, 1198 struct ieee80211_sta *sta, 1199 bool enabled) 1200 { 1201 struct mt7915_dev *dev = mt7915_hw_dev(hw); 1202 struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; 1203 1204 if (enabled) 1205 set_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags); 1206 else 1207 clear_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags); 1208 1209 mt76_connac_mcu_wtbl_update_hdr_trans(&dev->mt76, vif, sta); 1210 } 1211 1212 static int mt7915_sta_set_txpwr(struct ieee80211_hw *hw, 1213 struct ieee80211_vif *vif, 1214 struct ieee80211_sta *sta) 1215 { 1216 struct mt7915_phy *phy = mt7915_hw_phy(hw); 1217 struct mt7915_dev *dev = mt7915_hw_dev(hw); 1218 s16 txpower = sta->deflink.txpwr.power; 1219 int ret; 1220 1221 if (sta->deflink.txpwr.type == NL80211_TX_POWER_AUTOMATIC) 1222 txpower = 0; 1223 1224 mutex_lock(&dev->mt76.mutex); 1225 1226 /* NOTE: temporarily use 0 as minimum limit, which is a 1227 * global setting and will be applied to all stations. 1228 */ 1229 ret = mt7915_mcu_set_txpower_frame_min(phy, 0); 1230 if (ret) 1231 goto out; 1232 1233 /* This only applies to data frames while pushing traffic, 1234 * whereas the management frames or other packets that are 1235 * using fixed rate can be configured via TxD. 1236 */ 1237 ret = mt7915_mcu_set_txpower_frame(phy, vif, sta, txpower); 1238 1239 out: 1240 mutex_unlock(&dev->mt76.mutex); 1241 1242 return ret; 1243 } 1244 1245 static const char mt7915_gstrings_stats[][ETH_GSTRING_LEN] = { 1246 "tx_ampdu_cnt", 1247 "tx_stop_q_empty_cnt", 1248 "tx_mpdu_attempts", 1249 "tx_mpdu_success", 1250 "tx_rwp_fail_cnt", 1251 "tx_rwp_need_cnt", 1252 "tx_pkt_ebf_cnt", 1253 "tx_pkt_ibf_cnt", 1254 "tx_ampdu_len:0-1", 1255 "tx_ampdu_len:2-10", 1256 "tx_ampdu_len:11-19", 1257 "tx_ampdu_len:20-28", 1258 "tx_ampdu_len:29-37", 1259 "tx_ampdu_len:38-46", 1260 "tx_ampdu_len:47-55", 1261 "tx_ampdu_len:56-79", 1262 "tx_ampdu_len:80-103", 1263 "tx_ampdu_len:104-127", 1264 "tx_ampdu_len:128-151", 1265 "tx_ampdu_len:152-175", 1266 "tx_ampdu_len:176-199", 1267 "tx_ampdu_len:200-223", 1268 "tx_ampdu_len:224-247", 1269 "ba_miss_count", 1270 "tx_beamformer_ppdu_iBF", 1271 "tx_beamformer_ppdu_eBF", 1272 "tx_beamformer_rx_feedback_all", 1273 "tx_beamformer_rx_feedback_he", 1274 "tx_beamformer_rx_feedback_vht", 1275 "tx_beamformer_rx_feedback_ht", 1276 "tx_beamformer_rx_feedback_bw", /* zero based idx: 20, 40, 80, 160 */ 1277 "tx_beamformer_rx_feedback_nc", 1278 "tx_beamformer_rx_feedback_nr", 1279 "tx_beamformee_ok_feedback_pkts", 1280 "tx_beamformee_feedback_trig", 1281 "tx_mu_beamforming", 1282 "tx_mu_mpdu", 1283 "tx_mu_successful_mpdu", 1284 "tx_su_successful_mpdu", 1285 "tx_msdu_pack_1", 1286 "tx_msdu_pack_2", 1287 "tx_msdu_pack_3", 1288 "tx_msdu_pack_4", 1289 "tx_msdu_pack_5", 1290 "tx_msdu_pack_6", 1291 "tx_msdu_pack_7", 1292 "tx_msdu_pack_8", 1293 1294 /* rx counters */ 1295 "rx_fifo_full_cnt", 1296 "rx_mpdu_cnt", 1297 "channel_idle_cnt", 1298 "primary_cca_busy_time", 1299 "secondary_cca_busy_time", 1300 "primary_energy_detect_time", 1301 "cck_mdrdy_time", 1302 "ofdm_mdrdy_time", 1303 "green_mdrdy_time", 1304 "rx_vector_mismatch_cnt", 1305 "rx_delimiter_fail_cnt", 1306 "rx_mrdy_cnt", 1307 "rx_len_mismatch_cnt", 1308 "rx_ampdu_cnt", 1309 "rx_ampdu_bytes_cnt", 1310 "rx_ampdu_valid_subframe_cnt", 1311 "rx_ampdu_valid_subframe_b_cnt", 1312 "rx_pfdrop_cnt", 1313 "rx_vec_queue_overflow_drop_cnt", 1314 "rx_ba_cnt", 1315 1316 /* muru mu-mimo and ofdma related stats */ 1317 "dl_cck_cnt", 1318 "dl_ofdm_cnt", 1319 "dl_htmix_cnt", 1320 "dl_htgf_cnt", 1321 "dl_vht_su_cnt", 1322 "dl_vht_2mu_cnt", 1323 "dl_vht_3mu_cnt", 1324 "dl_vht_4mu_cnt", 1325 "dl_he_su_cnt", 1326 "dl_he_ext_su_cnt", 1327 "dl_he_2ru_cnt", 1328 "dl_he_2mu_cnt", 1329 "dl_he_3ru_cnt", 1330 "dl_he_3mu_cnt", 1331 "dl_he_4ru_cnt", 1332 "dl_he_4mu_cnt", 1333 "dl_he_5to8ru_cnt", 1334 "dl_he_9to16ru_cnt", 1335 "dl_he_gtr16ru_cnt", 1336 1337 "ul_hetrig_su_cnt", 1338 "ul_hetrig_2ru_cnt", 1339 "ul_hetrig_3ru_cnt", 1340 "ul_hetrig_4ru_cnt", 1341 "ul_hetrig_5to8ru_cnt", 1342 "ul_hetrig_9to16ru_cnt", 1343 "ul_hetrig_gtr16ru_cnt", 1344 "ul_hetrig_2mu_cnt", 1345 "ul_hetrig_3mu_cnt", 1346 "ul_hetrig_4mu_cnt", 1347 1348 /* per vif counters */ 1349 "v_tx_mode_cck", 1350 "v_tx_mode_ofdm", 1351 "v_tx_mode_ht", 1352 "v_tx_mode_ht_gf", 1353 "v_tx_mode_vht", 1354 "v_tx_mode_he_su", 1355 "v_tx_mode_he_ext_su", 1356 "v_tx_mode_he_tb", 1357 "v_tx_mode_he_mu", 1358 "v_tx_bw_20", 1359 "v_tx_bw_40", 1360 "v_tx_bw_80", 1361 "v_tx_bw_160", 1362 "v_tx_mcs_0", 1363 "v_tx_mcs_1", 1364 "v_tx_mcs_2", 1365 "v_tx_mcs_3", 1366 "v_tx_mcs_4", 1367 "v_tx_mcs_5", 1368 "v_tx_mcs_6", 1369 "v_tx_mcs_7", 1370 "v_tx_mcs_8", 1371 "v_tx_mcs_9", 1372 "v_tx_mcs_10", 1373 "v_tx_mcs_11", 1374 "v_tx_nss_1", 1375 "v_tx_nss_2", 1376 "v_tx_nss_3", 1377 "v_tx_nss_4", 1378 }; 1379 1380 #define MT7915_SSTATS_LEN ARRAY_SIZE(mt7915_gstrings_stats) 1381 1382 /* Ethtool related API */ 1383 static 1384 void mt7915_get_et_strings(struct ieee80211_hw *hw, 1385 struct ieee80211_vif *vif, 1386 u32 sset, u8 *data) 1387 { 1388 if (sset != ETH_SS_STATS) 1389 return; 1390 1391 memcpy(data, mt7915_gstrings_stats, sizeof(mt7915_gstrings_stats)); 1392 data += sizeof(mt7915_gstrings_stats); 1393 page_pool_ethtool_stats_get_strings(data); 1394 } 1395 1396 static 1397 int mt7915_get_et_sset_count(struct ieee80211_hw *hw, 1398 struct ieee80211_vif *vif, int sset) 1399 { 1400 if (sset != ETH_SS_STATS) 1401 return 0; 1402 1403 return MT7915_SSTATS_LEN + page_pool_ethtool_stats_get_count(); 1404 } 1405 1406 static void mt7915_ethtool_worker(void *wi_data, struct ieee80211_sta *sta) 1407 { 1408 struct mt76_ethtool_worker_info *wi = wi_data; 1409 struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; 1410 1411 if (msta->vif->mt76.idx != wi->idx) 1412 return; 1413 1414 mt76_ethtool_worker(wi, &msta->wcid.stats, false); 1415 } 1416 1417 static 1418 void mt7915_get_et_stats(struct ieee80211_hw *hw, 1419 struct ieee80211_vif *vif, 1420 struct ethtool_stats *stats, u64 *data) 1421 { 1422 struct mt7915_dev *dev = mt7915_hw_dev(hw); 1423 struct mt7915_phy *phy = mt7915_hw_phy(hw); 1424 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 1425 struct mt76_mib_stats *mib = &phy->mib; 1426 struct mt76_ethtool_worker_info wi = { 1427 .data = data, 1428 .idx = mvif->mt76.idx, 1429 }; 1430 /* See mt7915_ampdu_stat_read_phy, etc */ 1431 int i, ei = 0, stats_size; 1432 1433 mutex_lock(&dev->mt76.mutex); 1434 1435 mt7915_mac_update_stats(phy); 1436 1437 data[ei++] = mib->tx_ampdu_cnt; 1438 data[ei++] = mib->tx_stop_q_empty_cnt; 1439 data[ei++] = mib->tx_mpdu_attempts_cnt; 1440 data[ei++] = mib->tx_mpdu_success_cnt; 1441 data[ei++] = mib->tx_rwp_fail_cnt; 1442 data[ei++] = mib->tx_rwp_need_cnt; 1443 data[ei++] = mib->tx_pkt_ebf_cnt; 1444 data[ei++] = mib->tx_pkt_ibf_cnt; 1445 1446 /* Tx ampdu stat */ 1447 for (i = 0; i < 15 /*ARRAY_SIZE(bound)*/; i++) 1448 data[ei++] = phy->mt76->aggr_stats[i]; 1449 1450 data[ei++] = phy->mib.ba_miss_cnt; 1451 1452 /* Tx Beamformer monitor */ 1453 data[ei++] = mib->tx_bf_ibf_ppdu_cnt; 1454 data[ei++] = mib->tx_bf_ebf_ppdu_cnt; 1455 1456 /* Tx Beamformer Rx feedback monitor */ 1457 data[ei++] = mib->tx_bf_rx_fb_all_cnt; 1458 data[ei++] = mib->tx_bf_rx_fb_he_cnt; 1459 data[ei++] = mib->tx_bf_rx_fb_vht_cnt; 1460 data[ei++] = mib->tx_bf_rx_fb_ht_cnt; 1461 1462 data[ei++] = mib->tx_bf_rx_fb_bw; 1463 data[ei++] = mib->tx_bf_rx_fb_nc_cnt; 1464 data[ei++] = mib->tx_bf_rx_fb_nr_cnt; 1465 1466 /* Tx Beamformee Rx NDPA & Tx feedback report */ 1467 data[ei++] = mib->tx_bf_fb_cpl_cnt; 1468 data[ei++] = mib->tx_bf_fb_trig_cnt; 1469 1470 /* Tx SU & MU counters */ 1471 data[ei++] = mib->tx_bf_cnt; 1472 data[ei++] = mib->tx_mu_mpdu_cnt; 1473 data[ei++] = mib->tx_mu_acked_mpdu_cnt; 1474 data[ei++] = mib->tx_su_acked_mpdu_cnt; 1475 1476 /* Tx amsdu info (pack-count histogram) */ 1477 for (i = 0; i < ARRAY_SIZE(mib->tx_amsdu); i++) 1478 data[ei++] = mib->tx_amsdu[i]; 1479 1480 /* rx counters */ 1481 data[ei++] = mib->rx_fifo_full_cnt; 1482 data[ei++] = mib->rx_mpdu_cnt; 1483 data[ei++] = mib->channel_idle_cnt; 1484 data[ei++] = mib->primary_cca_busy_time; 1485 data[ei++] = mib->secondary_cca_busy_time; 1486 data[ei++] = mib->primary_energy_detect_time; 1487 data[ei++] = mib->cck_mdrdy_time; 1488 data[ei++] = mib->ofdm_mdrdy_time; 1489 data[ei++] = mib->green_mdrdy_time; 1490 data[ei++] = mib->rx_vector_mismatch_cnt; 1491 data[ei++] = mib->rx_delimiter_fail_cnt; 1492 data[ei++] = mib->rx_mrdy_cnt; 1493 data[ei++] = mib->rx_len_mismatch_cnt; 1494 data[ei++] = mib->rx_ampdu_cnt; 1495 data[ei++] = mib->rx_ampdu_bytes_cnt; 1496 data[ei++] = mib->rx_ampdu_valid_subframe_cnt; 1497 data[ei++] = mib->rx_ampdu_valid_subframe_bytes_cnt; 1498 data[ei++] = mib->rx_pfdrop_cnt; 1499 data[ei++] = mib->rx_vec_queue_overflow_drop_cnt; 1500 data[ei++] = mib->rx_ba_cnt; 1501 1502 data[ei++] = mib->dl_cck_cnt; 1503 data[ei++] = mib->dl_ofdm_cnt; 1504 data[ei++] = mib->dl_htmix_cnt; 1505 data[ei++] = mib->dl_htgf_cnt; 1506 data[ei++] = mib->dl_vht_su_cnt; 1507 data[ei++] = mib->dl_vht_2mu_cnt; 1508 data[ei++] = mib->dl_vht_3mu_cnt; 1509 data[ei++] = mib->dl_vht_4mu_cnt; 1510 data[ei++] = mib->dl_he_su_cnt; 1511 data[ei++] = mib->dl_he_ext_su_cnt; 1512 data[ei++] = mib->dl_he_2ru_cnt; 1513 data[ei++] = mib->dl_he_2mu_cnt; 1514 data[ei++] = mib->dl_he_3ru_cnt; 1515 data[ei++] = mib->dl_he_3mu_cnt; 1516 data[ei++] = mib->dl_he_4ru_cnt; 1517 data[ei++] = mib->dl_he_4mu_cnt; 1518 data[ei++] = mib->dl_he_5to8ru_cnt; 1519 data[ei++] = mib->dl_he_9to16ru_cnt; 1520 data[ei++] = mib->dl_he_gtr16ru_cnt; 1521 1522 data[ei++] = mib->ul_hetrig_su_cnt; 1523 data[ei++] = mib->ul_hetrig_2ru_cnt; 1524 data[ei++] = mib->ul_hetrig_3ru_cnt; 1525 data[ei++] = mib->ul_hetrig_4ru_cnt; 1526 data[ei++] = mib->ul_hetrig_5to8ru_cnt; 1527 data[ei++] = mib->ul_hetrig_9to16ru_cnt; 1528 data[ei++] = mib->ul_hetrig_gtr16ru_cnt; 1529 data[ei++] = mib->ul_hetrig_2mu_cnt; 1530 data[ei++] = mib->ul_hetrig_3mu_cnt; 1531 data[ei++] = mib->ul_hetrig_4mu_cnt; 1532 1533 /* Add values for all stations owned by this vif */ 1534 wi.initial_stat_idx = ei; 1535 ieee80211_iterate_stations_atomic(hw, mt7915_ethtool_worker, &wi); 1536 1537 mutex_unlock(&dev->mt76.mutex); 1538 1539 if (wi.sta_count == 0) 1540 return; 1541 1542 ei += wi.worker_stat_count; 1543 1544 mt76_ethtool_page_pool_stats(&dev->mt76, &data[ei], &ei); 1545 1546 stats_size = MT7915_SSTATS_LEN + page_pool_ethtool_stats_get_count(); 1547 if (ei != stats_size) 1548 dev_err(dev->mt76.dev, "ei: %d size: %d", ei, stats_size); 1549 } 1550 1551 static void 1552 mt7915_twt_teardown_request(struct ieee80211_hw *hw, 1553 struct ieee80211_sta *sta, 1554 u8 flowid) 1555 { 1556 struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; 1557 struct mt7915_dev *dev = mt7915_hw_dev(hw); 1558 1559 mutex_lock(&dev->mt76.mutex); 1560 mt7915_mac_twt_teardown_flow(dev, msta, flowid); 1561 mutex_unlock(&dev->mt76.mutex); 1562 } 1563 1564 static int 1565 mt7915_set_radar_background(struct ieee80211_hw *hw, 1566 struct cfg80211_chan_def *chandef) 1567 { 1568 struct mt7915_phy *phy = mt7915_hw_phy(hw); 1569 struct mt7915_dev *dev = phy->dev; 1570 int ret = -EINVAL; 1571 bool running; 1572 1573 mutex_lock(&dev->mt76.mutex); 1574 1575 if (dev->mt76.region == NL80211_DFS_UNSET) 1576 goto out; 1577 1578 if (dev->rdd2_phy && dev->rdd2_phy != phy) { 1579 /* rdd2 is already locked */ 1580 ret = -EBUSY; 1581 goto out; 1582 } 1583 1584 /* rdd2 already configured on a radar channel */ 1585 running = dev->rdd2_phy && 1586 cfg80211_chandef_valid(&dev->rdd2_chandef) && 1587 !!(dev->rdd2_chandef.chan->flags & IEEE80211_CHAN_RADAR); 1588 1589 if (!chandef || running || 1590 !(chandef->chan->flags & IEEE80211_CHAN_RADAR)) { 1591 ret = mt7915_mcu_rdd_background_enable(phy, NULL); 1592 if (ret) 1593 goto out; 1594 1595 if (!running) 1596 goto update_phy; 1597 } 1598 1599 ret = mt7915_mcu_rdd_background_enable(phy, chandef); 1600 if (ret) 1601 goto out; 1602 1603 update_phy: 1604 dev->rdd2_phy = chandef ? phy : NULL; 1605 if (chandef) 1606 dev->rdd2_chandef = *chandef; 1607 out: 1608 mutex_unlock(&dev->mt76.mutex); 1609 1610 return ret; 1611 } 1612 1613 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 1614 static int 1615 mt7915_net_fill_forward_path(struct ieee80211_hw *hw, 1616 struct ieee80211_vif *vif, 1617 struct ieee80211_sta *sta, 1618 struct net_device_path_ctx *ctx, 1619 struct net_device_path *path) 1620 { 1621 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; 1622 struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; 1623 struct mt7915_dev *dev = mt7915_hw_dev(hw); 1624 struct mt7915_phy *phy = mt7915_hw_phy(hw); 1625 struct mtk_wed_device *wed = &dev->mt76.mmio.wed; 1626 1627 if (!mtk_wed_device_active(wed)) 1628 return -ENODEV; 1629 1630 if (msta->wcid.idx > 0xff) 1631 return -EIO; 1632 1633 path->type = DEV_PATH_MTK_WDMA; 1634 path->dev = ctx->dev; 1635 path->mtk_wdma.wdma_idx = wed->wdma_idx; 1636 path->mtk_wdma.bss = mvif->mt76.idx; 1637 path->mtk_wdma.wcid = is_mt7915(&dev->mt76) ? msta->wcid.idx : 0x3ff; 1638 path->mtk_wdma.queue = phy != &dev->phy; 1639 1640 ctx->dev = NULL; 1641 1642 return 0; 1643 } 1644 #endif 1645 1646 const struct ieee80211_ops mt7915_ops = { 1647 .tx = mt7915_tx, 1648 .start = mt7915_start, 1649 .stop = mt7915_stop, 1650 .add_interface = mt7915_add_interface, 1651 .remove_interface = mt7915_remove_interface, 1652 .config = mt7915_config, 1653 .conf_tx = mt7915_conf_tx, 1654 .configure_filter = mt7915_configure_filter, 1655 .bss_info_changed = mt7915_bss_info_changed, 1656 .start_ap = mt7915_start_ap, 1657 .stop_ap = mt7915_stop_ap, 1658 .sta_add = mt7915_sta_add, 1659 .sta_remove = mt7915_sta_remove, 1660 .sta_pre_rcu_remove = mt76_sta_pre_rcu_remove, 1661 .sta_rc_update = mt7915_sta_rc_update, 1662 .set_key = mt7915_set_key, 1663 .ampdu_action = mt7915_ampdu_action, 1664 .set_rts_threshold = mt7915_set_rts_threshold, 1665 .wake_tx_queue = mt76_wake_tx_queue, 1666 .sw_scan_start = mt76_sw_scan, 1667 .sw_scan_complete = mt76_sw_scan_complete, 1668 .release_buffered_frames = mt76_release_buffered_frames, 1669 .get_txpower = mt76_get_txpower, 1670 .set_sar_specs = mt7915_set_sar_specs, 1671 .channel_switch_beacon = mt7915_channel_switch_beacon, 1672 .get_stats = mt7915_get_stats, 1673 .get_et_sset_count = mt7915_get_et_sset_count, 1674 .get_et_stats = mt7915_get_et_stats, 1675 .get_et_strings = mt7915_get_et_strings, 1676 .get_tsf = mt7915_get_tsf, 1677 .set_tsf = mt7915_set_tsf, 1678 .offset_tsf = mt7915_offset_tsf, 1679 .get_survey = mt76_get_survey, 1680 .get_antenna = mt76_get_antenna, 1681 .set_antenna = mt7915_set_antenna, 1682 .set_bitrate_mask = mt7915_set_bitrate_mask, 1683 .set_coverage_class = mt7915_set_coverage_class, 1684 .sta_statistics = mt7915_sta_statistics, 1685 .sta_set_txpwr = mt7915_sta_set_txpwr, 1686 .sta_set_4addr = mt7915_sta_set_4addr, 1687 .sta_set_decap_offload = mt7915_sta_set_decap_offload, 1688 .add_twt_setup = mt7915_mac_add_twt_setup, 1689 .twt_teardown_request = mt7915_twt_teardown_request, 1690 CFG80211_TESTMODE_CMD(mt76_testmode_cmd) 1691 CFG80211_TESTMODE_DUMP(mt76_testmode_dump) 1692 #ifdef CONFIG_MAC80211_DEBUGFS 1693 .sta_add_debugfs = mt7915_sta_add_debugfs, 1694 #endif 1695 .set_radar_background = mt7915_set_radar_background, 1696 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 1697 .net_fill_forward_path = mt7915_net_fill_forward_path, 1698 #endif 1699 }; 1700