1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. 4 * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. 5 */ 6 7 #include <crypto/hash.h> 8 #include "core.h" 9 #include "dp_tx.h" 10 #include "hal_tx.h" 11 #include "hif.h" 12 #include "debug.h" 13 #include "dp_rx.h" 14 #include "peer.h" 15 #include "dp_mon.h" 16 17 static void ath12k_dp_htt_htc_tx_complete(struct ath12k_base *ab, 18 struct sk_buff *skb) 19 { 20 dev_kfree_skb_any(skb); 21 } 22 23 void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr) 24 { 25 struct ath12k_base *ab = ar->ab; 26 struct ath12k_peer *peer; 27 28 /* TODO: Any other peer specific DP cleanup */ 29 30 spin_lock_bh(&ab->base_lock); 31 peer = ath12k_peer_find(ab, vdev_id, addr); 32 if (!peer) { 33 ath12k_warn(ab, "failed to lookup peer %pM on vdev %d\n", 34 addr, vdev_id); 35 spin_unlock_bh(&ab->base_lock); 36 return; 37 } 38 39 ath12k_dp_rx_peer_tid_cleanup(ar, peer); 40 crypto_free_shash(peer->tfm_mmic); 41 peer->dp_setup_done = false; 42 spin_unlock_bh(&ab->base_lock); 43 } 44 45 int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr) 46 { 47 struct ath12k_base *ab = ar->ab; 48 struct ath12k_peer *peer; 49 u32 reo_dest; 50 int ret = 0, tid; 51 52 /* NOTE: reo_dest ring id starts from 1 unlike mac_id which starts from 0 */ 53 reo_dest = ar->dp.mac_id + 1; 54 ret = ath12k_wmi_set_peer_param(ar, addr, vdev_id, 55 WMI_PEER_SET_DEFAULT_ROUTING, 56 DP_RX_HASH_ENABLE | (reo_dest << 1)); 57 58 if (ret) { 59 ath12k_warn(ab, "failed to set default routing %d peer :%pM vdev_id :%d\n", 60 ret, addr, vdev_id); 61 return ret; 62 } 63 64 for (tid = 0; tid <= IEEE80211_NUM_TIDS; tid++) { 65 ret = ath12k_dp_rx_peer_tid_setup(ar, addr, vdev_id, tid, 1, 0, 66 HAL_PN_TYPE_NONE); 67 if (ret) { 68 ath12k_warn(ab, "failed to setup rxd tid queue for tid %d: %d\n", 69 tid, ret); 70 goto peer_clean; 71 } 72 } 73 74 ret = ath12k_dp_rx_peer_frag_setup(ar, addr, vdev_id); 75 if (ret) { 76 ath12k_warn(ab, "failed to setup rx defrag context\n"); 77 goto peer_clean; 78 } 79 80 /* TODO: Setup other peer specific resource used in data path */ 81 82 return 0; 83 84 peer_clean: 85 spin_lock_bh(&ab->base_lock); 86 87 peer = ath12k_peer_find(ab, vdev_id, addr); 88 if (!peer) { 89 ath12k_warn(ab, "failed to find the peer to del rx tid\n"); 90 spin_unlock_bh(&ab->base_lock); 91 return -ENOENT; 92 } 93 94 for (; tid >= 0; tid--) 95 ath12k_dp_rx_peer_tid_delete(ar, peer, tid); 96 97 spin_unlock_bh(&ab->base_lock); 98 99 return ret; 100 } 101 102 void ath12k_dp_srng_cleanup(struct ath12k_base *ab, struct dp_srng *ring) 103 { 104 if (!ring->vaddr_unaligned) 105 return; 106 107 dma_free_coherent(ab->dev, ring->size, ring->vaddr_unaligned, 108 ring->paddr_unaligned); 109 110 ring->vaddr_unaligned = NULL; 111 } 112 113 static int ath12k_dp_srng_find_ring_in_mask(int ring_num, const u8 *grp_mask) 114 { 115 int ext_group_num; 116 u8 mask = 1 << ring_num; 117 118 for (ext_group_num = 0; ext_group_num < ATH12K_EXT_IRQ_GRP_NUM_MAX; 119 ext_group_num++) { 120 if (mask & grp_mask[ext_group_num]) 121 return ext_group_num; 122 } 123 124 return -ENOENT; 125 } 126 127 static int ath12k_dp_srng_calculate_msi_group(struct ath12k_base *ab, 128 enum hal_ring_type type, int ring_num) 129 { 130 const u8 *grp_mask; 131 132 switch (type) { 133 case HAL_WBM2SW_RELEASE: 134 if (ring_num == HAL_WBM2SW_REL_ERR_RING_NUM) { 135 grp_mask = &ab->hw_params->ring_mask->rx_wbm_rel[0]; 136 ring_num = 0; 137 } else { 138 grp_mask = &ab->hw_params->ring_mask->tx[0]; 139 } 140 break; 141 case HAL_REO_EXCEPTION: 142 grp_mask = &ab->hw_params->ring_mask->rx_err[0]; 143 break; 144 case HAL_REO_DST: 145 grp_mask = &ab->hw_params->ring_mask->rx[0]; 146 break; 147 case HAL_REO_STATUS: 148 grp_mask = &ab->hw_params->ring_mask->reo_status[0]; 149 break; 150 case HAL_RXDMA_MONITOR_STATUS: 151 case HAL_RXDMA_MONITOR_DST: 152 grp_mask = &ab->hw_params->ring_mask->rx_mon_dest[0]; 153 break; 154 case HAL_TX_MONITOR_DST: 155 grp_mask = &ab->hw_params->ring_mask->tx_mon_dest[0]; 156 break; 157 case HAL_RXDMA_BUF: 158 grp_mask = &ab->hw_params->ring_mask->host2rxdma[0]; 159 break; 160 case HAL_RXDMA_MONITOR_BUF: 161 case HAL_TCL_DATA: 162 case HAL_TCL_CMD: 163 case HAL_REO_CMD: 164 case HAL_SW2WBM_RELEASE: 165 case HAL_WBM_IDLE_LINK: 166 case HAL_TCL_STATUS: 167 case HAL_REO_REINJECT: 168 case HAL_CE_SRC: 169 case HAL_CE_DST: 170 case HAL_CE_DST_STATUS: 171 default: 172 return -ENOENT; 173 } 174 175 return ath12k_dp_srng_find_ring_in_mask(ring_num, grp_mask); 176 } 177 178 static void ath12k_dp_srng_msi_setup(struct ath12k_base *ab, 179 struct hal_srng_params *ring_params, 180 enum hal_ring_type type, int ring_num) 181 { 182 int msi_group_number, msi_data_count; 183 u32 msi_data_start, msi_irq_start, addr_lo, addr_hi; 184 int ret; 185 186 ret = ath12k_hif_get_user_msi_vector(ab, "DP", 187 &msi_data_count, &msi_data_start, 188 &msi_irq_start); 189 if (ret) 190 return; 191 192 msi_group_number = ath12k_dp_srng_calculate_msi_group(ab, type, 193 ring_num); 194 if (msi_group_number < 0) { 195 ath12k_dbg(ab, ATH12K_DBG_PCI, 196 "ring not part of an ext_group; ring_type: %d,ring_num %d", 197 type, ring_num); 198 ring_params->msi_addr = 0; 199 ring_params->msi_data = 0; 200 return; 201 } 202 203 if (msi_group_number > msi_data_count) { 204 ath12k_dbg(ab, ATH12K_DBG_PCI, 205 "multiple msi_groups share one msi, msi_group_num %d", 206 msi_group_number); 207 } 208 209 ath12k_hif_get_msi_address(ab, &addr_lo, &addr_hi); 210 211 ring_params->msi_addr = addr_lo; 212 ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32); 213 ring_params->msi_data = (msi_group_number % msi_data_count) 214 + msi_data_start; 215 ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR; 216 } 217 218 int ath12k_dp_srng_setup(struct ath12k_base *ab, struct dp_srng *ring, 219 enum hal_ring_type type, int ring_num, 220 int mac_id, int num_entries) 221 { 222 struct hal_srng_params params = { 0 }; 223 int entry_sz = ath12k_hal_srng_get_entrysize(ab, type); 224 int max_entries = ath12k_hal_srng_get_max_entries(ab, type); 225 int ret; 226 227 if (max_entries < 0 || entry_sz < 0) 228 return -EINVAL; 229 230 if (num_entries > max_entries) 231 num_entries = max_entries; 232 233 ring->size = (num_entries * entry_sz) + HAL_RING_BASE_ALIGN - 1; 234 ring->vaddr_unaligned = dma_alloc_coherent(ab->dev, ring->size, 235 &ring->paddr_unaligned, 236 GFP_KERNEL); 237 if (!ring->vaddr_unaligned) 238 return -ENOMEM; 239 240 ring->vaddr = PTR_ALIGN(ring->vaddr_unaligned, HAL_RING_BASE_ALIGN); 241 ring->paddr = ring->paddr_unaligned + ((unsigned long)ring->vaddr - 242 (unsigned long)ring->vaddr_unaligned); 243 244 params.ring_base_vaddr = ring->vaddr; 245 params.ring_base_paddr = ring->paddr; 246 params.num_entries = num_entries; 247 ath12k_dp_srng_msi_setup(ab, ¶ms, type, ring_num + mac_id); 248 249 switch (type) { 250 case HAL_REO_DST: 251 params.intr_batch_cntr_thres_entries = 252 HAL_SRNG_INT_BATCH_THRESHOLD_RX; 253 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 254 break; 255 case HAL_RXDMA_BUF: 256 case HAL_RXDMA_MONITOR_BUF: 257 case HAL_RXDMA_MONITOR_STATUS: 258 params.low_threshold = num_entries >> 3; 259 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN; 260 params.intr_batch_cntr_thres_entries = 0; 261 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 262 break; 263 case HAL_TX_MONITOR_DST: 264 params.low_threshold = DP_TX_MONITOR_BUF_SIZE_MAX >> 3; 265 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN; 266 params.intr_batch_cntr_thres_entries = 0; 267 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 268 break; 269 case HAL_WBM2SW_RELEASE: 270 if (ab->hw_params->hw_ops->dp_srng_is_tx_comp_ring(ring_num)) { 271 params.intr_batch_cntr_thres_entries = 272 HAL_SRNG_INT_BATCH_THRESHOLD_TX; 273 params.intr_timer_thres_us = 274 HAL_SRNG_INT_TIMER_THRESHOLD_TX; 275 break; 276 } 277 /* follow through when ring_num != HAL_WBM2SW_REL_ERR_RING_NUM */ 278 fallthrough; 279 case HAL_REO_EXCEPTION: 280 case HAL_REO_REINJECT: 281 case HAL_REO_CMD: 282 case HAL_REO_STATUS: 283 case HAL_TCL_DATA: 284 case HAL_TCL_CMD: 285 case HAL_TCL_STATUS: 286 case HAL_WBM_IDLE_LINK: 287 case HAL_SW2WBM_RELEASE: 288 case HAL_RXDMA_DST: 289 case HAL_RXDMA_MONITOR_DST: 290 case HAL_RXDMA_MONITOR_DESC: 291 params.intr_batch_cntr_thres_entries = 292 HAL_SRNG_INT_BATCH_THRESHOLD_OTHER; 293 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_OTHER; 294 break; 295 case HAL_RXDMA_DIR_BUF: 296 break; 297 default: 298 ath12k_warn(ab, "Not a valid ring type in dp :%d\n", type); 299 return -EINVAL; 300 } 301 302 ret = ath12k_hal_srng_setup(ab, type, ring_num, mac_id, ¶ms); 303 if (ret < 0) { 304 ath12k_warn(ab, "failed to setup srng: %d ring_id %d\n", 305 ret, ring_num); 306 return ret; 307 } 308 309 ring->ring_id = ret; 310 311 return 0; 312 } 313 314 static 315 u32 ath12k_dp_tx_get_vdev_bank_config(struct ath12k_base *ab, struct ath12k_vif *arvif) 316 { 317 u32 bank_config = 0; 318 319 /* Only valid for raw frames with HW crypto enabled. 320 * With SW crypto, mac80211 sets key per packet 321 */ 322 if (arvif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW && 323 test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags)) 324 bank_config |= 325 u32_encode_bits(ath12k_dp_tx_get_encrypt_type(arvif->key_cipher), 326 HAL_TX_BANK_CONFIG_ENCRYPT_TYPE); 327 328 bank_config |= u32_encode_bits(arvif->tx_encap_type, 329 HAL_TX_BANK_CONFIG_ENCAP_TYPE); 330 bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_SRC_BUFFER_SWAP) | 331 u32_encode_bits(0, HAL_TX_BANK_CONFIG_LINK_META_SWAP) | 332 u32_encode_bits(0, HAL_TX_BANK_CONFIG_EPD); 333 334 /* only valid if idx_lookup_override is not set in tcl_data_cmd */ 335 bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN); 336 337 bank_config |= u32_encode_bits(arvif->hal_addr_search_flags & HAL_TX_ADDRX_EN, 338 HAL_TX_BANK_CONFIG_ADDRX_EN) | 339 u32_encode_bits(!!(arvif->hal_addr_search_flags & 340 HAL_TX_ADDRY_EN), 341 HAL_TX_BANK_CONFIG_ADDRY_EN); 342 343 bank_config |= u32_encode_bits(ieee80211_vif_is_mesh(arvif->vif) ? 3 : 0, 344 HAL_TX_BANK_CONFIG_MESH_EN) | 345 u32_encode_bits(arvif->vdev_id_check_en, 346 HAL_TX_BANK_CONFIG_VDEV_ID_CHECK_EN); 347 348 bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID); 349 350 return bank_config; 351 } 352 353 static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab, struct ath12k_vif *arvif, 354 struct ath12k_dp *dp) 355 { 356 int bank_id = DP_INVALID_BANK_ID; 357 int i; 358 u32 bank_config; 359 bool configure_register = false; 360 361 /* convert vdev params into hal_tx_bank_config */ 362 bank_config = ath12k_dp_tx_get_vdev_bank_config(ab, arvif); 363 364 spin_lock_bh(&dp->tx_bank_lock); 365 /* TODO: implement using idr kernel framework*/ 366 for (i = 0; i < dp->num_bank_profiles; i++) { 367 if (dp->bank_profiles[i].is_configured && 368 (dp->bank_profiles[i].bank_config ^ bank_config) == 0) { 369 bank_id = i; 370 goto inc_ref_and_return; 371 } 372 if (!dp->bank_profiles[i].is_configured || 373 !dp->bank_profiles[i].num_users) { 374 bank_id = i; 375 goto configure_and_return; 376 } 377 } 378 379 if (bank_id == DP_INVALID_BANK_ID) { 380 spin_unlock_bh(&dp->tx_bank_lock); 381 ath12k_err(ab, "unable to find TX bank!"); 382 return bank_id; 383 } 384 385 configure_and_return: 386 dp->bank_profiles[bank_id].is_configured = true; 387 dp->bank_profiles[bank_id].bank_config = bank_config; 388 configure_register = true; 389 inc_ref_and_return: 390 dp->bank_profiles[bank_id].num_users++; 391 spin_unlock_bh(&dp->tx_bank_lock); 392 393 if (configure_register) 394 ath12k_hal_tx_configure_bank_register(ab, bank_config, bank_id); 395 396 ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt tcl bank_id %d input 0x%x match 0x%x num_users %u", 397 bank_id, bank_config, dp->bank_profiles[bank_id].bank_config, 398 dp->bank_profiles[bank_id].num_users); 399 400 return bank_id; 401 } 402 403 void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id) 404 { 405 spin_lock_bh(&dp->tx_bank_lock); 406 dp->bank_profiles[bank_id].num_users--; 407 spin_unlock_bh(&dp->tx_bank_lock); 408 } 409 410 static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab) 411 { 412 struct ath12k_dp *dp = &ab->dp; 413 414 kfree(dp->bank_profiles); 415 dp->bank_profiles = NULL; 416 } 417 418 static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab) 419 { 420 struct ath12k_dp *dp = &ab->dp; 421 u32 num_tcl_banks = ab->hw_params->num_tcl_banks; 422 int i; 423 424 dp->num_bank_profiles = num_tcl_banks; 425 dp->bank_profiles = kmalloc_array(num_tcl_banks, 426 sizeof(struct ath12k_dp_tx_bank_profile), 427 GFP_KERNEL); 428 if (!dp->bank_profiles) 429 return -ENOMEM; 430 431 spin_lock_init(&dp->tx_bank_lock); 432 433 for (i = 0; i < num_tcl_banks; i++) { 434 dp->bank_profiles[i].is_configured = false; 435 dp->bank_profiles[i].num_users = 0; 436 } 437 438 return 0; 439 } 440 441 static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab) 442 { 443 struct ath12k_dp *dp = &ab->dp; 444 int i; 445 446 ath12k_dp_srng_cleanup(ab, &dp->reo_status_ring); 447 ath12k_dp_srng_cleanup(ab, &dp->reo_cmd_ring); 448 ath12k_dp_srng_cleanup(ab, &dp->reo_except_ring); 449 ath12k_dp_srng_cleanup(ab, &dp->rx_rel_ring); 450 ath12k_dp_srng_cleanup(ab, &dp->reo_reinject_ring); 451 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 452 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_comp_ring); 453 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_data_ring); 454 } 455 ath12k_dp_srng_cleanup(ab, &dp->tcl_status_ring); 456 ath12k_dp_srng_cleanup(ab, &dp->tcl_cmd_ring); 457 ath12k_dp_srng_cleanup(ab, &dp->wbm_desc_rel_ring); 458 } 459 460 static int ath12k_dp_srng_common_setup(struct ath12k_base *ab) 461 { 462 struct ath12k_dp *dp = &ab->dp; 463 const struct ath12k_hal_tcl_to_wbm_rbm_map *map; 464 struct hal_srng *srng; 465 int i, ret, tx_comp_ring_num; 466 u32 ring_hash_map; 467 468 ret = ath12k_dp_srng_setup(ab, &dp->wbm_desc_rel_ring, 469 HAL_SW2WBM_RELEASE, 0, 0, 470 DP_WBM_RELEASE_RING_SIZE); 471 if (ret) { 472 ath12k_warn(ab, "failed to set up wbm2sw_release ring :%d\n", 473 ret); 474 goto err; 475 } 476 477 ret = ath12k_dp_srng_setup(ab, &dp->tcl_cmd_ring, HAL_TCL_CMD, 0, 0, 478 DP_TCL_CMD_RING_SIZE); 479 if (ret) { 480 ath12k_warn(ab, "failed to set up tcl_cmd ring :%d\n", ret); 481 goto err; 482 } 483 484 ret = ath12k_dp_srng_setup(ab, &dp->tcl_status_ring, HAL_TCL_STATUS, 485 0, 0, DP_TCL_STATUS_RING_SIZE); 486 if (ret) { 487 ath12k_warn(ab, "failed to set up tcl_status ring :%d\n", ret); 488 goto err; 489 } 490 491 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 492 map = ab->hw_params->hal_ops->tcl_to_wbm_rbm_map; 493 tx_comp_ring_num = map[i].wbm_ring_num; 494 495 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_data_ring, 496 HAL_TCL_DATA, i, 0, 497 DP_TCL_DATA_RING_SIZE); 498 if (ret) { 499 ath12k_warn(ab, "failed to set up tcl_data ring (%d) :%d\n", 500 i, ret); 501 goto err; 502 } 503 504 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_comp_ring, 505 HAL_WBM2SW_RELEASE, tx_comp_ring_num, 0, 506 DP_TX_COMP_RING_SIZE); 507 if (ret) { 508 ath12k_warn(ab, "failed to set up tcl_comp ring (%d) :%d\n", 509 tx_comp_ring_num, ret); 510 goto err; 511 } 512 } 513 514 ret = ath12k_dp_srng_setup(ab, &dp->reo_reinject_ring, HAL_REO_REINJECT, 515 0, 0, DP_REO_REINJECT_RING_SIZE); 516 if (ret) { 517 ath12k_warn(ab, "failed to set up reo_reinject ring :%d\n", 518 ret); 519 goto err; 520 } 521 522 ret = ath12k_dp_srng_setup(ab, &dp->rx_rel_ring, HAL_WBM2SW_RELEASE, 523 HAL_WBM2SW_REL_ERR_RING_NUM, 0, 524 DP_RX_RELEASE_RING_SIZE); 525 if (ret) { 526 ath12k_warn(ab, "failed to set up rx_rel ring :%d\n", ret); 527 goto err; 528 } 529 530 ret = ath12k_dp_srng_setup(ab, &dp->reo_except_ring, HAL_REO_EXCEPTION, 531 0, 0, DP_REO_EXCEPTION_RING_SIZE); 532 if (ret) { 533 ath12k_warn(ab, "failed to set up reo_exception ring :%d\n", 534 ret); 535 goto err; 536 } 537 538 ret = ath12k_dp_srng_setup(ab, &dp->reo_cmd_ring, HAL_REO_CMD, 539 0, 0, DP_REO_CMD_RING_SIZE); 540 if (ret) { 541 ath12k_warn(ab, "failed to set up reo_cmd ring :%d\n", ret); 542 goto err; 543 } 544 545 srng = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id]; 546 ath12k_hal_reo_init_cmd_ring(ab, srng); 547 548 ret = ath12k_dp_srng_setup(ab, &dp->reo_status_ring, HAL_REO_STATUS, 549 0, 0, DP_REO_STATUS_RING_SIZE); 550 if (ret) { 551 ath12k_warn(ab, "failed to set up reo_status ring :%d\n", ret); 552 goto err; 553 } 554 555 /* When hash based routing of rx packet is enabled, 32 entries to map 556 * the hash values to the ring will be configured. Each hash entry uses 557 * four bits to map to a particular ring. The ring mapping will be 558 * 0:TCL, 1:SW1, 2:SW2, 3:SW3, 4:SW4, 5:Release, 6:FW and 7:SW5 559 * 8:SW6, 9:SW7, 10:SW8, 11:Not used. 560 */ 561 ring_hash_map = HAL_HASH_ROUTING_RING_SW1 | 562 HAL_HASH_ROUTING_RING_SW2 << 4 | 563 HAL_HASH_ROUTING_RING_SW3 << 8 | 564 HAL_HASH_ROUTING_RING_SW4 << 12 | 565 HAL_HASH_ROUTING_RING_SW1 << 16 | 566 HAL_HASH_ROUTING_RING_SW2 << 20 | 567 HAL_HASH_ROUTING_RING_SW3 << 24 | 568 HAL_HASH_ROUTING_RING_SW4 << 28; 569 570 ath12k_hal_reo_hw_setup(ab, ring_hash_map); 571 572 return 0; 573 574 err: 575 ath12k_dp_srng_common_cleanup(ab); 576 577 return ret; 578 } 579 580 static void ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base *ab) 581 { 582 struct ath12k_dp *dp = &ab->dp; 583 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list; 584 int i; 585 586 for (i = 0; i < DP_IDLE_SCATTER_BUFS_MAX; i++) { 587 if (!slist[i].vaddr) 588 continue; 589 590 dma_free_coherent(ab->dev, HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX, 591 slist[i].vaddr, slist[i].paddr); 592 slist[i].vaddr = NULL; 593 } 594 } 595 596 static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab, 597 int size, 598 u32 n_link_desc_bank, 599 u32 n_link_desc, 600 u32 last_bank_sz) 601 { 602 struct ath12k_dp *dp = &ab->dp; 603 struct dp_link_desc_bank *link_desc_banks = dp->link_desc_banks; 604 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list; 605 u32 n_entries_per_buf; 606 int num_scatter_buf, scatter_idx; 607 struct hal_wbm_link_desc *scatter_buf; 608 int align_bytes, n_entries; 609 dma_addr_t paddr; 610 int rem_entries; 611 int i; 612 int ret = 0; 613 u32 end_offset, cookie; 614 615 n_entries_per_buf = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 616 ath12k_hal_srng_get_entrysize(ab, HAL_WBM_IDLE_LINK); 617 num_scatter_buf = DIV_ROUND_UP(size, HAL_WBM_IDLE_SCATTER_BUF_SIZE); 618 619 if (num_scatter_buf > DP_IDLE_SCATTER_BUFS_MAX) 620 return -EINVAL; 621 622 for (i = 0; i < num_scatter_buf; i++) { 623 slist[i].vaddr = dma_alloc_coherent(ab->dev, 624 HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX, 625 &slist[i].paddr, GFP_KERNEL); 626 if (!slist[i].vaddr) { 627 ret = -ENOMEM; 628 goto err; 629 } 630 } 631 632 scatter_idx = 0; 633 scatter_buf = slist[scatter_idx].vaddr; 634 rem_entries = n_entries_per_buf; 635 636 for (i = 0; i < n_link_desc_bank; i++) { 637 align_bytes = link_desc_banks[i].vaddr - 638 link_desc_banks[i].vaddr_unaligned; 639 n_entries = (DP_LINK_DESC_ALLOC_SIZE_THRESH - align_bytes) / 640 HAL_LINK_DESC_SIZE; 641 paddr = link_desc_banks[i].paddr; 642 while (n_entries) { 643 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i); 644 ath12k_hal_set_link_desc_addr(scatter_buf, cookie, paddr); 645 n_entries--; 646 paddr += HAL_LINK_DESC_SIZE; 647 if (rem_entries) { 648 rem_entries--; 649 scatter_buf++; 650 continue; 651 } 652 653 rem_entries = n_entries_per_buf; 654 scatter_idx++; 655 scatter_buf = slist[scatter_idx].vaddr; 656 } 657 } 658 659 end_offset = (scatter_buf - slist[scatter_idx].vaddr) * 660 sizeof(struct hal_wbm_link_desc); 661 ath12k_hal_setup_link_idle_list(ab, slist, num_scatter_buf, 662 n_link_desc, end_offset); 663 664 return 0; 665 666 err: 667 ath12k_dp_scatter_idle_link_desc_cleanup(ab); 668 669 return ret; 670 } 671 672 static void 673 ath12k_dp_link_desc_bank_free(struct ath12k_base *ab, 674 struct dp_link_desc_bank *link_desc_banks) 675 { 676 int i; 677 678 for (i = 0; i < DP_LINK_DESC_BANKS_MAX; i++) { 679 if (link_desc_banks[i].vaddr_unaligned) { 680 dma_free_coherent(ab->dev, 681 link_desc_banks[i].size, 682 link_desc_banks[i].vaddr_unaligned, 683 link_desc_banks[i].paddr_unaligned); 684 link_desc_banks[i].vaddr_unaligned = NULL; 685 } 686 } 687 } 688 689 static int ath12k_dp_link_desc_bank_alloc(struct ath12k_base *ab, 690 struct dp_link_desc_bank *desc_bank, 691 int n_link_desc_bank, 692 int last_bank_sz) 693 { 694 struct ath12k_dp *dp = &ab->dp; 695 int i; 696 int ret = 0; 697 int desc_sz = DP_LINK_DESC_ALLOC_SIZE_THRESH; 698 699 for (i = 0; i < n_link_desc_bank; i++) { 700 if (i == (n_link_desc_bank - 1) && last_bank_sz) 701 desc_sz = last_bank_sz; 702 703 desc_bank[i].vaddr_unaligned = 704 dma_alloc_coherent(ab->dev, desc_sz, 705 &desc_bank[i].paddr_unaligned, 706 GFP_KERNEL); 707 if (!desc_bank[i].vaddr_unaligned) { 708 ret = -ENOMEM; 709 goto err; 710 } 711 712 desc_bank[i].vaddr = PTR_ALIGN(desc_bank[i].vaddr_unaligned, 713 HAL_LINK_DESC_ALIGN); 714 desc_bank[i].paddr = desc_bank[i].paddr_unaligned + 715 ((unsigned long)desc_bank[i].vaddr - 716 (unsigned long)desc_bank[i].vaddr_unaligned); 717 desc_bank[i].size = desc_sz; 718 } 719 720 return 0; 721 722 err: 723 ath12k_dp_link_desc_bank_free(ab, dp->link_desc_banks); 724 725 return ret; 726 } 727 728 void ath12k_dp_link_desc_cleanup(struct ath12k_base *ab, 729 struct dp_link_desc_bank *desc_bank, 730 u32 ring_type, struct dp_srng *ring) 731 { 732 ath12k_dp_link_desc_bank_free(ab, desc_bank); 733 734 if (ring_type != HAL_RXDMA_MONITOR_DESC) { 735 ath12k_dp_srng_cleanup(ab, ring); 736 ath12k_dp_scatter_idle_link_desc_cleanup(ab); 737 } 738 } 739 740 static int ath12k_wbm_idle_ring_setup(struct ath12k_base *ab, u32 *n_link_desc) 741 { 742 struct ath12k_dp *dp = &ab->dp; 743 u32 n_mpdu_link_desc, n_mpdu_queue_desc; 744 u32 n_tx_msdu_link_desc, n_rx_msdu_link_desc; 745 int ret = 0; 746 747 n_mpdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX) / 748 HAL_NUM_MPDUS_PER_LINK_DESC; 749 750 n_mpdu_queue_desc = n_mpdu_link_desc / 751 HAL_NUM_MPDU_LINKS_PER_QUEUE_DESC; 752 753 n_tx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_FLOWS_PER_TID * 754 DP_AVG_MSDUS_PER_FLOW) / 755 HAL_NUM_TX_MSDUS_PER_LINK_DESC; 756 757 n_rx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX * 758 DP_AVG_MSDUS_PER_MPDU) / 759 HAL_NUM_RX_MSDUS_PER_LINK_DESC; 760 761 *n_link_desc = n_mpdu_link_desc + n_mpdu_queue_desc + 762 n_tx_msdu_link_desc + n_rx_msdu_link_desc; 763 764 if (*n_link_desc & (*n_link_desc - 1)) 765 *n_link_desc = 1 << fls(*n_link_desc); 766 767 ret = ath12k_dp_srng_setup(ab, &dp->wbm_idle_ring, 768 HAL_WBM_IDLE_LINK, 0, 0, *n_link_desc); 769 if (ret) { 770 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret); 771 return ret; 772 } 773 return ret; 774 } 775 776 int ath12k_dp_link_desc_setup(struct ath12k_base *ab, 777 struct dp_link_desc_bank *link_desc_banks, 778 u32 ring_type, struct hal_srng *srng, 779 u32 n_link_desc) 780 { 781 u32 tot_mem_sz; 782 u32 n_link_desc_bank, last_bank_sz; 783 u32 entry_sz, align_bytes, n_entries; 784 struct hal_wbm_link_desc *desc; 785 u32 paddr; 786 int i, ret; 787 u32 cookie; 788 789 tot_mem_sz = n_link_desc * HAL_LINK_DESC_SIZE; 790 tot_mem_sz += HAL_LINK_DESC_ALIGN; 791 792 if (tot_mem_sz <= DP_LINK_DESC_ALLOC_SIZE_THRESH) { 793 n_link_desc_bank = 1; 794 last_bank_sz = tot_mem_sz; 795 } else { 796 n_link_desc_bank = tot_mem_sz / 797 (DP_LINK_DESC_ALLOC_SIZE_THRESH - 798 HAL_LINK_DESC_ALIGN); 799 last_bank_sz = tot_mem_sz % 800 (DP_LINK_DESC_ALLOC_SIZE_THRESH - 801 HAL_LINK_DESC_ALIGN); 802 803 if (last_bank_sz) 804 n_link_desc_bank += 1; 805 } 806 807 if (n_link_desc_bank > DP_LINK_DESC_BANKS_MAX) 808 return -EINVAL; 809 810 ret = ath12k_dp_link_desc_bank_alloc(ab, link_desc_banks, 811 n_link_desc_bank, last_bank_sz); 812 if (ret) 813 return ret; 814 815 /* Setup link desc idle list for HW internal usage */ 816 entry_sz = ath12k_hal_srng_get_entrysize(ab, ring_type); 817 tot_mem_sz = entry_sz * n_link_desc; 818 819 /* Setup scatter desc list when the total memory requirement is more */ 820 if (tot_mem_sz > DP_LINK_DESC_ALLOC_SIZE_THRESH && 821 ring_type != HAL_RXDMA_MONITOR_DESC) { 822 ret = ath12k_dp_scatter_idle_link_desc_setup(ab, tot_mem_sz, 823 n_link_desc_bank, 824 n_link_desc, 825 last_bank_sz); 826 if (ret) { 827 ath12k_warn(ab, "failed to setup scatting idle list descriptor :%d\n", 828 ret); 829 goto fail_desc_bank_free; 830 } 831 832 return 0; 833 } 834 835 spin_lock_bh(&srng->lock); 836 837 ath12k_hal_srng_access_begin(ab, srng); 838 839 for (i = 0; i < n_link_desc_bank; i++) { 840 align_bytes = link_desc_banks[i].vaddr - 841 link_desc_banks[i].vaddr_unaligned; 842 n_entries = (link_desc_banks[i].size - align_bytes) / 843 HAL_LINK_DESC_SIZE; 844 paddr = link_desc_banks[i].paddr; 845 while (n_entries && 846 (desc = ath12k_hal_srng_src_get_next_entry(ab, srng))) { 847 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i); 848 ath12k_hal_set_link_desc_addr(desc, 849 cookie, paddr); 850 n_entries--; 851 paddr += HAL_LINK_DESC_SIZE; 852 } 853 } 854 855 ath12k_hal_srng_access_end(ab, srng); 856 857 spin_unlock_bh(&srng->lock); 858 859 return 0; 860 861 fail_desc_bank_free: 862 ath12k_dp_link_desc_bank_free(ab, link_desc_banks); 863 864 return ret; 865 } 866 867 int ath12k_dp_service_srng(struct ath12k_base *ab, 868 struct ath12k_ext_irq_grp *irq_grp, 869 int budget) 870 { 871 struct napi_struct *napi = &irq_grp->napi; 872 int grp_id = irq_grp->grp_id; 873 int work_done = 0; 874 int i = 0, j; 875 int tot_work_done = 0; 876 enum dp_monitor_mode monitor_mode; 877 u8 ring_mask; 878 879 while (i < ab->hw_params->max_tx_ring) { 880 if (ab->hw_params->ring_mask->tx[grp_id] & 881 BIT(ab->hw_params->hal_ops->tcl_to_wbm_rbm_map[i].wbm_ring_num)) 882 ath12k_dp_tx_completion_handler(ab, i); 883 i++; 884 } 885 886 if (ab->hw_params->ring_mask->rx_err[grp_id]) { 887 work_done = ath12k_dp_rx_process_err(ab, napi, budget); 888 budget -= work_done; 889 tot_work_done += work_done; 890 if (budget <= 0) 891 goto done; 892 } 893 894 if (ab->hw_params->ring_mask->rx_wbm_rel[grp_id]) { 895 work_done = ath12k_dp_rx_process_wbm_err(ab, 896 napi, 897 budget); 898 budget -= work_done; 899 tot_work_done += work_done; 900 901 if (budget <= 0) 902 goto done; 903 } 904 905 if (ab->hw_params->ring_mask->rx[grp_id]) { 906 i = fls(ab->hw_params->ring_mask->rx[grp_id]) - 1; 907 work_done = ath12k_dp_rx_process(ab, i, napi, 908 budget); 909 budget -= work_done; 910 tot_work_done += work_done; 911 if (budget <= 0) 912 goto done; 913 } 914 915 if (ab->hw_params->ring_mask->rx_mon_dest[grp_id]) { 916 monitor_mode = ATH12K_DP_RX_MONITOR_MODE; 917 ring_mask = ab->hw_params->ring_mask->rx_mon_dest[grp_id]; 918 for (i = 0; i < ab->num_radios; i++) { 919 for (j = 0; j < ab->hw_params->num_rxmda_per_pdev; j++) { 920 int id = i * ab->hw_params->num_rxmda_per_pdev + j; 921 922 if (ring_mask & BIT(id)) { 923 work_done = 924 ath12k_dp_mon_process_ring(ab, id, napi, budget, 925 monitor_mode); 926 budget -= work_done; 927 tot_work_done += work_done; 928 929 if (budget <= 0) 930 goto done; 931 } 932 } 933 } 934 } 935 936 if (ab->hw_params->ring_mask->tx_mon_dest[grp_id]) { 937 monitor_mode = ATH12K_DP_TX_MONITOR_MODE; 938 ring_mask = ab->hw_params->ring_mask->tx_mon_dest[grp_id]; 939 for (i = 0; i < ab->num_radios; i++) { 940 for (j = 0; j < ab->hw_params->num_rxmda_per_pdev; j++) { 941 int id = i * ab->hw_params->num_rxmda_per_pdev + j; 942 943 if (ring_mask & BIT(id)) { 944 work_done = 945 ath12k_dp_mon_process_ring(ab, id, napi, budget, 946 monitor_mode); 947 budget -= work_done; 948 tot_work_done += work_done; 949 950 if (budget <= 0) 951 goto done; 952 } 953 } 954 } 955 } 956 957 if (ab->hw_params->ring_mask->reo_status[grp_id]) 958 ath12k_dp_rx_process_reo_status(ab); 959 960 if (ab->hw_params->ring_mask->host2rxdma[grp_id]) { 961 struct ath12k_dp *dp = &ab->dp; 962 struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring; 963 964 ath12k_dp_rx_bufs_replenish(ab, 0, rx_ring, 0, 965 ab->hw_params->hal_params->rx_buf_rbm, 966 true); 967 } 968 969 /* TODO: Implement handler for other interrupts */ 970 971 done: 972 return tot_work_done; 973 } 974 975 void ath12k_dp_pdev_free(struct ath12k_base *ab) 976 { 977 int i; 978 979 del_timer_sync(&ab->mon_reap_timer); 980 981 for (i = 0; i < ab->num_radios; i++) 982 ath12k_dp_rx_pdev_free(ab, i); 983 } 984 985 void ath12k_dp_pdev_pre_alloc(struct ath12k_base *ab) 986 { 987 struct ath12k *ar; 988 struct ath12k_pdev_dp *dp; 989 int i; 990 991 for (i = 0; i < ab->num_radios; i++) { 992 ar = ab->pdevs[i].ar; 993 dp = &ar->dp; 994 dp->mac_id = i; 995 atomic_set(&dp->num_tx_pending, 0); 996 init_waitqueue_head(&dp->tx_empty_waitq); 997 998 /* TODO: Add any RXDMA setup required per pdev */ 999 } 1000 } 1001 1002 static void ath12k_dp_service_mon_ring(struct timer_list *t) 1003 { 1004 struct ath12k_base *ab = from_timer(ab, t, mon_reap_timer); 1005 int i; 1006 1007 for (i = 0; i < ab->hw_params->num_rxmda_per_pdev; i++) 1008 ath12k_dp_mon_process_ring(ab, i, NULL, DP_MON_SERVICE_BUDGET, 1009 ATH12K_DP_RX_MONITOR_MODE); 1010 1011 mod_timer(&ab->mon_reap_timer, jiffies + 1012 msecs_to_jiffies(ATH12K_MON_TIMER_INTERVAL)); 1013 } 1014 1015 static void ath12k_dp_mon_reap_timer_init(struct ath12k_base *ab) 1016 { 1017 if (ab->hw_params->rxdma1_enable) 1018 return; 1019 1020 timer_setup(&ab->mon_reap_timer, ath12k_dp_service_mon_ring, 0); 1021 } 1022 1023 int ath12k_dp_pdev_alloc(struct ath12k_base *ab) 1024 { 1025 struct ath12k *ar; 1026 int ret; 1027 int i; 1028 1029 ret = ath12k_dp_rx_htt_setup(ab); 1030 if (ret) 1031 goto out; 1032 1033 ath12k_dp_mon_reap_timer_init(ab); 1034 1035 /* TODO: Per-pdev rx ring unlike tx ring which is mapped to different AC's */ 1036 for (i = 0; i < ab->num_radios; i++) { 1037 ar = ab->pdevs[i].ar; 1038 ret = ath12k_dp_rx_pdev_alloc(ab, i); 1039 if (ret) { 1040 ath12k_warn(ab, "failed to allocate pdev rx for pdev_id :%d\n", 1041 i); 1042 goto err; 1043 } 1044 ret = ath12k_dp_rx_pdev_mon_attach(ar); 1045 if (ret) { 1046 ath12k_warn(ab, "failed to initialize mon pdev %d\n", i); 1047 goto err; 1048 } 1049 } 1050 1051 return 0; 1052 err: 1053 ath12k_dp_pdev_free(ab); 1054 out: 1055 return ret; 1056 } 1057 1058 int ath12k_dp_htt_connect(struct ath12k_dp *dp) 1059 { 1060 struct ath12k_htc_svc_conn_req conn_req = {0}; 1061 struct ath12k_htc_svc_conn_resp conn_resp = {0}; 1062 int status; 1063 1064 conn_req.ep_ops.ep_tx_complete = ath12k_dp_htt_htc_tx_complete; 1065 conn_req.ep_ops.ep_rx_complete = ath12k_dp_htt_htc_t2h_msg_handler; 1066 1067 /* connect to control service */ 1068 conn_req.service_id = ATH12K_HTC_SVC_ID_HTT_DATA_MSG; 1069 1070 status = ath12k_htc_connect_service(&dp->ab->htc, &conn_req, 1071 &conn_resp); 1072 1073 if (status) 1074 return status; 1075 1076 dp->eid = conn_resp.eid; 1077 1078 return 0; 1079 } 1080 1081 static void ath12k_dp_update_vdev_search(struct ath12k_vif *arvif) 1082 { 1083 switch (arvif->vdev_type) { 1084 case WMI_VDEV_TYPE_STA: 1085 /* TODO: Verify the search type and flags since ast hash 1086 * is not part of peer mapv3 1087 */ 1088 arvif->hal_addr_search_flags = HAL_TX_ADDRY_EN; 1089 arvif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; 1090 break; 1091 case WMI_VDEV_TYPE_AP: 1092 case WMI_VDEV_TYPE_IBSS: 1093 arvif->hal_addr_search_flags = HAL_TX_ADDRX_EN; 1094 arvif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; 1095 break; 1096 case WMI_VDEV_TYPE_MONITOR: 1097 default: 1098 return; 1099 } 1100 } 1101 1102 void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_vif *arvif) 1103 { 1104 struct ath12k_base *ab = ar->ab; 1105 1106 arvif->tcl_metadata |= u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) | 1107 u32_encode_bits(arvif->vdev_id, 1108 HTT_TCL_META_DATA_VDEV_ID) | 1109 u32_encode_bits(ar->pdev->pdev_id, 1110 HTT_TCL_META_DATA_PDEV_ID); 1111 1112 /* set HTT extension valid bit to 0 by default */ 1113 arvif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT; 1114 1115 ath12k_dp_update_vdev_search(arvif); 1116 arvif->vdev_id_check_en = true; 1117 arvif->bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, &ab->dp); 1118 1119 /* TODO: error path for bank id failure */ 1120 if (arvif->bank_id == DP_INVALID_BANK_ID) { 1121 ath12k_err(ar->ab, "Failed to initialize DP TX Banks"); 1122 return; 1123 } 1124 } 1125 1126 static void ath12k_dp_cc_cleanup(struct ath12k_base *ab) 1127 { 1128 struct ath12k_rx_desc_info *desc_info, *tmp; 1129 struct ath12k_tx_desc_info *tx_desc_info, *tmp1; 1130 struct ath12k_dp *dp = &ab->dp; 1131 struct sk_buff *skb; 1132 int i; 1133 u32 pool_id, tx_spt_page; 1134 1135 if (!dp->spt_info) 1136 return; 1137 1138 /* RX Descriptor cleanup */ 1139 spin_lock_bh(&dp->rx_desc_lock); 1140 1141 list_for_each_entry_safe(desc_info, tmp, &dp->rx_desc_used_list, list) { 1142 list_del(&desc_info->list); 1143 skb = desc_info->skb; 1144 1145 if (!skb) 1146 continue; 1147 1148 dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(skb)->paddr, 1149 skb->len + skb_tailroom(skb), DMA_FROM_DEVICE); 1150 dev_kfree_skb_any(skb); 1151 } 1152 1153 for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES; i++) { 1154 if (!dp->spt_info->rxbaddr[i]) 1155 continue; 1156 1157 kfree(dp->spt_info->rxbaddr[i]); 1158 dp->spt_info->rxbaddr[i] = NULL; 1159 } 1160 1161 spin_unlock_bh(&dp->rx_desc_lock); 1162 1163 /* TX Descriptor cleanup */ 1164 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) { 1165 spin_lock_bh(&dp->tx_desc_lock[i]); 1166 1167 list_for_each_entry_safe(tx_desc_info, tmp1, &dp->tx_desc_used_list[i], 1168 list) { 1169 list_del(&tx_desc_info->list); 1170 skb = tx_desc_info->skb; 1171 1172 if (!skb) 1173 continue; 1174 1175 dma_unmap_single(ab->dev, ATH12K_SKB_CB(skb)->paddr, 1176 skb->len, DMA_TO_DEVICE); 1177 dev_kfree_skb_any(skb); 1178 } 1179 1180 spin_unlock_bh(&dp->tx_desc_lock[i]); 1181 } 1182 1183 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) { 1184 spin_lock_bh(&dp->tx_desc_lock[pool_id]); 1185 1186 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL; i++) { 1187 tx_spt_page = i + pool_id * ATH12K_TX_SPT_PAGES_PER_POOL; 1188 if (!dp->spt_info->txbaddr[tx_spt_page]) 1189 continue; 1190 1191 kfree(dp->spt_info->txbaddr[tx_spt_page]); 1192 dp->spt_info->txbaddr[tx_spt_page] = NULL; 1193 } 1194 1195 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1196 } 1197 1198 /* unmap SPT pages */ 1199 for (i = 0; i < dp->num_spt_pages; i++) { 1200 if (!dp->spt_info[i].vaddr) 1201 continue; 1202 1203 dma_free_coherent(ab->dev, ATH12K_PAGE_SIZE, 1204 dp->spt_info[i].vaddr, dp->spt_info[i].paddr); 1205 dp->spt_info[i].vaddr = NULL; 1206 } 1207 1208 kfree(dp->spt_info); 1209 } 1210 1211 static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab) 1212 { 1213 struct ath12k_dp *dp = &ab->dp; 1214 1215 if (!ab->hw_params->reoq_lut_support) 1216 return; 1217 1218 if (!dp->reoq_lut.vaddr) 1219 return; 1220 1221 dma_free_coherent(ab->dev, DP_REOQ_LUT_SIZE, 1222 dp->reoq_lut.vaddr, dp->reoq_lut.paddr); 1223 dp->reoq_lut.vaddr = NULL; 1224 1225 ath12k_hif_write32(ab, 1226 HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab), 0); 1227 } 1228 1229 void ath12k_dp_free(struct ath12k_base *ab) 1230 { 1231 struct ath12k_dp *dp = &ab->dp; 1232 int i; 1233 1234 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks, 1235 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring); 1236 1237 ath12k_dp_cc_cleanup(ab); 1238 ath12k_dp_reoq_lut_cleanup(ab); 1239 ath12k_dp_deinit_bank_profiles(ab); 1240 ath12k_dp_srng_common_cleanup(ab); 1241 1242 ath12k_dp_rx_reo_cmd_list_cleanup(ab); 1243 1244 for (i = 0; i < ab->hw_params->max_tx_ring; i++) 1245 kfree(dp->tx_ring[i].tx_status); 1246 1247 ath12k_dp_rx_free(ab); 1248 /* Deinit any SOC level resource */ 1249 } 1250 1251 void ath12k_dp_cc_config(struct ath12k_base *ab) 1252 { 1253 u32 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start; 1254 u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG; 1255 u32 wbm_base = HAL_SEQ_WCSS_UMAC_WBM_REG; 1256 u32 val = 0; 1257 1258 ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG0(ab), cmem_base); 1259 1260 val |= u32_encode_bits(ATH12K_CMEM_ADDR_MSB, 1261 HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) | 1262 u32_encode_bits(ATH12K_CC_PPT_MSB, 1263 HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB) | 1264 u32_encode_bits(ATH12K_CC_SPT_MSB, 1265 HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB) | 1266 u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ALIGN) | 1267 u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ENABLE) | 1268 u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE); 1269 1270 ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG1(ab), val); 1271 1272 /* Enable HW CC for WBM */ 1273 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG0, cmem_base); 1274 1275 val = u32_encode_bits(ATH12K_CMEM_ADDR_MSB, 1276 HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) | 1277 u32_encode_bits(ATH12K_CC_PPT_MSB, 1278 HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB) | 1279 u32_encode_bits(ATH12K_CC_SPT_MSB, 1280 HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB) | 1281 u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ALIGN); 1282 1283 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG1, val); 1284 1285 /* Enable conversion complete indication */ 1286 val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2); 1287 val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN) | 1288 u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN) | 1289 u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN); 1290 1291 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2, val); 1292 1293 /* Enable Cookie conversion for WBM2SW Rings */ 1294 val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG); 1295 val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN) | 1296 ab->hw_params->hal_params->wbm2sw_cc_enable; 1297 1298 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG, val); 1299 } 1300 1301 static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx) 1302 { 1303 return (u32)ppt_idx << ATH12K_CC_PPT_SHIFT | spt_idx; 1304 } 1305 1306 static inline void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_base *ab, 1307 u16 ppt_idx, u16 spt_idx) 1308 { 1309 struct ath12k_dp *dp = &ab->dp; 1310 1311 return dp->spt_info[ppt_idx].vaddr + spt_idx; 1312 } 1313 1314 struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_base *ab, 1315 u32 cookie) 1316 { 1317 struct ath12k_rx_desc_info **desc_addr_ptr; 1318 u16 ppt_idx, spt_idx; 1319 1320 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT); 1321 spt_idx = u32_get_bits(cookie, ATH12k_DP_CC_COOKIE_SPT); 1322 1323 if (ppt_idx > ATH12K_NUM_RX_SPT_PAGES || 1324 spt_idx > ATH12K_MAX_SPT_ENTRIES) 1325 return NULL; 1326 1327 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx); 1328 1329 return *desc_addr_ptr; 1330 } 1331 1332 struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_base *ab, 1333 u32 cookie) 1334 { 1335 struct ath12k_tx_desc_info **desc_addr_ptr; 1336 u16 ppt_idx, spt_idx; 1337 1338 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT); 1339 spt_idx = u32_get_bits(cookie, ATH12k_DP_CC_COOKIE_SPT); 1340 1341 if (ppt_idx < ATH12K_NUM_RX_SPT_PAGES || 1342 ppt_idx > ab->dp.num_spt_pages || 1343 spt_idx > ATH12K_MAX_SPT_ENTRIES) 1344 return NULL; 1345 1346 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx); 1347 1348 return *desc_addr_ptr; 1349 } 1350 1351 static int ath12k_dp_cc_desc_init(struct ath12k_base *ab) 1352 { 1353 struct ath12k_dp *dp = &ab->dp; 1354 struct ath12k_rx_desc_info *rx_descs, **rx_desc_addr; 1355 struct ath12k_tx_desc_info *tx_descs, **tx_desc_addr; 1356 u32 i, j, pool_id, tx_spt_page; 1357 u32 ppt_idx; 1358 1359 spin_lock_bh(&dp->rx_desc_lock); 1360 1361 /* First ATH12K_NUM_RX_SPT_PAGES of allocated SPT pages are used for RX */ 1362 for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES; i++) { 1363 rx_descs = kcalloc(ATH12K_MAX_SPT_ENTRIES, sizeof(*rx_descs), 1364 GFP_ATOMIC); 1365 1366 if (!rx_descs) { 1367 spin_unlock_bh(&dp->rx_desc_lock); 1368 return -ENOMEM; 1369 } 1370 1371 dp->spt_info->rxbaddr[i] = &rx_descs[0]; 1372 1373 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) { 1374 rx_descs[j].cookie = ath12k_dp_cc_cookie_gen(i, j); 1375 rx_descs[j].magic = ATH12K_DP_RX_DESC_MAGIC; 1376 list_add_tail(&rx_descs[j].list, &dp->rx_desc_free_list); 1377 1378 /* Update descriptor VA in SPT */ 1379 rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(ab, i, j); 1380 *rx_desc_addr = &rx_descs[j]; 1381 } 1382 } 1383 1384 spin_unlock_bh(&dp->rx_desc_lock); 1385 1386 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) { 1387 spin_lock_bh(&dp->tx_desc_lock[pool_id]); 1388 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL; i++) { 1389 tx_descs = kcalloc(ATH12K_MAX_SPT_ENTRIES, sizeof(*tx_descs), 1390 GFP_ATOMIC); 1391 1392 if (!tx_descs) { 1393 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1394 /* Caller takes care of TX pending and RX desc cleanup */ 1395 return -ENOMEM; 1396 } 1397 1398 tx_spt_page = i + pool_id * ATH12K_TX_SPT_PAGES_PER_POOL; 1399 dp->spt_info->txbaddr[tx_spt_page] = &tx_descs[0]; 1400 1401 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) { 1402 ppt_idx = ATH12K_NUM_RX_SPT_PAGES + tx_spt_page; 1403 tx_descs[j].desc_id = ath12k_dp_cc_cookie_gen(ppt_idx, j); 1404 tx_descs[j].pool_id = pool_id; 1405 list_add_tail(&tx_descs[j].list, 1406 &dp->tx_desc_free_list[pool_id]); 1407 1408 /* Update descriptor VA in SPT */ 1409 tx_desc_addr = 1410 ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, j); 1411 *tx_desc_addr = &tx_descs[j]; 1412 } 1413 } 1414 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1415 } 1416 return 0; 1417 } 1418 1419 static int ath12k_dp_cc_init(struct ath12k_base *ab) 1420 { 1421 struct ath12k_dp *dp = &ab->dp; 1422 int i, ret = 0; 1423 u32 cmem_base; 1424 1425 INIT_LIST_HEAD(&dp->rx_desc_free_list); 1426 INIT_LIST_HEAD(&dp->rx_desc_used_list); 1427 spin_lock_init(&dp->rx_desc_lock); 1428 1429 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) { 1430 INIT_LIST_HEAD(&dp->tx_desc_free_list[i]); 1431 INIT_LIST_HEAD(&dp->tx_desc_used_list[i]); 1432 spin_lock_init(&dp->tx_desc_lock[i]); 1433 } 1434 1435 dp->num_spt_pages = ATH12K_NUM_SPT_PAGES; 1436 if (dp->num_spt_pages > ATH12K_MAX_PPT_ENTRIES) 1437 dp->num_spt_pages = ATH12K_MAX_PPT_ENTRIES; 1438 1439 dp->spt_info = kcalloc(dp->num_spt_pages, sizeof(struct ath12k_spt_info), 1440 GFP_KERNEL); 1441 1442 if (!dp->spt_info) { 1443 ath12k_warn(ab, "SPT page allocation failure"); 1444 return -ENOMEM; 1445 } 1446 1447 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start; 1448 1449 for (i = 0; i < dp->num_spt_pages; i++) { 1450 dp->spt_info[i].vaddr = dma_alloc_coherent(ab->dev, 1451 ATH12K_PAGE_SIZE, 1452 &dp->spt_info[i].paddr, 1453 GFP_KERNEL); 1454 1455 if (!dp->spt_info[i].vaddr) { 1456 ret = -ENOMEM; 1457 goto free; 1458 } 1459 1460 if (dp->spt_info[i].paddr & ATH12K_SPT_4K_ALIGN_CHECK) { 1461 ath12k_warn(ab, "SPT allocated memory is not 4K aligned"); 1462 ret = -EINVAL; 1463 goto free; 1464 } 1465 1466 /* Write to PPT in CMEM */ 1467 ath12k_hif_write32(ab, cmem_base + ATH12K_PPT_ADDR_OFFSET(i), 1468 dp->spt_info[i].paddr >> ATH12K_SPT_4K_ALIGN_OFFSET); 1469 } 1470 1471 ret = ath12k_dp_cc_desc_init(ab); 1472 if (ret) { 1473 ath12k_warn(ab, "HW CC desc init failed %d", ret); 1474 goto free; 1475 } 1476 1477 return 0; 1478 free: 1479 ath12k_dp_cc_cleanup(ab); 1480 return ret; 1481 } 1482 1483 static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab) 1484 { 1485 struct ath12k_dp *dp = &ab->dp; 1486 1487 if (!ab->hw_params->reoq_lut_support) 1488 return 0; 1489 1490 dp->reoq_lut.vaddr = dma_alloc_coherent(ab->dev, 1491 DP_REOQ_LUT_SIZE, 1492 &dp->reoq_lut.paddr, 1493 GFP_KERNEL | __GFP_ZERO); 1494 if (!dp->reoq_lut.vaddr) { 1495 ath12k_warn(ab, "failed to allocate memory for reoq table"); 1496 return -ENOMEM; 1497 } 1498 1499 ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab), 1500 dp->reoq_lut.paddr); 1501 return 0; 1502 } 1503 1504 int ath12k_dp_alloc(struct ath12k_base *ab) 1505 { 1506 struct ath12k_dp *dp = &ab->dp; 1507 struct hal_srng *srng = NULL; 1508 size_t size = 0; 1509 u32 n_link_desc = 0; 1510 int ret; 1511 int i; 1512 1513 dp->ab = ab; 1514 1515 INIT_LIST_HEAD(&dp->reo_cmd_list); 1516 INIT_LIST_HEAD(&dp->reo_cmd_cache_flush_list); 1517 spin_lock_init(&dp->reo_cmd_lock); 1518 1519 dp->reo_cmd_cache_flush_count = 0; 1520 1521 ret = ath12k_wbm_idle_ring_setup(ab, &n_link_desc); 1522 if (ret) { 1523 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret); 1524 return ret; 1525 } 1526 1527 srng = &ab->hal.srng_list[dp->wbm_idle_ring.ring_id]; 1528 1529 ret = ath12k_dp_link_desc_setup(ab, dp->link_desc_banks, 1530 HAL_WBM_IDLE_LINK, srng, n_link_desc); 1531 if (ret) { 1532 ath12k_warn(ab, "failed to setup link desc: %d\n", ret); 1533 return ret; 1534 } 1535 1536 ret = ath12k_dp_cc_init(ab); 1537 1538 if (ret) { 1539 ath12k_warn(ab, "failed to setup cookie converter %d\n", ret); 1540 goto fail_link_desc_cleanup; 1541 } 1542 ret = ath12k_dp_init_bank_profiles(ab); 1543 if (ret) { 1544 ath12k_warn(ab, "failed to setup bank profiles %d\n", ret); 1545 goto fail_hw_cc_cleanup; 1546 } 1547 1548 ret = ath12k_dp_srng_common_setup(ab); 1549 if (ret) 1550 goto fail_dp_bank_profiles_cleanup; 1551 1552 size = sizeof(struct hal_wbm_release_ring_tx) * DP_TX_COMP_RING_SIZE; 1553 1554 ret = ath12k_dp_reoq_lut_setup(ab); 1555 if (ret) { 1556 ath12k_warn(ab, "failed to setup reoq table %d\n", ret); 1557 goto fail_cmn_srng_cleanup; 1558 } 1559 1560 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 1561 dp->tx_ring[i].tcl_data_ring_id = i; 1562 1563 dp->tx_ring[i].tx_status_head = 0; 1564 dp->tx_ring[i].tx_status_tail = DP_TX_COMP_RING_SIZE - 1; 1565 dp->tx_ring[i].tx_status = kmalloc(size, GFP_KERNEL); 1566 if (!dp->tx_ring[i].tx_status) { 1567 ret = -ENOMEM; 1568 /* FIXME: The allocated tx status is not freed 1569 * properly here 1570 */ 1571 goto fail_cmn_reoq_cleanup; 1572 } 1573 } 1574 1575 for (i = 0; i < HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX; i++) 1576 ath12k_hal_tx_set_dscp_tid_map(ab, i); 1577 1578 ret = ath12k_dp_rx_alloc(ab); 1579 if (ret) 1580 goto fail_dp_rx_free; 1581 1582 /* Init any SOC level resource for DP */ 1583 1584 return 0; 1585 1586 fail_dp_rx_free: 1587 ath12k_dp_rx_free(ab); 1588 1589 fail_cmn_reoq_cleanup: 1590 ath12k_dp_reoq_lut_cleanup(ab); 1591 1592 fail_cmn_srng_cleanup: 1593 ath12k_dp_srng_common_cleanup(ab); 1594 1595 fail_dp_bank_profiles_cleanup: 1596 ath12k_dp_deinit_bank_profiles(ab); 1597 1598 fail_hw_cc_cleanup: 1599 ath12k_dp_cc_cleanup(ab); 1600 1601 fail_link_desc_cleanup: 1602 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks, 1603 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring); 1604 1605 return ret; 1606 } 1607