1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. 4 */ 5 #include <linux/dma-mapping.h> 6 #include "hal_tx.h" 7 #include "debug.h" 8 #include "hal_desc.h" 9 #include "hif.h" 10 11 static const struct hal_srng_config hw_srng_config_template[] = { 12 /* TODO: max_rings can populated by querying HW capabilities */ 13 { /* REO_DST */ 14 .start_ring_id = HAL_SRNG_RING_ID_REO2SW1, 15 .max_rings = 4, 16 .entry_size = sizeof(struct hal_reo_dest_ring) >> 2, 17 .lmac_ring = false, 18 .ring_dir = HAL_SRNG_DIR_DST, 19 .max_size = HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE, 20 }, 21 { /* REO_EXCEPTION */ 22 /* Designating REO2TCL ring as exception ring. This ring is 23 * similar to other REO2SW rings though it is named as REO2TCL. 24 * Any of theREO2SW rings can be used as exception ring. 25 */ 26 .start_ring_id = HAL_SRNG_RING_ID_REO2TCL, 27 .max_rings = 1, 28 .entry_size = sizeof(struct hal_reo_dest_ring) >> 2, 29 .lmac_ring = false, 30 .ring_dir = HAL_SRNG_DIR_DST, 31 .max_size = HAL_REO_REO2TCL_RING_BASE_MSB_RING_SIZE, 32 }, 33 { /* REO_REINJECT */ 34 .start_ring_id = HAL_SRNG_RING_ID_SW2REO, 35 .max_rings = 1, 36 .entry_size = sizeof(struct hal_reo_entrance_ring) >> 2, 37 .lmac_ring = false, 38 .ring_dir = HAL_SRNG_DIR_SRC, 39 .max_size = HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE, 40 }, 41 { /* REO_CMD */ 42 .start_ring_id = HAL_SRNG_RING_ID_REO_CMD, 43 .max_rings = 1, 44 .entry_size = (sizeof(struct hal_tlv_hdr) + 45 sizeof(struct hal_reo_get_queue_stats)) >> 2, 46 .lmac_ring = false, 47 .ring_dir = HAL_SRNG_DIR_SRC, 48 .max_size = HAL_REO_CMD_RING_BASE_MSB_RING_SIZE, 49 }, 50 { /* REO_STATUS */ 51 .start_ring_id = HAL_SRNG_RING_ID_REO_STATUS, 52 .max_rings = 1, 53 .entry_size = (sizeof(struct hal_tlv_hdr) + 54 sizeof(struct hal_reo_get_queue_stats_status)) >> 2, 55 .lmac_ring = false, 56 .ring_dir = HAL_SRNG_DIR_DST, 57 .max_size = HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE, 58 }, 59 { /* TCL_DATA */ 60 .start_ring_id = HAL_SRNG_RING_ID_SW2TCL1, 61 .max_rings = 3, 62 .entry_size = (sizeof(struct hal_tlv_hdr) + 63 sizeof(struct hal_tcl_data_cmd)) >> 2, 64 .lmac_ring = false, 65 .ring_dir = HAL_SRNG_DIR_SRC, 66 .max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE, 67 }, 68 { /* TCL_CMD */ 69 .start_ring_id = HAL_SRNG_RING_ID_SW2TCL_CMD, 70 .max_rings = 1, 71 .entry_size = (sizeof(struct hal_tlv_hdr) + 72 sizeof(struct hal_tcl_gse_cmd)) >> 2, 73 .lmac_ring = false, 74 .ring_dir = HAL_SRNG_DIR_SRC, 75 .max_size = HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE, 76 }, 77 { /* TCL_STATUS */ 78 .start_ring_id = HAL_SRNG_RING_ID_TCL_STATUS, 79 .max_rings = 1, 80 .entry_size = (sizeof(struct hal_tlv_hdr) + 81 sizeof(struct hal_tcl_status_ring)) >> 2, 82 .lmac_ring = false, 83 .ring_dir = HAL_SRNG_DIR_DST, 84 .max_size = HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE, 85 }, 86 { /* CE_SRC */ 87 .start_ring_id = HAL_SRNG_RING_ID_CE0_SRC, 88 .max_rings = 12, 89 .entry_size = sizeof(struct hal_ce_srng_src_desc) >> 2, 90 .lmac_ring = false, 91 .ring_dir = HAL_SRNG_DIR_SRC, 92 .max_size = HAL_CE_SRC_RING_BASE_MSB_RING_SIZE, 93 }, 94 { /* CE_DST */ 95 .start_ring_id = HAL_SRNG_RING_ID_CE0_DST, 96 .max_rings = 12, 97 .entry_size = sizeof(struct hal_ce_srng_dest_desc) >> 2, 98 .lmac_ring = false, 99 .ring_dir = HAL_SRNG_DIR_SRC, 100 .max_size = HAL_CE_DST_RING_BASE_MSB_RING_SIZE, 101 }, 102 { /* CE_DST_STATUS */ 103 .start_ring_id = HAL_SRNG_RING_ID_CE0_DST_STATUS, 104 .max_rings = 12, 105 .entry_size = sizeof(struct hal_ce_srng_dst_status_desc) >> 2, 106 .lmac_ring = false, 107 .ring_dir = HAL_SRNG_DIR_DST, 108 .max_size = HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE, 109 }, 110 { /* WBM_IDLE_LINK */ 111 .start_ring_id = HAL_SRNG_RING_ID_WBM_IDLE_LINK, 112 .max_rings = 1, 113 .entry_size = sizeof(struct hal_wbm_link_desc) >> 2, 114 .lmac_ring = false, 115 .ring_dir = HAL_SRNG_DIR_SRC, 116 .max_size = HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE, 117 }, 118 { /* SW2WBM_RELEASE */ 119 .start_ring_id = HAL_SRNG_RING_ID_WBM_SW_RELEASE, 120 .max_rings = 1, 121 .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, 122 .lmac_ring = false, 123 .ring_dir = HAL_SRNG_DIR_SRC, 124 .max_size = HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE, 125 }, 126 { /* WBM2SW_RELEASE */ 127 .start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE, 128 .max_rings = 4, 129 .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, 130 .lmac_ring = false, 131 .ring_dir = HAL_SRNG_DIR_DST, 132 .max_size = HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE, 133 }, 134 { /* RXDMA_BUF */ 135 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA0_BUF, 136 .max_rings = 2, 137 .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2, 138 .lmac_ring = true, 139 .ring_dir = HAL_SRNG_DIR_SRC, 140 .max_size = HAL_RXDMA_RING_MAX_SIZE, 141 }, 142 { /* RXDMA_DST */ 143 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW0, 144 .max_rings = 1, 145 .entry_size = sizeof(struct hal_reo_entrance_ring) >> 2, 146 .lmac_ring = true, 147 .ring_dir = HAL_SRNG_DIR_DST, 148 .max_size = HAL_RXDMA_RING_MAX_SIZE, 149 }, 150 { /* RXDMA_MONITOR_BUF */ 151 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA2_BUF, 152 .max_rings = 1, 153 .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2, 154 .lmac_ring = true, 155 .ring_dir = HAL_SRNG_DIR_SRC, 156 .max_size = HAL_RXDMA_RING_MAX_SIZE, 157 }, 158 { /* RXDMA_MONITOR_STATUS */ 159 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_STATBUF, 160 .max_rings = 1, 161 .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2, 162 .lmac_ring = true, 163 .ring_dir = HAL_SRNG_DIR_SRC, 164 .max_size = HAL_RXDMA_RING_MAX_SIZE, 165 }, 166 { /* RXDMA_MONITOR_DST */ 167 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW1, 168 .max_rings = 1, 169 .entry_size = sizeof(struct hal_reo_entrance_ring) >> 2, 170 .lmac_ring = true, 171 .ring_dir = HAL_SRNG_DIR_DST, 172 .max_size = HAL_RXDMA_RING_MAX_SIZE, 173 }, 174 { /* RXDMA_MONITOR_DESC */ 175 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_DESC, 176 .max_rings = 1, 177 .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2, 178 .lmac_ring = true, 179 .ring_dir = HAL_SRNG_DIR_SRC, 180 .max_size = HAL_RXDMA_RING_MAX_SIZE, 181 }, 182 { /* RXDMA DIR BUF */ 183 .start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF, 184 .max_rings = 1, 185 .entry_size = 8 >> 2, /* TODO: Define the struct */ 186 .lmac_ring = true, 187 .ring_dir = HAL_SRNG_DIR_SRC, 188 .max_size = HAL_RXDMA_RING_MAX_SIZE, 189 }, 190 }; 191 192 static int ath11k_hal_alloc_cont_rdp(struct ath11k_base *ab) 193 { 194 struct ath11k_hal *hal = &ab->hal; 195 size_t size; 196 197 size = sizeof(u32) * HAL_SRNG_RING_ID_MAX; 198 hal->rdp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->rdp.paddr, 199 GFP_KERNEL); 200 if (!hal->rdp.vaddr) 201 return -ENOMEM; 202 203 return 0; 204 } 205 206 static void ath11k_hal_free_cont_rdp(struct ath11k_base *ab) 207 { 208 struct ath11k_hal *hal = &ab->hal; 209 size_t size; 210 211 if (!hal->rdp.vaddr) 212 return; 213 214 size = sizeof(u32) * HAL_SRNG_RING_ID_MAX; 215 dma_free_coherent(ab->dev, size, 216 hal->rdp.vaddr, hal->rdp.paddr); 217 hal->rdp.vaddr = NULL; 218 } 219 220 static int ath11k_hal_alloc_cont_wrp(struct ath11k_base *ab) 221 { 222 struct ath11k_hal *hal = &ab->hal; 223 size_t size; 224 225 size = sizeof(u32) * HAL_SRNG_NUM_LMAC_RINGS; 226 hal->wrp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->wrp.paddr, 227 GFP_KERNEL); 228 if (!hal->wrp.vaddr) 229 return -ENOMEM; 230 231 return 0; 232 } 233 234 static void ath11k_hal_free_cont_wrp(struct ath11k_base *ab) 235 { 236 struct ath11k_hal *hal = &ab->hal; 237 size_t size; 238 239 if (!hal->wrp.vaddr) 240 return; 241 242 size = sizeof(u32) * HAL_SRNG_NUM_LMAC_RINGS; 243 dma_free_coherent(ab->dev, size, 244 hal->wrp.vaddr, hal->wrp.paddr); 245 hal->wrp.vaddr = NULL; 246 } 247 248 static void ath11k_hal_ce_dst_setup(struct ath11k_base *ab, 249 struct hal_srng *srng, int ring_num) 250 { 251 struct hal_srng_config *srng_config = &ab->hal.srng_config[HAL_CE_DST]; 252 u32 addr; 253 u32 val; 254 255 addr = HAL_CE_DST_RING_CTRL + 256 srng_config->reg_start[HAL_SRNG_REG_GRP_R0] + 257 ring_num * srng_config->reg_size[HAL_SRNG_REG_GRP_R0]; 258 259 val = ath11k_hif_read32(ab, addr); 260 val &= ~HAL_CE_DST_R0_DEST_CTRL_MAX_LEN; 261 val |= FIELD_PREP(HAL_CE_DST_R0_DEST_CTRL_MAX_LEN, 262 srng->u.dst_ring.max_buffer_length); 263 ath11k_hif_write32(ab, addr, val); 264 } 265 266 static void ath11k_hal_srng_dst_hw_init(struct ath11k_base *ab, 267 struct hal_srng *srng) 268 { 269 struct ath11k_hal *hal = &ab->hal; 270 u32 val; 271 u64 hp_addr; 272 u32 reg_base; 273 274 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; 275 276 if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) { 277 ath11k_hif_write32(ab, reg_base + 278 HAL_REO1_RING_MSI1_BASE_LSB_OFFSET(ab), 279 srng->msi_addr); 280 281 val = FIELD_PREP(HAL_REO1_RING_MSI1_BASE_MSB_ADDR, 282 ((u64)srng->msi_addr >> 283 HAL_ADDR_MSB_REG_SHIFT)) | 284 HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE; 285 ath11k_hif_write32(ab, reg_base + 286 HAL_REO1_RING_MSI1_BASE_MSB_OFFSET(ab), val); 287 288 ath11k_hif_write32(ab, 289 reg_base + HAL_REO1_RING_MSI1_DATA_OFFSET(ab), 290 srng->msi_data); 291 } 292 293 ath11k_hif_write32(ab, reg_base, srng->ring_base_paddr); 294 295 val = FIELD_PREP(HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB, 296 ((u64)srng->ring_base_paddr >> 297 HAL_ADDR_MSB_REG_SHIFT)) | 298 FIELD_PREP(HAL_REO1_RING_BASE_MSB_RING_SIZE, 299 (srng->entry_size * srng->num_entries)); 300 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_BASE_MSB_OFFSET(ab), val); 301 302 val = FIELD_PREP(HAL_REO1_RING_ID_RING_ID, srng->ring_id) | 303 FIELD_PREP(HAL_REO1_RING_ID_ENTRY_SIZE, srng->entry_size); 304 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_ID_OFFSET(ab), val); 305 306 /* interrupt setup */ 307 val = FIELD_PREP(HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD, 308 (srng->intr_timer_thres_us >> 3)); 309 310 val |= FIELD_PREP(HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD, 311 (srng->intr_batch_cntr_thres_entries * 312 srng->entry_size)); 313 314 ath11k_hif_write32(ab, 315 reg_base + HAL_REO1_RING_PRODUCER_INT_SETUP_OFFSET(ab), 316 val); 317 318 hp_addr = hal->rdp.paddr + 319 ((unsigned long)srng->u.dst_ring.hp_addr - 320 (unsigned long)hal->rdp.vaddr); 321 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_HP_ADDR_LSB_OFFSET(ab), 322 hp_addr & HAL_ADDR_LSB_REG_MASK); 323 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_HP_ADDR_MSB_OFFSET(ab), 324 hp_addr >> HAL_ADDR_MSB_REG_SHIFT); 325 326 /* Initialize head and tail pointers to indicate ring is empty */ 327 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2]; 328 ath11k_hif_write32(ab, reg_base, 0); 329 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_TP_OFFSET(ab), 0); 330 *srng->u.dst_ring.hp_addr = 0; 331 332 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; 333 val = 0; 334 if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP) 335 val |= HAL_REO1_RING_MISC_DATA_TLV_SWAP; 336 if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP) 337 val |= HAL_REO1_RING_MISC_HOST_FW_SWAP; 338 if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP) 339 val |= HAL_REO1_RING_MISC_MSI_SWAP; 340 val |= HAL_REO1_RING_MISC_SRNG_ENABLE; 341 342 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_MISC_OFFSET(ab), val); 343 } 344 345 static void ath11k_hal_srng_src_hw_init(struct ath11k_base *ab, 346 struct hal_srng *srng) 347 { 348 struct ath11k_hal *hal = &ab->hal; 349 u32 val; 350 u64 tp_addr; 351 u32 reg_base; 352 353 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; 354 355 if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) { 356 ath11k_hif_write32(ab, reg_base + 357 HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(ab), 358 srng->msi_addr); 359 360 val = FIELD_PREP(HAL_TCL1_RING_MSI1_BASE_MSB_ADDR, 361 ((u64)srng->msi_addr >> 362 HAL_ADDR_MSB_REG_SHIFT)) | 363 HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE; 364 ath11k_hif_write32(ab, reg_base + 365 HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(ab), 366 val); 367 368 ath11k_hif_write32(ab, reg_base + 369 HAL_TCL1_RING_MSI1_DATA_OFFSET(ab), 370 srng->msi_data); 371 } 372 373 ath11k_hif_write32(ab, reg_base, srng->ring_base_paddr); 374 375 val = FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB, 376 ((u64)srng->ring_base_paddr >> 377 HAL_ADDR_MSB_REG_SHIFT)) | 378 FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_SIZE, 379 (srng->entry_size * srng->num_entries)); 380 ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(ab), val); 381 382 val = FIELD_PREP(HAL_REO1_RING_ID_ENTRY_SIZE, srng->entry_size); 383 ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_ID_OFFSET(ab), val); 384 385 if (srng->ring_id == HAL_SRNG_RING_ID_WBM_IDLE_LINK) { 386 ath11k_hif_write32(ab, reg_base, (u32)srng->ring_base_paddr); 387 val = FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB, 388 ((u64)srng->ring_base_paddr >> 389 HAL_ADDR_MSB_REG_SHIFT)) | 390 FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_SIZE, 391 (srng->entry_size * srng->num_entries)); 392 ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(ab), val); 393 } 394 395 /* interrupt setup */ 396 /* NOTE: IPQ8074 v2 requires the interrupt timer threshold in the 397 * unit of 8 usecs instead of 1 usec (as required by v1). 398 */ 399 val = FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD, 400 srng->intr_timer_thres_us); 401 402 val |= FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD, 403 (srng->intr_batch_cntr_thres_entries * 404 srng->entry_size)); 405 406 ath11k_hif_write32(ab, 407 reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(ab), 408 val); 409 410 val = 0; 411 if (srng->flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) { 412 val |= FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD, 413 srng->u.src_ring.low_threshold); 414 } 415 ath11k_hif_write32(ab, 416 reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(ab), 417 val); 418 419 if (srng->ring_id != HAL_SRNG_RING_ID_WBM_IDLE_LINK) { 420 tp_addr = hal->rdp.paddr + 421 ((unsigned long)srng->u.src_ring.tp_addr - 422 (unsigned long)hal->rdp.vaddr); 423 ath11k_hif_write32(ab, 424 reg_base + HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(ab), 425 tp_addr & HAL_ADDR_LSB_REG_MASK); 426 ath11k_hif_write32(ab, 427 reg_base + HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(ab), 428 tp_addr >> HAL_ADDR_MSB_REG_SHIFT); 429 } 430 431 /* Initialize head and tail pointers to indicate ring is empty */ 432 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2]; 433 ath11k_hif_write32(ab, reg_base, 0); 434 ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_TP_OFFSET, 0); 435 *srng->u.src_ring.tp_addr = 0; 436 437 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; 438 val = 0; 439 if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP) 440 val |= HAL_TCL1_RING_MISC_DATA_TLV_SWAP; 441 if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP) 442 val |= HAL_TCL1_RING_MISC_HOST_FW_SWAP; 443 if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP) 444 val |= HAL_TCL1_RING_MISC_MSI_SWAP; 445 446 /* Loop count is not used for SRC rings */ 447 val |= HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE; 448 449 val |= HAL_TCL1_RING_MISC_SRNG_ENABLE; 450 451 ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_MISC_OFFSET(ab), val); 452 } 453 454 static void ath11k_hal_srng_hw_init(struct ath11k_base *ab, 455 struct hal_srng *srng) 456 { 457 if (srng->ring_dir == HAL_SRNG_DIR_SRC) 458 ath11k_hal_srng_src_hw_init(ab, srng); 459 else 460 ath11k_hal_srng_dst_hw_init(ab, srng); 461 } 462 463 static int ath11k_hal_srng_get_ring_id(struct ath11k_base *ab, 464 enum hal_ring_type type, 465 int ring_num, int mac_id) 466 { 467 struct hal_srng_config *srng_config = &ab->hal.srng_config[type]; 468 int ring_id; 469 470 if (ring_num >= srng_config->max_rings) { 471 ath11k_warn(ab, "invalid ring number :%d\n", ring_num); 472 return -EINVAL; 473 } 474 475 ring_id = srng_config->start_ring_id + ring_num; 476 if (srng_config->lmac_ring) 477 ring_id += mac_id * HAL_SRNG_RINGS_PER_LMAC; 478 479 if (WARN_ON(ring_id >= HAL_SRNG_RING_ID_MAX)) 480 return -EINVAL; 481 482 return ring_id; 483 } 484 485 int ath11k_hal_srng_get_entrysize(struct ath11k_base *ab, u32 ring_type) 486 { 487 struct hal_srng_config *srng_config; 488 489 if (WARN_ON(ring_type >= HAL_MAX_RING_TYPES)) 490 return -EINVAL; 491 492 srng_config = &ab->hal.srng_config[ring_type]; 493 494 return (srng_config->entry_size << 2); 495 } 496 497 int ath11k_hal_srng_get_max_entries(struct ath11k_base *ab, u32 ring_type) 498 { 499 struct hal_srng_config *srng_config; 500 501 if (WARN_ON(ring_type >= HAL_MAX_RING_TYPES)) 502 return -EINVAL; 503 504 srng_config = &ab->hal.srng_config[ring_type]; 505 506 return (srng_config->max_size / srng_config->entry_size); 507 } 508 509 void ath11k_hal_srng_get_params(struct ath11k_base *ab, struct hal_srng *srng, 510 struct hal_srng_params *params) 511 { 512 params->ring_base_paddr = srng->ring_base_paddr; 513 params->ring_base_vaddr = srng->ring_base_vaddr; 514 params->num_entries = srng->num_entries; 515 params->intr_timer_thres_us = srng->intr_timer_thres_us; 516 params->intr_batch_cntr_thres_entries = 517 srng->intr_batch_cntr_thres_entries; 518 params->low_threshold = srng->u.src_ring.low_threshold; 519 params->msi_addr = srng->msi_addr; 520 params->msi_data = srng->msi_data; 521 params->flags = srng->flags; 522 } 523 524 dma_addr_t ath11k_hal_srng_get_hp_addr(struct ath11k_base *ab, 525 struct hal_srng *srng) 526 { 527 if (!(srng->flags & HAL_SRNG_FLAGS_LMAC_RING)) 528 return 0; 529 530 if (srng->ring_dir == HAL_SRNG_DIR_SRC) 531 return ab->hal.wrp.paddr + 532 ((unsigned long)srng->u.src_ring.hp_addr - 533 (unsigned long)ab->hal.wrp.vaddr); 534 else 535 return ab->hal.rdp.paddr + 536 ((unsigned long)srng->u.dst_ring.hp_addr - 537 (unsigned long)ab->hal.rdp.vaddr); 538 } 539 540 dma_addr_t ath11k_hal_srng_get_tp_addr(struct ath11k_base *ab, 541 struct hal_srng *srng) 542 { 543 if (!(srng->flags & HAL_SRNG_FLAGS_LMAC_RING)) 544 return 0; 545 546 if (srng->ring_dir == HAL_SRNG_DIR_SRC) 547 return ab->hal.rdp.paddr + 548 ((unsigned long)srng->u.src_ring.tp_addr - 549 (unsigned long)ab->hal.rdp.vaddr); 550 else 551 return ab->hal.wrp.paddr + 552 ((unsigned long)srng->u.dst_ring.tp_addr - 553 (unsigned long)ab->hal.wrp.vaddr); 554 } 555 556 u32 ath11k_hal_ce_get_desc_size(enum hal_ce_desc type) 557 { 558 switch (type) { 559 case HAL_CE_DESC_SRC: 560 return sizeof(struct hal_ce_srng_src_desc); 561 case HAL_CE_DESC_DST: 562 return sizeof(struct hal_ce_srng_dest_desc); 563 case HAL_CE_DESC_DST_STATUS: 564 return sizeof(struct hal_ce_srng_dst_status_desc); 565 } 566 567 return 0; 568 } 569 570 void ath11k_hal_ce_src_set_desc(void *buf, dma_addr_t paddr, u32 len, u32 id, 571 u8 byte_swap_data) 572 { 573 struct hal_ce_srng_src_desc *desc = (struct hal_ce_srng_src_desc *)buf; 574 575 desc->buffer_addr_low = paddr & HAL_ADDR_LSB_REG_MASK; 576 desc->buffer_addr_info = 577 FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_ADDR_HI, 578 ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT)) | 579 FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_BYTE_SWAP, 580 byte_swap_data) | 581 FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_GATHER, 0) | 582 FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_LEN, len); 583 desc->meta_info = FIELD_PREP(HAL_CE_SRC_DESC_META_INFO_DATA, id); 584 } 585 586 void ath11k_hal_ce_dst_set_desc(void *buf, dma_addr_t paddr) 587 { 588 struct hal_ce_srng_dest_desc *desc = 589 (struct hal_ce_srng_dest_desc *)buf; 590 591 desc->buffer_addr_low = paddr & HAL_ADDR_LSB_REG_MASK; 592 desc->buffer_addr_info = 593 FIELD_PREP(HAL_CE_DEST_DESC_ADDR_INFO_ADDR_HI, 594 ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT)); 595 } 596 597 u32 ath11k_hal_ce_dst_status_get_length(void *buf) 598 { 599 struct hal_ce_srng_dst_status_desc *desc = 600 (struct hal_ce_srng_dst_status_desc *)buf; 601 u32 len; 602 603 len = FIELD_GET(HAL_CE_DST_STATUS_DESC_FLAGS_LEN, desc->flags); 604 desc->flags &= ~HAL_CE_DST_STATUS_DESC_FLAGS_LEN; 605 606 return len; 607 } 608 609 void ath11k_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc, u32 cookie, 610 dma_addr_t paddr) 611 { 612 desc->buf_addr_info.info0 = FIELD_PREP(BUFFER_ADDR_INFO0_ADDR, 613 (paddr & HAL_ADDR_LSB_REG_MASK)); 614 desc->buf_addr_info.info1 = FIELD_PREP(BUFFER_ADDR_INFO1_ADDR, 615 ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT)) | 616 FIELD_PREP(BUFFER_ADDR_INFO1_RET_BUF_MGR, 1) | 617 FIELD_PREP(BUFFER_ADDR_INFO1_SW_COOKIE, cookie); 618 } 619 620 u32 *ath11k_hal_srng_dst_peek(struct ath11k_base *ab, struct hal_srng *srng) 621 { 622 lockdep_assert_held(&srng->lock); 623 624 if (srng->u.dst_ring.tp != srng->u.dst_ring.cached_hp) 625 return (srng->ring_base_vaddr + srng->u.dst_ring.tp); 626 627 return NULL; 628 } 629 630 static void ath11k_hal_srng_prefetch_desc(struct ath11k_base *ab, 631 struct hal_srng *srng) 632 { 633 u32 *desc; 634 635 /* prefetch only if desc is available */ 636 desc = ath11k_hal_srng_dst_peek(ab, srng); 637 if (likely(desc)) { 638 dma_sync_single_for_cpu(ab->dev, virt_to_phys(desc), 639 (srng->entry_size * sizeof(u32)), 640 DMA_FROM_DEVICE); 641 prefetch(desc); 642 } 643 } 644 645 u32 *ath11k_hal_srng_dst_get_next_entry(struct ath11k_base *ab, 646 struct hal_srng *srng) 647 { 648 u32 *desc; 649 650 lockdep_assert_held(&srng->lock); 651 652 if (srng->u.dst_ring.tp == srng->u.dst_ring.cached_hp) 653 return NULL; 654 655 desc = srng->ring_base_vaddr + srng->u.dst_ring.tp; 656 657 srng->u.dst_ring.tp += srng->entry_size; 658 659 /* wrap around to start of ring*/ 660 if (srng->u.dst_ring.tp == srng->ring_size) 661 srng->u.dst_ring.tp = 0; 662 663 /* Try to prefetch the next descriptor in the ring */ 664 if (srng->flags & HAL_SRNG_FLAGS_CACHED) 665 ath11k_hal_srng_prefetch_desc(ab, srng); 666 667 return desc; 668 } 669 670 int ath11k_hal_srng_dst_num_free(struct ath11k_base *ab, struct hal_srng *srng, 671 bool sync_hw_ptr) 672 { 673 u32 tp, hp; 674 675 lockdep_assert_held(&srng->lock); 676 677 tp = srng->u.dst_ring.tp; 678 679 if (sync_hw_ptr) { 680 hp = *srng->u.dst_ring.hp_addr; 681 srng->u.dst_ring.cached_hp = hp; 682 } else { 683 hp = srng->u.dst_ring.cached_hp; 684 } 685 686 if (hp >= tp) 687 return (hp - tp) / srng->entry_size; 688 else 689 return (srng->ring_size - tp + hp) / srng->entry_size; 690 } 691 692 /* Returns number of available entries in src ring */ 693 int ath11k_hal_srng_src_num_free(struct ath11k_base *ab, struct hal_srng *srng, 694 bool sync_hw_ptr) 695 { 696 u32 tp, hp; 697 698 lockdep_assert_held(&srng->lock); 699 700 hp = srng->u.src_ring.hp; 701 702 if (sync_hw_ptr) { 703 tp = *srng->u.src_ring.tp_addr; 704 srng->u.src_ring.cached_tp = tp; 705 } else { 706 tp = srng->u.src_ring.cached_tp; 707 } 708 709 if (tp > hp) 710 return ((tp - hp) / srng->entry_size) - 1; 711 else 712 return ((srng->ring_size - hp + tp) / srng->entry_size) - 1; 713 } 714 715 u32 *ath11k_hal_srng_src_get_next_entry(struct ath11k_base *ab, 716 struct hal_srng *srng) 717 { 718 u32 *desc; 719 u32 next_hp; 720 721 lockdep_assert_held(&srng->lock); 722 723 /* TODO: Using % is expensive, but we have to do this since size of some 724 * SRNG rings is not power of 2 (due to descriptor sizes). Need to see 725 * if separate function is defined for rings having power of 2 ring size 726 * (TCL2SW, REO2SW, SW2RXDMA and CE rings) so that we can avoid the 727 * overhead of % by using mask (with &). 728 */ 729 next_hp = (srng->u.src_ring.hp + srng->entry_size) % srng->ring_size; 730 731 if (next_hp == srng->u.src_ring.cached_tp) 732 return NULL; 733 734 desc = srng->ring_base_vaddr + srng->u.src_ring.hp; 735 srng->u.src_ring.hp = next_hp; 736 737 /* TODO: Reap functionality is not used by all rings. If particular 738 * ring does not use reap functionality, we need not update reap_hp 739 * with next_hp pointer. Need to make sure a separate function is used 740 * before doing any optimization by removing below code updating 741 * reap_hp. 742 */ 743 srng->u.src_ring.reap_hp = next_hp; 744 745 return desc; 746 } 747 748 u32 *ath11k_hal_srng_src_reap_next(struct ath11k_base *ab, 749 struct hal_srng *srng) 750 { 751 u32 *desc; 752 u32 next_reap_hp; 753 754 lockdep_assert_held(&srng->lock); 755 756 next_reap_hp = (srng->u.src_ring.reap_hp + srng->entry_size) % 757 srng->ring_size; 758 759 if (next_reap_hp == srng->u.src_ring.cached_tp) 760 return NULL; 761 762 desc = srng->ring_base_vaddr + next_reap_hp; 763 srng->u.src_ring.reap_hp = next_reap_hp; 764 765 return desc; 766 } 767 768 u32 *ath11k_hal_srng_src_get_next_reaped(struct ath11k_base *ab, 769 struct hal_srng *srng) 770 { 771 u32 *desc; 772 773 lockdep_assert_held(&srng->lock); 774 775 if (srng->u.src_ring.hp == srng->u.src_ring.reap_hp) 776 return NULL; 777 778 desc = srng->ring_base_vaddr + srng->u.src_ring.hp; 779 srng->u.src_ring.hp = (srng->u.src_ring.hp + srng->entry_size) % 780 srng->ring_size; 781 782 return desc; 783 } 784 785 u32 *ath11k_hal_srng_src_peek(struct ath11k_base *ab, struct hal_srng *srng) 786 { 787 lockdep_assert_held(&srng->lock); 788 789 if (((srng->u.src_ring.hp + srng->entry_size) % srng->ring_size) == 790 srng->u.src_ring.cached_tp) 791 return NULL; 792 793 return srng->ring_base_vaddr + srng->u.src_ring.hp; 794 } 795 796 void ath11k_hal_srng_access_begin(struct ath11k_base *ab, struct hal_srng *srng) 797 { 798 lockdep_assert_held(&srng->lock); 799 800 if (srng->ring_dir == HAL_SRNG_DIR_SRC) { 801 srng->u.src_ring.cached_tp = 802 *(volatile u32 *)srng->u.src_ring.tp_addr; 803 } else { 804 srng->u.dst_ring.cached_hp = *srng->u.dst_ring.hp_addr; 805 806 /* Try to prefetch the next descriptor in the ring */ 807 if (srng->flags & HAL_SRNG_FLAGS_CACHED) 808 ath11k_hal_srng_prefetch_desc(ab, srng); 809 } 810 } 811 812 /* Update cached ring head/tail pointers to HW. ath11k_hal_srng_access_begin() 813 * should have been called before this. 814 */ 815 void ath11k_hal_srng_access_end(struct ath11k_base *ab, struct hal_srng *srng) 816 { 817 lockdep_assert_held(&srng->lock); 818 819 /* TODO: See if we need a write memory barrier here */ 820 if (srng->flags & HAL_SRNG_FLAGS_LMAC_RING) { 821 /* For LMAC rings, ring pointer updates are done through FW and 822 * hence written to a shared memory location that is read by FW 823 */ 824 if (srng->ring_dir == HAL_SRNG_DIR_SRC) { 825 srng->u.src_ring.last_tp = 826 *(volatile u32 *)srng->u.src_ring.tp_addr; 827 *srng->u.src_ring.hp_addr = srng->u.src_ring.hp; 828 } else { 829 srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr; 830 *srng->u.dst_ring.tp_addr = srng->u.dst_ring.tp; 831 } 832 } else { 833 if (srng->ring_dir == HAL_SRNG_DIR_SRC) { 834 srng->u.src_ring.last_tp = 835 *(volatile u32 *)srng->u.src_ring.tp_addr; 836 ath11k_hif_write32(ab, 837 (unsigned long)srng->u.src_ring.hp_addr - 838 (unsigned long)ab->mem, 839 srng->u.src_ring.hp); 840 } else { 841 srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr; 842 ath11k_hif_write32(ab, 843 (unsigned long)srng->u.dst_ring.tp_addr - 844 (unsigned long)ab->mem, 845 srng->u.dst_ring.tp); 846 } 847 } 848 849 srng->timestamp = jiffies; 850 } 851 852 void ath11k_hal_setup_link_idle_list(struct ath11k_base *ab, 853 struct hal_wbm_idle_scatter_list *sbuf, 854 u32 nsbufs, u32 tot_link_desc, 855 u32 end_offset) 856 { 857 struct ath11k_buffer_addr *link_addr; 858 int i; 859 u32 reg_scatter_buf_sz = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 64; 860 861 link_addr = (void *)sbuf[0].vaddr + HAL_WBM_IDLE_SCATTER_BUF_SIZE; 862 863 for (i = 1; i < nsbufs; i++) { 864 link_addr->info0 = sbuf[i].paddr & HAL_ADDR_LSB_REG_MASK; 865 link_addr->info1 = FIELD_PREP( 866 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32, 867 (u64)sbuf[i].paddr >> HAL_ADDR_MSB_REG_SHIFT) | 868 FIELD_PREP( 869 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG, 870 BASE_ADDR_MATCH_TAG_VAL); 871 872 link_addr = (void *)sbuf[i].vaddr + 873 HAL_WBM_IDLE_SCATTER_BUF_SIZE; 874 } 875 876 ath11k_hif_write32(ab, 877 HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR, 878 FIELD_PREP(HAL_WBM_SCATTER_BUFFER_SIZE, reg_scatter_buf_sz) | 879 FIELD_PREP(HAL_WBM_LINK_DESC_IDLE_LIST_MODE, 0x1)); 880 ath11k_hif_write32(ab, 881 HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_R0_IDLE_LIST_SIZE_ADDR, 882 FIELD_PREP(HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST, 883 reg_scatter_buf_sz * nsbufs)); 884 ath11k_hif_write32(ab, 885 HAL_SEQ_WCSS_UMAC_WBM_REG + 886 HAL_WBM_SCATTERED_RING_BASE_LSB, 887 FIELD_PREP(BUFFER_ADDR_INFO0_ADDR, 888 sbuf[0].paddr & HAL_ADDR_LSB_REG_MASK)); 889 ath11k_hif_write32(ab, 890 HAL_SEQ_WCSS_UMAC_WBM_REG + 891 HAL_WBM_SCATTERED_RING_BASE_MSB, 892 FIELD_PREP( 893 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32, 894 (u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT) | 895 FIELD_PREP( 896 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG, 897 BASE_ADDR_MATCH_TAG_VAL)); 898 899 /* Setup head and tail pointers for the idle list */ 900 ath11k_hif_write32(ab, 901 HAL_SEQ_WCSS_UMAC_WBM_REG + 902 HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0, 903 FIELD_PREP(BUFFER_ADDR_INFO0_ADDR, 904 sbuf[nsbufs - 1].paddr)); 905 ath11k_hif_write32(ab, 906 HAL_SEQ_WCSS_UMAC_WBM_REG + 907 HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1, 908 FIELD_PREP( 909 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32, 910 ((u64)sbuf[nsbufs - 1].paddr >> 911 HAL_ADDR_MSB_REG_SHIFT)) | 912 FIELD_PREP(HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1, 913 (end_offset >> 2))); 914 ath11k_hif_write32(ab, 915 HAL_SEQ_WCSS_UMAC_WBM_REG + 916 HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0, 917 FIELD_PREP(BUFFER_ADDR_INFO0_ADDR, 918 sbuf[0].paddr)); 919 920 ath11k_hif_write32(ab, 921 HAL_SEQ_WCSS_UMAC_WBM_REG + 922 HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0, 923 FIELD_PREP(BUFFER_ADDR_INFO0_ADDR, 924 sbuf[0].paddr)); 925 ath11k_hif_write32(ab, 926 HAL_SEQ_WCSS_UMAC_WBM_REG + 927 HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1, 928 FIELD_PREP( 929 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32, 930 ((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT)) | 931 FIELD_PREP(HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1, 932 0)); 933 ath11k_hif_write32(ab, 934 HAL_SEQ_WCSS_UMAC_WBM_REG + 935 HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR, 936 2 * tot_link_desc); 937 938 /* Enable the SRNG */ 939 ath11k_hif_write32(ab, 940 HAL_SEQ_WCSS_UMAC_WBM_REG + 941 HAL_WBM_IDLE_LINK_RING_MISC_ADDR(ab), 0x40); 942 } 943 944 int ath11k_hal_srng_setup(struct ath11k_base *ab, enum hal_ring_type type, 945 int ring_num, int mac_id, 946 struct hal_srng_params *params) 947 { 948 struct ath11k_hal *hal = &ab->hal; 949 struct hal_srng_config *srng_config = &ab->hal.srng_config[type]; 950 struct hal_srng *srng; 951 int ring_id; 952 u32 lmac_idx; 953 int i; 954 u32 reg_base; 955 956 ring_id = ath11k_hal_srng_get_ring_id(ab, type, ring_num, mac_id); 957 if (ring_id < 0) 958 return ring_id; 959 960 srng = &hal->srng_list[ring_id]; 961 962 srng->ring_id = ring_id; 963 srng->ring_dir = srng_config->ring_dir; 964 srng->ring_base_paddr = params->ring_base_paddr; 965 srng->ring_base_vaddr = params->ring_base_vaddr; 966 srng->entry_size = srng_config->entry_size; 967 srng->num_entries = params->num_entries; 968 srng->ring_size = srng->entry_size * srng->num_entries; 969 srng->intr_batch_cntr_thres_entries = 970 params->intr_batch_cntr_thres_entries; 971 srng->intr_timer_thres_us = params->intr_timer_thres_us; 972 srng->flags = params->flags; 973 srng->msi_addr = params->msi_addr; 974 srng->msi_data = params->msi_data; 975 srng->initialized = 1; 976 spin_lock_init(&srng->lock); 977 lockdep_set_class(&srng->lock, hal->srng_key + ring_id); 978 979 for (i = 0; i < HAL_SRNG_NUM_REG_GRP; i++) { 980 srng->hwreg_base[i] = srng_config->reg_start[i] + 981 (ring_num * srng_config->reg_size[i]); 982 } 983 984 memset(srng->ring_base_vaddr, 0, 985 (srng->entry_size * srng->num_entries) << 2); 986 987 /* TODO: Add comments on these swap configurations */ 988 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) 989 srng->flags |= HAL_SRNG_FLAGS_MSI_SWAP | HAL_SRNG_FLAGS_DATA_TLV_SWAP | 990 HAL_SRNG_FLAGS_RING_PTR_SWAP; 991 992 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2]; 993 994 if (srng->ring_dir == HAL_SRNG_DIR_SRC) { 995 srng->u.src_ring.hp = 0; 996 srng->u.src_ring.cached_tp = 0; 997 srng->u.src_ring.reap_hp = srng->ring_size - srng->entry_size; 998 srng->u.src_ring.tp_addr = (void *)(hal->rdp.vaddr + ring_id); 999 srng->u.src_ring.low_threshold = params->low_threshold * 1000 srng->entry_size; 1001 if (srng_config->lmac_ring) { 1002 lmac_idx = ring_id - HAL_SRNG_RING_ID_LMAC1_ID_START; 1003 srng->u.src_ring.hp_addr = (void *)(hal->wrp.vaddr + 1004 lmac_idx); 1005 srng->flags |= HAL_SRNG_FLAGS_LMAC_RING; 1006 } else { 1007 if (!ab->hw_params.supports_shadow_regs) 1008 srng->u.src_ring.hp_addr = 1009 (u32 *)((unsigned long)ab->mem + reg_base); 1010 else 1011 ath11k_dbg(ab, ATH11k_DBG_HAL, 1012 "hal type %d ring_num %d reg_base 0x%x shadow 0x%lx\n", 1013 type, ring_num, 1014 reg_base, 1015 (unsigned long)srng->u.src_ring.hp_addr - 1016 (unsigned long)ab->mem); 1017 } 1018 } else { 1019 /* During initialization loop count in all the descriptors 1020 * will be set to zero, and HW will set it to 1 on completing 1021 * descriptor update in first loop, and increments it by 1 on 1022 * subsequent loops (loop count wraps around after reaching 1023 * 0xffff). The 'loop_cnt' in SW ring state is the expected 1024 * loop count in descriptors updated by HW (to be processed 1025 * by SW). 1026 */ 1027 srng->u.dst_ring.loop_cnt = 1; 1028 srng->u.dst_ring.tp = 0; 1029 srng->u.dst_ring.cached_hp = 0; 1030 srng->u.dst_ring.hp_addr = (void *)(hal->rdp.vaddr + ring_id); 1031 if (srng_config->lmac_ring) { 1032 /* For LMAC rings, tail pointer updates will be done 1033 * through FW by writing to a shared memory location 1034 */ 1035 lmac_idx = ring_id - HAL_SRNG_RING_ID_LMAC1_ID_START; 1036 srng->u.dst_ring.tp_addr = (void *)(hal->wrp.vaddr + 1037 lmac_idx); 1038 srng->flags |= HAL_SRNG_FLAGS_LMAC_RING; 1039 } else { 1040 if (!ab->hw_params.supports_shadow_regs) 1041 srng->u.dst_ring.tp_addr = 1042 (u32 *)((unsigned long)ab->mem + reg_base + 1043 (HAL_REO1_RING_TP(ab) - HAL_REO1_RING_HP(ab))); 1044 else 1045 ath11k_dbg(ab, ATH11k_DBG_HAL, 1046 "type %d ring_num %d target_reg 0x%x shadow 0x%lx\n", 1047 type, ring_num, 1048 reg_base + (HAL_REO1_RING_TP(ab) - 1049 HAL_REO1_RING_HP(ab)), 1050 (unsigned long)srng->u.dst_ring.tp_addr - 1051 (unsigned long)ab->mem); 1052 } 1053 } 1054 1055 if (srng_config->lmac_ring) 1056 return ring_id; 1057 1058 ath11k_hal_srng_hw_init(ab, srng); 1059 1060 if (type == HAL_CE_DST) { 1061 srng->u.dst_ring.max_buffer_length = params->max_buffer_len; 1062 ath11k_hal_ce_dst_setup(ab, srng, ring_num); 1063 } 1064 1065 return ring_id; 1066 } 1067 1068 static void ath11k_hal_srng_update_hp_tp_addr(struct ath11k_base *ab, 1069 int shadow_cfg_idx, 1070 enum hal_ring_type ring_type, 1071 int ring_num) 1072 { 1073 struct hal_srng *srng; 1074 struct ath11k_hal *hal = &ab->hal; 1075 int ring_id; 1076 struct hal_srng_config *srng_config = &hal->srng_config[ring_type]; 1077 1078 ring_id = ath11k_hal_srng_get_ring_id(ab, ring_type, ring_num, 0); 1079 if (ring_id < 0) 1080 return; 1081 1082 srng = &hal->srng_list[ring_id]; 1083 1084 if (srng_config->ring_dir == HAL_SRNG_DIR_DST) 1085 srng->u.dst_ring.tp_addr = (u32 *)(HAL_SHADOW_REG(shadow_cfg_idx) + 1086 (unsigned long)ab->mem); 1087 else 1088 srng->u.src_ring.hp_addr = (u32 *)(HAL_SHADOW_REG(shadow_cfg_idx) + 1089 (unsigned long)ab->mem); 1090 } 1091 1092 int ath11k_hal_srng_update_shadow_config(struct ath11k_base *ab, 1093 enum hal_ring_type ring_type, 1094 int ring_num) 1095 { 1096 struct ath11k_hal *hal = &ab->hal; 1097 struct hal_srng_config *srng_config = &hal->srng_config[ring_type]; 1098 int shadow_cfg_idx = hal->num_shadow_reg_configured; 1099 u32 target_reg; 1100 1101 if (shadow_cfg_idx >= HAL_SHADOW_NUM_REGS) 1102 return -EINVAL; 1103 1104 hal->num_shadow_reg_configured++; 1105 1106 target_reg = srng_config->reg_start[HAL_HP_OFFSET_IN_REG_START]; 1107 target_reg += srng_config->reg_size[HAL_HP_OFFSET_IN_REG_START] * 1108 ring_num; 1109 1110 /* For destination ring, shadow the TP */ 1111 if (srng_config->ring_dir == HAL_SRNG_DIR_DST) 1112 target_reg += HAL_OFFSET_FROM_HP_TO_TP; 1113 1114 hal->shadow_reg_addr[shadow_cfg_idx] = target_reg; 1115 1116 /* update hp/tp addr to hal structure*/ 1117 ath11k_hal_srng_update_hp_tp_addr(ab, shadow_cfg_idx, ring_type, 1118 ring_num); 1119 1120 ath11k_dbg(ab, ATH11k_DBG_HAL, 1121 "target_reg %x, shadow reg 0x%x shadow_idx 0x%x, ring_type %d, ring num %d", 1122 target_reg, 1123 HAL_SHADOW_REG(shadow_cfg_idx), 1124 shadow_cfg_idx, 1125 ring_type, ring_num); 1126 1127 return 0; 1128 } 1129 1130 void ath11k_hal_srng_shadow_config(struct ath11k_base *ab) 1131 { 1132 struct ath11k_hal *hal = &ab->hal; 1133 int ring_type, ring_num; 1134 1135 /* update all the non-CE srngs. */ 1136 for (ring_type = 0; ring_type < HAL_MAX_RING_TYPES; ring_type++) { 1137 struct hal_srng_config *srng_config = &hal->srng_config[ring_type]; 1138 1139 if (ring_type == HAL_CE_SRC || 1140 ring_type == HAL_CE_DST || 1141 ring_type == HAL_CE_DST_STATUS) 1142 continue; 1143 1144 if (srng_config->lmac_ring) 1145 continue; 1146 1147 for (ring_num = 0; ring_num < srng_config->max_rings; ring_num++) 1148 ath11k_hal_srng_update_shadow_config(ab, ring_type, ring_num); 1149 } 1150 } 1151 1152 void ath11k_hal_srng_get_shadow_config(struct ath11k_base *ab, 1153 u32 **cfg, u32 *len) 1154 { 1155 struct ath11k_hal *hal = &ab->hal; 1156 1157 *len = hal->num_shadow_reg_configured; 1158 *cfg = hal->shadow_reg_addr; 1159 } 1160 1161 void ath11k_hal_srng_shadow_update_hp_tp(struct ath11k_base *ab, 1162 struct hal_srng *srng) 1163 { 1164 lockdep_assert_held(&srng->lock); 1165 1166 /* check whether the ring is emptry. Update the shadow 1167 * HP only when then ring isn't' empty. 1168 */ 1169 if (srng->ring_dir == HAL_SRNG_DIR_SRC && 1170 *srng->u.src_ring.tp_addr != srng->u.src_ring.hp) 1171 ath11k_hal_srng_access_end(ab, srng); 1172 } 1173 1174 static int ath11k_hal_srng_create_config(struct ath11k_base *ab) 1175 { 1176 struct ath11k_hal *hal = &ab->hal; 1177 struct hal_srng_config *s; 1178 1179 hal->srng_config = kmemdup(hw_srng_config_template, 1180 sizeof(hw_srng_config_template), 1181 GFP_KERNEL); 1182 if (!hal->srng_config) 1183 return -ENOMEM; 1184 1185 s = &hal->srng_config[HAL_REO_DST]; 1186 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(ab); 1187 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP(ab); 1188 s->reg_size[0] = HAL_REO2_RING_BASE_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab); 1189 s->reg_size[1] = HAL_REO2_RING_HP(ab) - HAL_REO1_RING_HP(ab); 1190 1191 s = &hal->srng_config[HAL_REO_EXCEPTION]; 1192 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_TCL_RING_BASE_LSB(ab); 1193 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_TCL_RING_HP(ab); 1194 1195 s = &hal->srng_config[HAL_REO_REINJECT]; 1196 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB; 1197 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP; 1198 1199 s = &hal->srng_config[HAL_REO_CMD]; 1200 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB; 1201 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP; 1202 1203 s = &hal->srng_config[HAL_REO_STATUS]; 1204 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(ab); 1205 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP(ab); 1206 1207 s = &hal->srng_config[HAL_TCL_DATA]; 1208 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(ab); 1209 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP; 1210 s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(ab) - HAL_TCL1_RING_BASE_LSB(ab); 1211 s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP; 1212 1213 s = &hal->srng_config[HAL_TCL_CMD]; 1214 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(ab); 1215 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP; 1216 1217 s = &hal->srng_config[HAL_TCL_STATUS]; 1218 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(ab); 1219 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP; 1220 1221 s = &hal->srng_config[HAL_CE_SRC]; 1222 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_BASE_LSB; 1223 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_HP; 1224 s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) - 1225 HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab); 1226 s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) - 1227 HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab); 1228 1229 s = &hal->srng_config[HAL_CE_DST]; 1230 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_BASE_LSB; 1231 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_HP; 1232 s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - 1233 HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); 1234 s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - 1235 HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); 1236 1237 s = &hal->srng_config[HAL_CE_DST_STATUS]; 1238 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + 1239 HAL_CE_DST_STATUS_RING_BASE_LSB; 1240 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_STATUS_RING_HP; 1241 s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - 1242 HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); 1243 s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - 1244 HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); 1245 1246 s = &hal->srng_config[HAL_WBM_IDLE_LINK]; 1247 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(ab); 1248 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP; 1249 1250 s = &hal->srng_config[HAL_SW2WBM_RELEASE]; 1251 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_RELEASE_RING_BASE_LSB(ab); 1252 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_RELEASE_RING_HP; 1253 1254 s = &hal->srng_config[HAL_WBM2SW_RELEASE]; 1255 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(ab); 1256 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP; 1257 s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(ab) - 1258 HAL_WBM0_RELEASE_RING_BASE_LSB(ab); 1259 s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP; 1260 1261 return 0; 1262 } 1263 1264 static void ath11k_hal_register_srng_key(struct ath11k_base *ab) 1265 { 1266 struct ath11k_hal *hal = &ab->hal; 1267 u32 ring_id; 1268 1269 for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++) 1270 lockdep_register_key(hal->srng_key + ring_id); 1271 } 1272 1273 static void ath11k_hal_unregister_srng_key(struct ath11k_base *ab) 1274 { 1275 struct ath11k_hal *hal = &ab->hal; 1276 u32 ring_id; 1277 1278 for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++) 1279 lockdep_unregister_key(hal->srng_key + ring_id); 1280 } 1281 1282 int ath11k_hal_srng_init(struct ath11k_base *ab) 1283 { 1284 struct ath11k_hal *hal = &ab->hal; 1285 int ret; 1286 1287 memset(hal, 0, sizeof(*hal)); 1288 1289 ret = ath11k_hal_srng_create_config(ab); 1290 if (ret) 1291 goto err_hal; 1292 1293 ret = ath11k_hal_alloc_cont_rdp(ab); 1294 if (ret) 1295 goto err_hal; 1296 1297 ret = ath11k_hal_alloc_cont_wrp(ab); 1298 if (ret) 1299 goto err_free_cont_rdp; 1300 1301 ath11k_hal_register_srng_key(ab); 1302 1303 return 0; 1304 1305 err_free_cont_rdp: 1306 ath11k_hal_free_cont_rdp(ab); 1307 1308 err_hal: 1309 return ret; 1310 } 1311 EXPORT_SYMBOL(ath11k_hal_srng_init); 1312 1313 void ath11k_hal_srng_deinit(struct ath11k_base *ab) 1314 { 1315 struct ath11k_hal *hal = &ab->hal; 1316 1317 ath11k_hal_unregister_srng_key(ab); 1318 ath11k_hal_free_cont_rdp(ab); 1319 ath11k_hal_free_cont_wrp(ab); 1320 kfree(hal->srng_config); 1321 } 1322 EXPORT_SYMBOL(ath11k_hal_srng_deinit); 1323 1324 void ath11k_hal_dump_srng_stats(struct ath11k_base *ab) 1325 { 1326 struct hal_srng *srng; 1327 struct ath11k_ext_irq_grp *irq_grp; 1328 struct ath11k_ce_pipe *ce_pipe; 1329 int i; 1330 1331 ath11k_err(ab, "Last interrupt received for each CE:\n"); 1332 for (i = 0; i < ab->hw_params.ce_count; i++) { 1333 ce_pipe = &ab->ce.ce_pipe[i]; 1334 1335 if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR) 1336 continue; 1337 1338 ath11k_err(ab, "CE_id %d pipe_num %d %ums before\n", 1339 i, ce_pipe->pipe_num, 1340 jiffies_to_msecs(jiffies - ce_pipe->timestamp)); 1341 } 1342 1343 ath11k_err(ab, "\nLast interrupt received for each group:\n"); 1344 for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) { 1345 irq_grp = &ab->ext_irq_grp[i]; 1346 ath11k_err(ab, "group_id %d %ums before\n", 1347 irq_grp->grp_id, 1348 jiffies_to_msecs(jiffies - irq_grp->timestamp)); 1349 } 1350 1351 for (i = 0; i < HAL_SRNG_RING_ID_MAX; i++) { 1352 srng = &ab->hal.srng_list[i]; 1353 1354 if (!srng->initialized) 1355 continue; 1356 1357 if (srng->ring_dir == HAL_SRNG_DIR_SRC) 1358 ath11k_err(ab, 1359 "src srng id %u hp %u, reap_hp %u, cur tp %u, cached tp %u last tp %u napi processed before %ums\n", 1360 srng->ring_id, srng->u.src_ring.hp, 1361 srng->u.src_ring.reap_hp, 1362 *srng->u.src_ring.tp_addr, srng->u.src_ring.cached_tp, 1363 srng->u.src_ring.last_tp, 1364 jiffies_to_msecs(jiffies - srng->timestamp)); 1365 else if (srng->ring_dir == HAL_SRNG_DIR_DST) 1366 ath11k_err(ab, 1367 "dst srng id %u tp %u, cur hp %u, cached hp %u last hp %u napi processed before %ums\n", 1368 srng->ring_id, srng->u.dst_ring.tp, 1369 *srng->u.dst_ring.hp_addr, 1370 srng->u.dst_ring.cached_hp, 1371 srng->u.dst_ring.last_hp, 1372 jiffies_to_msecs(jiffies - srng->timestamp)); 1373 } 1374 } 1375