1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include "en/params.h" 5 #include "en/txrx.h" 6 #include "en/port.h" 7 #include "en_accel/en_accel.h" 8 #include "en_accel/ipsec.h" 9 10 u16 mlx5e_mpwrq_umr_wqe_sz(u8 pages_per_wqe) 11 { 12 return sizeof(struct mlx5e_umr_wqe) + 13 ALIGN(pages_per_wqe * sizeof(struct mlx5_mtt), MLX5_UMR_MTT_ALIGNMENT); 14 } 15 16 u8 mlx5e_mpwrq_umr_wqebbs(u8 pages_per_wqe) 17 { 18 return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(pages_per_wqe), MLX5_SEND_WQE_BB); 19 } 20 21 u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, 22 struct mlx5e_xsk_param *xsk) 23 { 24 u16 headroom; 25 26 if (xsk) 27 return xsk->headroom; 28 29 headroom = NET_IP_ALIGN; 30 if (params->xdp_prog) 31 headroom += XDP_PACKET_HEADROOM; 32 else 33 headroom += MLX5_RX_HEADROOM; 34 35 return headroom; 36 } 37 38 static u32 mlx5e_rx_get_linear_sz_xsk(struct mlx5e_params *params, 39 struct mlx5e_xsk_param *xsk) 40 { 41 u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 42 43 return xsk->headroom + hw_mtu; 44 } 45 46 static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool xsk) 47 { 48 /* SKBs built on XDP_PASS on XSK RQs don't have headroom. */ 49 u16 headroom = xsk ? 0 : mlx5e_get_linear_rq_headroom(params, NULL); 50 u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 51 52 return MLX5_SKB_FRAG_SZ(headroom + hw_mtu); 53 } 54 55 static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5e_params *params, 56 struct mlx5e_xsk_param *xsk) 57 { 58 /* XSK frames are mapped as individual pages, because frames may come in 59 * an arbitrary order from random locations in the UMEM. 60 */ 61 if (xsk) 62 return PAGE_SIZE; 63 64 /* XDP in mlx5e doesn't support multiple packets per page. */ 65 if (params->xdp_prog) 66 return PAGE_SIZE; 67 68 return roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, false)); 69 } 70 71 static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params, 72 struct mlx5e_xsk_param *xsk) 73 { 74 u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(params, xsk); 75 76 return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_stride_sz); 77 } 78 79 bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params, 80 struct mlx5e_xsk_param *xsk) 81 { 82 if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) 83 return false; 84 85 /* Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data 86 * must fit into a CPU page. 87 */ 88 if (mlx5e_rx_get_linear_sz_skb(params, xsk) > PAGE_SIZE) 89 return false; 90 91 /* XSK frames must be big enough to hold the packet data. */ 92 if (xsk && mlx5e_rx_get_linear_sz_xsk(params, xsk) > xsk->chunk_size) 93 return false; 94 95 return true; 96 } 97 98 static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev, 99 u8 log_stride_sz, u8 log_num_strides) 100 { 101 if (log_stride_sz + log_num_strides != MLX5_MPWRQ_LOG_WQE_SZ) 102 return false; 103 104 if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE || 105 log_stride_sz > MLX5_MPWQE_LOG_STRIDE_SZ_MAX) 106 return false; 107 108 if (log_num_strides > MLX5_MPWQE_LOG_NUM_STRIDES_MAX) 109 return false; 110 111 if (MLX5_CAP_GEN(mdev, ext_stride_num_range)) 112 return log_num_strides >= MLX5_MPWQE_LOG_NUM_STRIDES_EXT_BASE; 113 114 return log_num_strides >= MLX5_MPWQE_LOG_NUM_STRIDES_BASE; 115 } 116 117 bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, 118 struct mlx5e_params *params, 119 struct mlx5e_xsk_param *xsk) 120 { 121 s8 log_num_strides; 122 u8 log_stride_sz; 123 124 if (!mlx5e_rx_is_linear_skb(params, xsk)) 125 return false; 126 127 log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(params, xsk)); 128 log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - log_stride_sz; 129 130 return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz, log_num_strides); 131 } 132 133 u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params, 134 struct mlx5e_xsk_param *xsk) 135 { 136 u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params, xsk); 137 138 /* Numbers are unsigned, don't subtract to avoid underflow. */ 139 if (params->log_rq_mtu_frames < 140 log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW) 141 return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW; 142 143 return params->log_rq_mtu_frames - log_pkts_per_wqe; 144 } 145 146 u8 mlx5e_shampo_get_log_hd_entry_size(struct mlx5_core_dev *mdev, 147 struct mlx5e_params *params) 148 { 149 return order_base_2(DIV_ROUND_UP(MLX5E_RX_MAX_HEAD, MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE)); 150 } 151 152 u8 mlx5e_shampo_get_log_rsrv_size(struct mlx5_core_dev *mdev, 153 struct mlx5e_params *params) 154 { 155 return order_base_2(MLX5E_SHAMPO_WQ_RESRV_SIZE / MLX5E_SHAMPO_WQ_BASE_RESRV_SIZE); 156 } 157 158 u8 mlx5e_shampo_get_log_pkt_per_rsrv(struct mlx5_core_dev *mdev, 159 struct mlx5e_params *params) 160 { 161 u32 resrv_size = BIT(mlx5e_shampo_get_log_rsrv_size(mdev, params)) * 162 PAGE_SIZE; 163 164 return order_base_2(DIV_ROUND_UP(resrv_size, params->sw_mtu)); 165 } 166 167 u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, 168 struct mlx5e_params *params, 169 struct mlx5e_xsk_param *xsk) 170 { 171 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) 172 return order_base_2(mlx5e_rx_get_linear_stride_sz(params, xsk)); 173 174 return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev); 175 } 176 177 u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, 178 struct mlx5e_params *params, 179 struct mlx5e_xsk_param *xsk) 180 { 181 return MLX5_MPWRQ_LOG_WQE_SZ - 182 mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); 183 } 184 185 u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz) 186 { 187 #define UMR_WQE_BULK (2) 188 return min_t(unsigned int, UMR_WQE_BULK, wq_sz / 2 - 1); 189 } 190 191 u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, 192 struct mlx5e_params *params, 193 struct mlx5e_xsk_param *xsk) 194 { 195 u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, xsk); 196 197 if (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) 198 return linear_headroom; 199 200 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) 201 return linear_headroom; 202 203 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) 204 return linear_headroom; 205 206 return 0; 207 } 208 209 u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 210 { 211 bool is_mpwqe = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE); 212 u16 stop_room; 213 214 stop_room = mlx5e_ktls_get_stop_room(mdev, params); 215 stop_room += mlx5e_stop_room_for_max_wqe(mdev); 216 if (is_mpwqe) 217 /* A MPWQE can take up to the maximum cacheline-aligned WQE + 218 * all the normal stop room can be taken if a new packet breaks 219 * the active MPWQE session and allocates its WQEs right away. 220 */ 221 stop_room += mlx5e_stop_room_for_mpwqe(mdev); 222 223 return stop_room; 224 } 225 226 int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 227 { 228 size_t sq_size = 1 << params->log_sq_size; 229 u16 stop_room; 230 231 stop_room = mlx5e_calc_sq_stop_room(mdev, params); 232 if (stop_room >= sq_size) { 233 mlx5_core_err(mdev, "Stop room %u is bigger than the SQ size %zu\n", 234 stop_room, sq_size); 235 return -EINVAL; 236 } 237 238 return 0; 239 } 240 241 static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) 242 { 243 struct dim_cq_moder moder = {}; 244 245 moder.cq_period_mode = cq_period_mode; 246 moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS; 247 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC; 248 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE) 249 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE; 250 251 return moder; 252 } 253 254 static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode) 255 { 256 struct dim_cq_moder moder = {}; 257 258 moder.cq_period_mode = cq_period_mode; 259 moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS; 260 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC; 261 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE) 262 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE; 263 264 return moder; 265 } 266 267 static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode) 268 { 269 return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ? 270 DIM_CQ_PERIOD_MODE_START_FROM_CQE : 271 DIM_CQ_PERIOD_MODE_START_FROM_EQE; 272 } 273 274 void mlx5e_reset_tx_moderation(struct mlx5e_params *params, u8 cq_period_mode) 275 { 276 if (params->tx_dim_enabled) { 277 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode); 278 279 params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode); 280 } else { 281 params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode); 282 } 283 } 284 285 void mlx5e_reset_rx_moderation(struct mlx5e_params *params, u8 cq_period_mode) 286 { 287 if (params->rx_dim_enabled) { 288 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode); 289 290 params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode); 291 } else { 292 params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode); 293 } 294 } 295 296 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) 297 { 298 mlx5e_reset_tx_moderation(params, cq_period_mode); 299 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER, 300 params->tx_cq_moderation.cq_period_mode == 301 MLX5_CQ_PERIOD_MODE_START_FROM_CQE); 302 } 303 304 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) 305 { 306 mlx5e_reset_rx_moderation(params, cq_period_mode); 307 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER, 308 params->rx_cq_moderation.cq_period_mode == 309 MLX5_CQ_PERIOD_MODE_START_FROM_CQE); 310 } 311 312 bool slow_pci_heuristic(struct mlx5_core_dev *mdev) 313 { 314 u32 link_speed = 0; 315 u32 pci_bw = 0; 316 317 mlx5e_port_max_linkspeed(mdev, &link_speed); 318 pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL); 319 mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n", 320 link_speed, pci_bw); 321 322 #define MLX5E_SLOW_PCI_RATIO (2) 323 324 return link_speed && pci_bw && 325 link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw; 326 } 327 328 int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 329 { 330 if (!mlx5e_check_fragmented_striding_rq_cap(mdev)) 331 return -EOPNOTSUPP; 332 333 if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) 334 return -EINVAL; 335 336 return 0; 337 } 338 339 int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params, 340 struct mlx5e_xsk_param *xsk) 341 { 342 if (!mlx5e_check_fragmented_striding_rq_cap(mdev)) 343 return -EOPNOTSUPP; 344 345 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) 346 return -EINVAL; 347 348 return 0; 349 } 350 351 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, 352 struct mlx5e_params *params) 353 { 354 params->log_rq_mtu_frames = is_kdump_kernel() ? 355 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE : 356 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE; 357 358 mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n", 359 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ, 360 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ? 361 BIT(mlx5e_mpwqe_get_log_rq_size(params, NULL)) : 362 BIT(params->log_rq_mtu_frames), 363 BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)), 364 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)); 365 } 366 367 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 368 { 369 params->rq_wq_type = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ? 370 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ : 371 MLX5_WQ_TYPE_CYCLIC; 372 } 373 374 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, 375 struct mlx5e_params *params) 376 { 377 /* Prefer Striding RQ, unless any of the following holds: 378 * - Striding RQ configuration is not possible/supported. 379 * - CQE compression is ON, and stride_index mini_cqe layout is not supported. 380 * - Legacy RQ would use linear SKB while Striding RQ would use non-linear. 381 * 382 * No XSK params: checking the availability of striding RQ in general. 383 */ 384 if ((!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) || 385 MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) && 386 !mlx5e_mpwrq_validate_regular(mdev, params) && 387 (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) || 388 !mlx5e_rx_is_linear_skb(params, NULL))) 389 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true); 390 mlx5e_set_rq_type(mdev, params); 391 mlx5e_init_rq_type_params(mdev, params); 392 } 393 394 /* Build queue parameters */ 395 396 void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c) 397 { 398 *ccp = (struct mlx5e_create_cq_param) { 399 .napi = &c->napi, 400 .ch_stats = c->stats, 401 .node = cpu_to_node(c->cpu), 402 .ix = c->ix, 403 }; 404 } 405 406 static int mlx5e_max_nonlinear_mtu(int first_frag_size, int frag_size, bool xdp) 407 { 408 if (xdp) 409 /* XDP requires all fragments to be of the same size. */ 410 return first_frag_size + (MLX5E_MAX_RX_FRAGS - 1) * frag_size; 411 412 /* Optimization for small packets: the last fragment is bigger than the others. */ 413 return first_frag_size + (MLX5E_MAX_RX_FRAGS - 2) * frag_size + PAGE_SIZE; 414 } 415 416 #define DEFAULT_FRAG_SIZE (2048) 417 418 static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, 419 struct mlx5e_params *params, 420 struct mlx5e_xsk_param *xsk, 421 struct mlx5e_rq_frags_info *info) 422 { 423 u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu); 424 int frag_size_max = DEFAULT_FRAG_SIZE; 425 int first_frag_size_max; 426 u32 buf_size = 0; 427 u16 headroom; 428 int max_mtu; 429 int i; 430 431 if (mlx5e_rx_is_linear_skb(params, xsk)) { 432 int frag_stride; 433 434 frag_stride = mlx5e_rx_get_linear_stride_sz(params, xsk); 435 436 info->arr[0].frag_size = byte_count; 437 info->arr[0].frag_stride = frag_stride; 438 info->num_frags = 1; 439 info->wqe_bulk = PAGE_SIZE / frag_stride; 440 goto out; 441 } 442 443 headroom = mlx5e_get_linear_rq_headroom(params, xsk); 444 first_frag_size_max = SKB_WITH_OVERHEAD(frag_size_max - headroom); 445 446 max_mtu = mlx5e_max_nonlinear_mtu(first_frag_size_max, frag_size_max, 447 params->xdp_prog); 448 if (byte_count > max_mtu || params->xdp_prog) { 449 frag_size_max = PAGE_SIZE; 450 first_frag_size_max = SKB_WITH_OVERHEAD(frag_size_max - headroom); 451 452 max_mtu = mlx5e_max_nonlinear_mtu(first_frag_size_max, frag_size_max, 453 params->xdp_prog); 454 if (byte_count > max_mtu) { 455 mlx5_core_err(mdev, "MTU %u is too big for non-linear legacy RQ (max %d)\n", 456 params->sw_mtu, max_mtu); 457 return -EINVAL; 458 } 459 } 460 461 i = 0; 462 while (buf_size < byte_count) { 463 int frag_size = byte_count - buf_size; 464 465 if (i == 0) 466 frag_size = min(frag_size, first_frag_size_max); 467 else if (i < MLX5E_MAX_RX_FRAGS - 1) 468 frag_size = min(frag_size, frag_size_max); 469 470 info->arr[i].frag_size = frag_size; 471 buf_size += frag_size; 472 473 if (params->xdp_prog) { 474 /* XDP multi buffer expects fragments of the same size. */ 475 info->arr[i].frag_stride = frag_size_max; 476 } else { 477 if (i == 0) { 478 /* Ensure that headroom and tailroom are included. */ 479 frag_size += headroom; 480 frag_size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 481 } 482 info->arr[i].frag_stride = roundup_pow_of_two(frag_size); 483 } 484 485 i++; 486 } 487 info->num_frags = i; 488 /* number of different wqes sharing a page */ 489 info->wqe_bulk = 1 + (info->num_frags % 2); 490 491 out: 492 info->wqe_bulk = max_t(u8, info->wqe_bulk, 8); 493 info->log_num_frags = order_base_2(info->num_frags); 494 495 return 0; 496 } 497 498 static u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs) 499 { 500 int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs; 501 502 switch (wq_type) { 503 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 504 sz += sizeof(struct mlx5e_rx_wqe_ll); 505 break; 506 default: /* MLX5_WQ_TYPE_CYCLIC */ 507 sz += sizeof(struct mlx5e_rx_wqe_cyc); 508 } 509 510 return order_base_2(sz); 511 } 512 513 static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev, 514 struct mlx5e_cq_param *param) 515 { 516 void *cqc = param->cqc; 517 518 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); 519 if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128) 520 MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD); 521 } 522 523 static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev, 524 struct mlx5e_params *params, 525 struct mlx5e_xsk_param *xsk) 526 { 527 int rsrv_size = BIT(mlx5e_shampo_get_log_rsrv_size(mdev, params)) * PAGE_SIZE; 528 u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); 529 int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(mdev, params)); 530 u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); 531 int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(params, xsk)); 532 int wqe_size = BIT(log_stride_sz) * num_strides; 533 534 /* +1 is for the case that the pkt_per_rsrv dont consume the reservation 535 * so we get a filler cqe for the rest of the reservation. 536 */ 537 return order_base_2((wqe_size / rsrv_size) * wq_size * (pkt_per_rsrv + 1)); 538 } 539 540 static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev, 541 struct mlx5e_params *params, 542 struct mlx5e_xsk_param *xsk, 543 struct mlx5e_cq_param *param) 544 { 545 bool hw_stridx = false; 546 void *cqc = param->cqc; 547 u8 log_cq_size; 548 549 switch (params->rq_wq_type) { 550 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 551 hw_stridx = MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index); 552 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) 553 log_cq_size = mlx5e_shampo_get_log_cq_size(mdev, params, xsk); 554 else 555 log_cq_size = mlx5e_mpwqe_get_log_rq_size(params, xsk) + 556 mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); 557 break; 558 default: /* MLX5_WQ_TYPE_CYCLIC */ 559 log_cq_size = params->log_rq_mtu_frames; 560 } 561 562 MLX5_SET(cqc, cqc, log_cq_size, log_cq_size); 563 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 564 MLX5_SET(cqc, cqc, mini_cqe_res_format, hw_stridx ? 565 MLX5_CQE_FORMAT_CSUM_STRIDX : MLX5_CQE_FORMAT_CSUM); 566 MLX5_SET(cqc, cqc, cqe_comp_en, 1); 567 } 568 569 mlx5e_build_common_cq_param(mdev, param); 570 param->cq_period_mode = params->rx_cq_moderation.cq_period_mode; 571 } 572 573 static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 574 { 575 bool lro_en = params->packet_merge.type == MLX5E_PACKET_MERGE_LRO; 576 bool ro = pcie_relaxed_ordering_enabled(mdev->pdev) && 577 MLX5_CAP_GEN(mdev, relaxed_ordering_write); 578 579 return ro && lro_en ? 580 MLX5_WQ_END_PAD_MODE_NONE : MLX5_WQ_END_PAD_MODE_ALIGN; 581 } 582 583 int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, 584 struct mlx5e_params *params, 585 struct mlx5e_xsk_param *xsk, 586 u16 q_counter, 587 struct mlx5e_rq_param *param) 588 { 589 void *rqc = param->rqc; 590 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 591 int ndsegs = 1; 592 int err; 593 594 switch (params->rq_wq_type) { 595 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: { 596 u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); 597 u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); 598 599 if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, 600 log_wqe_num_of_strides)) { 601 mlx5_core_err(mdev, 602 "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u\n", 603 log_wqe_stride_size, log_wqe_num_of_strides); 604 return -EINVAL; 605 } 606 607 MLX5_SET(wq, wq, log_wqe_num_of_strides, 608 log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE); 609 MLX5_SET(wq, wq, log_wqe_stride_size, 610 log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE); 611 MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params, xsk)); 612 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) { 613 MLX5_SET(wq, wq, shampo_enable, true); 614 MLX5_SET(wq, wq, log_reservation_size, 615 mlx5e_shampo_get_log_rsrv_size(mdev, params)); 616 MLX5_SET(wq, wq, 617 log_max_num_of_packets_per_reservation, 618 mlx5e_shampo_get_log_pkt_per_rsrv(mdev, params)); 619 MLX5_SET(wq, wq, log_headers_entry_size, 620 mlx5e_shampo_get_log_hd_entry_size(mdev, params)); 621 MLX5_SET(rqc, rqc, reservation_timeout, 622 params->packet_merge.timeout); 623 MLX5_SET(rqc, rqc, shampo_match_criteria_type, 624 params->packet_merge.shampo.match_criteria_type); 625 MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity, 626 params->packet_merge.shampo.alignment_granularity); 627 } 628 break; 629 } 630 default: /* MLX5_WQ_TYPE_CYCLIC */ 631 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames); 632 err = mlx5e_build_rq_frags_info(mdev, params, xsk, ¶m->frags_info); 633 if (err) 634 return err; 635 ndsegs = param->frags_info.num_frags; 636 } 637 638 MLX5_SET(wq, wq, wq_type, params->rq_wq_type); 639 MLX5_SET(wq, wq, end_padding_mode, rq_end_pad_mode(mdev, params)); 640 MLX5_SET(wq, wq, log_wq_stride, 641 mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs)); 642 MLX5_SET(wq, wq, pd, mdev->mlx5e_res.hw_objs.pdn); 643 MLX5_SET(rqc, rqc, counter_set_id, q_counter); 644 MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); 645 MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); 646 647 param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 648 mlx5e_build_rx_cq_param(mdev, params, xsk, ¶m->cqp); 649 650 return 0; 651 } 652 653 void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, 654 u16 q_counter, 655 struct mlx5e_rq_param *param) 656 { 657 void *rqc = param->rqc; 658 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 659 660 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 661 MLX5_SET(wq, wq, log_wq_stride, 662 mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1)); 663 MLX5_SET(rqc, rqc, counter_set_id, q_counter); 664 665 param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 666 } 667 668 void mlx5e_build_tx_cq_param(struct mlx5_core_dev *mdev, 669 struct mlx5e_params *params, 670 struct mlx5e_cq_param *param) 671 { 672 void *cqc = param->cqc; 673 674 MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size); 675 676 mlx5e_build_common_cq_param(mdev, param); 677 param->cq_period_mode = params->tx_cq_moderation.cq_period_mode; 678 } 679 680 void mlx5e_build_sq_param_common(struct mlx5_core_dev *mdev, 681 struct mlx5e_sq_param *param) 682 { 683 void *sqc = param->sqc; 684 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 685 686 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); 687 MLX5_SET(wq, wq, pd, mdev->mlx5e_res.hw_objs.pdn); 688 689 param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 690 } 691 692 void mlx5e_build_sq_param(struct mlx5_core_dev *mdev, 693 struct mlx5e_params *params, 694 struct mlx5e_sq_param *param) 695 { 696 void *sqc = param->sqc; 697 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 698 bool allow_swp; 699 700 allow_swp = 701 mlx5_geneve_tx_allowed(mdev) || !!mlx5_ipsec_device_caps(mdev); 702 mlx5e_build_sq_param_common(mdev, param); 703 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); 704 MLX5_SET(sqc, sqc, allow_swp, allow_swp); 705 param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE); 706 param->stop_room = mlx5e_calc_sq_stop_room(mdev, params); 707 mlx5e_build_tx_cq_param(mdev, params, ¶m->cqp); 708 } 709 710 static void mlx5e_build_ico_cq_param(struct mlx5_core_dev *mdev, 711 u8 log_wq_size, 712 struct mlx5e_cq_param *param) 713 { 714 void *cqc = param->cqc; 715 716 MLX5_SET(cqc, cqc, log_cq_size, log_wq_size); 717 718 mlx5e_build_common_cq_param(mdev, param); 719 720 param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 721 } 722 723 static u8 mlx5e_get_rq_log_wq_sz(void *rqc) 724 { 725 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 726 727 return MLX5_GET(wq, wq, log_wq_sz); 728 } 729 730 /* This function calculates the maximum number of headers entries that are needed 731 * per WQE, the formula is based on the size of the reservations and the 732 * restriction we have about max packets for reservation that is equal to max 733 * headers per reservation. 734 */ 735 u32 mlx5e_shampo_hd_per_wqe(struct mlx5_core_dev *mdev, 736 struct mlx5e_params *params, 737 struct mlx5e_rq_param *rq_param) 738 { 739 int resv_size = BIT(mlx5e_shampo_get_log_rsrv_size(mdev, params)) * PAGE_SIZE; 740 u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, NULL)); 741 int pkt_per_resv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(mdev, params)); 742 u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL); 743 int wqe_size = BIT(log_stride_sz) * num_strides; 744 u32 hd_per_wqe; 745 746 /* Assumption: hd_per_wqe % 8 == 0. */ 747 hd_per_wqe = (wqe_size / resv_size) * pkt_per_resv; 748 mlx5_core_dbg(mdev, "%s hd_per_wqe = %d rsrv_size = %d wqe_size = %d pkt_per_resv = %d\n", 749 __func__, hd_per_wqe, resv_size, wqe_size, pkt_per_resv); 750 return hd_per_wqe; 751 } 752 753 /* This function calculates the maximum number of headers entries that are needed 754 * for the WQ, this value is uesed to allocate the header buffer in HW, thus 755 * must be a pow of 2. 756 */ 757 u32 mlx5e_shampo_hd_per_wq(struct mlx5_core_dev *mdev, 758 struct mlx5e_params *params, 759 struct mlx5e_rq_param *rq_param) 760 { 761 void *wqc = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); 762 int wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); 763 u32 hd_per_wqe, hd_per_wq; 764 765 hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rq_param); 766 hd_per_wq = roundup_pow_of_two(hd_per_wqe * wq_size); 767 return hd_per_wq; 768 } 769 770 static u32 mlx5e_shampo_icosq_sz(struct mlx5_core_dev *mdev, 771 struct mlx5e_params *params, 772 struct mlx5e_rq_param *rq_param) 773 { 774 int max_num_of_umr_per_wqe, max_hd_per_wqe, max_klm_per_umr, rest; 775 void *wqc = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); 776 int wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); 777 u32 wqebbs; 778 779 max_klm_per_umr = MLX5E_MAX_KLM_PER_WQE(mdev); 780 max_hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rq_param); 781 max_num_of_umr_per_wqe = max_hd_per_wqe / max_klm_per_umr; 782 rest = max_hd_per_wqe % max_klm_per_umr; 783 wqebbs = MLX5E_KLM_UMR_WQEBBS(max_klm_per_umr) * max_num_of_umr_per_wqe; 784 if (rest) 785 wqebbs += MLX5E_KLM_UMR_WQEBBS(rest); 786 wqebbs *= wq_size; 787 return wqebbs; 788 } 789 790 static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, 791 struct mlx5e_params *params, 792 struct mlx5e_rq_param *rqp) 793 { 794 u32 wqebbs; 795 796 /* MLX5_WQ_TYPE_CYCLIC */ 797 if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) 798 return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; 799 800 wqebbs = mlx5e_mpwrq_umr_wqebbs(MLX5_MPWRQ_PAGES_PER_WQE) * 801 (1 << mlx5e_get_rq_log_wq_sz(rqp->rqc)); 802 803 /* If XDP program is attached, XSK may be turned on at any time without 804 * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of 805 * both regular RQ and XSK RQ. 806 * Although mlx5e_mpwqe_get_log_rq_size accepts mlx5e_xsk_param, it 807 * doesn't affect its return value, as long as params->xdp_prog != NULL, 808 * so we can just multiply by 2. 809 */ 810 if (params->xdp_prog) 811 wqebbs *= 2; 812 813 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) 814 wqebbs += mlx5e_shampo_icosq_sz(mdev, params, rqp); 815 816 return max_t(u8, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE, order_base_2(wqebbs)); 817 } 818 819 static u8 mlx5e_build_async_icosq_log_wq_sz(struct mlx5_core_dev *mdev) 820 { 821 if (mlx5e_is_ktls_rx(mdev)) 822 return MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 823 824 return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; 825 } 826 827 static void mlx5e_build_icosq_param(struct mlx5_core_dev *mdev, 828 u8 log_wq_size, 829 struct mlx5e_sq_param *param) 830 { 831 void *sqc = param->sqc; 832 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 833 834 mlx5e_build_sq_param_common(mdev, param); 835 836 MLX5_SET(wq, wq, log_wq_sz, log_wq_size); 837 MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(mdev, reg_umr_sq)); 838 mlx5e_build_ico_cq_param(mdev, log_wq_size, ¶m->cqp); 839 } 840 841 static void mlx5e_build_async_icosq_param(struct mlx5_core_dev *mdev, 842 u8 log_wq_size, 843 struct mlx5e_sq_param *param) 844 { 845 void *sqc = param->sqc; 846 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 847 848 mlx5e_build_sq_param_common(mdev, param); 849 param->stop_room = mlx5e_stop_room_for_wqe(mdev, 1); /* for XSK NOP */ 850 param->is_tls = mlx5e_is_ktls_rx(mdev); 851 if (param->is_tls) 852 param->stop_room += mlx5e_stop_room_for_wqe(mdev, 1); /* for TLS RX resync NOP */ 853 MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(mdev, reg_umr_sq)); 854 MLX5_SET(wq, wq, log_wq_sz, log_wq_size); 855 mlx5e_build_ico_cq_param(mdev, log_wq_size, ¶m->cqp); 856 } 857 858 void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, 859 struct mlx5e_params *params, 860 struct mlx5e_xsk_param *xsk, 861 struct mlx5e_sq_param *param) 862 { 863 void *sqc = param->sqc; 864 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 865 866 mlx5e_build_sq_param_common(mdev, param); 867 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); 868 param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE); 869 param->is_xdp_mb = !mlx5e_rx_is_linear_skb(params, xsk); 870 mlx5e_build_tx_cq_param(mdev, params, ¶m->cqp); 871 } 872 873 int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, 874 struct mlx5e_params *params, 875 u16 q_counter, 876 struct mlx5e_channel_param *cparam) 877 { 878 u8 icosq_log_wq_sz, async_icosq_log_wq_sz; 879 int err; 880 881 err = mlx5e_build_rq_param(mdev, params, NULL, q_counter, &cparam->rq); 882 if (err) 883 return err; 884 885 icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(mdev, params, &cparam->rq); 886 async_icosq_log_wq_sz = mlx5e_build_async_icosq_log_wq_sz(mdev); 887 888 mlx5e_build_sq_param(mdev, params, &cparam->txq_sq); 889 mlx5e_build_xdpsq_param(mdev, params, NULL, &cparam->xdp_sq); 890 mlx5e_build_icosq_param(mdev, icosq_log_wq_sz, &cparam->icosq); 891 mlx5e_build_async_icosq_param(mdev, async_icosq_log_wq_sz, &cparam->async_icosq); 892 893 return 0; 894 } 895