1 /* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <net/tc_act/tc_gact.h> 34 #include <linux/mlx5/fs.h> 35 #include <net/vxlan.h> 36 #include <net/geneve.h> 37 #include <linux/bpf.h> 38 #include <linux/if_bridge.h> 39 #include <linux/filter.h> 40 #include <net/page_pool.h> 41 #include <net/xdp_sock_drv.h> 42 #include "eswitch.h" 43 #include "en.h" 44 #include "en/txrx.h" 45 #include "en_tc.h" 46 #include "en_rep.h" 47 #include "en_accel/ipsec.h" 48 #include "en_accel/macsec.h" 49 #include "en_accel/en_accel.h" 50 #include "en_accel/ktls.h" 51 #include "lib/vxlan.h" 52 #include "lib/clock.h" 53 #include "en/port.h" 54 #include "en/xdp.h" 55 #include "lib/eq.h" 56 #include "en/monitor_stats.h" 57 #include "en/health.h" 58 #include "en/params.h" 59 #include "en/xsk/pool.h" 60 #include "en/xsk/setup.h" 61 #include "en/xsk/rx.h" 62 #include "en/xsk/tx.h" 63 #include "en/hv_vhca_stats.h" 64 #include "en/devlink.h" 65 #include "lib/mlx5.h" 66 #include "en/ptp.h" 67 #include "en/htb.h" 68 #include "qos.h" 69 #include "en/trap.h" 70 71 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, 72 enum mlx5e_mpwrq_umr_mode umr_mode) 73 { 74 u16 umr_wqebbs, max_wqebbs; 75 bool striding_rq_umr; 76 77 striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) && 78 MLX5_CAP_ETH(mdev, reg_umr_sq); 79 if (!striding_rq_umr) 80 return false; 81 82 umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode); 83 max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 84 /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is 85 * calculated from mlx5e_get_max_sq_aligned_wqebbs. 86 */ 87 if (WARN_ON(umr_wqebbs > max_wqebbs)) 88 return false; 89 90 return true; 91 } 92 93 void mlx5e_update_carrier(struct mlx5e_priv *priv) 94 { 95 struct mlx5_core_dev *mdev = priv->mdev; 96 u8 port_state; 97 bool up; 98 99 port_state = mlx5_query_vport_state(mdev, 100 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 101 0); 102 103 up = port_state == VPORT_STATE_UP; 104 if (up == netif_carrier_ok(priv->netdev)) 105 netif_carrier_event(priv->netdev); 106 if (up) { 107 netdev_info(priv->netdev, "Link up\n"); 108 netif_carrier_on(priv->netdev); 109 } else { 110 netdev_info(priv->netdev, "Link down\n"); 111 netif_carrier_off(priv->netdev); 112 } 113 } 114 115 static void mlx5e_update_carrier_work(struct work_struct *work) 116 { 117 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 118 update_carrier_work); 119 120 mutex_lock(&priv->state_lock); 121 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 122 if (priv->profile->update_carrier) 123 priv->profile->update_carrier(priv); 124 mutex_unlock(&priv->state_lock); 125 } 126 127 static void mlx5e_update_stats_work(struct work_struct *work) 128 { 129 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 130 update_stats_work); 131 132 mutex_lock(&priv->state_lock); 133 priv->profile->update_stats(priv); 134 mutex_unlock(&priv->state_lock); 135 } 136 137 void mlx5e_queue_update_stats(struct mlx5e_priv *priv) 138 { 139 if (!priv->profile->update_stats) 140 return; 141 142 if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state))) 143 return; 144 145 queue_work(priv->wq, &priv->update_stats_work); 146 } 147 148 static int async_event(struct notifier_block *nb, unsigned long event, void *data) 149 { 150 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 151 struct mlx5_eqe *eqe = data; 152 153 if (event != MLX5_EVENT_TYPE_PORT_CHANGE) 154 return NOTIFY_DONE; 155 156 switch (eqe->sub_type) { 157 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 158 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 159 queue_work(priv->wq, &priv->update_carrier_work); 160 break; 161 default: 162 return NOTIFY_DONE; 163 } 164 165 return NOTIFY_OK; 166 } 167 168 static void mlx5e_enable_async_events(struct mlx5e_priv *priv) 169 { 170 priv->events_nb.notifier_call = async_event; 171 mlx5_notifier_register(priv->mdev, &priv->events_nb); 172 } 173 174 static void mlx5e_disable_async_events(struct mlx5e_priv *priv) 175 { 176 mlx5_notifier_unregister(priv->mdev, &priv->events_nb); 177 } 178 179 static int blocking_event(struct notifier_block *nb, unsigned long event, void *data) 180 { 181 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb); 182 int err; 183 184 switch (event) { 185 case MLX5_DRIVER_EVENT_TYPE_TRAP: 186 err = mlx5e_handle_trap_event(priv, data); 187 break; 188 default: 189 netdev_warn(priv->netdev, "Sync event: Unknown event %ld\n", event); 190 err = -EINVAL; 191 } 192 return err; 193 } 194 195 static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv) 196 { 197 priv->blocking_events_nb.notifier_call = blocking_event; 198 mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb); 199 } 200 201 static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv) 202 { 203 mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb); 204 } 205 206 static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode) 207 { 208 u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode); 209 u32 sz; 210 211 sz = ALIGN(entries * umr_entry_size, MLX5_UMR_FLEX_ALIGNMENT); 212 213 return sz / MLX5_OCTWORD; 214 } 215 216 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, 217 struct mlx5e_icosq *sq, 218 struct mlx5e_umr_wqe *wqe) 219 { 220 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 221 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; 222 u16 octowords; 223 u8 ds_cnt; 224 225 ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift, 226 rq->mpwqe.umr_mode), 227 MLX5_SEND_WQE_DS); 228 229 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | 230 ds_cnt); 231 cseg->umr_mkey = rq->mpwqe.umr_mkey_be; 232 233 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; 234 octowords = mlx5e_mpwrq_umr_octowords(rq->mpwqe.pages_per_wqe, rq->mpwqe.umr_mode); 235 ucseg->xlt_octowords = cpu_to_be16(octowords); 236 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 237 } 238 239 static int mlx5e_rq_shampo_hd_alloc(struct mlx5e_rq *rq, int node) 240 { 241 rq->mpwqe.shampo = kvzalloc_node(sizeof(*rq->mpwqe.shampo), 242 GFP_KERNEL, node); 243 if (!rq->mpwqe.shampo) 244 return -ENOMEM; 245 return 0; 246 } 247 248 static void mlx5e_rq_shampo_hd_free(struct mlx5e_rq *rq) 249 { 250 kvfree(rq->mpwqe.shampo); 251 } 252 253 static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, int node) 254 { 255 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 256 257 shampo->bitmap = bitmap_zalloc_node(shampo->hd_per_wq, GFP_KERNEL, 258 node); 259 if (!shampo->bitmap) 260 return -ENOMEM; 261 262 shampo->info = kvzalloc_node(array_size(shampo->hd_per_wq, 263 sizeof(*shampo->info)), 264 GFP_KERNEL, node); 265 if (!shampo->info) { 266 kvfree(shampo->bitmap); 267 return -ENOMEM; 268 } 269 return 0; 270 } 271 272 static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq) 273 { 274 kvfree(rq->mpwqe.shampo->bitmap); 275 kvfree(rq->mpwqe.shampo->info); 276 } 277 278 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) 279 { 280 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 281 size_t alloc_size; 282 283 alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, alloc_units, 284 rq->mpwqe.pages_per_wqe)); 285 286 rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node); 287 if (!rq->mpwqe.info) 288 return -ENOMEM; 289 290 mlx5e_build_umr_wqe(rq, rq->icosq, &rq->mpwqe.umr_wqe); 291 292 return 0; 293 } 294 295 296 static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode) 297 { 298 switch (umr_mode) { 299 case MLX5E_MPWRQ_UMR_MODE_ALIGNED: 300 return MLX5_MKC_ACCESS_MODE_MTT; 301 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: 302 return MLX5_MKC_ACCESS_MODE_KSM; 303 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: 304 return MLX5_MKC_ACCESS_MODE_KLMS; 305 case MLX5E_MPWRQ_UMR_MODE_TRIPLE: 306 return MLX5_MKC_ACCESS_MODE_KSM; 307 } 308 WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode); 309 return 0; 310 } 311 312 static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, 313 u32 npages, u8 page_shift, u32 *umr_mkey, 314 dma_addr_t filler_addr, 315 enum mlx5e_mpwrq_umr_mode umr_mode, 316 u32 xsk_chunk_size) 317 { 318 struct mlx5_mtt *mtt; 319 struct mlx5_ksm *ksm; 320 struct mlx5_klm *klm; 321 u32 octwords; 322 int inlen; 323 void *mkc; 324 u32 *in; 325 int err; 326 int i; 327 328 if ((umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED || 329 umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) && 330 !MLX5_CAP_GEN(mdev, fixed_buffer_size)) { 331 mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n"); 332 return -EINVAL; 333 } 334 335 octwords = mlx5e_mpwrq_umr_octowords(npages, umr_mode); 336 337 inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), 338 MLX5_OCTWORD, octwords); 339 if (inlen < 0) 340 return inlen; 341 342 in = kvzalloc(inlen, GFP_KERNEL); 343 if (!in) 344 return -ENOMEM; 345 346 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 347 348 MLX5_SET(mkc, mkc, free, 1); 349 MLX5_SET(mkc, mkc, umr_en, 1); 350 MLX5_SET(mkc, mkc, lw, 1); 351 MLX5_SET(mkc, mkc, lr, 1); 352 MLX5_SET(mkc, mkc, access_mode_1_0, mlx5e_mpwrq_access_mode(umr_mode)); 353 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 354 MLX5_SET(mkc, mkc, qpn, 0xffffff); 355 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 356 MLX5_SET64(mkc, mkc, len, npages << page_shift); 357 MLX5_SET(mkc, mkc, translations_octword_size, octwords); 358 if (umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) 359 MLX5_SET(mkc, mkc, log_page_size, page_shift - 2); 360 else if (umr_mode != MLX5E_MPWRQ_UMR_MODE_OVERSIZED) 361 MLX5_SET(mkc, mkc, log_page_size, page_shift); 362 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords); 363 364 /* Initialize the mkey with all MTTs pointing to a default 365 * page (filler_addr). When the channels are activated, UMR 366 * WQEs will redirect the RX WQEs to the actual memory from 367 * the RQ's pool, while the gaps (wqe_overflow) remain mapped 368 * to the default page. 369 */ 370 switch (umr_mode) { 371 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: 372 klm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 373 for (i = 0; i < npages; i++) { 374 klm[i << 1] = (struct mlx5_klm) { 375 .va = cpu_to_be64(filler_addr), 376 .bcount = cpu_to_be32(xsk_chunk_size), 377 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 378 }; 379 klm[(i << 1) + 1] = (struct mlx5_klm) { 380 .va = cpu_to_be64(filler_addr), 381 .bcount = cpu_to_be32((1 << page_shift) - xsk_chunk_size), 382 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 383 }; 384 } 385 break; 386 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: 387 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 388 for (i = 0; i < npages; i++) 389 ksm[i] = (struct mlx5_ksm) { 390 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 391 .va = cpu_to_be64(filler_addr), 392 }; 393 break; 394 case MLX5E_MPWRQ_UMR_MODE_ALIGNED: 395 mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 396 for (i = 0; i < npages; i++) 397 mtt[i] = (struct mlx5_mtt) { 398 .ptag = cpu_to_be64(filler_addr), 399 }; 400 break; 401 case MLX5E_MPWRQ_UMR_MODE_TRIPLE: 402 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 403 for (i = 0; i < npages * 4; i++) { 404 ksm[i] = (struct mlx5_ksm) { 405 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 406 .va = cpu_to_be64(filler_addr), 407 }; 408 } 409 break; 410 } 411 412 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 413 414 kvfree(in); 415 return err; 416 } 417 418 static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev, 419 u64 nentries, 420 u32 *umr_mkey) 421 { 422 int inlen; 423 void *mkc; 424 u32 *in; 425 int err; 426 427 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 428 429 in = kvzalloc(inlen, GFP_KERNEL); 430 if (!in) 431 return -ENOMEM; 432 433 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 434 435 MLX5_SET(mkc, mkc, free, 1); 436 MLX5_SET(mkc, mkc, umr_en, 1); 437 MLX5_SET(mkc, mkc, lw, 1); 438 MLX5_SET(mkc, mkc, lr, 1); 439 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS); 440 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 441 MLX5_SET(mkc, mkc, qpn, 0xffffff); 442 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 443 MLX5_SET(mkc, mkc, translations_octword_size, nentries); 444 MLX5_SET(mkc, mkc, length64, 1); 445 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 446 447 kvfree(in); 448 return err; 449 } 450 451 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) 452 { 453 u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0; 454 u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 455 u32 num_entries, max_num_entries; 456 u32 umr_mkey; 457 int err; 458 459 max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.umr_mode); 460 461 /* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */ 462 if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe, 463 &num_entries) || 464 num_entries > max_num_entries)) 465 mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n", 466 __func__, wq_size, rq->mpwqe.mtts_per_wqe, 467 max_num_entries); 468 469 err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift, 470 &umr_mkey, rq->wqe_overflow.addr, 471 rq->mpwqe.umr_mode, xsk_chunk_size); 472 rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey); 473 return err; 474 } 475 476 static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev, 477 struct mlx5e_rq *rq) 478 { 479 u32 max_klm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size)); 480 481 if (max_klm_size < rq->mpwqe.shampo->hd_per_wq) { 482 mlx5_core_err(mdev, "max klm list size 0x%x is smaller than shampo header buffer list size 0x%x\n", 483 max_klm_size, rq->mpwqe.shampo->hd_per_wq); 484 return -EINVAL; 485 } 486 return mlx5e_create_umr_klm_mkey(mdev, rq->mpwqe.shampo->hd_per_wq, 487 &rq->mpwqe.shampo->mkey); 488 } 489 490 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) 491 { 492 struct mlx5e_wqe_frag_info next_frag = {}; 493 struct mlx5e_wqe_frag_info *prev = NULL; 494 int i; 495 496 if (rq->xsk_pool) { 497 /* Assumptions used by XSK batched allocator. */ 498 WARN_ON(rq->wqe.info.num_frags != 1); 499 WARN_ON(rq->wqe.info.log_num_frags != 0); 500 WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE); 501 } 502 503 next_frag.au = &rq->wqe.alloc_units[0]; 504 505 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 506 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; 507 struct mlx5e_wqe_frag_info *frag = 508 &rq->wqe.frags[i << rq->wqe.info.log_num_frags]; 509 int f; 510 511 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { 512 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) { 513 next_frag.au++; 514 next_frag.offset = 0; 515 if (prev) 516 prev->last_in_page = true; 517 } 518 *frag = next_frag; 519 520 /* prepare next */ 521 next_frag.offset += frag_info[f].frag_stride; 522 prev = frag; 523 } 524 } 525 526 if (prev) 527 prev->last_in_page = true; 528 } 529 530 static int mlx5e_init_au_list(struct mlx5e_rq *rq, int wq_sz, int node) 531 { 532 int len = wq_sz << rq->wqe.info.log_num_frags; 533 534 rq->wqe.alloc_units = kvzalloc_node(array_size(len, sizeof(*rq->wqe.alloc_units)), 535 GFP_KERNEL, node); 536 if (!rq->wqe.alloc_units) 537 return -ENOMEM; 538 539 mlx5e_init_frags_partition(rq); 540 541 return 0; 542 } 543 544 static void mlx5e_free_au_list(struct mlx5e_rq *rq) 545 { 546 kvfree(rq->wqe.alloc_units); 547 } 548 549 static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work) 550 { 551 struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work); 552 553 mlx5e_reporter_rq_cqe_err(rq); 554 } 555 556 static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) 557 { 558 rq->wqe_overflow.page = alloc_page(GFP_KERNEL); 559 if (!rq->wqe_overflow.page) 560 return -ENOMEM; 561 562 rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0, 563 PAGE_SIZE, rq->buff.map_dir); 564 if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) { 565 __free_page(rq->wqe_overflow.page); 566 return -ENOMEM; 567 } 568 return 0; 569 } 570 571 static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq) 572 { 573 dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE, 574 rq->buff.map_dir); 575 __free_page(rq->wqe_overflow.page); 576 } 577 578 static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 579 struct mlx5e_rq *rq) 580 { 581 struct mlx5_core_dev *mdev = c->mdev; 582 int err; 583 584 rq->wq_type = params->rq_wq_type; 585 rq->pdev = c->pdev; 586 rq->netdev = c->netdev; 587 rq->priv = c->priv; 588 rq->tstamp = c->tstamp; 589 rq->clock = &mdev->clock; 590 rq->icosq = &c->icosq; 591 rq->ix = c->ix; 592 rq->channel = c; 593 rq->mdev = mdev; 594 rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 595 rq->xdpsq = &c->rq_xdpsq; 596 rq->stats = &c->priv->channel_stats[c->ix]->rq; 597 rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev); 598 err = mlx5e_rq_set_handlers(rq, params, NULL); 599 if (err) 600 return err; 601 602 return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, c->napi.napi_id); 603 } 604 605 static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, 606 struct mlx5e_params *params, 607 struct mlx5e_rq_param *rqp, 608 struct mlx5e_rq *rq, 609 u32 *pool_size, 610 int node) 611 { 612 void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq); 613 int wq_size; 614 int err; 615 616 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 617 return 0; 618 err = mlx5e_rq_shampo_hd_alloc(rq, node); 619 if (err) 620 goto out; 621 rq->mpwqe.shampo->hd_per_wq = 622 mlx5e_shampo_hd_per_wq(mdev, params, rqp); 623 err = mlx5e_create_rq_hd_umr_mkey(mdev, rq); 624 if (err) 625 goto err_shampo_hd; 626 err = mlx5e_rq_shampo_hd_info_alloc(rq, node); 627 if (err) 628 goto err_shampo_info; 629 rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node); 630 if (!rq->hw_gro_data) { 631 err = -ENOMEM; 632 goto err_hw_gro_data; 633 } 634 rq->mpwqe.shampo->key = 635 cpu_to_be32(rq->mpwqe.shampo->mkey); 636 rq->mpwqe.shampo->hd_per_wqe = 637 mlx5e_shampo_hd_per_wqe(mdev, params, rqp); 638 wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); 639 *pool_size += (rq->mpwqe.shampo->hd_per_wqe * wq_size) / 640 MLX5E_SHAMPO_WQ_HEADER_PER_PAGE; 641 return 0; 642 643 err_hw_gro_data: 644 mlx5e_rq_shampo_hd_info_free(rq); 645 err_shampo_info: 646 mlx5_core_destroy_mkey(mdev, rq->mpwqe.shampo->mkey); 647 err_shampo_hd: 648 mlx5e_rq_shampo_hd_free(rq); 649 out: 650 return err; 651 } 652 653 static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) 654 { 655 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 656 return; 657 658 kvfree(rq->hw_gro_data); 659 mlx5e_rq_shampo_hd_info_free(rq); 660 mlx5_core_destroy_mkey(rq->mdev, rq->mpwqe.shampo->mkey); 661 mlx5e_rq_shampo_hd_free(rq); 662 } 663 664 static int mlx5e_alloc_rq(struct mlx5e_params *params, 665 struct mlx5e_xsk_param *xsk, 666 struct mlx5e_rq_param *rqp, 667 int node, struct mlx5e_rq *rq) 668 { 669 struct page_pool_params pp_params = { 0 }; 670 struct mlx5_core_dev *mdev = rq->mdev; 671 void *rqc = rqp->rqc; 672 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 673 u32 pool_size; 674 int wq_sz; 675 int err; 676 int i; 677 678 rqp->wq.db_numa_node = node; 679 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work); 680 681 if (params->xdp_prog) 682 bpf_prog_inc(params->xdp_prog); 683 RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog); 684 685 rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 686 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk); 687 pool_size = 1 << params->log_rq_mtu_frames; 688 689 rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); 690 691 switch (rq->wq_type) { 692 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 693 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, 694 &rq->wq_ctrl); 695 if (err) 696 goto err_rq_xdp_prog; 697 698 err = mlx5e_alloc_mpwqe_rq_drop_page(rq); 699 if (err) 700 goto err_rq_wq_destroy; 701 702 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; 703 704 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 705 706 rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); 707 rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); 708 rq->mpwqe.pages_per_wqe = 709 mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift, 710 rq->mpwqe.umr_mode); 711 rq->mpwqe.umr_wqebbs = 712 mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift, 713 rq->mpwqe.umr_mode); 714 rq->mpwqe.mtts_per_wqe = 715 mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift, 716 rq->mpwqe.umr_mode); 717 718 pool_size = rq->mpwqe.pages_per_wqe << 719 mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk); 720 721 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); 722 rq->mpwqe.num_strides = 723 BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); 724 rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz); 725 726 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz); 727 728 err = mlx5e_create_rq_umr_mkey(mdev, rq); 729 if (err) 730 goto err_rq_drop_page; 731 732 err = mlx5e_rq_alloc_mpwqe_info(rq, node); 733 if (err) 734 goto err_rq_mkey; 735 736 err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node); 737 if (err) 738 goto err_free_mpwqe_info; 739 740 break; 741 default: /* MLX5_WQ_TYPE_CYCLIC */ 742 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, 743 &rq->wq_ctrl); 744 if (err) 745 goto err_rq_xdp_prog; 746 747 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; 748 749 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 750 751 rq->wqe.info = rqp->frags_info; 752 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride; 753 754 rq->wqe.frags = 755 kvzalloc_node(array_size(sizeof(*rq->wqe.frags), 756 (wq_sz << rq->wqe.info.log_num_frags)), 757 GFP_KERNEL, node); 758 if (!rq->wqe.frags) { 759 err = -ENOMEM; 760 goto err_rq_wq_destroy; 761 } 762 763 err = mlx5e_init_au_list(rq, wq_sz, node); 764 if (err) 765 goto err_rq_frags; 766 } 767 768 if (xsk) { 769 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 770 MEM_TYPE_XSK_BUFF_POOL, NULL); 771 xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq); 772 } else { 773 /* Create a page_pool and register it with rxq */ 774 pp_params.order = 0; 775 pp_params.flags = 0; /* No-internal DMA mapping in page_pool */ 776 pp_params.pool_size = pool_size; 777 pp_params.nid = node; 778 pp_params.dev = rq->pdev; 779 pp_params.dma_dir = rq->buff.map_dir; 780 781 /* page_pool can be used even when there is no rq->xdp_prog, 782 * given page_pool does not handle DMA mapping there is no 783 * required state to clear. And page_pool gracefully handle 784 * elevated refcnt. 785 */ 786 rq->page_pool = page_pool_create(&pp_params); 787 if (IS_ERR(rq->page_pool)) { 788 err = PTR_ERR(rq->page_pool); 789 rq->page_pool = NULL; 790 goto err_free_by_rq_type; 791 } 792 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) 793 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 794 MEM_TYPE_PAGE_POOL, rq->page_pool); 795 } 796 if (err) 797 goto err_destroy_page_pool; 798 799 for (i = 0; i < wq_sz; i++) { 800 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 801 struct mlx5e_rx_wqe_ll *wqe = 802 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i); 803 u32 byte_count = 804 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz; 805 u64 dma_offset = mul_u32_u32(i, rq->mpwqe.mtts_per_wqe) << 806 rq->mpwqe.page_shift; 807 u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ? 808 0 : rq->buff.headroom; 809 810 wqe->data[0].addr = cpu_to_be64(dma_offset + headroom); 811 wqe->data[0].byte_count = cpu_to_be32(byte_count); 812 wqe->data[0].lkey = rq->mpwqe.umr_mkey_be; 813 } else { 814 struct mlx5e_rx_wqe_cyc *wqe = 815 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); 816 int f; 817 818 for (f = 0; f < rq->wqe.info.num_frags; f++) { 819 u32 frag_size = rq->wqe.info.arr[f].frag_size | 820 MLX5_HW_START_PADDING; 821 822 wqe->data[f].byte_count = cpu_to_be32(frag_size); 823 wqe->data[f].lkey = rq->mkey_be; 824 } 825 /* check if num_frags is not a pow of two */ 826 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { 827 wqe->data[f].byte_count = 0; 828 wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 829 wqe->data[f].addr = 0; 830 } 831 } 832 } 833 834 INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work); 835 836 switch (params->rx_cq_moderation.cq_period_mode) { 837 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE: 838 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE; 839 break; 840 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE: 841 default: 842 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 843 } 844 845 rq->page_cache.head = 0; 846 rq->page_cache.tail = 0; 847 848 return 0; 849 850 err_destroy_page_pool: 851 page_pool_destroy(rq->page_pool); 852 err_free_by_rq_type: 853 switch (rq->wq_type) { 854 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 855 mlx5e_rq_free_shampo(rq); 856 err_free_mpwqe_info: 857 kvfree(rq->mpwqe.info); 858 err_rq_mkey: 859 mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); 860 err_rq_drop_page: 861 mlx5e_free_mpwqe_rq_drop_page(rq); 862 break; 863 default: /* MLX5_WQ_TYPE_CYCLIC */ 864 mlx5e_free_au_list(rq); 865 err_rq_frags: 866 kvfree(rq->wqe.frags); 867 } 868 err_rq_wq_destroy: 869 mlx5_wq_destroy(&rq->wq_ctrl); 870 err_rq_xdp_prog: 871 if (params->xdp_prog) 872 bpf_prog_put(params->xdp_prog); 873 874 return err; 875 } 876 877 static void mlx5e_free_rq(struct mlx5e_rq *rq) 878 { 879 struct bpf_prog *old_prog; 880 int i; 881 882 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) { 883 old_prog = rcu_dereference_protected(rq->xdp_prog, 884 lockdep_is_held(&rq->priv->state_lock)); 885 if (old_prog) 886 bpf_prog_put(old_prog); 887 } 888 889 switch (rq->wq_type) { 890 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 891 kvfree(rq->mpwqe.info); 892 mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); 893 mlx5e_free_mpwqe_rq_drop_page(rq); 894 mlx5e_rq_free_shampo(rq); 895 break; 896 default: /* MLX5_WQ_TYPE_CYCLIC */ 897 kvfree(rq->wqe.frags); 898 mlx5e_free_au_list(rq); 899 } 900 901 for (i = rq->page_cache.head; i != rq->page_cache.tail; 902 i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) { 903 /* With AF_XDP, page_cache is not used, so this loop is not 904 * entered, and it's safe to call mlx5e_page_release_dynamic 905 * directly. 906 */ 907 mlx5e_page_release_dynamic(rq, rq->page_cache.page_cache[i], false); 908 } 909 910 xdp_rxq_info_unreg(&rq->xdp_rxq); 911 page_pool_destroy(rq->page_pool); 912 mlx5_wq_destroy(&rq->wq_ctrl); 913 } 914 915 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param) 916 { 917 struct mlx5_core_dev *mdev = rq->mdev; 918 u8 ts_format; 919 void *in; 920 void *rqc; 921 void *wq; 922 int inlen; 923 int err; 924 925 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + 926 sizeof(u64) * rq->wq_ctrl.buf.npages; 927 in = kvzalloc(inlen, GFP_KERNEL); 928 if (!in) 929 return -ENOMEM; 930 931 ts_format = mlx5_is_real_time_rq(mdev) ? 932 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 933 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 934 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 935 wq = MLX5_ADDR_OF(rqc, rqc, wq); 936 937 memcpy(rqc, param->rqc, sizeof(param->rqc)); 938 939 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); 940 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 941 MLX5_SET(rqc, rqc, ts_format, ts_format); 942 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - 943 MLX5_ADAPTER_PAGE_SHIFT); 944 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); 945 946 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 947 MLX5_SET(wq, wq, log_headers_buffer_entry_num, 948 order_base_2(rq->mpwqe.shampo->hd_per_wq)); 949 MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey); 950 } 951 952 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, 953 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 954 955 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn); 956 957 kvfree(in); 958 959 return err; 960 } 961 962 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state) 963 { 964 struct mlx5_core_dev *mdev = rq->mdev; 965 966 void *in; 967 void *rqc; 968 int inlen; 969 int err; 970 971 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 972 in = kvzalloc(inlen, GFP_KERNEL); 973 if (!in) 974 return -ENOMEM; 975 976 if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY) 977 mlx5e_rqwq_reset(rq); 978 979 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 980 981 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 982 MLX5_SET(rqc, rqc, state, next_state); 983 984 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 985 986 kvfree(in); 987 988 return err; 989 } 990 991 static int mlx5e_rq_to_ready(struct mlx5e_rq *rq, int curr_state) 992 { 993 struct net_device *dev = rq->netdev; 994 int err; 995 996 err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST); 997 if (err) { 998 netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn); 999 return err; 1000 } 1001 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 1002 if (err) { 1003 netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn); 1004 return err; 1005 } 1006 1007 return 0; 1008 } 1009 1010 int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state) 1011 { 1012 mlx5e_free_rx_descs(rq); 1013 1014 return mlx5e_rq_to_ready(rq, curr_state); 1015 } 1016 1017 static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable) 1018 { 1019 struct mlx5_core_dev *mdev = rq->mdev; 1020 1021 void *in; 1022 void *rqc; 1023 int inlen; 1024 int err; 1025 1026 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 1027 in = kvzalloc(inlen, GFP_KERNEL); 1028 if (!in) 1029 return -ENOMEM; 1030 1031 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 1032 1033 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 1034 MLX5_SET64(modify_rq_in, in, modify_bitmask, 1035 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS); 1036 MLX5_SET(rqc, rqc, scatter_fcs, enable); 1037 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 1038 1039 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 1040 1041 kvfree(in); 1042 1043 return err; 1044 } 1045 1046 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd) 1047 { 1048 struct mlx5_core_dev *mdev = rq->mdev; 1049 void *in; 1050 void *rqc; 1051 int inlen; 1052 int err; 1053 1054 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 1055 in = kvzalloc(inlen, GFP_KERNEL); 1056 if (!in) 1057 return -ENOMEM; 1058 1059 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 1060 1061 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 1062 MLX5_SET64(modify_rq_in, in, modify_bitmask, 1063 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD); 1064 MLX5_SET(rqc, rqc, vsd, vsd); 1065 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 1066 1067 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 1068 1069 kvfree(in); 1070 1071 return err; 1072 } 1073 1074 void mlx5e_destroy_rq(struct mlx5e_rq *rq) 1075 { 1076 mlx5_core_destroy_rq(rq->mdev, rq->rqn); 1077 } 1078 1079 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time) 1080 { 1081 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time); 1082 1083 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq)); 1084 1085 do { 1086 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes) 1087 return 0; 1088 1089 msleep(20); 1090 } while (time_before(jiffies, exp_time)); 1091 1092 netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n", 1093 rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes); 1094 1095 mlx5e_reporter_rx_timeout(rq); 1096 return -ETIMEDOUT; 1097 } 1098 1099 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq) 1100 { 1101 struct mlx5_wq_ll *wq; 1102 u16 head; 1103 int i; 1104 1105 if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) 1106 return; 1107 1108 wq = &rq->mpwqe.wq; 1109 head = wq->head; 1110 1111 /* Outstanding UMR WQEs (in progress) start at wq->head */ 1112 for (i = 0; i < rq->mpwqe.umr_in_progress; i++) { 1113 rq->dealloc_wqe(rq, head); 1114 head = mlx5_wq_ll_get_wqe_next_ix(wq, head); 1115 } 1116 1117 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 1118 u16 len; 1119 1120 len = (rq->mpwqe.shampo->pi - rq->mpwqe.shampo->ci) & 1121 (rq->mpwqe.shampo->hd_per_wq - 1); 1122 mlx5e_shampo_dealloc_hd(rq, len, rq->mpwqe.shampo->ci, false); 1123 rq->mpwqe.shampo->pi = rq->mpwqe.shampo->ci; 1124 } 1125 1126 rq->mpwqe.actual_wq_head = wq->head; 1127 rq->mpwqe.umr_in_progress = 0; 1128 rq->mpwqe.umr_completed = 0; 1129 } 1130 1131 void mlx5e_free_rx_descs(struct mlx5e_rq *rq) 1132 { 1133 __be16 wqe_ix_be; 1134 u16 wqe_ix; 1135 1136 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 1137 struct mlx5_wq_ll *wq = &rq->mpwqe.wq; 1138 1139 mlx5e_free_rx_in_progress_descs(rq); 1140 1141 while (!mlx5_wq_ll_is_empty(wq)) { 1142 struct mlx5e_rx_wqe_ll *wqe; 1143 1144 wqe_ix_be = *wq->tail_next; 1145 wqe_ix = be16_to_cpu(wqe_ix_be); 1146 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix); 1147 rq->dealloc_wqe(rq, wqe_ix); 1148 mlx5_wq_ll_pop(wq, wqe_ix_be, 1149 &wqe->next.next_wqe_index); 1150 } 1151 1152 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 1153 mlx5e_shampo_dealloc_hd(rq, rq->mpwqe.shampo->hd_per_wq, 1154 0, true); 1155 } else { 1156 struct mlx5_wq_cyc *wq = &rq->wqe.wq; 1157 1158 while (!mlx5_wq_cyc_is_empty(wq)) { 1159 wqe_ix = mlx5_wq_cyc_get_tail(wq); 1160 rq->dealloc_wqe(rq, wqe_ix); 1161 mlx5_wq_cyc_pop(wq); 1162 } 1163 } 1164 1165 } 1166 1167 int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, 1168 struct mlx5e_xsk_param *xsk, int node, 1169 struct mlx5e_rq *rq) 1170 { 1171 struct mlx5_core_dev *mdev = rq->mdev; 1172 int err; 1173 1174 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) 1175 __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state); 1176 1177 err = mlx5e_alloc_rq(params, xsk, param, node, rq); 1178 if (err) 1179 return err; 1180 1181 err = mlx5e_create_rq(rq, param); 1182 if (err) 1183 goto err_free_rq; 1184 1185 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 1186 if (err) 1187 goto err_destroy_rq; 1188 1189 if (MLX5_CAP_ETH(mdev, cqe_checksum_full)) 1190 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state); 1191 1192 if (params->rx_dim_enabled) 1193 __set_bit(MLX5E_RQ_STATE_AM, &rq->state); 1194 1195 /* We disable csum_complete when XDP is enabled since 1196 * XDP programs might manipulate packets which will render 1197 * skb->checksum incorrect. 1198 */ 1199 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || params->xdp_prog) 1200 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state); 1201 1202 /* For CQE compression on striding RQ, use stride index provided by 1203 * HW if capability is supported. 1204 */ 1205 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) && 1206 MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) 1207 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state); 1208 1209 /* For enhanced CQE compression packet processing. decompress 1210 * session according to the enhanced layout. 1211 */ 1212 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) && 1213 MLX5_CAP_GEN(mdev, enhanced_cqe_compression)) 1214 __set_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state); 1215 1216 return 0; 1217 1218 err_destroy_rq: 1219 mlx5e_destroy_rq(rq); 1220 err_free_rq: 1221 mlx5e_free_rq(rq); 1222 1223 return err; 1224 } 1225 1226 void mlx5e_activate_rq(struct mlx5e_rq *rq) 1227 { 1228 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 1229 } 1230 1231 void mlx5e_deactivate_rq(struct mlx5e_rq *rq) 1232 { 1233 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 1234 synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */ 1235 } 1236 1237 void mlx5e_close_rq(struct mlx5e_rq *rq) 1238 { 1239 cancel_work_sync(&rq->dim.work); 1240 cancel_work_sync(&rq->recover_work); 1241 mlx5e_destroy_rq(rq); 1242 mlx5e_free_rx_descs(rq); 1243 mlx5e_free_rq(rq); 1244 } 1245 1246 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq) 1247 { 1248 kvfree(sq->db.xdpi_fifo.xi); 1249 kvfree(sq->db.wqe_info); 1250 } 1251 1252 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa) 1253 { 1254 struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; 1255 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1256 int dsegs_per_wq = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 1257 size_t size; 1258 1259 size = array_size(sizeof(*xdpi_fifo->xi), dsegs_per_wq); 1260 xdpi_fifo->xi = kvzalloc_node(size, GFP_KERNEL, numa); 1261 if (!xdpi_fifo->xi) 1262 return -ENOMEM; 1263 1264 xdpi_fifo->pc = &sq->xdpi_fifo_pc; 1265 xdpi_fifo->cc = &sq->xdpi_fifo_cc; 1266 xdpi_fifo->mask = dsegs_per_wq - 1; 1267 1268 return 0; 1269 } 1270 1271 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) 1272 { 1273 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1274 size_t size; 1275 int err; 1276 1277 size = array_size(sizeof(*sq->db.wqe_info), wq_sz); 1278 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1279 if (!sq->db.wqe_info) 1280 return -ENOMEM; 1281 1282 err = mlx5e_alloc_xdpsq_fifo(sq, numa); 1283 if (err) { 1284 mlx5e_free_xdpsq_db(sq); 1285 return err; 1286 } 1287 1288 return 0; 1289 } 1290 1291 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, 1292 struct mlx5e_params *params, 1293 struct xsk_buff_pool *xsk_pool, 1294 struct mlx5e_sq_param *param, 1295 struct mlx5e_xdpsq *sq, 1296 bool is_redirect) 1297 { 1298 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1299 struct mlx5_core_dev *mdev = c->mdev; 1300 struct mlx5_wq_cyc *wq = &sq->wq; 1301 int err; 1302 1303 sq->pdev = c->pdev; 1304 sq->mkey_be = c->mkey_be; 1305 sq->channel = c; 1306 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; 1307 sq->min_inline_mode = params->tx_min_inline_mode; 1308 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 1309 sq->xsk_pool = xsk_pool; 1310 1311 sq->stats = sq->xsk_pool ? 1312 &c->priv->channel_stats[c->ix]->xsksq : 1313 is_redirect ? 1314 &c->priv->channel_stats[c->ix]->xdpsq : 1315 &c->priv->channel_stats[c->ix]->rq_xdpsq; 1316 sq->stop_room = param->is_mpw ? mlx5e_stop_room_for_mpwqe(mdev) : 1317 mlx5e_stop_room_for_max_wqe(mdev); 1318 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 1319 1320 param->wq.db_numa_node = cpu_to_node(c->cpu); 1321 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1322 if (err) 1323 return err; 1324 wq->db = &wq->db[MLX5_SND_DBR]; 1325 1326 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu)); 1327 if (err) 1328 goto err_sq_wq_destroy; 1329 1330 return 0; 1331 1332 err_sq_wq_destroy: 1333 mlx5_wq_destroy(&sq->wq_ctrl); 1334 1335 return err; 1336 } 1337 1338 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq) 1339 { 1340 mlx5e_free_xdpsq_db(sq); 1341 mlx5_wq_destroy(&sq->wq_ctrl); 1342 } 1343 1344 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) 1345 { 1346 kvfree(sq->db.wqe_info); 1347 } 1348 1349 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa) 1350 { 1351 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1352 size_t size; 1353 1354 size = array_size(wq_sz, sizeof(*sq->db.wqe_info)); 1355 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1356 if (!sq->db.wqe_info) 1357 return -ENOMEM; 1358 1359 return 0; 1360 } 1361 1362 static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work) 1363 { 1364 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1365 recover_work); 1366 1367 mlx5e_reporter_icosq_cqe_err(sq); 1368 } 1369 1370 static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work) 1371 { 1372 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1373 recover_work); 1374 1375 /* Not implemented yet. */ 1376 1377 netdev_warn(sq->channel->netdev, "async_icosq recovery is not implemented\n"); 1378 } 1379 1380 static int mlx5e_alloc_icosq(struct mlx5e_channel *c, 1381 struct mlx5e_sq_param *param, 1382 struct mlx5e_icosq *sq, 1383 work_func_t recover_work_func) 1384 { 1385 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1386 struct mlx5_core_dev *mdev = c->mdev; 1387 struct mlx5_wq_cyc *wq = &sq->wq; 1388 int err; 1389 1390 sq->channel = c; 1391 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; 1392 sq->reserved_room = param->stop_room; 1393 1394 param->wq.db_numa_node = cpu_to_node(c->cpu); 1395 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1396 if (err) 1397 return err; 1398 wq->db = &wq->db[MLX5_SND_DBR]; 1399 1400 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu)); 1401 if (err) 1402 goto err_sq_wq_destroy; 1403 1404 INIT_WORK(&sq->recover_work, recover_work_func); 1405 1406 return 0; 1407 1408 err_sq_wq_destroy: 1409 mlx5_wq_destroy(&sq->wq_ctrl); 1410 1411 return err; 1412 } 1413 1414 static void mlx5e_free_icosq(struct mlx5e_icosq *sq) 1415 { 1416 mlx5e_free_icosq_db(sq); 1417 mlx5_wq_destroy(&sq->wq_ctrl); 1418 } 1419 1420 void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq) 1421 { 1422 kvfree(sq->db.wqe_info); 1423 kvfree(sq->db.skb_fifo.fifo); 1424 kvfree(sq->db.dma_fifo); 1425 } 1426 1427 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa) 1428 { 1429 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1430 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 1431 1432 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz, 1433 sizeof(*sq->db.dma_fifo)), 1434 GFP_KERNEL, numa); 1435 sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz, 1436 sizeof(*sq->db.skb_fifo.fifo)), 1437 GFP_KERNEL, numa); 1438 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz, 1439 sizeof(*sq->db.wqe_info)), 1440 GFP_KERNEL, numa); 1441 if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) { 1442 mlx5e_free_txqsq_db(sq); 1443 return -ENOMEM; 1444 } 1445 1446 sq->dma_fifo_mask = df_sz - 1; 1447 1448 sq->db.skb_fifo.pc = &sq->skb_fifo_pc; 1449 sq->db.skb_fifo.cc = &sq->skb_fifo_cc; 1450 sq->db.skb_fifo.mask = df_sz - 1; 1451 1452 return 0; 1453 } 1454 1455 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 1456 int txq_ix, 1457 struct mlx5e_params *params, 1458 struct mlx5e_sq_param *param, 1459 struct mlx5e_txqsq *sq, 1460 int tc) 1461 { 1462 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1463 struct mlx5_core_dev *mdev = c->mdev; 1464 struct mlx5_wq_cyc *wq = &sq->wq; 1465 int err; 1466 1467 sq->pdev = c->pdev; 1468 sq->clock = &mdev->clock; 1469 sq->mkey_be = c->mkey_be; 1470 sq->netdev = c->netdev; 1471 sq->mdev = c->mdev; 1472 sq->priv = c->priv; 1473 sq->ch_ix = c->ix; 1474 sq->txq_ix = txq_ix; 1475 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; 1476 sq->min_inline_mode = params->tx_min_inline_mode; 1477 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 1478 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 1479 INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work); 1480 if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert)) 1481 set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state); 1482 if (mlx5_ipsec_device_caps(c->priv->mdev)) 1483 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state); 1484 if (param->is_mpw) 1485 set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state); 1486 sq->stop_room = param->stop_room; 1487 sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev); 1488 1489 param->wq.db_numa_node = cpu_to_node(c->cpu); 1490 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1491 if (err) 1492 return err; 1493 wq->db = &wq->db[MLX5_SND_DBR]; 1494 1495 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu)); 1496 if (err) 1497 goto err_sq_wq_destroy; 1498 1499 INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work); 1500 sq->dim.mode = params->tx_cq_moderation.cq_period_mode; 1501 1502 return 0; 1503 1504 err_sq_wq_destroy: 1505 mlx5_wq_destroy(&sq->wq_ctrl); 1506 1507 return err; 1508 } 1509 1510 void mlx5e_free_txqsq(struct mlx5e_txqsq *sq) 1511 { 1512 mlx5e_free_txqsq_db(sq); 1513 mlx5_wq_destroy(&sq->wq_ctrl); 1514 } 1515 1516 static int mlx5e_create_sq(struct mlx5_core_dev *mdev, 1517 struct mlx5e_sq_param *param, 1518 struct mlx5e_create_sq_param *csp, 1519 u32 *sqn) 1520 { 1521 u8 ts_format; 1522 void *in; 1523 void *sqc; 1524 void *wq; 1525 int inlen; 1526 int err; 1527 1528 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + 1529 sizeof(u64) * csp->wq_ctrl->buf.npages; 1530 in = kvzalloc(inlen, GFP_KERNEL); 1531 if (!in) 1532 return -ENOMEM; 1533 1534 ts_format = mlx5_is_real_time_sq(mdev) ? 1535 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 1536 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 1537 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 1538 wq = MLX5_ADDR_OF(sqc, sqc, wq); 1539 1540 memcpy(sqc, param->sqc, sizeof(param->sqc)); 1541 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz); 1542 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn); 1543 MLX5_SET(sqc, sqc, cqn, csp->cqn); 1544 MLX5_SET(sqc, sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn); 1545 MLX5_SET(sqc, sqc, ts_format, ts_format); 1546 1547 1548 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) 1549 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode); 1550 1551 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 1552 MLX5_SET(sqc, sqc, flush_in_error_en, 1); 1553 1554 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1555 MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index); 1556 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift - 1557 MLX5_ADAPTER_PAGE_SHIFT); 1558 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma); 1559 1560 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf, 1561 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1562 1563 err = mlx5_core_create_sq(mdev, in, inlen, sqn); 1564 1565 kvfree(in); 1566 1567 return err; 1568 } 1569 1570 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, 1571 struct mlx5e_modify_sq_param *p) 1572 { 1573 u64 bitmask = 0; 1574 void *in; 1575 void *sqc; 1576 int inlen; 1577 int err; 1578 1579 inlen = MLX5_ST_SZ_BYTES(modify_sq_in); 1580 in = kvzalloc(inlen, GFP_KERNEL); 1581 if (!in) 1582 return -ENOMEM; 1583 1584 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 1585 1586 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state); 1587 MLX5_SET(sqc, sqc, state, p->next_state); 1588 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) { 1589 bitmask |= 1; 1590 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index); 1591 } 1592 if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) { 1593 bitmask |= 1 << 2; 1594 MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id); 1595 } 1596 MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask); 1597 1598 err = mlx5_core_modify_sq(mdev, sqn, in); 1599 1600 kvfree(in); 1601 1602 return err; 1603 } 1604 1605 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) 1606 { 1607 mlx5_core_destroy_sq(mdev, sqn); 1608 } 1609 1610 int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, 1611 struct mlx5e_sq_param *param, 1612 struct mlx5e_create_sq_param *csp, 1613 u16 qos_queue_group_id, 1614 u32 *sqn) 1615 { 1616 struct mlx5e_modify_sq_param msp = {0}; 1617 int err; 1618 1619 err = mlx5e_create_sq(mdev, param, csp, sqn); 1620 if (err) 1621 return err; 1622 1623 msp.curr_state = MLX5_SQC_STATE_RST; 1624 msp.next_state = MLX5_SQC_STATE_RDY; 1625 if (qos_queue_group_id) { 1626 msp.qos_update = true; 1627 msp.qos_queue_group_id = qos_queue_group_id; 1628 } 1629 err = mlx5e_modify_sq(mdev, *sqn, &msp); 1630 if (err) 1631 mlx5e_destroy_sq(mdev, *sqn); 1632 1633 return err; 1634 } 1635 1636 static int mlx5e_set_sq_maxrate(struct net_device *dev, 1637 struct mlx5e_txqsq *sq, u32 rate); 1638 1639 int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix, 1640 struct mlx5e_params *params, struct mlx5e_sq_param *param, 1641 struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id, 1642 struct mlx5e_sq_stats *sq_stats) 1643 { 1644 struct mlx5e_create_sq_param csp = {}; 1645 u32 tx_rate; 1646 int err; 1647 1648 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc); 1649 if (err) 1650 return err; 1651 1652 sq->stats = sq_stats; 1653 1654 csp.tisn = tisn; 1655 csp.tis_lst_sz = 1; 1656 csp.cqn = sq->cq.mcq.cqn; 1657 csp.wq_ctrl = &sq->wq_ctrl; 1658 csp.min_inline_mode = sq->min_inline_mode; 1659 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn); 1660 if (err) 1661 goto err_free_txqsq; 1662 1663 tx_rate = c->priv->tx_rates[sq->txq_ix]; 1664 if (tx_rate) 1665 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate); 1666 1667 if (params->tx_dim_enabled) 1668 sq->state |= BIT(MLX5E_SQ_STATE_AM); 1669 1670 return 0; 1671 1672 err_free_txqsq: 1673 mlx5e_free_txqsq(sq); 1674 1675 return err; 1676 } 1677 1678 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq) 1679 { 1680 sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix); 1681 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1682 netdev_tx_reset_queue(sq->txq); 1683 netif_tx_start_queue(sq->txq); 1684 } 1685 1686 void mlx5e_tx_disable_queue(struct netdev_queue *txq) 1687 { 1688 __netif_tx_lock_bh(txq); 1689 netif_tx_stop_queue(txq); 1690 __netif_tx_unlock_bh(txq); 1691 } 1692 1693 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) 1694 { 1695 struct mlx5_wq_cyc *wq = &sq->wq; 1696 1697 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1698 synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */ 1699 1700 mlx5e_tx_disable_queue(sq->txq); 1701 1702 /* last doorbell out, godspeed .. */ 1703 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) { 1704 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1705 struct mlx5e_tx_wqe *nop; 1706 1707 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) { 1708 .num_wqebbs = 1, 1709 }; 1710 1711 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc); 1712 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl); 1713 } 1714 } 1715 1716 void mlx5e_close_txqsq(struct mlx5e_txqsq *sq) 1717 { 1718 struct mlx5_core_dev *mdev = sq->mdev; 1719 struct mlx5_rate_limit rl = {0}; 1720 1721 cancel_work_sync(&sq->dim.work); 1722 cancel_work_sync(&sq->recover_work); 1723 mlx5e_destroy_sq(mdev, sq->sqn); 1724 if (sq->rate_limit) { 1725 rl.rate = sq->rate_limit; 1726 mlx5_rl_remove_rate(mdev, &rl); 1727 } 1728 mlx5e_free_txqsq_descs(sq); 1729 mlx5e_free_txqsq(sq); 1730 } 1731 1732 void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) 1733 { 1734 struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, 1735 recover_work); 1736 1737 mlx5e_reporter_tx_err_cqe(sq); 1738 } 1739 1740 static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params, 1741 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq, 1742 work_func_t recover_work_func) 1743 { 1744 struct mlx5e_create_sq_param csp = {}; 1745 int err; 1746 1747 err = mlx5e_alloc_icosq(c, param, sq, recover_work_func); 1748 if (err) 1749 return err; 1750 1751 csp.cqn = sq->cq.mcq.cqn; 1752 csp.wq_ctrl = &sq->wq_ctrl; 1753 csp.min_inline_mode = params->tx_min_inline_mode; 1754 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); 1755 if (err) 1756 goto err_free_icosq; 1757 1758 if (param->is_tls) { 1759 sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list(); 1760 if (IS_ERR(sq->ktls_resync)) { 1761 err = PTR_ERR(sq->ktls_resync); 1762 goto err_destroy_icosq; 1763 } 1764 } 1765 return 0; 1766 1767 err_destroy_icosq: 1768 mlx5e_destroy_sq(c->mdev, sq->sqn); 1769 err_free_icosq: 1770 mlx5e_free_icosq(sq); 1771 1772 return err; 1773 } 1774 1775 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq) 1776 { 1777 set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 1778 } 1779 1780 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq) 1781 { 1782 clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 1783 synchronize_net(); /* Sync with NAPI. */ 1784 } 1785 1786 static void mlx5e_close_icosq(struct mlx5e_icosq *sq) 1787 { 1788 struct mlx5e_channel *c = sq->channel; 1789 1790 if (sq->ktls_resync) 1791 mlx5e_ktls_rx_resync_destroy_resp_list(sq->ktls_resync); 1792 mlx5e_destroy_sq(c->mdev, sq->sqn); 1793 mlx5e_free_icosq_descs(sq); 1794 mlx5e_free_icosq(sq); 1795 } 1796 1797 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, 1798 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool, 1799 struct mlx5e_xdpsq *sq, bool is_redirect) 1800 { 1801 struct mlx5e_create_sq_param csp = {}; 1802 int err; 1803 1804 err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect); 1805 if (err) 1806 return err; 1807 1808 csp.tis_lst_sz = 1; 1809 csp.tisn = c->priv->tisn[c->lag_port][0]; /* tc = 0 */ 1810 csp.cqn = sq->cq.mcq.cqn; 1811 csp.wq_ctrl = &sq->wq_ctrl; 1812 csp.min_inline_mode = sq->min_inline_mode; 1813 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1814 1815 /* Don't enable multi buffer on XDP_REDIRECT SQ, as it's not yet 1816 * supported by upstream, and there is no defined trigger to allow 1817 * transmitting redirected multi-buffer frames. 1818 */ 1819 if (param->is_xdp_mb && !is_redirect) 1820 set_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state); 1821 1822 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); 1823 if (err) 1824 goto err_free_xdpsq; 1825 1826 mlx5e_set_xmit_fp(sq, param->is_mpw); 1827 1828 if (!param->is_mpw && !test_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state)) { 1829 unsigned int ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT + 1; 1830 unsigned int inline_hdr_sz = 0; 1831 int i; 1832 1833 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) { 1834 inline_hdr_sz = MLX5E_XDP_MIN_INLINE; 1835 ds_cnt++; 1836 } 1837 1838 /* Pre initialize fixed WQE fields */ 1839 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) { 1840 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, i); 1841 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 1842 struct mlx5_wqe_eth_seg *eseg = &wqe->eth; 1843 struct mlx5_wqe_data_seg *dseg; 1844 1845 sq->db.wqe_info[i] = (struct mlx5e_xdp_wqe_info) { 1846 .num_wqebbs = 1, 1847 .num_pkts = 1, 1848 }; 1849 1850 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 1851 eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz); 1852 1853 dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1); 1854 dseg->lkey = sq->mkey_be; 1855 } 1856 } 1857 1858 return 0; 1859 1860 err_free_xdpsq: 1861 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1862 mlx5e_free_xdpsq(sq); 1863 1864 return err; 1865 } 1866 1867 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq) 1868 { 1869 struct mlx5e_channel *c = sq->channel; 1870 1871 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1872 synchronize_net(); /* Sync with NAPI. */ 1873 1874 mlx5e_destroy_sq(c->mdev, sq->sqn); 1875 mlx5e_free_xdpsq_descs(sq); 1876 mlx5e_free_xdpsq(sq); 1877 } 1878 1879 static int mlx5e_alloc_cq_common(struct mlx5e_priv *priv, 1880 struct mlx5e_cq_param *param, 1881 struct mlx5e_cq *cq) 1882 { 1883 struct mlx5_core_dev *mdev = priv->mdev; 1884 struct mlx5_core_cq *mcq = &cq->mcq; 1885 int err; 1886 u32 i; 1887 1888 err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq, 1889 &cq->wq_ctrl); 1890 if (err) 1891 return err; 1892 1893 mcq->cqe_sz = 64; 1894 mcq->set_ci_db = cq->wq_ctrl.db.db; 1895 mcq->arm_db = cq->wq_ctrl.db.db + 1; 1896 *mcq->set_ci_db = 0; 1897 *mcq->arm_db = 0; 1898 mcq->vector = param->eq_ix; 1899 mcq->comp = mlx5e_completion_event; 1900 mcq->event = mlx5e_cq_error_event; 1901 1902 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 1903 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 1904 1905 cqe->op_own = 0xf1; 1906 cqe->validity_iteration_count = 0xff; 1907 } 1908 1909 cq->mdev = mdev; 1910 cq->netdev = priv->netdev; 1911 cq->priv = priv; 1912 1913 return 0; 1914 } 1915 1916 static int mlx5e_alloc_cq(struct mlx5e_priv *priv, 1917 struct mlx5e_cq_param *param, 1918 struct mlx5e_create_cq_param *ccp, 1919 struct mlx5e_cq *cq) 1920 { 1921 int err; 1922 1923 param->wq.buf_numa_node = ccp->node; 1924 param->wq.db_numa_node = ccp->node; 1925 param->eq_ix = ccp->ix; 1926 1927 err = mlx5e_alloc_cq_common(priv, param, cq); 1928 1929 cq->napi = ccp->napi; 1930 cq->ch_stats = ccp->ch_stats; 1931 1932 return err; 1933 } 1934 1935 static void mlx5e_free_cq(struct mlx5e_cq *cq) 1936 { 1937 mlx5_wq_destroy(&cq->wq_ctrl); 1938 } 1939 1940 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 1941 { 1942 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 1943 struct mlx5_core_dev *mdev = cq->mdev; 1944 struct mlx5_core_cq *mcq = &cq->mcq; 1945 1946 void *in; 1947 void *cqc; 1948 int inlen; 1949 int eqn; 1950 int err; 1951 1952 err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn); 1953 if (err) 1954 return err; 1955 1956 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 1957 sizeof(u64) * cq->wq_ctrl.buf.npages; 1958 in = kvzalloc(inlen, GFP_KERNEL); 1959 if (!in) 1960 return -ENOMEM; 1961 1962 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 1963 1964 memcpy(cqc, param->cqc, sizeof(param->cqc)); 1965 1966 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, 1967 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); 1968 1969 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode); 1970 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 1971 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); 1972 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 1973 MLX5_ADAPTER_PAGE_SHIFT); 1974 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 1975 1976 err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out)); 1977 1978 kvfree(in); 1979 1980 if (err) 1981 return err; 1982 1983 mlx5e_cq_arm(cq); 1984 1985 return 0; 1986 } 1987 1988 static void mlx5e_destroy_cq(struct mlx5e_cq *cq) 1989 { 1990 mlx5_core_destroy_cq(cq->mdev, &cq->mcq); 1991 } 1992 1993 int mlx5e_open_cq(struct mlx5e_priv *priv, struct dim_cq_moder moder, 1994 struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp, 1995 struct mlx5e_cq *cq) 1996 { 1997 struct mlx5_core_dev *mdev = priv->mdev; 1998 int err; 1999 2000 err = mlx5e_alloc_cq(priv, param, ccp, cq); 2001 if (err) 2002 return err; 2003 2004 err = mlx5e_create_cq(cq, param); 2005 if (err) 2006 goto err_free_cq; 2007 2008 if (MLX5_CAP_GEN(mdev, cq_moderation)) 2009 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts); 2010 return 0; 2011 2012 err_free_cq: 2013 mlx5e_free_cq(cq); 2014 2015 return err; 2016 } 2017 2018 void mlx5e_close_cq(struct mlx5e_cq *cq) 2019 { 2020 mlx5e_destroy_cq(cq); 2021 mlx5e_free_cq(cq); 2022 } 2023 2024 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, 2025 struct mlx5e_params *params, 2026 struct mlx5e_create_cq_param *ccp, 2027 struct mlx5e_channel_param *cparam) 2028 { 2029 int err; 2030 int tc; 2031 2032 for (tc = 0; tc < c->num_tc; tc++) { 2033 err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->txq_sq.cqp, 2034 ccp, &c->sq[tc].cq); 2035 if (err) 2036 goto err_close_tx_cqs; 2037 } 2038 2039 return 0; 2040 2041 err_close_tx_cqs: 2042 for (tc--; tc >= 0; tc--) 2043 mlx5e_close_cq(&c->sq[tc].cq); 2044 2045 return err; 2046 } 2047 2048 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c) 2049 { 2050 int tc; 2051 2052 for (tc = 0; tc < c->num_tc; tc++) 2053 mlx5e_close_cq(&c->sq[tc].cq); 2054 } 2055 2056 static int mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq *tc_to_txq, unsigned int txq) 2057 { 2058 int tc; 2059 2060 for (tc = 0; tc < TC_MAX_QUEUE; tc++) 2061 if (txq - tc_to_txq[tc].offset < tc_to_txq[tc].count) 2062 return tc; 2063 2064 WARN(1, "Unexpected TCs configuration. No match found for txq %u", txq); 2065 return -ENOENT; 2066 } 2067 2068 static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, 2069 u32 *hw_id) 2070 { 2071 int tc; 2072 2073 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) { 2074 *hw_id = 0; 2075 return 0; 2076 } 2077 2078 tc = mlx5e_mqprio_txq_to_tc(params->mqprio.tc_to_txq, txq_ix); 2079 if (tc < 0) 2080 return tc; 2081 2082 if (tc >= params->mqprio.num_tc) { 2083 WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u", 2084 tc, params->mqprio.num_tc); 2085 return -EINVAL; 2086 } 2087 2088 *hw_id = params->mqprio.channel.hw_id[tc]; 2089 return 0; 2090 } 2091 2092 static int mlx5e_open_sqs(struct mlx5e_channel *c, 2093 struct mlx5e_params *params, 2094 struct mlx5e_channel_param *cparam) 2095 { 2096 int err, tc; 2097 2098 for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) { 2099 int txq_ix = c->ix + tc * params->num_channels; 2100 u32 qos_queue_group_id; 2101 2102 err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id); 2103 if (err) 2104 goto err_close_sqs; 2105 2106 err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix, 2107 params, &cparam->txq_sq, &c->sq[tc], tc, 2108 qos_queue_group_id, 2109 &c->priv->channel_stats[c->ix]->sq[tc]); 2110 if (err) 2111 goto err_close_sqs; 2112 } 2113 2114 return 0; 2115 2116 err_close_sqs: 2117 for (tc--; tc >= 0; tc--) 2118 mlx5e_close_txqsq(&c->sq[tc]); 2119 2120 return err; 2121 } 2122 2123 static void mlx5e_close_sqs(struct mlx5e_channel *c) 2124 { 2125 int tc; 2126 2127 for (tc = 0; tc < c->num_tc; tc++) 2128 mlx5e_close_txqsq(&c->sq[tc]); 2129 } 2130 2131 static int mlx5e_set_sq_maxrate(struct net_device *dev, 2132 struct mlx5e_txqsq *sq, u32 rate) 2133 { 2134 struct mlx5e_priv *priv = netdev_priv(dev); 2135 struct mlx5_core_dev *mdev = priv->mdev; 2136 struct mlx5e_modify_sq_param msp = {0}; 2137 struct mlx5_rate_limit rl = {0}; 2138 u16 rl_index = 0; 2139 int err; 2140 2141 if (rate == sq->rate_limit) 2142 /* nothing to do */ 2143 return 0; 2144 2145 if (sq->rate_limit) { 2146 rl.rate = sq->rate_limit; 2147 /* remove current rl index to free space to next ones */ 2148 mlx5_rl_remove_rate(mdev, &rl); 2149 } 2150 2151 sq->rate_limit = 0; 2152 2153 if (rate) { 2154 rl.rate = rate; 2155 err = mlx5_rl_add_rate(mdev, &rl_index, &rl); 2156 if (err) { 2157 netdev_err(dev, "Failed configuring rate %u: %d\n", 2158 rate, err); 2159 return err; 2160 } 2161 } 2162 2163 msp.curr_state = MLX5_SQC_STATE_RDY; 2164 msp.next_state = MLX5_SQC_STATE_RDY; 2165 msp.rl_index = rl_index; 2166 msp.rl_update = true; 2167 err = mlx5e_modify_sq(mdev, sq->sqn, &msp); 2168 if (err) { 2169 netdev_err(dev, "Failed configuring rate %u: %d\n", 2170 rate, err); 2171 /* remove the rate from the table */ 2172 if (rate) 2173 mlx5_rl_remove_rate(mdev, &rl); 2174 return err; 2175 } 2176 2177 sq->rate_limit = rate; 2178 return 0; 2179 } 2180 2181 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 2182 { 2183 struct mlx5e_priv *priv = netdev_priv(dev); 2184 struct mlx5_core_dev *mdev = priv->mdev; 2185 struct mlx5e_txqsq *sq = priv->txq2sq[index]; 2186 int err = 0; 2187 2188 if (!mlx5_rl_is_supported(mdev)) { 2189 netdev_err(dev, "Rate limiting is not supported on this device\n"); 2190 return -EINVAL; 2191 } 2192 2193 /* rate is given in Mb/sec, HW config is in Kb/sec */ 2194 rate = rate << 10; 2195 2196 /* Check whether rate in valid range, 0 is always valid */ 2197 if (rate && !mlx5_rl_is_in_range(mdev, rate)) { 2198 netdev_err(dev, "TX rate %u, is not in range\n", rate); 2199 return -ERANGE; 2200 } 2201 2202 mutex_lock(&priv->state_lock); 2203 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 2204 err = mlx5e_set_sq_maxrate(dev, sq, rate); 2205 if (!err) 2206 priv->tx_rates[index] = rate; 2207 mutex_unlock(&priv->state_lock); 2208 2209 return err; 2210 } 2211 2212 static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 2213 struct mlx5e_rq_param *rq_params) 2214 { 2215 int err; 2216 2217 err = mlx5e_init_rxq_rq(c, params, &c->rq); 2218 if (err) 2219 return err; 2220 2221 return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), &c->rq); 2222 } 2223 2224 static int mlx5e_open_queues(struct mlx5e_channel *c, 2225 struct mlx5e_params *params, 2226 struct mlx5e_channel_param *cparam) 2227 { 2228 struct dim_cq_moder icocq_moder = {0, 0}; 2229 struct mlx5e_create_cq_param ccp; 2230 int err; 2231 2232 mlx5e_build_create_cq_param(&ccp, c); 2233 2234 err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->async_icosq.cqp, &ccp, 2235 &c->async_icosq.cq); 2236 if (err) 2237 return err; 2238 2239 err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->icosq.cqp, &ccp, 2240 &c->icosq.cq); 2241 if (err) 2242 goto err_close_async_icosq_cq; 2243 2244 err = mlx5e_open_tx_cqs(c, params, &ccp, cparam); 2245 if (err) 2246 goto err_close_icosq_cq; 2247 2248 err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp, 2249 &c->xdpsq.cq); 2250 if (err) 2251 goto err_close_tx_cqs; 2252 2253 err = mlx5e_open_cq(c->priv, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, 2254 &c->rq.cq); 2255 if (err) 2256 goto err_close_xdp_tx_cqs; 2257 2258 err = c->xdp ? mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp, 2259 &ccp, &c->rq_xdpsq.cq) : 0; 2260 if (err) 2261 goto err_close_rx_cq; 2262 2263 spin_lock_init(&c->async_icosq_lock); 2264 2265 err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq, 2266 mlx5e_async_icosq_err_cqe_work); 2267 if (err) 2268 goto err_close_xdpsq_cq; 2269 2270 mutex_init(&c->icosq_recovery_lock); 2271 2272 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq, 2273 mlx5e_icosq_err_cqe_work); 2274 if (err) 2275 goto err_close_async_icosq; 2276 2277 err = mlx5e_open_sqs(c, params, cparam); 2278 if (err) 2279 goto err_close_icosq; 2280 2281 err = mlx5e_open_rxq_rq(c, params, &cparam->rq); 2282 if (err) 2283 goto err_close_sqs; 2284 2285 if (c->xdp) { 2286 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, 2287 &c->rq_xdpsq, false); 2288 if (err) 2289 goto err_close_rq; 2290 } 2291 2292 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true); 2293 if (err) 2294 goto err_close_xdp_sq; 2295 2296 return 0; 2297 2298 err_close_xdp_sq: 2299 if (c->xdp) 2300 mlx5e_close_xdpsq(&c->rq_xdpsq); 2301 2302 err_close_rq: 2303 mlx5e_close_rq(&c->rq); 2304 2305 err_close_sqs: 2306 mlx5e_close_sqs(c); 2307 2308 err_close_icosq: 2309 mlx5e_close_icosq(&c->icosq); 2310 2311 err_close_async_icosq: 2312 mlx5e_close_icosq(&c->async_icosq); 2313 2314 err_close_xdpsq_cq: 2315 if (c->xdp) 2316 mlx5e_close_cq(&c->rq_xdpsq.cq); 2317 2318 err_close_rx_cq: 2319 mlx5e_close_cq(&c->rq.cq); 2320 2321 err_close_xdp_tx_cqs: 2322 mlx5e_close_cq(&c->xdpsq.cq); 2323 2324 err_close_tx_cqs: 2325 mlx5e_close_tx_cqs(c); 2326 2327 err_close_icosq_cq: 2328 mlx5e_close_cq(&c->icosq.cq); 2329 2330 err_close_async_icosq_cq: 2331 mlx5e_close_cq(&c->async_icosq.cq); 2332 2333 return err; 2334 } 2335 2336 static void mlx5e_close_queues(struct mlx5e_channel *c) 2337 { 2338 mlx5e_close_xdpsq(&c->xdpsq); 2339 if (c->xdp) 2340 mlx5e_close_xdpsq(&c->rq_xdpsq); 2341 /* The same ICOSQ is used for UMRs for both RQ and XSKRQ. */ 2342 cancel_work_sync(&c->icosq.recover_work); 2343 mlx5e_close_rq(&c->rq); 2344 mlx5e_close_sqs(c); 2345 mlx5e_close_icosq(&c->icosq); 2346 mutex_destroy(&c->icosq_recovery_lock); 2347 mlx5e_close_icosq(&c->async_icosq); 2348 if (c->xdp) 2349 mlx5e_close_cq(&c->rq_xdpsq.cq); 2350 mlx5e_close_cq(&c->rq.cq); 2351 mlx5e_close_cq(&c->xdpsq.cq); 2352 mlx5e_close_tx_cqs(c); 2353 mlx5e_close_cq(&c->icosq.cq); 2354 mlx5e_close_cq(&c->async_icosq.cq); 2355 } 2356 2357 static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix) 2358 { 2359 u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id); 2360 2361 return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev); 2362 } 2363 2364 static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu) 2365 { 2366 if (ix > priv->stats_nch) { 2367 netdev_warn(priv->netdev, "Unexpected channel stats index %d > %d\n", ix, 2368 priv->stats_nch); 2369 return -EINVAL; 2370 } 2371 2372 if (priv->channel_stats[ix]) 2373 return 0; 2374 2375 /* Asymmetric dynamic memory allocation. 2376 * Freed in mlx5e_priv_arrays_free, not on channel closure. 2377 */ 2378 mlx5e_dbg(DRV, priv, "Creating channel stats %d\n", ix); 2379 priv->channel_stats[ix] = kvzalloc_node(sizeof(**priv->channel_stats), 2380 GFP_KERNEL, cpu_to_node(cpu)); 2381 if (!priv->channel_stats[ix]) 2382 return -ENOMEM; 2383 priv->stats_nch++; 2384 2385 return 0; 2386 } 2387 2388 void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c) 2389 { 2390 spin_lock_bh(&c->async_icosq_lock); 2391 mlx5e_trigger_irq(&c->async_icosq); 2392 spin_unlock_bh(&c->async_icosq_lock); 2393 } 2394 2395 void mlx5e_trigger_napi_sched(struct napi_struct *napi) 2396 { 2397 local_bh_disable(); 2398 napi_schedule(napi); 2399 local_bh_enable(); 2400 } 2401 2402 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 2403 struct mlx5e_params *params, 2404 struct mlx5e_channel_param *cparam, 2405 struct xsk_buff_pool *xsk_pool, 2406 struct mlx5e_channel **cp) 2407 { 2408 int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix)); 2409 struct net_device *netdev = priv->netdev; 2410 struct mlx5e_xsk_param xsk; 2411 struct mlx5e_channel *c; 2412 unsigned int irq; 2413 int err; 2414 2415 err = mlx5_vector2irqn(priv->mdev, ix, &irq); 2416 if (err) 2417 return err; 2418 2419 err = mlx5e_channel_stats_alloc(priv, ix, cpu); 2420 if (err) 2421 return err; 2422 2423 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu)); 2424 if (!c) 2425 return -ENOMEM; 2426 2427 c->priv = priv; 2428 c->mdev = priv->mdev; 2429 c->tstamp = &priv->tstamp; 2430 c->ix = ix; 2431 c->cpu = cpu; 2432 c->pdev = mlx5_core_dma_dev(priv->mdev); 2433 c->netdev = priv->netdev; 2434 c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey); 2435 c->num_tc = mlx5e_get_dcb_num_tc(params); 2436 c->xdp = !!params->xdp_prog; 2437 c->stats = &priv->channel_stats[ix]->ch; 2438 c->aff_mask = irq_get_effective_affinity_mask(irq); 2439 c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix); 2440 2441 netif_napi_add(netdev, &c->napi, mlx5e_napi_poll); 2442 2443 err = mlx5e_open_queues(c, params, cparam); 2444 if (unlikely(err)) 2445 goto err_napi_del; 2446 2447 if (xsk_pool) { 2448 mlx5e_build_xsk_param(xsk_pool, &xsk); 2449 err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c); 2450 if (unlikely(err)) 2451 goto err_close_queues; 2452 } 2453 2454 *cp = c; 2455 2456 return 0; 2457 2458 err_close_queues: 2459 mlx5e_close_queues(c); 2460 2461 err_napi_del: 2462 netif_napi_del(&c->napi); 2463 2464 kvfree(c); 2465 2466 return err; 2467 } 2468 2469 static void mlx5e_activate_channel(struct mlx5e_channel *c) 2470 { 2471 int tc; 2472 2473 napi_enable(&c->napi); 2474 2475 for (tc = 0; tc < c->num_tc; tc++) 2476 mlx5e_activate_txqsq(&c->sq[tc]); 2477 mlx5e_activate_icosq(&c->icosq); 2478 mlx5e_activate_icosq(&c->async_icosq); 2479 2480 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2481 mlx5e_activate_xsk(c); 2482 else 2483 mlx5e_activate_rq(&c->rq); 2484 2485 mlx5e_trigger_napi_icosq(c); 2486 } 2487 2488 static void mlx5e_deactivate_channel(struct mlx5e_channel *c) 2489 { 2490 int tc; 2491 2492 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2493 mlx5e_deactivate_xsk(c); 2494 else 2495 mlx5e_deactivate_rq(&c->rq); 2496 2497 mlx5e_deactivate_icosq(&c->async_icosq); 2498 mlx5e_deactivate_icosq(&c->icosq); 2499 for (tc = 0; tc < c->num_tc; tc++) 2500 mlx5e_deactivate_txqsq(&c->sq[tc]); 2501 mlx5e_qos_deactivate_queues(c); 2502 2503 napi_disable(&c->napi); 2504 } 2505 2506 static void mlx5e_close_channel(struct mlx5e_channel *c) 2507 { 2508 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2509 mlx5e_close_xsk(c); 2510 mlx5e_close_queues(c); 2511 mlx5e_qos_close_queues(c); 2512 netif_napi_del(&c->napi); 2513 2514 kvfree(c); 2515 } 2516 2517 int mlx5e_open_channels(struct mlx5e_priv *priv, 2518 struct mlx5e_channels *chs) 2519 { 2520 struct mlx5e_channel_param *cparam; 2521 int err = -ENOMEM; 2522 int i; 2523 2524 chs->num = chs->params.num_channels; 2525 2526 chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL); 2527 cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL); 2528 if (!chs->c || !cparam) 2529 goto err_free; 2530 2531 err = mlx5e_build_channel_param(priv->mdev, &chs->params, priv->q_counter, cparam); 2532 if (err) 2533 goto err_free; 2534 2535 for (i = 0; i < chs->num; i++) { 2536 struct xsk_buff_pool *xsk_pool = NULL; 2537 2538 if (chs->params.xdp_prog) 2539 xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i); 2540 2541 err = mlx5e_open_channel(priv, i, &chs->params, cparam, xsk_pool, &chs->c[i]); 2542 if (err) 2543 goto err_close_channels; 2544 } 2545 2546 if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || chs->params.ptp_rx) { 2547 err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp); 2548 if (err) 2549 goto err_close_channels; 2550 } 2551 2552 if (priv->htb) { 2553 err = mlx5e_qos_open_queues(priv, chs); 2554 if (err) 2555 goto err_close_ptp; 2556 } 2557 2558 mlx5e_health_channels_update(priv); 2559 kvfree(cparam); 2560 return 0; 2561 2562 err_close_ptp: 2563 if (chs->ptp) 2564 mlx5e_ptp_close(chs->ptp); 2565 2566 err_close_channels: 2567 for (i--; i >= 0; i--) 2568 mlx5e_close_channel(chs->c[i]); 2569 2570 err_free: 2571 kfree(chs->c); 2572 kvfree(cparam); 2573 chs->num = 0; 2574 return err; 2575 } 2576 2577 static void mlx5e_activate_channels(struct mlx5e_channels *chs) 2578 { 2579 int i; 2580 2581 for (i = 0; i < chs->num; i++) 2582 mlx5e_activate_channel(chs->c[i]); 2583 2584 if (chs->ptp) 2585 mlx5e_ptp_activate_channel(chs->ptp); 2586 } 2587 2588 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) 2589 { 2590 int err = 0; 2591 int i; 2592 2593 for (i = 0; i < chs->num; i++) { 2594 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT; 2595 struct mlx5e_channel *c = chs->c[i]; 2596 2597 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2598 continue; 2599 2600 err |= mlx5e_wait_for_min_rx_wqes(&c->rq, timeout); 2601 2602 /* Don't wait on the XSK RQ, because the newer xdpsock sample 2603 * doesn't provide any Fill Ring entries at the setup stage. 2604 */ 2605 } 2606 2607 return err ? -ETIMEDOUT : 0; 2608 } 2609 2610 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs) 2611 { 2612 int i; 2613 2614 if (chs->ptp) 2615 mlx5e_ptp_deactivate_channel(chs->ptp); 2616 2617 for (i = 0; i < chs->num; i++) 2618 mlx5e_deactivate_channel(chs->c[i]); 2619 } 2620 2621 void mlx5e_close_channels(struct mlx5e_channels *chs) 2622 { 2623 int i; 2624 2625 if (chs->ptp) { 2626 mlx5e_ptp_close(chs->ptp); 2627 chs->ptp = NULL; 2628 } 2629 for (i = 0; i < chs->num; i++) 2630 mlx5e_close_channel(chs->c[i]); 2631 2632 kfree(chs->c); 2633 chs->num = 0; 2634 } 2635 2636 static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv) 2637 { 2638 struct mlx5e_rx_res *res = priv->rx_res; 2639 2640 return mlx5e_rx_res_packet_merge_set_param(res, &priv->channels.params.packet_merge); 2641 } 2642 2643 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge); 2644 2645 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev, 2646 struct mlx5e_params *params, u16 mtu) 2647 { 2648 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu); 2649 int err; 2650 2651 err = mlx5_set_port_mtu(mdev, hw_mtu, 1); 2652 if (err) 2653 return err; 2654 2655 /* Update vport context MTU */ 2656 mlx5_modify_nic_vport_mtu(mdev, hw_mtu); 2657 return 0; 2658 } 2659 2660 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev, 2661 struct mlx5e_params *params, u16 *mtu) 2662 { 2663 u16 hw_mtu = 0; 2664 int err; 2665 2666 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2667 if (err || !hw_mtu) /* fallback to port oper mtu */ 2668 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1); 2669 2670 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu); 2671 } 2672 2673 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) 2674 { 2675 struct mlx5e_params *params = &priv->channels.params; 2676 struct net_device *netdev = priv->netdev; 2677 struct mlx5_core_dev *mdev = priv->mdev; 2678 u16 mtu; 2679 int err; 2680 2681 err = mlx5e_set_mtu(mdev, params, params->sw_mtu); 2682 if (err) 2683 return err; 2684 2685 mlx5e_query_mtu(mdev, params, &mtu); 2686 if (mtu != params->sw_mtu) 2687 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n", 2688 __func__, mtu, params->sw_mtu); 2689 2690 params->sw_mtu = mtu; 2691 return 0; 2692 } 2693 2694 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu); 2695 2696 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv) 2697 { 2698 struct mlx5e_params *params = &priv->channels.params; 2699 struct net_device *netdev = priv->netdev; 2700 struct mlx5_core_dev *mdev = priv->mdev; 2701 u16 max_mtu; 2702 2703 /* MTU range: 68 - hw-specific max */ 2704 netdev->min_mtu = ETH_MIN_MTU; 2705 2706 mlx5_query_port_max_mtu(mdev, &max_mtu, 1); 2707 netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu), 2708 ETH_MAX_MTU); 2709 } 2710 2711 static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc, 2712 struct netdev_tc_txq *tc_to_txq) 2713 { 2714 int tc, err; 2715 2716 netdev_reset_tc(netdev); 2717 2718 if (ntc == 1) 2719 return 0; 2720 2721 err = netdev_set_num_tc(netdev, ntc); 2722 if (err) { 2723 netdev_WARN(netdev, "netdev_set_num_tc failed (%d), ntc = %d\n", err, ntc); 2724 return err; 2725 } 2726 2727 for (tc = 0; tc < ntc; tc++) { 2728 u16 count, offset; 2729 2730 count = tc_to_txq[tc].count; 2731 offset = tc_to_txq[tc].offset; 2732 netdev_set_tc_queue(netdev, tc, count, offset); 2733 } 2734 2735 return 0; 2736 } 2737 2738 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv) 2739 { 2740 int nch, ntc, num_txqs, err; 2741 int qos_queues = 0; 2742 2743 if (priv->htb) 2744 qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb); 2745 2746 nch = priv->channels.params.num_channels; 2747 ntc = mlx5e_get_dcb_num_tc(&priv->channels.params); 2748 num_txqs = nch * ntc + qos_queues; 2749 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS)) 2750 num_txqs += ntc; 2751 2752 mlx5e_dbg(DRV, priv, "Setting num_txqs %d\n", num_txqs); 2753 err = netif_set_real_num_tx_queues(priv->netdev, num_txqs); 2754 if (err) 2755 netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err); 2756 2757 return err; 2758 } 2759 2760 static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) 2761 { 2762 struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq; 2763 struct net_device *netdev = priv->netdev; 2764 int old_num_txqs, old_ntc; 2765 int nch, ntc; 2766 int err; 2767 int i; 2768 2769 old_num_txqs = netdev->real_num_tx_queues; 2770 old_ntc = netdev->num_tc ? : 1; 2771 for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++) 2772 old_tc_to_txq[i] = netdev->tc_to_txq[i]; 2773 2774 nch = priv->channels.params.num_channels; 2775 ntc = priv->channels.params.mqprio.num_tc; 2776 tc_to_txq = priv->channels.params.mqprio.tc_to_txq; 2777 2778 err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq); 2779 if (err) 2780 goto err_out; 2781 err = mlx5e_update_tx_netdev_queues(priv); 2782 if (err) 2783 goto err_tcs; 2784 err = netif_set_real_num_rx_queues(netdev, nch); 2785 if (err) { 2786 netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err); 2787 goto err_txqs; 2788 } 2789 2790 return 0; 2791 2792 err_txqs: 2793 /* netif_set_real_num_rx_queues could fail only when nch increased. Only 2794 * one of nch and ntc is changed in this function. That means, the call 2795 * to netif_set_real_num_tx_queues below should not fail, because it 2796 * decreases the number of TX queues. 2797 */ 2798 WARN_ON_ONCE(netif_set_real_num_tx_queues(netdev, old_num_txqs)); 2799 2800 err_tcs: 2801 WARN_ON_ONCE(mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc, 2802 old_tc_to_txq)); 2803 err_out: 2804 return err; 2805 } 2806 2807 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_netdev_queues); 2808 2809 static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv, 2810 struct mlx5e_params *params) 2811 { 2812 struct mlx5_core_dev *mdev = priv->mdev; 2813 int num_comp_vectors, ix, irq; 2814 2815 num_comp_vectors = mlx5_comp_vectors_count(mdev); 2816 2817 for (ix = 0; ix < params->num_channels; ix++) { 2818 cpumask_clear(priv->scratchpad.cpumask); 2819 2820 for (irq = ix; irq < num_comp_vectors; irq += params->num_channels) { 2821 int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(mdev, irq)); 2822 2823 cpumask_set_cpu(cpu, priv->scratchpad.cpumask); 2824 } 2825 2826 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix); 2827 } 2828 } 2829 2830 static int mlx5e_num_channels_changed(struct mlx5e_priv *priv) 2831 { 2832 u16 count = priv->channels.params.num_channels; 2833 int err; 2834 2835 err = mlx5e_update_netdev_queues(priv); 2836 if (err) 2837 return err; 2838 2839 mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params); 2840 2841 /* This function may be called on attach, before priv->rx_res is created. */ 2842 if (!netif_is_rxfh_configured(priv->netdev) && priv->rx_res) 2843 mlx5e_rx_res_rss_set_indir_uniform(priv->rx_res, count); 2844 2845 return 0; 2846 } 2847 2848 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed); 2849 2850 static void mlx5e_build_txq_maps(struct mlx5e_priv *priv) 2851 { 2852 int i, ch, tc, num_tc; 2853 2854 ch = priv->channels.num; 2855 num_tc = mlx5e_get_dcb_num_tc(&priv->channels.params); 2856 2857 for (i = 0; i < ch; i++) { 2858 for (tc = 0; tc < num_tc; tc++) { 2859 struct mlx5e_channel *c = priv->channels.c[i]; 2860 struct mlx5e_txqsq *sq = &c->sq[tc]; 2861 2862 priv->txq2sq[sq->txq_ix] = sq; 2863 } 2864 } 2865 2866 if (!priv->channels.ptp) 2867 goto out; 2868 2869 if (!test_bit(MLX5E_PTP_STATE_TX, priv->channels.ptp->state)) 2870 goto out; 2871 2872 for (tc = 0; tc < num_tc; tc++) { 2873 struct mlx5e_ptp *c = priv->channels.ptp; 2874 struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq; 2875 2876 priv->txq2sq[sq->txq_ix] = sq; 2877 } 2878 2879 out: 2880 /* Make the change to txq2sq visible before the queue is started. 2881 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 2882 * which pairs with this barrier. 2883 */ 2884 smp_wmb(); 2885 } 2886 2887 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) 2888 { 2889 mlx5e_build_txq_maps(priv); 2890 mlx5e_activate_channels(&priv->channels); 2891 if (priv->htb) 2892 mlx5e_qos_activate_queues(priv); 2893 mlx5e_xdp_tx_enable(priv); 2894 2895 /* dev_watchdog() wants all TX queues to be started when the carrier is 2896 * OK, including the ones in range real_num_tx_queues..num_tx_queues-1. 2897 * Make it happy to avoid TX timeout false alarms. 2898 */ 2899 netif_tx_start_all_queues(priv->netdev); 2900 2901 if (mlx5e_is_vport_rep(priv)) 2902 mlx5e_rep_activate_channels(priv); 2903 2904 mlx5e_wait_channels_min_rx_wqes(&priv->channels); 2905 2906 if (priv->rx_res) 2907 mlx5e_rx_res_channels_activate(priv->rx_res, &priv->channels); 2908 } 2909 2910 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv) 2911 { 2912 if (priv->rx_res) 2913 mlx5e_rx_res_channels_deactivate(priv->rx_res); 2914 2915 if (mlx5e_is_vport_rep(priv)) 2916 mlx5e_rep_deactivate_channels(priv); 2917 2918 /* The results of ndo_select_queue are unreliable, while netdev config 2919 * is being changed (real_num_tx_queues, num_tc). Stop all queues to 2920 * prevent ndo_start_xmit from being called, so that it can assume that 2921 * the selected queue is always valid. 2922 */ 2923 netif_tx_disable(priv->netdev); 2924 2925 mlx5e_xdp_tx_disable(priv); 2926 mlx5e_deactivate_channels(&priv->channels); 2927 } 2928 2929 static int mlx5e_switch_priv_params(struct mlx5e_priv *priv, 2930 struct mlx5e_params *new_params, 2931 mlx5e_fp_preactivate preactivate, 2932 void *context) 2933 { 2934 struct mlx5e_params old_params; 2935 2936 old_params = priv->channels.params; 2937 priv->channels.params = *new_params; 2938 2939 if (preactivate) { 2940 int err; 2941 2942 err = preactivate(priv, context); 2943 if (err) { 2944 priv->channels.params = old_params; 2945 return err; 2946 } 2947 } 2948 2949 return 0; 2950 } 2951 2952 static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv, 2953 struct mlx5e_channels *new_chs, 2954 mlx5e_fp_preactivate preactivate, 2955 void *context) 2956 { 2957 struct net_device *netdev = priv->netdev; 2958 struct mlx5e_channels old_chs; 2959 int carrier_ok; 2960 int err = 0; 2961 2962 carrier_ok = netif_carrier_ok(netdev); 2963 netif_carrier_off(netdev); 2964 2965 mlx5e_deactivate_priv_channels(priv); 2966 2967 old_chs = priv->channels; 2968 priv->channels = *new_chs; 2969 2970 /* New channels are ready to roll, call the preactivate hook if needed 2971 * to modify HW settings or update kernel parameters. 2972 */ 2973 if (preactivate) { 2974 err = preactivate(priv, context); 2975 if (err) { 2976 priv->channels = old_chs; 2977 goto out; 2978 } 2979 } 2980 2981 mlx5e_close_channels(&old_chs); 2982 priv->profile->update_rx(priv); 2983 2984 mlx5e_selq_apply(&priv->selq); 2985 out: 2986 mlx5e_activate_priv_channels(priv); 2987 2988 /* return carrier back if needed */ 2989 if (carrier_ok) 2990 netif_carrier_on(netdev); 2991 2992 return err; 2993 } 2994 2995 int mlx5e_safe_switch_params(struct mlx5e_priv *priv, 2996 struct mlx5e_params *params, 2997 mlx5e_fp_preactivate preactivate, 2998 void *context, bool reset) 2999 { 3000 struct mlx5e_channels new_chs = {}; 3001 int err; 3002 3003 reset &= test_bit(MLX5E_STATE_OPENED, &priv->state); 3004 if (!reset) 3005 return mlx5e_switch_priv_params(priv, params, preactivate, context); 3006 3007 new_chs.params = *params; 3008 3009 mlx5e_selq_prepare_params(&priv->selq, &new_chs.params); 3010 3011 err = mlx5e_open_channels(priv, &new_chs); 3012 if (err) 3013 goto err_cancel_selq; 3014 3015 err = mlx5e_switch_priv_channels(priv, &new_chs, preactivate, context); 3016 if (err) 3017 goto err_close; 3018 3019 return 0; 3020 3021 err_close: 3022 mlx5e_close_channels(&new_chs); 3023 3024 err_cancel_selq: 3025 mlx5e_selq_cancel(&priv->selq); 3026 return err; 3027 } 3028 3029 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv) 3030 { 3031 return mlx5e_safe_switch_params(priv, &priv->channels.params, NULL, NULL, true); 3032 } 3033 3034 void mlx5e_timestamp_init(struct mlx5e_priv *priv) 3035 { 3036 priv->tstamp.tx_type = HWTSTAMP_TX_OFF; 3037 priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE; 3038 } 3039 3040 static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev, 3041 enum mlx5_port_status state) 3042 { 3043 struct mlx5_eswitch *esw = mdev->priv.eswitch; 3044 int vport_admin_state; 3045 3046 mlx5_set_port_admin_status(mdev, state); 3047 3048 if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS || 3049 !MLX5_CAP_GEN(mdev, uplink_follow)) 3050 return; 3051 3052 if (state == MLX5_PORT_UP) 3053 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO; 3054 else 3055 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3056 3057 mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state); 3058 } 3059 3060 int mlx5e_open_locked(struct net_device *netdev) 3061 { 3062 struct mlx5e_priv *priv = netdev_priv(netdev); 3063 int err; 3064 3065 mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params); 3066 3067 set_bit(MLX5E_STATE_OPENED, &priv->state); 3068 3069 err = mlx5e_open_channels(priv, &priv->channels); 3070 if (err) 3071 goto err_clear_state_opened_flag; 3072 3073 err = priv->profile->update_rx(priv); 3074 if (err) 3075 goto err_close_channels; 3076 3077 mlx5e_selq_apply(&priv->selq); 3078 mlx5e_activate_priv_channels(priv); 3079 mlx5e_apply_traps(priv, true); 3080 if (priv->profile->update_carrier) 3081 priv->profile->update_carrier(priv); 3082 3083 mlx5e_queue_update_stats(priv); 3084 return 0; 3085 3086 err_close_channels: 3087 mlx5e_close_channels(&priv->channels); 3088 err_clear_state_opened_flag: 3089 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3090 mlx5e_selq_cancel(&priv->selq); 3091 return err; 3092 } 3093 3094 int mlx5e_open(struct net_device *netdev) 3095 { 3096 struct mlx5e_priv *priv = netdev_priv(netdev); 3097 int err; 3098 3099 mutex_lock(&priv->state_lock); 3100 err = mlx5e_open_locked(netdev); 3101 if (!err) 3102 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP); 3103 mutex_unlock(&priv->state_lock); 3104 3105 return err; 3106 } 3107 3108 int mlx5e_close_locked(struct net_device *netdev) 3109 { 3110 struct mlx5e_priv *priv = netdev_priv(netdev); 3111 3112 /* May already be CLOSED in case a previous configuration operation 3113 * (e.g RX/TX queue size change) that involves close&open failed. 3114 */ 3115 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3116 return 0; 3117 3118 mlx5e_apply_traps(priv, false); 3119 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3120 3121 netif_carrier_off(priv->netdev); 3122 mlx5e_deactivate_priv_channels(priv); 3123 mlx5e_close_channels(&priv->channels); 3124 3125 return 0; 3126 } 3127 3128 int mlx5e_close(struct net_device *netdev) 3129 { 3130 struct mlx5e_priv *priv = netdev_priv(netdev); 3131 int err; 3132 3133 if (!netif_device_present(netdev)) 3134 return -ENODEV; 3135 3136 mutex_lock(&priv->state_lock); 3137 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN); 3138 err = mlx5e_close_locked(netdev); 3139 mutex_unlock(&priv->state_lock); 3140 3141 return err; 3142 } 3143 3144 static void mlx5e_free_drop_rq(struct mlx5e_rq *rq) 3145 { 3146 mlx5_wq_destroy(&rq->wq_ctrl); 3147 } 3148 3149 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev, 3150 struct mlx5e_rq *rq, 3151 struct mlx5e_rq_param *param) 3152 { 3153 void *rqc = param->rqc; 3154 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 3155 int err; 3156 3157 param->wq.db_numa_node = param->wq.buf_numa_node; 3158 3159 err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq, 3160 &rq->wq_ctrl); 3161 if (err) 3162 return err; 3163 3164 /* Mark as unused given "Drop-RQ" packets never reach XDP */ 3165 xdp_rxq_info_unused(&rq->xdp_rxq); 3166 3167 rq->mdev = mdev; 3168 3169 return 0; 3170 } 3171 3172 static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv, 3173 struct mlx5e_cq *cq, 3174 struct mlx5e_cq_param *param) 3175 { 3176 struct mlx5_core_dev *mdev = priv->mdev; 3177 3178 param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 3179 param->wq.db_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 3180 3181 return mlx5e_alloc_cq_common(priv, param, cq); 3182 } 3183 3184 int mlx5e_open_drop_rq(struct mlx5e_priv *priv, 3185 struct mlx5e_rq *drop_rq) 3186 { 3187 struct mlx5_core_dev *mdev = priv->mdev; 3188 struct mlx5e_cq_param cq_param = {}; 3189 struct mlx5e_rq_param rq_param = {}; 3190 struct mlx5e_cq *cq = &drop_rq->cq; 3191 int err; 3192 3193 mlx5e_build_drop_rq_param(mdev, priv->drop_rq_q_counter, &rq_param); 3194 3195 err = mlx5e_alloc_drop_cq(priv, cq, &cq_param); 3196 if (err) 3197 return err; 3198 3199 err = mlx5e_create_cq(cq, &cq_param); 3200 if (err) 3201 goto err_free_cq; 3202 3203 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param); 3204 if (err) 3205 goto err_destroy_cq; 3206 3207 err = mlx5e_create_rq(drop_rq, &rq_param); 3208 if (err) 3209 goto err_free_rq; 3210 3211 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 3212 if (err) 3213 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err); 3214 3215 return 0; 3216 3217 err_free_rq: 3218 mlx5e_free_drop_rq(drop_rq); 3219 3220 err_destroy_cq: 3221 mlx5e_destroy_cq(cq); 3222 3223 err_free_cq: 3224 mlx5e_free_cq(cq); 3225 3226 return err; 3227 } 3228 3229 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq) 3230 { 3231 mlx5e_destroy_rq(drop_rq); 3232 mlx5e_free_drop_rq(drop_rq); 3233 mlx5e_destroy_cq(&drop_rq->cq); 3234 mlx5e_free_cq(&drop_rq->cq); 3235 } 3236 3237 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn) 3238 { 3239 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3240 3241 MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn); 3242 3243 if (MLX5_GET(tisc, tisc, tls_en)) 3244 MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.hw_objs.pdn); 3245 3246 if (mlx5_lag_is_lacp_owner(mdev)) 3247 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1); 3248 3249 return mlx5_core_create_tis(mdev, in, tisn); 3250 } 3251 3252 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn) 3253 { 3254 mlx5_core_destroy_tis(mdev, tisn); 3255 } 3256 3257 void mlx5e_destroy_tises(struct mlx5e_priv *priv) 3258 { 3259 int tc, i; 3260 3261 for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) 3262 for (tc = 0; tc < priv->profile->max_tc; tc++) 3263 mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]); 3264 } 3265 3266 static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev *mdev) 3267 { 3268 return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1; 3269 } 3270 3271 int mlx5e_create_tises(struct mlx5e_priv *priv) 3272 { 3273 int tc, i; 3274 int err; 3275 3276 for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) { 3277 for (tc = 0; tc < priv->profile->max_tc; tc++) { 3278 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 3279 void *tisc; 3280 3281 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3282 3283 MLX5_SET(tisc, tisc, prio, tc << 1); 3284 3285 if (mlx5e_lag_should_assign_affinity(priv->mdev)) 3286 MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1); 3287 3288 err = mlx5e_create_tis(priv->mdev, in, &priv->tisn[i][tc]); 3289 if (err) 3290 goto err_close_tises; 3291 } 3292 } 3293 3294 return 0; 3295 3296 err_close_tises: 3297 for (; i >= 0; i--) { 3298 for (tc--; tc >= 0; tc--) 3299 mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]); 3300 tc = priv->profile->max_tc; 3301 } 3302 3303 return err; 3304 } 3305 3306 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) 3307 { 3308 if (priv->mqprio_rl) { 3309 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3310 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3311 priv->mqprio_rl = NULL; 3312 } 3313 mlx5e_accel_cleanup_tx(priv); 3314 mlx5e_destroy_tises(priv); 3315 } 3316 3317 static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable) 3318 { 3319 int err = 0; 3320 int i; 3321 3322 for (i = 0; i < chs->num; i++) { 3323 err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable); 3324 if (err) 3325 return err; 3326 } 3327 3328 return 0; 3329 } 3330 3331 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) 3332 { 3333 int err; 3334 int i; 3335 3336 for (i = 0; i < chs->num; i++) { 3337 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd); 3338 if (err) 3339 return err; 3340 } 3341 if (chs->ptp && test_bit(MLX5E_PTP_STATE_RX, chs->ptp->state)) 3342 return mlx5e_modify_rq_vsd(&chs->ptp->rq, vsd); 3343 3344 return 0; 3345 } 3346 3347 static void mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq *tc_to_txq, 3348 int ntc, int nch) 3349 { 3350 int tc; 3351 3352 memset(tc_to_txq, 0, sizeof(*tc_to_txq) * TC_MAX_QUEUE); 3353 3354 /* Map netdev TCs to offset 0. 3355 * We have our own UP to TXQ mapping for DCB mode of QoS 3356 */ 3357 for (tc = 0; tc < ntc; tc++) { 3358 tc_to_txq[tc] = (struct netdev_tc_txq) { 3359 .count = nch, 3360 .offset = 0, 3361 }; 3362 } 3363 } 3364 3365 static void mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq *tc_to_txq, 3366 struct tc_mqprio_qopt *qopt) 3367 { 3368 int tc; 3369 3370 for (tc = 0; tc < TC_MAX_QUEUE; tc++) { 3371 tc_to_txq[tc] = (struct netdev_tc_txq) { 3372 .count = qopt->count[tc], 3373 .offset = qopt->offset[tc], 3374 }; 3375 } 3376 } 3377 3378 static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc) 3379 { 3380 params->mqprio.mode = TC_MQPRIO_MODE_DCB; 3381 params->mqprio.num_tc = num_tc; 3382 mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc, 3383 params->num_channels); 3384 } 3385 3386 static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params, 3387 struct mlx5e_mqprio_rl *rl) 3388 { 3389 int tc; 3390 3391 for (tc = 0; tc < TC_MAX_QUEUE; tc++) { 3392 u32 hw_id = 0; 3393 3394 if (rl) 3395 mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id); 3396 params->mqprio.channel.hw_id[tc] = hw_id; 3397 } 3398 } 3399 3400 static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params, 3401 struct tc_mqprio_qopt_offload *mqprio, 3402 struct mlx5e_mqprio_rl *rl) 3403 { 3404 int tc; 3405 3406 params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL; 3407 params->mqprio.num_tc = mqprio->qopt.num_tc; 3408 3409 for (tc = 0; tc < TC_MAX_QUEUE; tc++) 3410 params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc]; 3411 3412 mlx5e_mqprio_rl_update_params(params, rl); 3413 mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt); 3414 } 3415 3416 static void mlx5e_params_mqprio_reset(struct mlx5e_params *params) 3417 { 3418 mlx5e_params_mqprio_dcb_set(params, 1); 3419 } 3420 3421 static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv, 3422 struct tc_mqprio_qopt *mqprio) 3423 { 3424 struct mlx5e_params new_params; 3425 u8 tc = mqprio->num_tc; 3426 int err; 3427 3428 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 3429 3430 if (tc && tc != MLX5E_MAX_NUM_TC) 3431 return -EINVAL; 3432 3433 new_params = priv->channels.params; 3434 mlx5e_params_mqprio_dcb_set(&new_params, tc ? tc : 1); 3435 3436 err = mlx5e_safe_switch_params(priv, &new_params, 3437 mlx5e_num_channels_changed_ctx, NULL, true); 3438 3439 if (!err && priv->mqprio_rl) { 3440 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3441 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3442 priv->mqprio_rl = NULL; 3443 } 3444 3445 priv->max_opened_tc = max_t(u8, priv->max_opened_tc, 3446 mlx5e_get_dcb_num_tc(&priv->channels.params)); 3447 return err; 3448 } 3449 3450 static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv, 3451 struct tc_mqprio_qopt_offload *mqprio) 3452 { 3453 struct net_device *netdev = priv->netdev; 3454 struct mlx5e_ptp *ptp_channel; 3455 int agg_count = 0; 3456 int i; 3457 3458 ptp_channel = priv->channels.ptp; 3459 if (ptp_channel && test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state)) { 3460 netdev_err(netdev, 3461 "Cannot activate MQPRIO mode channel since it conflicts with TX port TS\n"); 3462 return -EINVAL; 3463 } 3464 3465 if (mqprio->qopt.offset[0] != 0 || mqprio->qopt.num_tc < 1 || 3466 mqprio->qopt.num_tc > MLX5E_MAX_NUM_MQPRIO_CH_TC) 3467 return -EINVAL; 3468 3469 for (i = 0; i < mqprio->qopt.num_tc; i++) { 3470 if (!mqprio->qopt.count[i]) { 3471 netdev_err(netdev, "Zero size for queue-group (%d) is not supported\n", i); 3472 return -EINVAL; 3473 } 3474 if (mqprio->min_rate[i]) { 3475 netdev_err(netdev, "Min tx rate is not supported\n"); 3476 return -EINVAL; 3477 } 3478 3479 if (mqprio->max_rate[i]) { 3480 int err; 3481 3482 err = mlx5e_qos_bytes_rate_check(priv->mdev, mqprio->max_rate[i]); 3483 if (err) 3484 return err; 3485 } 3486 3487 if (mqprio->qopt.offset[i] != agg_count) { 3488 netdev_err(netdev, "Discontinuous queues config is not supported\n"); 3489 return -EINVAL; 3490 } 3491 agg_count += mqprio->qopt.count[i]; 3492 } 3493 3494 if (priv->channels.params.num_channels != agg_count) { 3495 netdev_err(netdev, "Num of queues (%d) does not match available (%d)\n", 3496 agg_count, priv->channels.params.num_channels); 3497 return -EINVAL; 3498 } 3499 3500 return 0; 3501 } 3502 3503 static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[]) 3504 { 3505 int tc; 3506 3507 for (tc = 0; tc < num_tc; tc++) 3508 if (max_rate[tc]) 3509 return true; 3510 return false; 3511 } 3512 3513 static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev, 3514 u8 num_tc, u64 max_rate[]) 3515 { 3516 struct mlx5e_mqprio_rl *rl; 3517 int err; 3518 3519 if (!mlx5e_mqprio_rate_limit(num_tc, max_rate)) 3520 return NULL; 3521 3522 rl = mlx5e_mqprio_rl_alloc(); 3523 if (!rl) 3524 return ERR_PTR(-ENOMEM); 3525 3526 err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate); 3527 if (err) { 3528 mlx5e_mqprio_rl_free(rl); 3529 return ERR_PTR(err); 3530 } 3531 3532 return rl; 3533 } 3534 3535 static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, 3536 struct tc_mqprio_qopt_offload *mqprio) 3537 { 3538 mlx5e_fp_preactivate preactivate; 3539 struct mlx5e_params new_params; 3540 struct mlx5e_mqprio_rl *rl; 3541 bool nch_changed; 3542 int err; 3543 3544 err = mlx5e_mqprio_channel_validate(priv, mqprio); 3545 if (err) 3546 return err; 3547 3548 rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate); 3549 if (IS_ERR(rl)) 3550 return PTR_ERR(rl); 3551 3552 new_params = priv->channels.params; 3553 mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl); 3554 3555 nch_changed = mlx5e_get_dcb_num_tc(&priv->channels.params) > 1; 3556 preactivate = nch_changed ? mlx5e_num_channels_changed_ctx : 3557 mlx5e_update_netdev_queues_ctx; 3558 err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, true); 3559 if (err) { 3560 if (rl) { 3561 mlx5e_mqprio_rl_cleanup(rl); 3562 mlx5e_mqprio_rl_free(rl); 3563 } 3564 return err; 3565 } 3566 3567 if (priv->mqprio_rl) { 3568 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3569 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3570 } 3571 priv->mqprio_rl = rl; 3572 3573 return 0; 3574 } 3575 3576 static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, 3577 struct tc_mqprio_qopt_offload *mqprio) 3578 { 3579 /* MQPRIO is another toplevel qdisc that can't be attached 3580 * simultaneously with the offloaded HTB. 3581 */ 3582 if (WARN_ON(mlx5e_selq_is_htb_enabled(&priv->selq))) 3583 return -EINVAL; 3584 3585 switch (mqprio->mode) { 3586 case TC_MQPRIO_MODE_DCB: 3587 return mlx5e_setup_tc_mqprio_dcb(priv, &mqprio->qopt); 3588 case TC_MQPRIO_MODE_CHANNEL: 3589 return mlx5e_setup_tc_mqprio_channel(priv, mqprio); 3590 default: 3591 return -EOPNOTSUPP; 3592 } 3593 } 3594 3595 static LIST_HEAD(mlx5e_block_cb_list); 3596 3597 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, 3598 void *type_data) 3599 { 3600 struct mlx5e_priv *priv = netdev_priv(dev); 3601 bool tc_unbind = false; 3602 int err; 3603 3604 if (type == TC_SETUP_BLOCK && 3605 ((struct flow_block_offload *)type_data)->command == FLOW_BLOCK_UNBIND) 3606 tc_unbind = true; 3607 3608 if (!netif_device_present(dev) && !tc_unbind) 3609 return -ENODEV; 3610 3611 switch (type) { 3612 case TC_SETUP_BLOCK: { 3613 struct flow_block_offload *f = type_data; 3614 3615 f->unlocked_driver_cb = true; 3616 return flow_block_cb_setup_simple(type_data, 3617 &mlx5e_block_cb_list, 3618 mlx5e_setup_tc_block_cb, 3619 priv, priv, true); 3620 } 3621 case TC_SETUP_QDISC_MQPRIO: 3622 mutex_lock(&priv->state_lock); 3623 err = mlx5e_setup_tc_mqprio(priv, type_data); 3624 mutex_unlock(&priv->state_lock); 3625 return err; 3626 case TC_SETUP_QDISC_HTB: 3627 mutex_lock(&priv->state_lock); 3628 err = mlx5e_htb_setup_tc(priv, type_data); 3629 mutex_unlock(&priv->state_lock); 3630 return err; 3631 default: 3632 return -EOPNOTSUPP; 3633 } 3634 } 3635 3636 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s) 3637 { 3638 int i; 3639 3640 for (i = 0; i < priv->stats_nch; i++) { 3641 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; 3642 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq; 3643 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq; 3644 int j; 3645 3646 s->rx_packets += rq_stats->packets + xskrq_stats->packets; 3647 s->rx_bytes += rq_stats->bytes + xskrq_stats->bytes; 3648 s->multicast += rq_stats->mcast_packets + xskrq_stats->mcast_packets; 3649 3650 for (j = 0; j < priv->max_opened_tc; j++) { 3651 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j]; 3652 3653 s->tx_packets += sq_stats->packets; 3654 s->tx_bytes += sq_stats->bytes; 3655 s->tx_dropped += sq_stats->dropped; 3656 } 3657 } 3658 if (priv->tx_ptp_opened) { 3659 for (i = 0; i < priv->max_opened_tc; i++) { 3660 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[i]; 3661 3662 s->tx_packets += sq_stats->packets; 3663 s->tx_bytes += sq_stats->bytes; 3664 s->tx_dropped += sq_stats->dropped; 3665 } 3666 } 3667 if (priv->rx_ptp_opened) { 3668 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq; 3669 3670 s->rx_packets += rq_stats->packets; 3671 s->rx_bytes += rq_stats->bytes; 3672 s->multicast += rq_stats->mcast_packets; 3673 } 3674 } 3675 3676 void 3677 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 3678 { 3679 struct mlx5e_priv *priv = netdev_priv(dev); 3680 struct mlx5e_pport_stats *pstats = &priv->stats.pport; 3681 3682 if (!netif_device_present(dev)) 3683 return; 3684 3685 /* In switchdev mode, monitor counters doesn't monitor 3686 * rx/tx stats of 802_3. The update stats mechanism 3687 * should keep the 802_3 layout counters updated 3688 */ 3689 if (!mlx5e_monitor_counter_supported(priv) || 3690 mlx5e_is_uplink_rep(priv)) { 3691 /* update HW stats in background for next time */ 3692 mlx5e_queue_update_stats(priv); 3693 } 3694 3695 if (mlx5e_is_uplink_rep(priv)) { 3696 struct mlx5e_vport_stats *vstats = &priv->stats.vport; 3697 3698 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok); 3699 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok); 3700 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok); 3701 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok); 3702 3703 /* vport multicast also counts packets that are dropped due to steering 3704 * or rx out of buffer 3705 */ 3706 stats->multicast = VPORT_COUNTER_GET(vstats, received_eth_multicast.packets); 3707 } else { 3708 mlx5e_fold_sw_stats64(priv, stats); 3709 } 3710 3711 stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer; 3712 3713 stats->rx_length_errors = 3714 PPORT_802_3_GET(pstats, a_in_range_length_errors) + 3715 PPORT_802_3_GET(pstats, a_out_of_range_length_field) + 3716 PPORT_802_3_GET(pstats, a_frame_too_long_errors) + 3717 VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small); 3718 stats->rx_crc_errors = 3719 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors); 3720 stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors); 3721 stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards); 3722 stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors + 3723 stats->rx_frame_errors; 3724 stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors; 3725 } 3726 3727 static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) 3728 { 3729 if (mlx5e_is_uplink_rep(priv)) 3730 return; /* no rx mode for uplink rep */ 3731 3732 queue_work(priv->wq, &priv->set_rx_mode_work); 3733 } 3734 3735 static void mlx5e_set_rx_mode(struct net_device *dev) 3736 { 3737 struct mlx5e_priv *priv = netdev_priv(dev); 3738 3739 mlx5e_nic_set_rx_mode(priv); 3740 } 3741 3742 static int mlx5e_set_mac(struct net_device *netdev, void *addr) 3743 { 3744 struct mlx5e_priv *priv = netdev_priv(netdev); 3745 struct sockaddr *saddr = addr; 3746 3747 if (!is_valid_ether_addr(saddr->sa_data)) 3748 return -EADDRNOTAVAIL; 3749 3750 netif_addr_lock_bh(netdev); 3751 eth_hw_addr_set(netdev, saddr->sa_data); 3752 netif_addr_unlock_bh(netdev); 3753 3754 mlx5e_nic_set_rx_mode(priv); 3755 3756 return 0; 3757 } 3758 3759 #define MLX5E_SET_FEATURE(features, feature, enable) \ 3760 do { \ 3761 if (enable) \ 3762 *features |= feature; \ 3763 else \ 3764 *features &= ~feature; \ 3765 } while (0) 3766 3767 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable); 3768 3769 static int set_feature_lro(struct net_device *netdev, bool enable) 3770 { 3771 struct mlx5e_priv *priv = netdev_priv(netdev); 3772 struct mlx5_core_dev *mdev = priv->mdev; 3773 struct mlx5e_params *cur_params; 3774 struct mlx5e_params new_params; 3775 bool reset = true; 3776 int err = 0; 3777 3778 mutex_lock(&priv->state_lock); 3779 3780 cur_params = &priv->channels.params; 3781 new_params = *cur_params; 3782 3783 if (enable) 3784 new_params.packet_merge.type = MLX5E_PACKET_MERGE_LRO; 3785 else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO) 3786 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE; 3787 else 3788 goto out; 3789 3790 if (!(cur_params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO && 3791 new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)) { 3792 if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 3793 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) == 3794 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL)) 3795 reset = false; 3796 } 3797 } 3798 3799 err = mlx5e_safe_switch_params(priv, &new_params, 3800 mlx5e_modify_tirs_packet_merge_ctx, NULL, reset); 3801 out: 3802 mutex_unlock(&priv->state_lock); 3803 return err; 3804 } 3805 3806 static int set_feature_hw_gro(struct net_device *netdev, bool enable) 3807 { 3808 struct mlx5e_priv *priv = netdev_priv(netdev); 3809 struct mlx5e_params new_params; 3810 bool reset = true; 3811 int err = 0; 3812 3813 mutex_lock(&priv->state_lock); 3814 new_params = priv->channels.params; 3815 3816 if (enable) { 3817 new_params.packet_merge.type = MLX5E_PACKET_MERGE_SHAMPO; 3818 new_params.packet_merge.shampo.match_criteria_type = 3819 MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED; 3820 new_params.packet_merge.shampo.alignment_granularity = 3821 MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE; 3822 } else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) { 3823 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE; 3824 } else { 3825 goto out; 3826 } 3827 3828 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); 3829 out: 3830 mutex_unlock(&priv->state_lock); 3831 return err; 3832 } 3833 3834 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) 3835 { 3836 struct mlx5e_priv *priv = netdev_priv(netdev); 3837 3838 if (enable) 3839 mlx5e_enable_cvlan_filter(priv->fs, 3840 !!(priv->netdev->flags & IFF_PROMISC)); 3841 else 3842 mlx5e_disable_cvlan_filter(priv->fs, 3843 !!(priv->netdev->flags & IFF_PROMISC)); 3844 3845 return 0; 3846 } 3847 3848 static int set_feature_hw_tc(struct net_device *netdev, bool enable) 3849 { 3850 struct mlx5e_priv *priv = netdev_priv(netdev); 3851 int err = 0; 3852 3853 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 3854 int tc_flag = mlx5e_is_uplink_rep(priv) ? MLX5_TC_FLAG(ESW_OFFLOAD) : 3855 MLX5_TC_FLAG(NIC_OFFLOAD); 3856 if (!enable && mlx5e_tc_num_filters(priv, tc_flag)) { 3857 netdev_err(netdev, 3858 "Active offloaded tc filters, can't turn hw_tc_offload off\n"); 3859 return -EINVAL; 3860 } 3861 #endif 3862 3863 mutex_lock(&priv->state_lock); 3864 if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) { 3865 netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n"); 3866 err = -EINVAL; 3867 } 3868 mutex_unlock(&priv->state_lock); 3869 3870 return err; 3871 } 3872 3873 static int set_feature_rx_all(struct net_device *netdev, bool enable) 3874 { 3875 struct mlx5e_priv *priv = netdev_priv(netdev); 3876 struct mlx5_core_dev *mdev = priv->mdev; 3877 3878 return mlx5_set_port_fcs(mdev, !enable); 3879 } 3880 3881 static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable) 3882 { 3883 u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {}; 3884 bool supported, curr_state; 3885 int err; 3886 3887 if (!MLX5_CAP_GEN(mdev, ports_check)) 3888 return 0; 3889 3890 err = mlx5_query_ports_check(mdev, in, sizeof(in)); 3891 if (err) 3892 return err; 3893 3894 supported = MLX5_GET(pcmr_reg, in, rx_ts_over_crc_cap); 3895 curr_state = MLX5_GET(pcmr_reg, in, rx_ts_over_crc); 3896 3897 if (!supported || enable == curr_state) 3898 return 0; 3899 3900 MLX5_SET(pcmr_reg, in, local_port, 1); 3901 MLX5_SET(pcmr_reg, in, rx_ts_over_crc, enable); 3902 3903 return mlx5_set_ports_check(mdev, in, sizeof(in)); 3904 } 3905 3906 static int set_feature_rx_fcs(struct net_device *netdev, bool enable) 3907 { 3908 struct mlx5e_priv *priv = netdev_priv(netdev); 3909 struct mlx5e_channels *chs = &priv->channels; 3910 struct mlx5_core_dev *mdev = priv->mdev; 3911 int err; 3912 3913 mutex_lock(&priv->state_lock); 3914 3915 if (enable) { 3916 err = mlx5e_set_rx_port_ts(mdev, false); 3917 if (err) 3918 goto out; 3919 3920 chs->params.scatter_fcs_en = true; 3921 err = mlx5e_modify_channels_scatter_fcs(chs, true); 3922 if (err) { 3923 chs->params.scatter_fcs_en = false; 3924 mlx5e_set_rx_port_ts(mdev, true); 3925 } 3926 } else { 3927 chs->params.scatter_fcs_en = false; 3928 err = mlx5e_modify_channels_scatter_fcs(chs, false); 3929 if (err) { 3930 chs->params.scatter_fcs_en = true; 3931 goto out; 3932 } 3933 err = mlx5e_set_rx_port_ts(mdev, true); 3934 if (err) { 3935 mlx5_core_warn(mdev, "Failed to set RX port timestamp %d\n", err); 3936 err = 0; 3937 } 3938 } 3939 3940 out: 3941 mutex_unlock(&priv->state_lock); 3942 return err; 3943 } 3944 3945 static int set_feature_rx_vlan(struct net_device *netdev, bool enable) 3946 { 3947 struct mlx5e_priv *priv = netdev_priv(netdev); 3948 int err = 0; 3949 3950 mutex_lock(&priv->state_lock); 3951 3952 mlx5e_fs_set_vlan_strip_disable(priv->fs, !enable); 3953 priv->channels.params.vlan_strip_disable = !enable; 3954 3955 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3956 goto unlock; 3957 3958 err = mlx5e_modify_channels_vsd(&priv->channels, !enable); 3959 if (err) { 3960 mlx5e_fs_set_vlan_strip_disable(priv->fs, enable); 3961 priv->channels.params.vlan_strip_disable = enable; 3962 } 3963 unlock: 3964 mutex_unlock(&priv->state_lock); 3965 3966 return err; 3967 } 3968 3969 int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) 3970 { 3971 struct mlx5e_priv *priv = netdev_priv(dev); 3972 struct mlx5e_flow_steering *fs = priv->fs; 3973 3974 if (mlx5e_is_uplink_rep(priv)) 3975 return 0; /* no vlan table for uplink rep */ 3976 3977 return mlx5e_fs_vlan_rx_add_vid(fs, dev, proto, vid); 3978 } 3979 3980 int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) 3981 { 3982 struct mlx5e_priv *priv = netdev_priv(dev); 3983 struct mlx5e_flow_steering *fs = priv->fs; 3984 3985 if (mlx5e_is_uplink_rep(priv)) 3986 return 0; /* no vlan table for uplink rep */ 3987 3988 return mlx5e_fs_vlan_rx_kill_vid(fs, dev, proto, vid); 3989 } 3990 3991 #ifdef CONFIG_MLX5_EN_ARFS 3992 static int set_feature_arfs(struct net_device *netdev, bool enable) 3993 { 3994 struct mlx5e_priv *priv = netdev_priv(netdev); 3995 int err; 3996 3997 if (enable) 3998 err = mlx5e_arfs_enable(priv->fs); 3999 else 4000 err = mlx5e_arfs_disable(priv->fs); 4001 4002 return err; 4003 } 4004 #endif 4005 4006 static int mlx5e_handle_feature(struct net_device *netdev, 4007 netdev_features_t *features, 4008 netdev_features_t feature, 4009 mlx5e_feature_handler feature_handler) 4010 { 4011 netdev_features_t changes = *features ^ netdev->features; 4012 bool enable = !!(*features & feature); 4013 int err; 4014 4015 if (!(changes & feature)) 4016 return 0; 4017 4018 err = feature_handler(netdev, enable); 4019 if (err) { 4020 MLX5E_SET_FEATURE(features, feature, !enable); 4021 netdev_err(netdev, "%s feature %pNF failed, err %d\n", 4022 enable ? "Enable" : "Disable", &feature, err); 4023 return err; 4024 } 4025 4026 return 0; 4027 } 4028 4029 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) 4030 { 4031 netdev_features_t oper_features = features; 4032 int err = 0; 4033 4034 #define MLX5E_HANDLE_FEATURE(feature, handler) \ 4035 mlx5e_handle_feature(netdev, &oper_features, feature, handler) 4036 4037 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4038 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4039 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER, 4040 set_feature_cvlan_filter); 4041 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc); 4042 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all); 4043 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4044 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan); 4045 #ifdef CONFIG_MLX5_EN_ARFS 4046 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); 4047 #endif 4048 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx); 4049 4050 if (err) { 4051 netdev->features = oper_features; 4052 return -EINVAL; 4053 } 4054 4055 return 0; 4056 } 4057 4058 static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev, 4059 netdev_features_t features) 4060 { 4061 features &= ~NETIF_F_HW_TLS_RX; 4062 if (netdev->features & NETIF_F_HW_TLS_RX) 4063 netdev_warn(netdev, "Disabling hw_tls_rx, not supported in switchdev mode\n"); 4064 4065 features &= ~NETIF_F_HW_TLS_TX; 4066 if (netdev->features & NETIF_F_HW_TLS_TX) 4067 netdev_warn(netdev, "Disabling hw_tls_tx, not supported in switchdev mode\n"); 4068 4069 features &= ~NETIF_F_NTUPLE; 4070 if (netdev->features & NETIF_F_NTUPLE) 4071 netdev_warn(netdev, "Disabling ntuple, not supported in switchdev mode\n"); 4072 4073 features &= ~NETIF_F_GRO_HW; 4074 if (netdev->features & NETIF_F_GRO_HW) 4075 netdev_warn(netdev, "Disabling HW_GRO, not supported in switchdev mode\n"); 4076 4077 return features; 4078 } 4079 4080 static netdev_features_t mlx5e_fix_features(struct net_device *netdev, 4081 netdev_features_t features) 4082 { 4083 struct mlx5e_priv *priv = netdev_priv(netdev); 4084 struct mlx5e_vlan_table *vlan; 4085 struct mlx5e_params *params; 4086 4087 vlan = mlx5e_fs_get_vlan(priv->fs); 4088 mutex_lock(&priv->state_lock); 4089 params = &priv->channels.params; 4090 if (!vlan || 4091 !bitmap_empty(mlx5e_vlan_get_active_svlans(vlan), VLAN_N_VID)) { 4092 /* HW strips the outer C-tag header, this is a problem 4093 * for S-tag traffic. 4094 */ 4095 features &= ~NETIF_F_HW_VLAN_CTAG_RX; 4096 if (!params->vlan_strip_disable) 4097 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n"); 4098 } 4099 4100 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 4101 if (features & NETIF_F_LRO) { 4102 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n"); 4103 features &= ~NETIF_F_LRO; 4104 } 4105 if (features & NETIF_F_GRO_HW) { 4106 netdev_warn(netdev, "Disabling HW-GRO, not supported in legacy RQ\n"); 4107 features &= ~NETIF_F_GRO_HW; 4108 } 4109 } 4110 4111 if (params->xdp_prog) { 4112 if (features & NETIF_F_LRO) { 4113 netdev_warn(netdev, "LRO is incompatible with XDP\n"); 4114 features &= ~NETIF_F_LRO; 4115 } 4116 if (features & NETIF_F_GRO_HW) { 4117 netdev_warn(netdev, "HW GRO is incompatible with XDP\n"); 4118 features &= ~NETIF_F_GRO_HW; 4119 } 4120 } 4121 4122 if (priv->xsk.refcnt) { 4123 if (features & NETIF_F_LRO) { 4124 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n", 4125 priv->xsk.refcnt); 4126 features &= ~NETIF_F_LRO; 4127 } 4128 if (features & NETIF_F_GRO_HW) { 4129 netdev_warn(netdev, "HW GRO is incompatible with AF_XDP (%u XSKs are active)\n", 4130 priv->xsk.refcnt); 4131 features &= ~NETIF_F_GRO_HW; 4132 } 4133 } 4134 4135 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 4136 features &= ~NETIF_F_RXHASH; 4137 if (netdev->features & NETIF_F_RXHASH) 4138 netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n"); 4139 4140 if (features & NETIF_F_GRO_HW) { 4141 netdev_warn(netdev, "Disabling HW-GRO, not supported when CQE compress is active\n"); 4142 features &= ~NETIF_F_GRO_HW; 4143 } 4144 } 4145 4146 if (mlx5e_is_uplink_rep(priv)) 4147 features = mlx5e_fix_uplink_rep_features(netdev, features); 4148 4149 mutex_unlock(&priv->state_lock); 4150 4151 return features; 4152 } 4153 4154 static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, 4155 struct mlx5e_channels *chs, 4156 struct mlx5e_params *new_params, 4157 struct mlx5_core_dev *mdev) 4158 { 4159 u16 ix; 4160 4161 for (ix = 0; ix < chs->params.num_channels; ix++) { 4162 struct xsk_buff_pool *xsk_pool = 4163 mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix); 4164 struct mlx5e_xsk_param xsk; 4165 4166 if (!xsk_pool) 4167 continue; 4168 4169 mlx5e_build_xsk_param(xsk_pool, &xsk); 4170 4171 if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev)) { 4172 u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk); 4173 int max_mtu_frame, max_mtu_page, max_mtu; 4174 4175 /* Two criteria must be met: 4176 * 1. HW MTU + all headrooms <= XSK frame size. 4177 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE. 4178 */ 4179 max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr); 4180 max_mtu_page = MLX5E_HW2SW_MTU(new_params, SKB_MAX_HEAD(0)); 4181 max_mtu = min(max_mtu_frame, max_mtu_page); 4182 4183 netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u. Try MTU <= %d\n", 4184 new_params->sw_mtu, ix, max_mtu); 4185 return false; 4186 } 4187 } 4188 4189 return true; 4190 } 4191 4192 static bool mlx5e_params_validate_xdp(struct net_device *netdev, 4193 struct mlx5_core_dev *mdev, 4194 struct mlx5e_params *params) 4195 { 4196 bool is_linear; 4197 4198 /* No XSK params: AF_XDP can't be enabled yet at the point of setting 4199 * the XDP program. 4200 */ 4201 is_linear = mlx5e_rx_is_linear_skb(mdev, params, NULL); 4202 4203 if (!is_linear && params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) { 4204 netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n", 4205 params->sw_mtu, 4206 mlx5e_xdp_max_mtu(params, NULL)); 4207 return false; 4208 } 4209 if (!is_linear && !params->xdp_prog->aux->xdp_has_frags) { 4210 netdev_warn(netdev, "MTU(%d) > %d, too big for an XDP program not aware of multi buffer\n", 4211 params->sw_mtu, 4212 mlx5e_xdp_max_mtu(params, NULL)); 4213 return false; 4214 } 4215 4216 return true; 4217 } 4218 4219 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, 4220 mlx5e_fp_preactivate preactivate) 4221 { 4222 struct mlx5e_priv *priv = netdev_priv(netdev); 4223 struct mlx5e_params new_params; 4224 struct mlx5e_params *params; 4225 bool reset = true; 4226 int err = 0; 4227 4228 mutex_lock(&priv->state_lock); 4229 4230 params = &priv->channels.params; 4231 4232 new_params = *params; 4233 new_params.sw_mtu = new_mtu; 4234 err = mlx5e_validate_params(priv->mdev, &new_params); 4235 if (err) 4236 goto out; 4237 4238 if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev, 4239 &new_params)) { 4240 err = -EINVAL; 4241 goto out; 4242 } 4243 4244 if (priv->xsk.refcnt && 4245 !mlx5e_xsk_validate_mtu(netdev, &priv->channels, 4246 &new_params, priv->mdev)) { 4247 err = -EINVAL; 4248 goto out; 4249 } 4250 4251 if (params->packet_merge.type == MLX5E_PACKET_MERGE_LRO) 4252 reset = false; 4253 4254 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && 4255 params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) { 4256 bool is_linear_old = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, params, NULL); 4257 bool is_linear_new = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, 4258 &new_params, NULL); 4259 u8 sz_old = mlx5e_mpwqe_get_log_rq_size(priv->mdev, params, NULL); 4260 u8 sz_new = mlx5e_mpwqe_get_log_rq_size(priv->mdev, &new_params, NULL); 4261 4262 /* Always reset in linear mode - hw_mtu is used in data path. 4263 * Check that the mode was non-linear and didn't change. 4264 * If XSK is active, XSK RQs are linear. 4265 * Reset if the RQ size changed, even if it's non-linear. 4266 */ 4267 if (!is_linear_old && !is_linear_new && !priv->xsk.refcnt && 4268 sz_old == sz_new) 4269 reset = false; 4270 } 4271 4272 err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, reset); 4273 4274 out: 4275 netdev->mtu = params->sw_mtu; 4276 mutex_unlock(&priv->state_lock); 4277 return err; 4278 } 4279 4280 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu) 4281 { 4282 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx); 4283 } 4284 4285 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx) 4286 { 4287 bool set = *(bool *)ctx; 4288 4289 return mlx5e_ptp_rx_manage_fs(priv, set); 4290 } 4291 4292 static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter) 4293 { 4294 bool rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; 4295 int err; 4296 4297 if (!rx_filter) 4298 /* Reset CQE compression to Admin default */ 4299 return mlx5e_modify_rx_cqe_compression_locked(priv, rx_cqe_compress_def, false); 4300 4301 if (!MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) 4302 return 0; 4303 4304 /* Disable CQE compression */ 4305 netdev_warn(priv->netdev, "Disabling RX cqe compression\n"); 4306 err = mlx5e_modify_rx_cqe_compression_locked(priv, false, true); 4307 if (err) 4308 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err); 4309 4310 return err; 4311 } 4312 4313 static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx) 4314 { 4315 struct mlx5e_params new_params; 4316 4317 if (ptp_rx == priv->channels.params.ptp_rx) 4318 return 0; 4319 4320 new_params = priv->channels.params; 4321 new_params.ptp_rx = ptp_rx; 4322 return mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx, 4323 &new_params.ptp_rx, true); 4324 } 4325 4326 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr) 4327 { 4328 struct hwtstamp_config config; 4329 bool rx_cqe_compress_def; 4330 bool ptp_rx; 4331 int err; 4332 4333 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) || 4334 (mlx5_clock_get_ptp_index(priv->mdev) == -1)) 4335 return -EOPNOTSUPP; 4336 4337 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 4338 return -EFAULT; 4339 4340 /* TX HW timestamp */ 4341 switch (config.tx_type) { 4342 case HWTSTAMP_TX_OFF: 4343 case HWTSTAMP_TX_ON: 4344 break; 4345 default: 4346 return -ERANGE; 4347 } 4348 4349 mutex_lock(&priv->state_lock); 4350 rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; 4351 4352 /* RX HW timestamp */ 4353 switch (config.rx_filter) { 4354 case HWTSTAMP_FILTER_NONE: 4355 ptp_rx = false; 4356 break; 4357 case HWTSTAMP_FILTER_ALL: 4358 case HWTSTAMP_FILTER_SOME: 4359 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 4360 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 4361 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 4362 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 4363 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 4364 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 4365 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 4366 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 4367 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 4368 case HWTSTAMP_FILTER_PTP_V2_EVENT: 4369 case HWTSTAMP_FILTER_PTP_V2_SYNC: 4370 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 4371 case HWTSTAMP_FILTER_NTP_ALL: 4372 config.rx_filter = HWTSTAMP_FILTER_ALL; 4373 /* ptp_rx is set if both HW TS is set and CQE 4374 * compression is set 4375 */ 4376 ptp_rx = rx_cqe_compress_def; 4377 break; 4378 default: 4379 err = -ERANGE; 4380 goto err_unlock; 4381 } 4382 4383 if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX)) 4384 err = mlx5e_hwstamp_config_no_ptp_rx(priv, 4385 config.rx_filter != HWTSTAMP_FILTER_NONE); 4386 else 4387 err = mlx5e_hwstamp_config_ptp_rx(priv, ptp_rx); 4388 if (err) 4389 goto err_unlock; 4390 4391 memcpy(&priv->tstamp, &config, sizeof(config)); 4392 mutex_unlock(&priv->state_lock); 4393 4394 /* might need to fix some features */ 4395 netdev_update_features(priv->netdev); 4396 4397 return copy_to_user(ifr->ifr_data, &config, 4398 sizeof(config)) ? -EFAULT : 0; 4399 err_unlock: 4400 mutex_unlock(&priv->state_lock); 4401 return err; 4402 } 4403 4404 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr) 4405 { 4406 struct hwtstamp_config *cfg = &priv->tstamp; 4407 4408 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz)) 4409 return -EOPNOTSUPP; 4410 4411 return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0; 4412 } 4413 4414 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 4415 { 4416 struct mlx5e_priv *priv = netdev_priv(dev); 4417 4418 switch (cmd) { 4419 case SIOCSHWTSTAMP: 4420 return mlx5e_hwstamp_set(priv, ifr); 4421 case SIOCGHWTSTAMP: 4422 return mlx5e_hwstamp_get(priv, ifr); 4423 default: 4424 return -EOPNOTSUPP; 4425 } 4426 } 4427 4428 #ifdef CONFIG_MLX5_ESWITCH 4429 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 4430 { 4431 struct mlx5e_priv *priv = netdev_priv(dev); 4432 struct mlx5_core_dev *mdev = priv->mdev; 4433 4434 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac); 4435 } 4436 4437 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 4438 __be16 vlan_proto) 4439 { 4440 struct mlx5e_priv *priv = netdev_priv(dev); 4441 struct mlx5_core_dev *mdev = priv->mdev; 4442 4443 if (vlan_proto != htons(ETH_P_8021Q)) 4444 return -EPROTONOSUPPORT; 4445 4446 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1, 4447 vlan, qos); 4448 } 4449 4450 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 4451 { 4452 struct mlx5e_priv *priv = netdev_priv(dev); 4453 struct mlx5_core_dev *mdev = priv->mdev; 4454 4455 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting); 4456 } 4457 4458 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting) 4459 { 4460 struct mlx5e_priv *priv = netdev_priv(dev); 4461 struct mlx5_core_dev *mdev = priv->mdev; 4462 4463 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting); 4464 } 4465 4466 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 4467 int max_tx_rate) 4468 { 4469 struct mlx5e_priv *priv = netdev_priv(dev); 4470 struct mlx5_core_dev *mdev = priv->mdev; 4471 4472 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1, 4473 max_tx_rate, min_tx_rate); 4474 } 4475 4476 static int mlx5_vport_link2ifla(u8 esw_link) 4477 { 4478 switch (esw_link) { 4479 case MLX5_VPORT_ADMIN_STATE_DOWN: 4480 return IFLA_VF_LINK_STATE_DISABLE; 4481 case MLX5_VPORT_ADMIN_STATE_UP: 4482 return IFLA_VF_LINK_STATE_ENABLE; 4483 } 4484 return IFLA_VF_LINK_STATE_AUTO; 4485 } 4486 4487 static int mlx5_ifla_link2vport(u8 ifla_link) 4488 { 4489 switch (ifla_link) { 4490 case IFLA_VF_LINK_STATE_DISABLE: 4491 return MLX5_VPORT_ADMIN_STATE_DOWN; 4492 case IFLA_VF_LINK_STATE_ENABLE: 4493 return MLX5_VPORT_ADMIN_STATE_UP; 4494 } 4495 return MLX5_VPORT_ADMIN_STATE_AUTO; 4496 } 4497 4498 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf, 4499 int link_state) 4500 { 4501 struct mlx5e_priv *priv = netdev_priv(dev); 4502 struct mlx5_core_dev *mdev = priv->mdev; 4503 4504 if (mlx5e_is_uplink_rep(priv)) 4505 return -EOPNOTSUPP; 4506 4507 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1, 4508 mlx5_ifla_link2vport(link_state)); 4509 } 4510 4511 int mlx5e_get_vf_config(struct net_device *dev, 4512 int vf, struct ifla_vf_info *ivi) 4513 { 4514 struct mlx5e_priv *priv = netdev_priv(dev); 4515 struct mlx5_core_dev *mdev = priv->mdev; 4516 int err; 4517 4518 if (!netif_device_present(dev)) 4519 return -EOPNOTSUPP; 4520 4521 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi); 4522 if (err) 4523 return err; 4524 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate); 4525 return 0; 4526 } 4527 4528 int mlx5e_get_vf_stats(struct net_device *dev, 4529 int vf, struct ifla_vf_stats *vf_stats) 4530 { 4531 struct mlx5e_priv *priv = netdev_priv(dev); 4532 struct mlx5_core_dev *mdev = priv->mdev; 4533 4534 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1, 4535 vf_stats); 4536 } 4537 4538 static bool 4539 mlx5e_has_offload_stats(const struct net_device *dev, int attr_id) 4540 { 4541 struct mlx5e_priv *priv = netdev_priv(dev); 4542 4543 if (!netif_device_present(dev)) 4544 return false; 4545 4546 if (!mlx5e_is_uplink_rep(priv)) 4547 return false; 4548 4549 return mlx5e_rep_has_offload_stats(dev, attr_id); 4550 } 4551 4552 static int 4553 mlx5e_get_offload_stats(int attr_id, const struct net_device *dev, 4554 void *sp) 4555 { 4556 struct mlx5e_priv *priv = netdev_priv(dev); 4557 4558 if (!mlx5e_is_uplink_rep(priv)) 4559 return -EOPNOTSUPP; 4560 4561 return mlx5e_rep_get_offload_stats(attr_id, dev, sp); 4562 } 4563 #endif 4564 4565 static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type) 4566 { 4567 switch (proto_type) { 4568 case IPPROTO_GRE: 4569 return MLX5_CAP_ETH(mdev, tunnel_stateless_gre); 4570 case IPPROTO_IPIP: 4571 case IPPROTO_IPV6: 4572 return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) || 4573 MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_tx)); 4574 default: 4575 return false; 4576 } 4577 } 4578 4579 static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev, 4580 struct sk_buff *skb) 4581 { 4582 switch (skb->inner_protocol) { 4583 case htons(ETH_P_IP): 4584 case htons(ETH_P_IPV6): 4585 case htons(ETH_P_TEB): 4586 return true; 4587 case htons(ETH_P_MPLS_UC): 4588 case htons(ETH_P_MPLS_MC): 4589 return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre); 4590 } 4591 return false; 4592 } 4593 4594 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, 4595 struct sk_buff *skb, 4596 netdev_features_t features) 4597 { 4598 unsigned int offset = 0; 4599 struct udphdr *udph; 4600 u8 proto; 4601 u16 port; 4602 4603 switch (vlan_get_protocol(skb)) { 4604 case htons(ETH_P_IP): 4605 proto = ip_hdr(skb)->protocol; 4606 break; 4607 case htons(ETH_P_IPV6): 4608 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL); 4609 break; 4610 default: 4611 goto out; 4612 } 4613 4614 switch (proto) { 4615 case IPPROTO_GRE: 4616 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb)) 4617 return features; 4618 break; 4619 case IPPROTO_IPIP: 4620 case IPPROTO_IPV6: 4621 if (mlx5e_tunnel_proto_supported_tx(priv->mdev, IPPROTO_IPIP)) 4622 return features; 4623 break; 4624 case IPPROTO_UDP: 4625 udph = udp_hdr(skb); 4626 port = be16_to_cpu(udph->dest); 4627 4628 /* Verify if UDP port is being offloaded by HW */ 4629 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port)) 4630 return features; 4631 4632 #if IS_ENABLED(CONFIG_GENEVE) 4633 /* Support Geneve offload for default UDP port */ 4634 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev)) 4635 return features; 4636 #endif 4637 break; 4638 #ifdef CONFIG_MLX5_EN_IPSEC 4639 case IPPROTO_ESP: 4640 return mlx5e_ipsec_feature_check(skb, features); 4641 #endif 4642 } 4643 4644 out: 4645 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */ 4646 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 4647 } 4648 4649 netdev_features_t mlx5e_features_check(struct sk_buff *skb, 4650 struct net_device *netdev, 4651 netdev_features_t features) 4652 { 4653 struct mlx5e_priv *priv = netdev_priv(netdev); 4654 4655 features = vlan_features_check(skb, features); 4656 features = vxlan_features_check(skb, features); 4657 4658 /* Validate if the tunneled packet is being offloaded by HW */ 4659 if (skb->encapsulation && 4660 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK)) 4661 return mlx5e_tunnel_features_check(priv, skb, features); 4662 4663 return features; 4664 } 4665 4666 static void mlx5e_tx_timeout_work(struct work_struct *work) 4667 { 4668 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 4669 tx_timeout_work); 4670 struct net_device *netdev = priv->netdev; 4671 int i; 4672 4673 rtnl_lock(); 4674 mutex_lock(&priv->state_lock); 4675 4676 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4677 goto unlock; 4678 4679 for (i = 0; i < netdev->real_num_tx_queues; i++) { 4680 struct netdev_queue *dev_queue = 4681 netdev_get_tx_queue(netdev, i); 4682 struct mlx5e_txqsq *sq = priv->txq2sq[i]; 4683 4684 if (!netif_xmit_stopped(dev_queue)) 4685 continue; 4686 4687 if (mlx5e_reporter_tx_timeout(sq)) 4688 /* break if tried to reopened channels */ 4689 break; 4690 } 4691 4692 unlock: 4693 mutex_unlock(&priv->state_lock); 4694 rtnl_unlock(); 4695 } 4696 4697 static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue) 4698 { 4699 struct mlx5e_priv *priv = netdev_priv(dev); 4700 4701 netdev_err(dev, "TX timeout detected\n"); 4702 queue_work(priv->wq, &priv->tx_timeout_work); 4703 } 4704 4705 static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog) 4706 { 4707 struct net_device *netdev = priv->netdev; 4708 struct mlx5e_params new_params; 4709 4710 if (priv->channels.params.packet_merge.type != MLX5E_PACKET_MERGE_NONE) { 4711 netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n"); 4712 return -EINVAL; 4713 } 4714 4715 new_params = priv->channels.params; 4716 new_params.xdp_prog = prog; 4717 4718 if (!mlx5e_params_validate_xdp(netdev, priv->mdev, &new_params)) 4719 return -EINVAL; 4720 4721 return 0; 4722 } 4723 4724 static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog) 4725 { 4726 struct bpf_prog *old_prog; 4727 4728 old_prog = rcu_replace_pointer(rq->xdp_prog, prog, 4729 lockdep_is_held(&rq->priv->state_lock)); 4730 if (old_prog) 4731 bpf_prog_put(old_prog); 4732 } 4733 4734 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) 4735 { 4736 struct mlx5e_priv *priv = netdev_priv(netdev); 4737 struct mlx5e_params new_params; 4738 struct bpf_prog *old_prog; 4739 int err = 0; 4740 bool reset; 4741 int i; 4742 4743 mutex_lock(&priv->state_lock); 4744 4745 if (prog) { 4746 err = mlx5e_xdp_allowed(priv, prog); 4747 if (err) 4748 goto unlock; 4749 } 4750 4751 /* no need for full reset when exchanging programs */ 4752 reset = (!priv->channels.params.xdp_prog || !prog); 4753 4754 new_params = priv->channels.params; 4755 new_params.xdp_prog = prog; 4756 4757 /* XDP affects striding RQ parameters. Block XDP if striding RQ won't be 4758 * supported with the new parameters: if PAGE_SIZE is bigger than 4759 * MLX5_MPWQE_LOG_STRIDE_SZ_MAX, striding RQ can't be used, even though 4760 * the MTU is small enough for the linear mode, because XDP uses strides 4761 * of PAGE_SIZE on regular RQs. 4762 */ 4763 if (reset && MLX5E_GET_PFLAG(&new_params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 4764 /* Checking for regular RQs here; XSK RQs were checked on XSK bind. */ 4765 err = mlx5e_mpwrq_validate_regular(priv->mdev, &new_params); 4766 if (err) 4767 goto unlock; 4768 } 4769 4770 old_prog = priv->channels.params.xdp_prog; 4771 4772 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); 4773 if (err) 4774 goto unlock; 4775 4776 if (old_prog) 4777 bpf_prog_put(old_prog); 4778 4779 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset) 4780 goto unlock; 4781 4782 /* exchanging programs w/o reset, we update ref counts on behalf 4783 * of the channels RQs here. 4784 */ 4785 bpf_prog_add(prog, priv->channels.num); 4786 for (i = 0; i < priv->channels.num; i++) { 4787 struct mlx5e_channel *c = priv->channels.c[i]; 4788 4789 mlx5e_rq_replace_xdp_prog(&c->rq, prog); 4790 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) { 4791 bpf_prog_inc(prog); 4792 mlx5e_rq_replace_xdp_prog(&c->xskrq, prog); 4793 } 4794 } 4795 4796 unlock: 4797 mutex_unlock(&priv->state_lock); 4798 4799 /* Need to fix some features. */ 4800 if (!err) 4801 netdev_update_features(netdev); 4802 4803 return err; 4804 } 4805 4806 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp) 4807 { 4808 switch (xdp->command) { 4809 case XDP_SETUP_PROG: 4810 return mlx5e_xdp_set(dev, xdp->prog); 4811 case XDP_SETUP_XSK_POOL: 4812 return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool, 4813 xdp->xsk.queue_id); 4814 default: 4815 return -EINVAL; 4816 } 4817 } 4818 4819 #ifdef CONFIG_MLX5_ESWITCH 4820 static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 4821 struct net_device *dev, u32 filter_mask, 4822 int nlflags) 4823 { 4824 struct mlx5e_priv *priv = netdev_priv(dev); 4825 struct mlx5_core_dev *mdev = priv->mdev; 4826 u8 mode, setting; 4827 int err; 4828 4829 err = mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting); 4830 if (err) 4831 return err; 4832 mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB; 4833 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 4834 mode, 4835 0, 0, nlflags, filter_mask, NULL); 4836 } 4837 4838 static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4839 u16 flags, struct netlink_ext_ack *extack) 4840 { 4841 struct mlx5e_priv *priv = netdev_priv(dev); 4842 struct mlx5_core_dev *mdev = priv->mdev; 4843 struct nlattr *attr, *br_spec; 4844 u16 mode = BRIDGE_MODE_UNDEF; 4845 u8 setting; 4846 int rem; 4847 4848 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4849 if (!br_spec) 4850 return -EINVAL; 4851 4852 nla_for_each_nested(attr, br_spec, rem) { 4853 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4854 continue; 4855 4856 if (nla_len(attr) < sizeof(mode)) 4857 return -EINVAL; 4858 4859 mode = nla_get_u16(attr); 4860 if (mode > BRIDGE_MODE_VEPA) 4861 return -EINVAL; 4862 4863 break; 4864 } 4865 4866 if (mode == BRIDGE_MODE_UNDEF) 4867 return -EINVAL; 4868 4869 setting = (mode == BRIDGE_MODE_VEPA) ? 1 : 0; 4870 return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting); 4871 } 4872 #endif 4873 4874 const struct net_device_ops mlx5e_netdev_ops = { 4875 .ndo_open = mlx5e_open, 4876 .ndo_stop = mlx5e_close, 4877 .ndo_start_xmit = mlx5e_xmit, 4878 .ndo_setup_tc = mlx5e_setup_tc, 4879 .ndo_select_queue = mlx5e_select_queue, 4880 .ndo_get_stats64 = mlx5e_get_stats, 4881 .ndo_set_rx_mode = mlx5e_set_rx_mode, 4882 .ndo_set_mac_address = mlx5e_set_mac, 4883 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid, 4884 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid, 4885 .ndo_set_features = mlx5e_set_features, 4886 .ndo_fix_features = mlx5e_fix_features, 4887 .ndo_change_mtu = mlx5e_change_nic_mtu, 4888 .ndo_eth_ioctl = mlx5e_ioctl, 4889 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate, 4890 .ndo_features_check = mlx5e_features_check, 4891 .ndo_tx_timeout = mlx5e_tx_timeout, 4892 .ndo_bpf = mlx5e_xdp, 4893 .ndo_xdp_xmit = mlx5e_xdp_xmit, 4894 .ndo_xsk_wakeup = mlx5e_xsk_wakeup, 4895 #ifdef CONFIG_MLX5_EN_ARFS 4896 .ndo_rx_flow_steer = mlx5e_rx_flow_steer, 4897 #endif 4898 #ifdef CONFIG_MLX5_ESWITCH 4899 .ndo_bridge_setlink = mlx5e_bridge_setlink, 4900 .ndo_bridge_getlink = mlx5e_bridge_getlink, 4901 4902 /* SRIOV E-Switch NDOs */ 4903 .ndo_set_vf_mac = mlx5e_set_vf_mac, 4904 .ndo_set_vf_vlan = mlx5e_set_vf_vlan, 4905 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk, 4906 .ndo_set_vf_trust = mlx5e_set_vf_trust, 4907 .ndo_set_vf_rate = mlx5e_set_vf_rate, 4908 .ndo_get_vf_config = mlx5e_get_vf_config, 4909 .ndo_set_vf_link_state = mlx5e_set_vf_link_state, 4910 .ndo_get_vf_stats = mlx5e_get_vf_stats, 4911 .ndo_has_offload_stats = mlx5e_has_offload_stats, 4912 .ndo_get_offload_stats = mlx5e_get_offload_stats, 4913 #endif 4914 }; 4915 4916 static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) 4917 { 4918 int i; 4919 4920 /* The supported periods are organized in ascending order */ 4921 for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++) 4922 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout) 4923 break; 4924 4925 return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]); 4926 } 4927 4928 void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu) 4929 { 4930 struct mlx5e_params *params = &priv->channels.params; 4931 struct mlx5_core_dev *mdev = priv->mdev; 4932 u8 rx_cq_period_mode; 4933 4934 params->sw_mtu = mtu; 4935 params->hard_mtu = MLX5E_ETH_HARD_MTU; 4936 params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2, 4937 priv->max_nch); 4938 mlx5e_params_mqprio_reset(params); 4939 4940 /* SQ */ 4941 params->log_sq_size = is_kdump_kernel() ? 4942 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : 4943 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 4944 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); 4945 4946 /* XDP SQ */ 4947 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); 4948 4949 /* set CQE compression */ 4950 params->rx_cqe_compress_def = false; 4951 if (MLX5_CAP_GEN(mdev, cqe_compression) && 4952 MLX5_CAP_GEN(mdev, vport_group_manager)) 4953 params->rx_cqe_compress_def = slow_pci_heuristic(mdev); 4954 4955 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def); 4956 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false); 4957 4958 /* RQ */ 4959 mlx5e_build_rq_params(mdev, params); 4960 4961 params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); 4962 4963 /* CQ moderation params */ 4964 rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 4965 MLX5_CQ_PERIOD_MODE_START_FROM_CQE : 4966 MLX5_CQ_PERIOD_MODE_START_FROM_EQE; 4967 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 4968 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 4969 mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode); 4970 mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE); 4971 4972 /* TX inline */ 4973 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); 4974 4975 params->tunneled_offload_en = mlx5_tunnel_inner_ft_supported(mdev); 4976 4977 /* AF_XDP */ 4978 params->xsk = xsk; 4979 4980 /* Do not update netdev->features directly in here 4981 * on mlx5e_attach_netdev() we will call mlx5e_update_features() 4982 * To update netdev->features please modify mlx5e_fix_features() 4983 */ 4984 } 4985 4986 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev) 4987 { 4988 struct mlx5e_priv *priv = netdev_priv(netdev); 4989 u8 addr[ETH_ALEN]; 4990 4991 mlx5_query_mac_address(priv->mdev, addr); 4992 if (is_zero_ether_addr(addr) && 4993 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) { 4994 eth_hw_addr_random(netdev); 4995 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr); 4996 return; 4997 } 4998 4999 eth_hw_addr_set(netdev, addr); 5000 } 5001 5002 static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table, 5003 unsigned int entry, struct udp_tunnel_info *ti) 5004 { 5005 struct mlx5e_priv *priv = netdev_priv(netdev); 5006 5007 return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port)); 5008 } 5009 5010 static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table, 5011 unsigned int entry, struct udp_tunnel_info *ti) 5012 { 5013 struct mlx5e_priv *priv = netdev_priv(netdev); 5014 5015 return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port)); 5016 } 5017 5018 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv) 5019 { 5020 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 5021 return; 5022 5023 priv->nic_info.set_port = mlx5e_vxlan_set_port; 5024 priv->nic_info.unset_port = mlx5e_vxlan_unset_port; 5025 priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 5026 UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN; 5027 priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN; 5028 /* Don't count the space hard-coded to the IANA port */ 5029 priv->nic_info.tables[0].n_entries = 5030 mlx5_vxlan_max_udp_ports(priv->mdev) - 1; 5031 5032 priv->netdev->udp_tunnel_nic_info = &priv->nic_info; 5033 } 5034 5035 static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev) 5036 { 5037 int tt; 5038 5039 for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) { 5040 if (mlx5e_tunnel_proto_supported_tx(mdev, mlx5_get_proto_by_tunnel_type(tt))) 5041 return true; 5042 } 5043 return (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)); 5044 } 5045 5046 static void mlx5e_build_nic_netdev(struct net_device *netdev) 5047 { 5048 struct mlx5e_priv *priv = netdev_priv(netdev); 5049 struct mlx5_core_dev *mdev = priv->mdev; 5050 bool fcs_supported; 5051 bool fcs_enabled; 5052 5053 SET_NETDEV_DEV(netdev, mdev->device); 5054 5055 netdev->netdev_ops = &mlx5e_netdev_ops; 5056 5057 mlx5e_dcbnl_build_netdev(netdev); 5058 5059 netdev->watchdog_timeo = 15 * HZ; 5060 5061 netdev->ethtool_ops = &mlx5e_ethtool_ops; 5062 5063 netdev->vlan_features |= NETIF_F_SG; 5064 netdev->vlan_features |= NETIF_F_HW_CSUM; 5065 netdev->vlan_features |= NETIF_F_GRO; 5066 netdev->vlan_features |= NETIF_F_TSO; 5067 netdev->vlan_features |= NETIF_F_TSO6; 5068 netdev->vlan_features |= NETIF_F_RXCSUM; 5069 netdev->vlan_features |= NETIF_F_RXHASH; 5070 netdev->vlan_features |= NETIF_F_GSO_PARTIAL; 5071 5072 netdev->mpls_features |= NETIF_F_SG; 5073 netdev->mpls_features |= NETIF_F_HW_CSUM; 5074 netdev->mpls_features |= NETIF_F_TSO; 5075 netdev->mpls_features |= NETIF_F_TSO6; 5076 5077 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX; 5078 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX; 5079 5080 /* Tunneled LRO is not supported in the driver, and the same RQs are 5081 * shared between inner and outer TIRs, so the driver can't disable LRO 5082 * for inner TIRs while having it enabled for outer TIRs. Due to this, 5083 * block LRO altogether if the firmware declares tunneled LRO support. 5084 */ 5085 if (!!MLX5_CAP_ETH(mdev, lro_cap) && 5086 !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) && 5087 !MLX5_CAP_ETH(mdev, tunnel_lro_gre) && 5088 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, 5089 MLX5E_MPWRQ_UMR_MODE_ALIGNED)) 5090 netdev->vlan_features |= NETIF_F_LRO; 5091 5092 netdev->hw_features = netdev->vlan_features; 5093 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 5094 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 5095 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 5096 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 5097 5098 if (mlx5e_tunnel_any_tx_proto_supported(mdev)) { 5099 netdev->hw_enc_features |= NETIF_F_HW_CSUM; 5100 netdev->hw_enc_features |= NETIF_F_TSO; 5101 netdev->hw_enc_features |= NETIF_F_TSO6; 5102 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL; 5103 } 5104 5105 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) { 5106 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 5107 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5108 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 5109 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5110 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 5111 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL | 5112 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5113 } 5114 5115 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) { 5116 netdev->hw_features |= NETIF_F_GSO_GRE | 5117 NETIF_F_GSO_GRE_CSUM; 5118 netdev->hw_enc_features |= NETIF_F_GSO_GRE | 5119 NETIF_F_GSO_GRE_CSUM; 5120 netdev->gso_partial_features |= NETIF_F_GSO_GRE | 5121 NETIF_F_GSO_GRE_CSUM; 5122 } 5123 5124 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) { 5125 netdev->hw_features |= NETIF_F_GSO_IPXIP4 | 5126 NETIF_F_GSO_IPXIP6; 5127 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 | 5128 NETIF_F_GSO_IPXIP6; 5129 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 | 5130 NETIF_F_GSO_IPXIP6; 5131 } 5132 5133 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4; 5134 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 5135 netdev->features |= NETIF_F_GSO_UDP_L4; 5136 5137 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled); 5138 5139 if (fcs_supported) 5140 netdev->hw_features |= NETIF_F_RXALL; 5141 5142 if (MLX5_CAP_ETH(mdev, scatter_fcs)) 5143 netdev->hw_features |= NETIF_F_RXFCS; 5144 5145 if (mlx5_qos_is_supported(mdev)) 5146 netdev->hw_features |= NETIF_F_HW_TC; 5147 5148 netdev->features = netdev->hw_features; 5149 5150 /* Defaults */ 5151 if (fcs_enabled) 5152 netdev->features &= ~NETIF_F_RXALL; 5153 netdev->features &= ~NETIF_F_LRO; 5154 netdev->features &= ~NETIF_F_GRO_HW; 5155 netdev->features &= ~NETIF_F_RXFCS; 5156 5157 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f) 5158 if (FT_CAP(flow_modify_en) && 5159 FT_CAP(modify_root) && 5160 FT_CAP(identified_miss_table_mode) && 5161 FT_CAP(flow_table_modify)) { 5162 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 5163 netdev->hw_features |= NETIF_F_HW_TC; 5164 #endif 5165 #ifdef CONFIG_MLX5_EN_ARFS 5166 netdev->hw_features |= NETIF_F_NTUPLE; 5167 #endif 5168 } 5169 5170 netdev->features |= NETIF_F_HIGHDMA; 5171 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 5172 5173 netdev->priv_flags |= IFF_UNICAST_FLT; 5174 5175 netif_set_tso_max_size(netdev, GSO_MAX_SIZE); 5176 mlx5e_set_netdev_dev_addr(netdev); 5177 mlx5e_macsec_build_netdev(priv); 5178 mlx5e_ipsec_build_netdev(priv); 5179 mlx5e_ktls_build_netdev(priv); 5180 } 5181 5182 void mlx5e_create_q_counters(struct mlx5e_priv *priv) 5183 { 5184 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 5185 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 5186 struct mlx5_core_dev *mdev = priv->mdev; 5187 int err; 5188 5189 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 5190 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); 5191 if (!err) 5192 priv->q_counter = 5193 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5194 5195 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); 5196 if (!err) 5197 priv->drop_rq_q_counter = 5198 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5199 } 5200 5201 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv) 5202 { 5203 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 5204 5205 MLX5_SET(dealloc_q_counter_in, in, opcode, 5206 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 5207 if (priv->q_counter) { 5208 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 5209 priv->q_counter); 5210 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in); 5211 } 5212 5213 if (priv->drop_rq_q_counter) { 5214 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 5215 priv->drop_rq_q_counter); 5216 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in); 5217 } 5218 } 5219 5220 static int mlx5e_nic_init(struct mlx5_core_dev *mdev, 5221 struct net_device *netdev) 5222 { 5223 struct mlx5e_priv *priv = netdev_priv(netdev); 5224 struct mlx5e_flow_steering *fs; 5225 int err; 5226 5227 mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu); 5228 mlx5e_vxlan_set_netdev_info(priv); 5229 5230 mlx5e_timestamp_init(priv); 5231 5232 fs = mlx5e_fs_init(priv->profile, mdev, 5233 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 5234 if (!fs) { 5235 err = -ENOMEM; 5236 mlx5_core_err(mdev, "FS initialization failed, %d\n", err); 5237 return err; 5238 } 5239 priv->fs = fs; 5240 5241 err = mlx5e_ktls_init(priv); 5242 if (err) 5243 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); 5244 5245 mlx5e_health_create_reporters(priv); 5246 return 0; 5247 } 5248 5249 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) 5250 { 5251 mlx5e_health_destroy_reporters(priv); 5252 mlx5e_ktls_cleanup(priv); 5253 mlx5e_fs_cleanup(priv->fs); 5254 } 5255 5256 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) 5257 { 5258 struct mlx5_core_dev *mdev = priv->mdev; 5259 enum mlx5e_rx_res_features features; 5260 int err; 5261 5262 priv->rx_res = mlx5e_rx_res_alloc(); 5263 if (!priv->rx_res) 5264 return -ENOMEM; 5265 5266 mlx5e_create_q_counters(priv); 5267 5268 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 5269 if (err) { 5270 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 5271 goto err_destroy_q_counters; 5272 } 5273 5274 features = MLX5E_RX_RES_FEATURE_PTP; 5275 if (priv->channels.params.tunneled_offload_en) 5276 features |= MLX5E_RX_RES_FEATURE_INNER_FT; 5277 err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, features, 5278 priv->max_nch, priv->drop_rq.rqn, 5279 &priv->channels.params.packet_merge, 5280 priv->channels.params.num_channels); 5281 if (err) 5282 goto err_close_drop_rq; 5283 5284 err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile, 5285 priv->netdev); 5286 if (err) { 5287 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); 5288 goto err_destroy_rx_res; 5289 } 5290 5291 err = mlx5e_tc_nic_init(priv); 5292 if (err) 5293 goto err_destroy_flow_steering; 5294 5295 err = mlx5e_accel_init_rx(priv); 5296 if (err) 5297 goto err_tc_nic_cleanup; 5298 5299 #ifdef CONFIG_MLX5_EN_ARFS 5300 priv->netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(priv->mdev); 5301 #endif 5302 5303 return 0; 5304 5305 err_tc_nic_cleanup: 5306 mlx5e_tc_nic_cleanup(priv); 5307 err_destroy_flow_steering: 5308 mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE), 5309 priv->profile); 5310 err_destroy_rx_res: 5311 mlx5e_rx_res_destroy(priv->rx_res); 5312 err_close_drop_rq: 5313 mlx5e_close_drop_rq(&priv->drop_rq); 5314 err_destroy_q_counters: 5315 mlx5e_destroy_q_counters(priv); 5316 mlx5e_rx_res_free(priv->rx_res); 5317 priv->rx_res = NULL; 5318 return err; 5319 } 5320 5321 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) 5322 { 5323 mlx5e_accel_cleanup_rx(priv); 5324 mlx5e_tc_nic_cleanup(priv); 5325 mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE), 5326 priv->profile); 5327 mlx5e_rx_res_destroy(priv->rx_res); 5328 mlx5e_close_drop_rq(&priv->drop_rq); 5329 mlx5e_destroy_q_counters(priv); 5330 mlx5e_rx_res_free(priv->rx_res); 5331 priv->rx_res = NULL; 5332 } 5333 5334 static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv) 5335 { 5336 struct mlx5e_params *params; 5337 struct mlx5e_mqprio_rl *rl; 5338 5339 params = &priv->channels.params; 5340 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) 5341 return; 5342 5343 rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc, 5344 params->mqprio.channel.max_rate); 5345 if (IS_ERR(rl)) 5346 rl = NULL; 5347 priv->mqprio_rl = rl; 5348 mlx5e_mqprio_rl_update_params(params, rl); 5349 } 5350 5351 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) 5352 { 5353 int err; 5354 5355 err = mlx5e_create_tises(priv); 5356 if (err) { 5357 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); 5358 return err; 5359 } 5360 5361 err = mlx5e_accel_init_tx(priv); 5362 if (err) 5363 goto err_destroy_tises; 5364 5365 mlx5e_set_mqprio_rl(priv); 5366 mlx5e_dcbnl_initialize(priv); 5367 return 0; 5368 5369 err_destroy_tises: 5370 mlx5e_destroy_tises(priv); 5371 return err; 5372 } 5373 5374 static void mlx5e_nic_enable(struct mlx5e_priv *priv) 5375 { 5376 struct net_device *netdev = priv->netdev; 5377 struct mlx5_core_dev *mdev = priv->mdev; 5378 int err; 5379 5380 mlx5e_fs_init_l2_addr(priv->fs, netdev); 5381 mlx5e_ipsec_init(priv); 5382 5383 err = mlx5e_macsec_init(priv); 5384 if (err) 5385 mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err); 5386 5387 /* Marking the link as currently not needed by the Driver */ 5388 if (!netif_running(netdev)) 5389 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN); 5390 5391 mlx5e_set_netdev_mtu_boundaries(priv); 5392 mlx5e_set_dev_port_mtu(priv); 5393 5394 mlx5_lag_add_netdev(mdev, netdev); 5395 5396 mlx5e_enable_async_events(priv); 5397 mlx5e_enable_blocking_events(priv); 5398 if (mlx5e_monitor_counter_supported(priv)) 5399 mlx5e_monitor_counter_init(priv); 5400 5401 mlx5e_hv_vhca_stats_create(priv); 5402 if (netdev->reg_state != NETREG_REGISTERED) 5403 return; 5404 mlx5e_dcbnl_init_app(priv); 5405 5406 mlx5e_nic_set_rx_mode(priv); 5407 5408 rtnl_lock(); 5409 if (netif_running(netdev)) 5410 mlx5e_open(netdev); 5411 udp_tunnel_nic_reset_ntf(priv->netdev); 5412 netif_device_attach(netdev); 5413 rtnl_unlock(); 5414 } 5415 5416 static void mlx5e_nic_disable(struct mlx5e_priv *priv) 5417 { 5418 struct mlx5_core_dev *mdev = priv->mdev; 5419 5420 if (priv->netdev->reg_state == NETREG_REGISTERED) 5421 mlx5e_dcbnl_delete_app(priv); 5422 5423 rtnl_lock(); 5424 if (netif_running(priv->netdev)) 5425 mlx5e_close(priv->netdev); 5426 netif_device_detach(priv->netdev); 5427 rtnl_unlock(); 5428 5429 mlx5e_nic_set_rx_mode(priv); 5430 5431 mlx5e_hv_vhca_stats_destroy(priv); 5432 if (mlx5e_monitor_counter_supported(priv)) 5433 mlx5e_monitor_counter_cleanup(priv); 5434 5435 mlx5e_disable_blocking_events(priv); 5436 if (priv->en_trap) { 5437 mlx5e_deactivate_trap(priv); 5438 mlx5e_close_trap(priv->en_trap); 5439 priv->en_trap = NULL; 5440 } 5441 mlx5e_disable_async_events(priv); 5442 mlx5_lag_remove_netdev(mdev, priv->netdev); 5443 mlx5_vxlan_reset_to_default(mdev->vxlan); 5444 mlx5e_macsec_cleanup(priv); 5445 mlx5e_ipsec_cleanup(priv); 5446 } 5447 5448 int mlx5e_update_nic_rx(struct mlx5e_priv *priv) 5449 { 5450 return mlx5e_refresh_tirs(priv, false, false); 5451 } 5452 5453 static const struct mlx5e_profile mlx5e_nic_profile = { 5454 .init = mlx5e_nic_init, 5455 .cleanup = mlx5e_nic_cleanup, 5456 .init_rx = mlx5e_init_nic_rx, 5457 .cleanup_rx = mlx5e_cleanup_nic_rx, 5458 .init_tx = mlx5e_init_nic_tx, 5459 .cleanup_tx = mlx5e_cleanup_nic_tx, 5460 .enable = mlx5e_nic_enable, 5461 .disable = mlx5e_nic_disable, 5462 .update_rx = mlx5e_update_nic_rx, 5463 .update_stats = mlx5e_stats_update_ndo_stats, 5464 .update_carrier = mlx5e_update_carrier, 5465 .rx_handlers = &mlx5e_rx_handlers_nic, 5466 .max_tc = MLX5E_MAX_NUM_TC, 5467 .stats_grps = mlx5e_nic_stats_grps, 5468 .stats_grps_num = mlx5e_nic_stats_grps_num, 5469 .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) | 5470 BIT(MLX5E_PROFILE_FEATURE_PTP_TX) | 5471 BIT(MLX5E_PROFILE_FEATURE_QOS_HTB) | 5472 BIT(MLX5E_PROFILE_FEATURE_FS_VLAN) | 5473 BIT(MLX5E_PROFILE_FEATURE_FS_TC), 5474 }; 5475 5476 static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev, 5477 const struct mlx5e_profile *profile) 5478 { 5479 int nch; 5480 5481 nch = mlx5e_get_max_num_channels(mdev); 5482 5483 if (profile->max_nch_limit) 5484 nch = min_t(int, nch, profile->max_nch_limit(mdev)); 5485 return nch; 5486 } 5487 5488 static unsigned int 5489 mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev, 5490 const struct mlx5e_profile *profile) 5491 5492 { 5493 unsigned int max_nch, tmp; 5494 5495 /* core resources */ 5496 max_nch = mlx5e_profile_max_num_channels(mdev, profile); 5497 5498 /* netdev rx queues */ 5499 max_nch = min_t(unsigned int, max_nch, netdev->num_rx_queues); 5500 5501 /* netdev tx queues */ 5502 tmp = netdev->num_tx_queues; 5503 if (mlx5_qos_is_supported(mdev)) 5504 tmp -= mlx5e_qos_max_leaf_nodes(mdev); 5505 if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn)) 5506 tmp -= profile->max_tc; 5507 tmp = tmp / profile->max_tc; 5508 max_nch = min_t(unsigned int, max_nch, tmp); 5509 5510 return max_nch; 5511 } 5512 5513 int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev) 5514 { 5515 /* Indirect TIRS: 2 sets of TTCs (inner + outer steering) 5516 * and 1 set of direct TIRS 5517 */ 5518 return 2 * MLX5E_NUM_INDIR_TIRS 5519 + mlx5e_profile_max_num_channels(mdev, &mlx5e_nic_profile); 5520 } 5521 5522 void mlx5e_set_rx_mode_work(struct work_struct *work) 5523 { 5524 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 5525 set_rx_mode_work); 5526 5527 return mlx5e_fs_set_rx_mode_work(priv->fs, priv->netdev); 5528 } 5529 5530 /* mlx5e generic netdev management API (move to en_common.c) */ 5531 int mlx5e_priv_init(struct mlx5e_priv *priv, 5532 const struct mlx5e_profile *profile, 5533 struct net_device *netdev, 5534 struct mlx5_core_dev *mdev) 5535 { 5536 int nch, num_txqs, node; 5537 int err; 5538 5539 num_txqs = netdev->num_tx_queues; 5540 nch = mlx5e_calc_max_nch(mdev, netdev, profile); 5541 node = dev_to_node(mlx5_core_dma_dev(mdev)); 5542 5543 /* priv init */ 5544 priv->mdev = mdev; 5545 priv->netdev = netdev; 5546 priv->msglevel = MLX5E_MSG_LEVEL; 5547 priv->max_nch = nch; 5548 priv->max_opened_tc = 1; 5549 5550 if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL)) 5551 return -ENOMEM; 5552 5553 mutex_init(&priv->state_lock); 5554 5555 err = mlx5e_selq_init(&priv->selq, &priv->state_lock); 5556 if (err) 5557 goto err_free_cpumask; 5558 5559 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); 5560 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); 5561 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); 5562 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); 5563 5564 priv->wq = create_singlethread_workqueue("mlx5e"); 5565 if (!priv->wq) 5566 goto err_free_selq; 5567 5568 priv->txq2sq = kcalloc_node(num_txqs, sizeof(*priv->txq2sq), GFP_KERNEL, node); 5569 if (!priv->txq2sq) 5570 goto err_destroy_workqueue; 5571 5572 priv->tx_rates = kcalloc_node(num_txqs, sizeof(*priv->tx_rates), GFP_KERNEL, node); 5573 if (!priv->tx_rates) 5574 goto err_free_txq2sq; 5575 5576 priv->channel_stats = 5577 kcalloc_node(nch, sizeof(*priv->channel_stats), GFP_KERNEL, node); 5578 if (!priv->channel_stats) 5579 goto err_free_tx_rates; 5580 5581 return 0; 5582 5583 err_free_tx_rates: 5584 kfree(priv->tx_rates); 5585 err_free_txq2sq: 5586 kfree(priv->txq2sq); 5587 err_destroy_workqueue: 5588 destroy_workqueue(priv->wq); 5589 err_free_selq: 5590 mlx5e_selq_cleanup(&priv->selq); 5591 err_free_cpumask: 5592 free_cpumask_var(priv->scratchpad.cpumask); 5593 return -ENOMEM; 5594 } 5595 5596 void mlx5e_priv_cleanup(struct mlx5e_priv *priv) 5597 { 5598 int i; 5599 5600 /* bail if change profile failed and also rollback failed */ 5601 if (!priv->mdev) 5602 return; 5603 5604 for (i = 0; i < priv->stats_nch; i++) 5605 kvfree(priv->channel_stats[i]); 5606 kfree(priv->channel_stats); 5607 kfree(priv->tx_rates); 5608 kfree(priv->txq2sq); 5609 destroy_workqueue(priv->wq); 5610 mutex_lock(&priv->state_lock); 5611 mlx5e_selq_cleanup(&priv->selq); 5612 mutex_unlock(&priv->state_lock); 5613 free_cpumask_var(priv->scratchpad.cpumask); 5614 5615 for (i = 0; i < priv->htb_max_qos_sqs; i++) 5616 kfree(priv->htb_qos_sq_stats[i]); 5617 kvfree(priv->htb_qos_sq_stats); 5618 5619 memset(priv, 0, sizeof(*priv)); 5620 } 5621 5622 static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev, 5623 const struct mlx5e_profile *profile) 5624 { 5625 unsigned int nch, ptp_txqs, qos_txqs; 5626 5627 nch = mlx5e_profile_max_num_channels(mdev, profile); 5628 5629 ptp_txqs = MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn) && 5630 mlx5e_profile_feature_cap(profile, PTP_TX) ? 5631 profile->max_tc : 0; 5632 5633 qos_txqs = mlx5_qos_is_supported(mdev) && 5634 mlx5e_profile_feature_cap(profile, QOS_HTB) ? 5635 mlx5e_qos_max_leaf_nodes(mdev) : 0; 5636 5637 return nch * profile->max_tc + ptp_txqs + qos_txqs; 5638 } 5639 5640 static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev, 5641 const struct mlx5e_profile *profile) 5642 { 5643 return mlx5e_profile_max_num_channels(mdev, profile); 5644 } 5645 5646 struct net_device * 5647 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile) 5648 { 5649 struct net_device *netdev; 5650 unsigned int txqs, rxqs; 5651 int err; 5652 5653 txqs = mlx5e_get_max_num_txqs(mdev, profile); 5654 rxqs = mlx5e_get_max_num_rxqs(mdev, profile); 5655 5656 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), txqs, rxqs); 5657 if (!netdev) { 5658 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n"); 5659 return NULL; 5660 } 5661 5662 err = mlx5e_priv_init(netdev_priv(netdev), profile, netdev, mdev); 5663 if (err) { 5664 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err); 5665 goto err_free_netdev; 5666 } 5667 5668 netif_carrier_off(netdev); 5669 netif_tx_disable(netdev); 5670 dev_net_set(netdev, mlx5_core_net(mdev)); 5671 5672 return netdev; 5673 5674 err_free_netdev: 5675 free_netdev(netdev); 5676 5677 return NULL; 5678 } 5679 5680 static void mlx5e_update_features(struct net_device *netdev) 5681 { 5682 if (netdev->reg_state != NETREG_REGISTERED) 5683 return; /* features will be updated on netdev registration */ 5684 5685 rtnl_lock(); 5686 netdev_update_features(netdev); 5687 rtnl_unlock(); 5688 } 5689 5690 static void mlx5e_reset_channels(struct net_device *netdev) 5691 { 5692 netdev_reset_tc(netdev); 5693 } 5694 5695 int mlx5e_attach_netdev(struct mlx5e_priv *priv) 5696 { 5697 const bool take_rtnl = priv->netdev->reg_state == NETREG_REGISTERED; 5698 const struct mlx5e_profile *profile = priv->profile; 5699 int max_nch; 5700 int err; 5701 5702 clear_bit(MLX5E_STATE_DESTROYING, &priv->state); 5703 if (priv->fs) 5704 mlx5e_fs_set_state_destroy(priv->fs, 5705 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 5706 5707 /* Validate the max_wqe_size_sq capability. */ 5708 if (WARN_ON_ONCE(mlx5e_get_max_sq_wqebbs(priv->mdev) < MLX5E_MAX_TX_WQEBBS)) { 5709 mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %lu\n", 5710 mlx5e_get_max_sq_wqebbs(priv->mdev), MLX5E_MAX_TX_WQEBBS); 5711 return -EIO; 5712 } 5713 5714 /* max number of channels may have changed */ 5715 max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile); 5716 if (priv->channels.params.num_channels > max_nch) { 5717 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch); 5718 /* Reducing the number of channels - RXFH has to be reset, and 5719 * mlx5e_num_channels_changed below will build the RQT. 5720 */ 5721 priv->netdev->priv_flags &= ~IFF_RXFH_CONFIGURED; 5722 priv->channels.params.num_channels = max_nch; 5723 if (priv->channels.params.mqprio.mode == TC_MQPRIO_MODE_CHANNEL) { 5724 mlx5_core_warn(priv->mdev, "MLX5E: Disabling MQPRIO channel mode\n"); 5725 mlx5e_params_mqprio_reset(&priv->channels.params); 5726 } 5727 } 5728 if (max_nch != priv->max_nch) { 5729 mlx5_core_warn(priv->mdev, 5730 "MLX5E: Updating max number of channels from %u to %u\n", 5731 priv->max_nch, max_nch); 5732 priv->max_nch = max_nch; 5733 } 5734 5735 /* 1. Set the real number of queues in the kernel the first time. 5736 * 2. Set our default XPS cpumask. 5737 * 3. Build the RQT. 5738 * 5739 * rtnl_lock is required by netif_set_real_num_*_queues in case the 5740 * netdev has been registered by this point (if this function was called 5741 * in the reload or resume flow). 5742 */ 5743 if (take_rtnl) 5744 rtnl_lock(); 5745 err = mlx5e_num_channels_changed(priv); 5746 if (take_rtnl) 5747 rtnl_unlock(); 5748 if (err) 5749 goto out; 5750 5751 err = profile->init_tx(priv); 5752 if (err) 5753 goto out; 5754 5755 err = profile->init_rx(priv); 5756 if (err) 5757 goto err_cleanup_tx; 5758 5759 if (profile->enable) 5760 profile->enable(priv); 5761 5762 mlx5e_update_features(priv->netdev); 5763 5764 return 0; 5765 5766 err_cleanup_tx: 5767 profile->cleanup_tx(priv); 5768 5769 out: 5770 mlx5e_reset_channels(priv->netdev); 5771 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 5772 if (priv->fs) 5773 mlx5e_fs_set_state_destroy(priv->fs, 5774 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 5775 cancel_work_sync(&priv->update_stats_work); 5776 return err; 5777 } 5778 5779 void mlx5e_detach_netdev(struct mlx5e_priv *priv) 5780 { 5781 const struct mlx5e_profile *profile = priv->profile; 5782 5783 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 5784 if (priv->fs) 5785 mlx5e_fs_set_state_destroy(priv->fs, 5786 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 5787 5788 if (profile->disable) 5789 profile->disable(priv); 5790 flush_workqueue(priv->wq); 5791 5792 profile->cleanup_rx(priv); 5793 profile->cleanup_tx(priv); 5794 mlx5e_reset_channels(priv->netdev); 5795 cancel_work_sync(&priv->update_stats_work); 5796 } 5797 5798 static int 5799 mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mdev, 5800 const struct mlx5e_profile *new_profile, void *new_ppriv) 5801 { 5802 struct mlx5e_priv *priv = netdev_priv(netdev); 5803 int err; 5804 5805 err = mlx5e_priv_init(priv, new_profile, netdev, mdev); 5806 if (err) { 5807 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err); 5808 return err; 5809 } 5810 netif_carrier_off(netdev); 5811 priv->profile = new_profile; 5812 priv->ppriv = new_ppriv; 5813 err = new_profile->init(priv->mdev, priv->netdev); 5814 if (err) 5815 goto priv_cleanup; 5816 err = mlx5e_attach_netdev(priv); 5817 if (err) 5818 goto profile_cleanup; 5819 return err; 5820 5821 profile_cleanup: 5822 new_profile->cleanup(priv); 5823 priv_cleanup: 5824 mlx5e_priv_cleanup(priv); 5825 return err; 5826 } 5827 5828 int mlx5e_netdev_change_profile(struct mlx5e_priv *priv, 5829 const struct mlx5e_profile *new_profile, void *new_ppriv) 5830 { 5831 const struct mlx5e_profile *orig_profile = priv->profile; 5832 struct net_device *netdev = priv->netdev; 5833 struct mlx5_core_dev *mdev = priv->mdev; 5834 void *orig_ppriv = priv->ppriv; 5835 int err, rollback_err; 5836 5837 /* cleanup old profile */ 5838 mlx5e_detach_netdev(priv); 5839 priv->profile->cleanup(priv); 5840 mlx5e_priv_cleanup(priv); 5841 5842 err = mlx5e_netdev_attach_profile(netdev, mdev, new_profile, new_ppriv); 5843 if (err) { /* roll back to original profile */ 5844 netdev_warn(netdev, "%s: new profile init failed, %d\n", __func__, err); 5845 goto rollback; 5846 } 5847 5848 return 0; 5849 5850 rollback: 5851 rollback_err = mlx5e_netdev_attach_profile(netdev, mdev, orig_profile, orig_ppriv); 5852 if (rollback_err) 5853 netdev_err(netdev, "%s: failed to rollback to orig profile, %d\n", 5854 __func__, rollback_err); 5855 return err; 5856 } 5857 5858 void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv) 5859 { 5860 mlx5e_netdev_change_profile(priv, &mlx5e_nic_profile, NULL); 5861 } 5862 5863 void mlx5e_destroy_netdev(struct mlx5e_priv *priv) 5864 { 5865 struct net_device *netdev = priv->netdev; 5866 5867 mlx5e_priv_cleanup(priv); 5868 free_netdev(netdev); 5869 } 5870 5871 static int mlx5e_resume(struct auxiliary_device *adev) 5872 { 5873 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 5874 struct mlx5e_priv *priv = auxiliary_get_drvdata(adev); 5875 struct net_device *netdev = priv->netdev; 5876 struct mlx5_core_dev *mdev = edev->mdev; 5877 int err; 5878 5879 if (netif_device_present(netdev)) 5880 return 0; 5881 5882 err = mlx5e_create_mdev_resources(mdev); 5883 if (err) 5884 return err; 5885 5886 err = mlx5e_attach_netdev(priv); 5887 if (err) { 5888 mlx5e_destroy_mdev_resources(mdev); 5889 return err; 5890 } 5891 5892 return 0; 5893 } 5894 5895 static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) 5896 { 5897 struct mlx5e_priv *priv = auxiliary_get_drvdata(adev); 5898 struct net_device *netdev = priv->netdev; 5899 struct mlx5_core_dev *mdev = priv->mdev; 5900 5901 if (!netif_device_present(netdev)) 5902 return -ENODEV; 5903 5904 mlx5e_detach_netdev(priv); 5905 mlx5e_destroy_mdev_resources(mdev); 5906 return 0; 5907 } 5908 5909 static int mlx5e_probe(struct auxiliary_device *adev, 5910 const struct auxiliary_device_id *id) 5911 { 5912 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 5913 const struct mlx5e_profile *profile = &mlx5e_nic_profile; 5914 struct mlx5_core_dev *mdev = edev->mdev; 5915 struct net_device *netdev; 5916 pm_message_t state = {}; 5917 struct mlx5e_priv *priv; 5918 int err; 5919 5920 netdev = mlx5e_create_netdev(mdev, profile); 5921 if (!netdev) { 5922 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n"); 5923 return -ENOMEM; 5924 } 5925 5926 mlx5e_build_nic_netdev(netdev); 5927 5928 priv = netdev_priv(netdev); 5929 auxiliary_set_drvdata(adev, priv); 5930 5931 priv->profile = profile; 5932 priv->ppriv = NULL; 5933 5934 err = mlx5e_devlink_port_register(priv); 5935 if (err) { 5936 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err); 5937 goto err_destroy_netdev; 5938 } 5939 5940 err = profile->init(mdev, netdev); 5941 if (err) { 5942 mlx5_core_err(mdev, "mlx5e_nic_profile init failed, %d\n", err); 5943 goto err_devlink_cleanup; 5944 } 5945 5946 err = mlx5e_resume(adev); 5947 if (err) { 5948 mlx5_core_err(mdev, "mlx5e_resume failed, %d\n", err); 5949 goto err_profile_cleanup; 5950 } 5951 5952 SET_NETDEV_DEVLINK_PORT(netdev, mlx5e_devlink_get_dl_port(priv)); 5953 err = register_netdev(netdev); 5954 if (err) { 5955 mlx5_core_err(mdev, "register_netdev failed, %d\n", err); 5956 goto err_resume; 5957 } 5958 5959 mlx5e_dcbnl_init_app(priv); 5960 mlx5_uplink_netdev_set(mdev, netdev); 5961 mlx5e_params_print_info(mdev, &priv->channels.params); 5962 return 0; 5963 5964 err_resume: 5965 mlx5e_suspend(adev, state); 5966 err_profile_cleanup: 5967 profile->cleanup(priv); 5968 err_devlink_cleanup: 5969 mlx5e_devlink_port_unregister(priv); 5970 err_destroy_netdev: 5971 mlx5e_destroy_netdev(priv); 5972 return err; 5973 } 5974 5975 static void mlx5e_remove(struct auxiliary_device *adev) 5976 { 5977 struct mlx5e_priv *priv = auxiliary_get_drvdata(adev); 5978 pm_message_t state = {}; 5979 5980 mlx5e_dcbnl_delete_app(priv); 5981 unregister_netdev(priv->netdev); 5982 mlx5e_suspend(adev, state); 5983 priv->profile->cleanup(priv); 5984 mlx5e_devlink_port_unregister(priv); 5985 mlx5e_destroy_netdev(priv); 5986 } 5987 5988 static const struct auxiliary_device_id mlx5e_id_table[] = { 5989 { .name = MLX5_ADEV_NAME ".eth", }, 5990 {}, 5991 }; 5992 5993 MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table); 5994 5995 static struct auxiliary_driver mlx5e_driver = { 5996 .name = "eth", 5997 .probe = mlx5e_probe, 5998 .remove = mlx5e_remove, 5999 .suspend = mlx5e_suspend, 6000 .resume = mlx5e_resume, 6001 .id_table = mlx5e_id_table, 6002 }; 6003 6004 int mlx5e_init(void) 6005 { 6006 int ret; 6007 6008 mlx5e_build_ptys2ethtool_map(); 6009 ret = auxiliary_driver_register(&mlx5e_driver); 6010 if (ret) 6011 return ret; 6012 6013 ret = mlx5e_rep_init(); 6014 if (ret) 6015 auxiliary_driver_unregister(&mlx5e_driver); 6016 return ret; 6017 } 6018 6019 void mlx5e_cleanup(void) 6020 { 6021 mlx5e_rep_cleanup(); 6022 auxiliary_driver_unregister(&mlx5e_driver); 6023 } 6024