1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include "setup.h" 5 #include "en/params.h" 6 7 bool mlx5e_validate_xsk_param(struct mlx5e_params *params, 8 struct mlx5e_xsk_param *xsk, 9 struct mlx5_core_dev *mdev) 10 { 11 /* AF_XDP doesn't support frames larger than PAGE_SIZE, and the current 12 * mlx5e XDP implementation doesn't support multiple packets per page. 13 */ 14 if (xsk->chunk_size != PAGE_SIZE) 15 return false; 16 17 /* Current MTU and XSK headroom don't allow packets to fit the frames. */ 18 if (mlx5e_rx_get_linear_frag_sz(params, xsk) > xsk->chunk_size) 19 return false; 20 21 /* frag_sz is different for regular and XSK RQs, so ensure that linear 22 * SKB mode is possible. 23 */ 24 switch (params->rq_wq_type) { 25 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 26 return mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk); 27 default: /* MLX5_WQ_TYPE_CYCLIC */ 28 return mlx5e_rx_is_linear_skb(params, xsk); 29 } 30 } 31 32 static void mlx5e_build_xskicosq_param(struct mlx5e_priv *priv, 33 u8 log_wq_size, 34 struct mlx5e_sq_param *param) 35 { 36 void *sqc = param->sqc; 37 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 38 39 mlx5e_build_sq_param_common(priv, param); 40 41 MLX5_SET(wq, wq, log_wq_sz, log_wq_size); 42 } 43 44 static void mlx5e_build_xsk_cparam(struct mlx5e_priv *priv, 45 struct mlx5e_params *params, 46 struct mlx5e_xsk_param *xsk, 47 struct mlx5e_channel_param *cparam) 48 { 49 const u8 xskicosq_size = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; 50 51 mlx5e_build_rq_param(priv, params, xsk, &cparam->rq); 52 mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); 53 mlx5e_build_xskicosq_param(priv, xskicosq_size, &cparam->icosq); 54 mlx5e_build_rx_cq_param(priv, params, xsk, &cparam->rx_cq); 55 mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq); 56 mlx5e_build_ico_cq_param(priv, xskicosq_size, &cparam->icosq_cq); 57 } 58 59 int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, 60 struct mlx5e_xsk_param *xsk, struct xdp_umem *umem, 61 struct mlx5e_channel *c) 62 { 63 struct mlx5e_channel_param *cparam; 64 struct dim_cq_moder icocq_moder = {}; 65 int err; 66 67 if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev)) 68 return -EINVAL; 69 70 cparam = kvzalloc(sizeof(*cparam), GFP_KERNEL); 71 if (!cparam) 72 return -ENOMEM; 73 74 mlx5e_build_xsk_cparam(priv, params, xsk, cparam); 75 76 err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->xskrq.cq); 77 if (unlikely(err)) 78 goto err_free_cparam; 79 80 err = mlx5e_open_rq(c, params, &cparam->rq, xsk, umem, &c->xskrq); 81 if (unlikely(err)) 82 goto err_close_rx_cq; 83 84 err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xsksq.cq); 85 if (unlikely(err)) 86 goto err_close_rq; 87 88 /* Create a separate SQ, so that when the UMEM is disabled, we could 89 * close this SQ safely and stop receiving CQEs. In other case, e.g., if 90 * the XDPSQ was used instead, we might run into trouble when the UMEM 91 * is disabled and then reenabled, but the SQ continues receiving CQEs 92 * from the old UMEM. 93 */ 94 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, umem, &c->xsksq, true); 95 if (unlikely(err)) 96 goto err_close_tx_cq; 97 98 err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->xskicosq.cq); 99 if (unlikely(err)) 100 goto err_close_sq; 101 102 /* Create a dedicated SQ for posting NOPs whenever we need an IRQ to be 103 * triggered and NAPI to be called on the correct CPU. 104 */ 105 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->xskicosq); 106 if (unlikely(err)) 107 goto err_close_icocq; 108 109 kvfree(cparam); 110 111 spin_lock_init(&c->xskicosq_lock); 112 113 set_bit(MLX5E_CHANNEL_STATE_XSK, c->state); 114 115 return 0; 116 117 err_close_icocq: 118 mlx5e_close_cq(&c->xskicosq.cq); 119 120 err_close_sq: 121 mlx5e_close_xdpsq(&c->xsksq); 122 123 err_close_tx_cq: 124 mlx5e_close_cq(&c->xsksq.cq); 125 126 err_close_rq: 127 mlx5e_close_rq(&c->xskrq); 128 129 err_close_rx_cq: 130 mlx5e_close_cq(&c->xskrq.cq); 131 132 err_free_cparam: 133 kvfree(cparam); 134 135 return err; 136 } 137 138 void mlx5e_close_xsk(struct mlx5e_channel *c) 139 { 140 clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state); 141 napi_synchronize(&c->napi); 142 143 mlx5e_close_rq(&c->xskrq); 144 mlx5e_close_cq(&c->xskrq.cq); 145 mlx5e_close_icosq(&c->xskicosq); 146 mlx5e_close_cq(&c->xskicosq.cq); 147 mlx5e_close_xdpsq(&c->xsksq); 148 mlx5e_close_cq(&c->xsksq.cq); 149 } 150 151 void mlx5e_activate_xsk(struct mlx5e_channel *c) 152 { 153 set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state); 154 /* TX queue is created active. */ 155 mlx5e_trigger_irq(&c->xskicosq); 156 } 157 158 void mlx5e_deactivate_xsk(struct mlx5e_channel *c) 159 { 160 mlx5e_deactivate_rq(&c->xskrq); 161 /* TX queue is disabled on close. */ 162 } 163 164 static int mlx5e_redirect_xsk_rqt(struct mlx5e_priv *priv, u16 ix, u32 rqn) 165 { 166 struct mlx5e_redirect_rqt_param direct_rrp = { 167 .is_rss = false, 168 { 169 .rqn = rqn, 170 }, 171 }; 172 173 u32 rqtn = priv->xsk_tir[ix].rqt.rqtn; 174 175 return mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp); 176 } 177 178 int mlx5e_xsk_redirect_rqt_to_channel(struct mlx5e_priv *priv, struct mlx5e_channel *c) 179 { 180 return mlx5e_redirect_xsk_rqt(priv, c->ix, c->xskrq.rqn); 181 } 182 183 int mlx5e_xsk_redirect_rqt_to_drop(struct mlx5e_priv *priv, u16 ix) 184 { 185 return mlx5e_redirect_xsk_rqt(priv, ix, priv->drop_rq.rqn); 186 } 187 188 int mlx5e_xsk_redirect_rqts_to_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 189 { 190 int err, i; 191 192 if (!priv->xsk.refcnt) 193 return 0; 194 195 for (i = 0; i < chs->num; i++) { 196 struct mlx5e_channel *c = chs->c[i]; 197 198 if (!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 199 continue; 200 201 err = mlx5e_xsk_redirect_rqt_to_channel(priv, c); 202 if (unlikely(err)) 203 goto err_stop; 204 } 205 206 return 0; 207 208 err_stop: 209 for (i--; i >= 0; i--) { 210 if (!test_bit(MLX5E_CHANNEL_STATE_XSK, chs->c[i]->state)) 211 continue; 212 213 mlx5e_xsk_redirect_rqt_to_drop(priv, i); 214 } 215 216 return err; 217 } 218 219 void mlx5e_xsk_redirect_rqts_to_drop(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 220 { 221 int i; 222 223 if (!priv->xsk.refcnt) 224 return; 225 226 for (i = 0; i < chs->num; i++) { 227 if (!test_bit(MLX5E_CHANNEL_STATE_XSK, chs->c[i]->state)) 228 continue; 229 230 mlx5e_xsk_redirect_rqt_to_drop(priv, i); 231 } 232 } 233