1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved. */
3 
4 #include <net/xdp_sock_drv.h>
5 #include "pool.h"
6 #include "setup.h"
7 #include "en/params.h"
8 
mlx5e_xsk_map_pool(struct mlx5e_priv * priv,struct xsk_buff_pool * pool)9 static int mlx5e_xsk_map_pool(struct mlx5e_priv *priv,
10 			      struct xsk_buff_pool *pool)
11 {
12 	struct device *dev = mlx5_core_dma_dev(priv->mdev);
13 
14 	return xsk_pool_dma_map(pool, dev, DMA_ATTR_SKIP_CPU_SYNC);
15 }
16 
mlx5e_xsk_unmap_pool(struct mlx5e_priv * priv,struct xsk_buff_pool * pool)17 static void mlx5e_xsk_unmap_pool(struct mlx5e_priv *priv,
18 				 struct xsk_buff_pool *pool)
19 {
20 	return xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC);
21 }
22 
mlx5e_xsk_get_pools(struct mlx5e_xsk * xsk)23 static int mlx5e_xsk_get_pools(struct mlx5e_xsk *xsk)
24 {
25 	if (!xsk->pools) {
26 		xsk->pools = kcalloc(MLX5E_MAX_NUM_CHANNELS,
27 				     sizeof(*xsk->pools), GFP_KERNEL);
28 		if (unlikely(!xsk->pools))
29 			return -ENOMEM;
30 	}
31 
32 	xsk->refcnt++;
33 	xsk->ever_used = true;
34 
35 	return 0;
36 }
37 
mlx5e_xsk_put_pools(struct mlx5e_xsk * xsk)38 static void mlx5e_xsk_put_pools(struct mlx5e_xsk *xsk)
39 {
40 	if (!--xsk->refcnt) {
41 		kfree(xsk->pools);
42 		xsk->pools = NULL;
43 	}
44 }
45 
mlx5e_xsk_add_pool(struct mlx5e_xsk * xsk,struct xsk_buff_pool * pool,u16 ix)46 static int mlx5e_xsk_add_pool(struct mlx5e_xsk *xsk, struct xsk_buff_pool *pool, u16 ix)
47 {
48 	int err;
49 
50 	err = mlx5e_xsk_get_pools(xsk);
51 	if (unlikely(err))
52 		return err;
53 
54 	xsk->pools[ix] = pool;
55 	return 0;
56 }
57 
mlx5e_xsk_remove_pool(struct mlx5e_xsk * xsk,u16 ix)58 static void mlx5e_xsk_remove_pool(struct mlx5e_xsk *xsk, u16 ix)
59 {
60 	xsk->pools[ix] = NULL;
61 
62 	mlx5e_xsk_put_pools(xsk);
63 }
64 
mlx5e_xsk_is_pool_sane(struct xsk_buff_pool * pool)65 static bool mlx5e_xsk_is_pool_sane(struct xsk_buff_pool *pool)
66 {
67 	return xsk_pool_get_headroom(pool) <= 0xffff &&
68 		xsk_pool_get_chunk_size(pool) <= 0xffff;
69 }
70 
mlx5e_build_xsk_param(struct xsk_buff_pool * pool,struct mlx5e_xsk_param * xsk)71 void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *xsk)
72 {
73 	xsk->headroom = xsk_pool_get_headroom(pool);
74 	xsk->chunk_size = xsk_pool_get_chunk_size(pool);
75 	xsk->unaligned = pool->unaligned;
76 }
77 
mlx5e_xsk_enable_locked(struct mlx5e_priv * priv,struct xsk_buff_pool * pool,u16 ix)78 static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
79 				   struct xsk_buff_pool *pool, u16 ix)
80 {
81 	struct mlx5e_params *params = &priv->channels.params;
82 	struct mlx5e_xsk_param xsk;
83 	struct mlx5e_channel *c;
84 	int err;
85 
86 	if (unlikely(mlx5e_xsk_get_pool(&priv->channels.params, &priv->xsk, ix)))
87 		return -EBUSY;
88 
89 	if (unlikely(!mlx5e_xsk_is_pool_sane(pool)))
90 		return -EINVAL;
91 
92 	err = mlx5e_xsk_map_pool(priv, pool);
93 	if (unlikely(err))
94 		return err;
95 
96 	err = mlx5e_xsk_add_pool(&priv->xsk, pool, ix);
97 	if (unlikely(err))
98 		goto err_unmap_pool;
99 
100 	mlx5e_build_xsk_param(pool, &xsk);
101 
102 	if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
103 	    mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
104 		const char *recommendation = is_power_of_2(xsk.chunk_size) ?
105 			"Upgrade firmware" : "Disable striding RQ";
106 
107 		mlx5_core_warn(priv->mdev, "Expected slowdown with XSK frame size %u. %s for better performance.\n",
108 			       xsk.chunk_size, recommendation);
109 	}
110 
111 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
112 		/* XSK objects will be created on open. */
113 		goto validate_closed;
114 	}
115 
116 	if (!params->xdp_prog) {
117 		/* XSK objects will be created when an XDP program is set,
118 		 * and the channels are reopened.
119 		 */
120 		goto validate_closed;
121 	}
122 
123 	c = priv->channels.c[ix];
124 
125 	err = mlx5e_open_xsk(priv, params, &xsk, pool, c);
126 	if (unlikely(err))
127 		goto err_remove_pool;
128 
129 	mlx5e_activate_xsk(c);
130 	mlx5e_trigger_napi_icosq(c);
131 
132 	/* Don't wait for WQEs, because the newer xdpsock sample doesn't provide
133 	 * any Fill Ring entries at the setup stage.
134 	 */
135 
136 	mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, true);
137 
138 	mlx5e_deactivate_rq(&c->rq);
139 	mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY);
140 
141 	return 0;
142 
143 err_remove_pool:
144 	mlx5e_xsk_remove_pool(&priv->xsk, ix);
145 
146 err_unmap_pool:
147 	mlx5e_xsk_unmap_pool(priv, pool);
148 
149 	return err;
150 
151 validate_closed:
152 	/* Check the configuration in advance, rather than fail at a later stage
153 	 * (in mlx5e_xdp_set or on open) and end up with no channels.
154 	 */
155 	if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) {
156 		err = -EINVAL;
157 		goto err_remove_pool;
158 	}
159 
160 	return 0;
161 }
162 
mlx5e_xsk_disable_locked(struct mlx5e_priv * priv,u16 ix)163 static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix)
164 {
165 	struct xsk_buff_pool *pool = mlx5e_xsk_get_pool(&priv->channels.params,
166 						   &priv->xsk, ix);
167 	struct mlx5e_channel *c;
168 
169 	if (unlikely(!pool))
170 		return -EINVAL;
171 
172 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
173 		goto remove_pool;
174 
175 	/* XSK RQ and SQ are only created if XDP program is set. */
176 	if (!priv->channels.params.xdp_prog)
177 		goto remove_pool;
178 
179 	c = priv->channels.c[ix];
180 
181 	mlx5e_activate_rq(&c->rq);
182 	mlx5e_trigger_napi_icosq(c);
183 	mlx5e_wait_for_min_rx_wqes(&c->rq, MLX5E_RQ_WQES_TIMEOUT);
184 
185 	mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, false);
186 
187 	mlx5e_deactivate_xsk(c);
188 	mlx5e_close_xsk(c);
189 
190 remove_pool:
191 	mlx5e_xsk_remove_pool(&priv->xsk, ix);
192 	mlx5e_xsk_unmap_pool(priv, pool);
193 
194 	return 0;
195 }
196 
mlx5e_xsk_enable_pool(struct mlx5e_priv * priv,struct xsk_buff_pool * pool,u16 ix)197 static int mlx5e_xsk_enable_pool(struct mlx5e_priv *priv, struct xsk_buff_pool *pool,
198 				 u16 ix)
199 {
200 	int err;
201 
202 	mutex_lock(&priv->state_lock);
203 	err = mlx5e_xsk_enable_locked(priv, pool, ix);
204 	mutex_unlock(&priv->state_lock);
205 
206 	return err;
207 }
208 
mlx5e_xsk_disable_pool(struct mlx5e_priv * priv,u16 ix)209 static int mlx5e_xsk_disable_pool(struct mlx5e_priv *priv, u16 ix)
210 {
211 	int err;
212 
213 	mutex_lock(&priv->state_lock);
214 	err = mlx5e_xsk_disable_locked(priv, ix);
215 	mutex_unlock(&priv->state_lock);
216 
217 	return err;
218 }
219 
mlx5e_xsk_setup_pool(struct net_device * dev,struct xsk_buff_pool * pool,u16 qid)220 int mlx5e_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid)
221 {
222 	struct mlx5e_priv *priv = netdev_priv(dev);
223 	struct mlx5e_params *params = &priv->channels.params;
224 
225 	if (unlikely(qid >= params->num_channels))
226 		return -EINVAL;
227 
228 	return pool ? mlx5e_xsk_enable_pool(priv, pool, qid) :
229 		      mlx5e_xsk_disable_pool(priv, qid);
230 }
231