11742b3d5SMagnus Karlsson // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
21742b3d5SMagnus Karlsson /* Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved. */
31742b3d5SMagnus Karlsson
41742b3d5SMagnus Karlsson #include <net/xdp_sock_drv.h>
51742b3d5SMagnus Karlsson #include "pool.h"
61742b3d5SMagnus Karlsson #include "setup.h"
71742b3d5SMagnus Karlsson #include "en/params.h"
81742b3d5SMagnus Karlsson
mlx5e_xsk_map_pool(struct mlx5e_priv * priv,struct xsk_buff_pool * pool)91742b3d5SMagnus Karlsson static int mlx5e_xsk_map_pool(struct mlx5e_priv *priv,
101742b3d5SMagnus Karlsson struct xsk_buff_pool *pool)
111742b3d5SMagnus Karlsson {
127be3412aSParav Pandit struct device *dev = mlx5_core_dma_dev(priv->mdev);
131742b3d5SMagnus Karlsson
140b7cfa40SAya Levin return xsk_pool_dma_map(pool, dev, DMA_ATTR_SKIP_CPU_SYNC);
151742b3d5SMagnus Karlsson }
161742b3d5SMagnus Karlsson
mlx5e_xsk_unmap_pool(struct mlx5e_priv * priv,struct xsk_buff_pool * pool)171742b3d5SMagnus Karlsson static void mlx5e_xsk_unmap_pool(struct mlx5e_priv *priv,
181742b3d5SMagnus Karlsson struct xsk_buff_pool *pool)
191742b3d5SMagnus Karlsson {
200b7cfa40SAya Levin return xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC);
211742b3d5SMagnus Karlsson }
221742b3d5SMagnus Karlsson
mlx5e_xsk_get_pools(struct mlx5e_xsk * xsk)231742b3d5SMagnus Karlsson static int mlx5e_xsk_get_pools(struct mlx5e_xsk *xsk)
241742b3d5SMagnus Karlsson {
251742b3d5SMagnus Karlsson if (!xsk->pools) {
261742b3d5SMagnus Karlsson xsk->pools = kcalloc(MLX5E_MAX_NUM_CHANNELS,
271742b3d5SMagnus Karlsson sizeof(*xsk->pools), GFP_KERNEL);
281742b3d5SMagnus Karlsson if (unlikely(!xsk->pools))
291742b3d5SMagnus Karlsson return -ENOMEM;
301742b3d5SMagnus Karlsson }
311742b3d5SMagnus Karlsson
321742b3d5SMagnus Karlsson xsk->refcnt++;
331742b3d5SMagnus Karlsson xsk->ever_used = true;
341742b3d5SMagnus Karlsson
351742b3d5SMagnus Karlsson return 0;
361742b3d5SMagnus Karlsson }
371742b3d5SMagnus Karlsson
mlx5e_xsk_put_pools(struct mlx5e_xsk * xsk)381742b3d5SMagnus Karlsson static void mlx5e_xsk_put_pools(struct mlx5e_xsk *xsk)
391742b3d5SMagnus Karlsson {
401742b3d5SMagnus Karlsson if (!--xsk->refcnt) {
411742b3d5SMagnus Karlsson kfree(xsk->pools);
421742b3d5SMagnus Karlsson xsk->pools = NULL;
431742b3d5SMagnus Karlsson }
441742b3d5SMagnus Karlsson }
451742b3d5SMagnus Karlsson
mlx5e_xsk_add_pool(struct mlx5e_xsk * xsk,struct xsk_buff_pool * pool,u16 ix)461742b3d5SMagnus Karlsson static int mlx5e_xsk_add_pool(struct mlx5e_xsk *xsk, struct xsk_buff_pool *pool, u16 ix)
471742b3d5SMagnus Karlsson {
481742b3d5SMagnus Karlsson int err;
491742b3d5SMagnus Karlsson
501742b3d5SMagnus Karlsson err = mlx5e_xsk_get_pools(xsk);
511742b3d5SMagnus Karlsson if (unlikely(err))
521742b3d5SMagnus Karlsson return err;
531742b3d5SMagnus Karlsson
541742b3d5SMagnus Karlsson xsk->pools[ix] = pool;
551742b3d5SMagnus Karlsson return 0;
561742b3d5SMagnus Karlsson }
571742b3d5SMagnus Karlsson
mlx5e_xsk_remove_pool(struct mlx5e_xsk * xsk,u16 ix)581742b3d5SMagnus Karlsson static void mlx5e_xsk_remove_pool(struct mlx5e_xsk *xsk, u16 ix)
591742b3d5SMagnus Karlsson {
601742b3d5SMagnus Karlsson xsk->pools[ix] = NULL;
611742b3d5SMagnus Karlsson
621742b3d5SMagnus Karlsson mlx5e_xsk_put_pools(xsk);
631742b3d5SMagnus Karlsson }
641742b3d5SMagnus Karlsson
mlx5e_xsk_is_pool_sane(struct xsk_buff_pool * pool)651742b3d5SMagnus Karlsson static bool mlx5e_xsk_is_pool_sane(struct xsk_buff_pool *pool)
661742b3d5SMagnus Karlsson {
67c4655761SMagnus Karlsson return xsk_pool_get_headroom(pool) <= 0xffff &&
68c4655761SMagnus Karlsson xsk_pool_get_chunk_size(pool) <= 0xffff;
691742b3d5SMagnus Karlsson }
701742b3d5SMagnus Karlsson
mlx5e_build_xsk_param(struct xsk_buff_pool * pool,struct mlx5e_xsk_param * xsk)711742b3d5SMagnus Karlsson void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *xsk)
721742b3d5SMagnus Karlsson {
73c4655761SMagnus Karlsson xsk->headroom = xsk_pool_get_headroom(pool);
74c4655761SMagnus Karlsson xsk->chunk_size = xsk_pool_get_chunk_size(pool);
756470d2e7SMaxim Mikityanskiy xsk->unaligned = pool->unaligned;
761742b3d5SMagnus Karlsson }
771742b3d5SMagnus Karlsson
mlx5e_xsk_enable_locked(struct mlx5e_priv * priv,struct xsk_buff_pool * pool,u16 ix)781742b3d5SMagnus Karlsson static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
791742b3d5SMagnus Karlsson struct xsk_buff_pool *pool, u16 ix)
801742b3d5SMagnus Karlsson {
811742b3d5SMagnus Karlsson struct mlx5e_params *params = &priv->channels.params;
821742b3d5SMagnus Karlsson struct mlx5e_xsk_param xsk;
831742b3d5SMagnus Karlsson struct mlx5e_channel *c;
841742b3d5SMagnus Karlsson int err;
851742b3d5SMagnus Karlsson
861742b3d5SMagnus Karlsson if (unlikely(mlx5e_xsk_get_pool(&priv->channels.params, &priv->xsk, ix)))
871742b3d5SMagnus Karlsson return -EBUSY;
881742b3d5SMagnus Karlsson
891742b3d5SMagnus Karlsson if (unlikely(!mlx5e_xsk_is_pool_sane(pool)))
901742b3d5SMagnus Karlsson return -EINVAL;
911742b3d5SMagnus Karlsson
921742b3d5SMagnus Karlsson err = mlx5e_xsk_map_pool(priv, pool);
931742b3d5SMagnus Karlsson if (unlikely(err))
941742b3d5SMagnus Karlsson return err;
951742b3d5SMagnus Karlsson
961742b3d5SMagnus Karlsson err = mlx5e_xsk_add_pool(&priv->xsk, pool, ix);
971742b3d5SMagnus Karlsson if (unlikely(err))
981742b3d5SMagnus Karlsson goto err_unmap_pool;
991742b3d5SMagnus Karlsson
1001742b3d5SMagnus Karlsson mlx5e_build_xsk_param(pool, &xsk);
1011742b3d5SMagnus Karlsson
102*c6f04204SMaxim Mikityanskiy if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
103*c6f04204SMaxim Mikityanskiy mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
104*c6f04204SMaxim Mikityanskiy const char *recommendation = is_power_of_2(xsk.chunk_size) ?
105*c6f04204SMaxim Mikityanskiy "Upgrade firmware" : "Disable striding RQ";
106*c6f04204SMaxim Mikityanskiy
107*c6f04204SMaxim Mikityanskiy mlx5_core_warn(priv->mdev, "Expected slowdown with XSK frame size %u. %s for better performance.\n",
108*c6f04204SMaxim Mikityanskiy xsk.chunk_size, recommendation);
109*c6f04204SMaxim Mikityanskiy }
110*c6f04204SMaxim Mikityanskiy
1111742b3d5SMagnus Karlsson if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
1121742b3d5SMagnus Karlsson /* XSK objects will be created on open. */
1131742b3d5SMagnus Karlsson goto validate_closed;
1141742b3d5SMagnus Karlsson }
1151742b3d5SMagnus Karlsson
1161742b3d5SMagnus Karlsson if (!params->xdp_prog) {
1171742b3d5SMagnus Karlsson /* XSK objects will be created when an XDP program is set,
1181742b3d5SMagnus Karlsson * and the channels are reopened.
1191742b3d5SMagnus Karlsson */
1201742b3d5SMagnus Karlsson goto validate_closed;
1211742b3d5SMagnus Karlsson }
1221742b3d5SMagnus Karlsson
1231742b3d5SMagnus Karlsson c = priv->channels.c[ix];
1241742b3d5SMagnus Karlsson
1251742b3d5SMagnus Karlsson err = mlx5e_open_xsk(priv, params, &xsk, pool, c);
1261742b3d5SMagnus Karlsson if (unlikely(err))
1271742b3d5SMagnus Karlsson goto err_remove_pool;
1281742b3d5SMagnus Karlsson
1291742b3d5SMagnus Karlsson mlx5e_activate_xsk(c);
1302e642afbSMaxim Mikityanskiy mlx5e_trigger_napi_icosq(c);
1311742b3d5SMagnus Karlsson
1321742b3d5SMagnus Karlsson /* Don't wait for WQEs, because the newer xdpsock sample doesn't provide
1331742b3d5SMagnus Karlsson * any Fill Ring entries at the setup stage.
1341742b3d5SMagnus Karlsson */
1351742b3d5SMagnus Karlsson
1363db4c85cSMaxim Mikityanskiy mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, true);
1371742b3d5SMagnus Karlsson
138082a9edfSMaxim Mikityanskiy mlx5e_deactivate_rq(&c->rq);
139082a9edfSMaxim Mikityanskiy mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY);
140082a9edfSMaxim Mikityanskiy
1411742b3d5SMagnus Karlsson return 0;
1421742b3d5SMagnus Karlsson
1431742b3d5SMagnus Karlsson err_remove_pool:
1441742b3d5SMagnus Karlsson mlx5e_xsk_remove_pool(&priv->xsk, ix);
1451742b3d5SMagnus Karlsson
1461742b3d5SMagnus Karlsson err_unmap_pool:
1471742b3d5SMagnus Karlsson mlx5e_xsk_unmap_pool(priv, pool);
1481742b3d5SMagnus Karlsson
1491742b3d5SMagnus Karlsson return err;
1501742b3d5SMagnus Karlsson
1511742b3d5SMagnus Karlsson validate_closed:
1521742b3d5SMagnus Karlsson /* Check the configuration in advance, rather than fail at a later stage
1531742b3d5SMagnus Karlsson * (in mlx5e_xdp_set or on open) and end up with no channels.
1541742b3d5SMagnus Karlsson */
1551742b3d5SMagnus Karlsson if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) {
1561742b3d5SMagnus Karlsson err = -EINVAL;
1571742b3d5SMagnus Karlsson goto err_remove_pool;
1581742b3d5SMagnus Karlsson }
1591742b3d5SMagnus Karlsson
1601742b3d5SMagnus Karlsson return 0;
1611742b3d5SMagnus Karlsson }
1621742b3d5SMagnus Karlsson
mlx5e_xsk_disable_locked(struct mlx5e_priv * priv,u16 ix)1631742b3d5SMagnus Karlsson static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix)
1641742b3d5SMagnus Karlsson {
1651742b3d5SMagnus Karlsson struct xsk_buff_pool *pool = mlx5e_xsk_get_pool(&priv->channels.params,
1661742b3d5SMagnus Karlsson &priv->xsk, ix);
1671742b3d5SMagnus Karlsson struct mlx5e_channel *c;
1681742b3d5SMagnus Karlsson
1691742b3d5SMagnus Karlsson if (unlikely(!pool))
1701742b3d5SMagnus Karlsson return -EINVAL;
1711742b3d5SMagnus Karlsson
1721742b3d5SMagnus Karlsson if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
1731742b3d5SMagnus Karlsson goto remove_pool;
1741742b3d5SMagnus Karlsson
1751742b3d5SMagnus Karlsson /* XSK RQ and SQ are only created if XDP program is set. */
1761742b3d5SMagnus Karlsson if (!priv->channels.params.xdp_prog)
1771742b3d5SMagnus Karlsson goto remove_pool;
1781742b3d5SMagnus Karlsson
1791742b3d5SMagnus Karlsson c = priv->channels.c[ix];
180082a9edfSMaxim Mikityanskiy
181082a9edfSMaxim Mikityanskiy mlx5e_activate_rq(&c->rq);
182082a9edfSMaxim Mikityanskiy mlx5e_trigger_napi_icosq(c);
183082a9edfSMaxim Mikityanskiy mlx5e_wait_for_min_rx_wqes(&c->rq, MLX5E_RQ_WQES_TIMEOUT);
184082a9edfSMaxim Mikityanskiy
1853db4c85cSMaxim Mikityanskiy mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, false);
186082a9edfSMaxim Mikityanskiy
1871742b3d5SMagnus Karlsson mlx5e_deactivate_xsk(c);
1881742b3d5SMagnus Karlsson mlx5e_close_xsk(c);
1891742b3d5SMagnus Karlsson
1901742b3d5SMagnus Karlsson remove_pool:
1911742b3d5SMagnus Karlsson mlx5e_xsk_remove_pool(&priv->xsk, ix);
1921742b3d5SMagnus Karlsson mlx5e_xsk_unmap_pool(priv, pool);
1931742b3d5SMagnus Karlsson
1941742b3d5SMagnus Karlsson return 0;
1951742b3d5SMagnus Karlsson }
1961742b3d5SMagnus Karlsson
mlx5e_xsk_enable_pool(struct mlx5e_priv * priv,struct xsk_buff_pool * pool,u16 ix)1971742b3d5SMagnus Karlsson static int mlx5e_xsk_enable_pool(struct mlx5e_priv *priv, struct xsk_buff_pool *pool,
1981742b3d5SMagnus Karlsson u16 ix)
1991742b3d5SMagnus Karlsson {
2001742b3d5SMagnus Karlsson int err;
2011742b3d5SMagnus Karlsson
2021742b3d5SMagnus Karlsson mutex_lock(&priv->state_lock);
2031742b3d5SMagnus Karlsson err = mlx5e_xsk_enable_locked(priv, pool, ix);
2041742b3d5SMagnus Karlsson mutex_unlock(&priv->state_lock);
2051742b3d5SMagnus Karlsson
2061742b3d5SMagnus Karlsson return err;
2071742b3d5SMagnus Karlsson }
2081742b3d5SMagnus Karlsson
mlx5e_xsk_disable_pool(struct mlx5e_priv * priv,u16 ix)2091742b3d5SMagnus Karlsson static int mlx5e_xsk_disable_pool(struct mlx5e_priv *priv, u16 ix)
2101742b3d5SMagnus Karlsson {
2111742b3d5SMagnus Karlsson int err;
2121742b3d5SMagnus Karlsson
2131742b3d5SMagnus Karlsson mutex_lock(&priv->state_lock);
2141742b3d5SMagnus Karlsson err = mlx5e_xsk_disable_locked(priv, ix);
2151742b3d5SMagnus Karlsson mutex_unlock(&priv->state_lock);
2161742b3d5SMagnus Karlsson
2171742b3d5SMagnus Karlsson return err;
2181742b3d5SMagnus Karlsson }
2191742b3d5SMagnus Karlsson
mlx5e_xsk_setup_pool(struct net_device * dev,struct xsk_buff_pool * pool,u16 qid)2201742b3d5SMagnus Karlsson int mlx5e_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid)
2211742b3d5SMagnus Karlsson {
2221742b3d5SMagnus Karlsson struct mlx5e_priv *priv = netdev_priv(dev);
2231742b3d5SMagnus Karlsson struct mlx5e_params *params = &priv->channels.params;
2241742b3d5SMagnus Karlsson
2253db4c85cSMaxim Mikityanskiy if (unlikely(qid >= params->num_channels))
2261742b3d5SMagnus Karlsson return -EINVAL;
2271742b3d5SMagnus Karlsson
2283db4c85cSMaxim Mikityanskiy return pool ? mlx5e_xsk_enable_pool(priv, pool, qid) :
2293db4c85cSMaxim Mikityanskiy mlx5e_xsk_disable_pool(priv, qid);
2301742b3d5SMagnus Karlsson }
231