1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include "en/params.h" 5 #include "en/txrx.h" 6 #include "en_accel/tls_rxtx.h" 7 8 static inline bool mlx5e_rx_is_xdp(struct mlx5e_params *params, 9 struct mlx5e_xsk_param *xsk) 10 { 11 return params->xdp_prog || xsk; 12 } 13 14 u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, 15 struct mlx5e_xsk_param *xsk) 16 { 17 u16 headroom; 18 19 if (xsk) 20 return xsk->headroom; 21 22 headroom = NET_IP_ALIGN; 23 if (mlx5e_rx_is_xdp(params, xsk)) 24 headroom += XDP_PACKET_HEADROOM; 25 else 26 headroom += MLX5_RX_HEADROOM; 27 28 return headroom; 29 } 30 31 u32 mlx5e_rx_get_min_frag_sz(struct mlx5e_params *params, 32 struct mlx5e_xsk_param *xsk) 33 { 34 u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 35 u16 linear_rq_headroom = mlx5e_get_linear_rq_headroom(params, xsk); 36 37 return linear_rq_headroom + hw_mtu; 38 } 39 40 u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params, 41 struct mlx5e_xsk_param *xsk) 42 { 43 u32 frag_sz = mlx5e_rx_get_min_frag_sz(params, xsk); 44 45 /* AF_XDP doesn't build SKBs in place. */ 46 if (!xsk) 47 frag_sz = MLX5_SKB_FRAG_SZ(frag_sz); 48 49 /* XDP in mlx5e doesn't support multiple packets per page. AF_XDP is a 50 * special case. It can run with frames smaller than a page, as it 51 * doesn't allocate pages dynamically. However, here we pretend that 52 * fragments are page-sized: it allows to treat XSK frames like pages 53 * by redirecting alloc and free operations to XSK rings and by using 54 * the fact there are no multiple packets per "page" (which is a frame). 55 * The latter is important, because frames may come in a random order, 56 * and we will have trouble assemblying a real page of multiple frames. 57 */ 58 if (mlx5e_rx_is_xdp(params, xsk)) 59 frag_sz = max_t(u32, frag_sz, PAGE_SIZE); 60 61 /* Even if we can go with a smaller fragment size, we must not put 62 * multiple packets into a single frame. 63 */ 64 if (xsk) 65 frag_sz = max_t(u32, frag_sz, xsk->chunk_size); 66 67 return frag_sz; 68 } 69 70 u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params, 71 struct mlx5e_xsk_param *xsk) 72 { 73 u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk); 74 75 return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz); 76 } 77 78 bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params, 79 struct mlx5e_xsk_param *xsk) 80 { 81 /* AF_XDP allocates SKBs on XDP_PASS - ensure they don't occupy more 82 * than one page. For this, check both with and without xsk. 83 */ 84 u32 linear_frag_sz = max(mlx5e_rx_get_linear_frag_sz(params, xsk), 85 mlx5e_rx_get_linear_frag_sz(params, NULL)); 86 87 return !params->lro_en && linear_frag_sz <= PAGE_SIZE; 88 } 89 90 #define MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ ((BIT(__mlx5_bit_sz(wq, log_wqe_stride_size)) - 1) + \ 91 MLX5_MPWQE_LOG_STRIDE_SZ_BASE) 92 bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, 93 struct mlx5e_params *params, 94 struct mlx5e_xsk_param *xsk) 95 { 96 u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk); 97 s8 signed_log_num_strides_param; 98 u8 log_num_strides; 99 100 if (!mlx5e_rx_is_linear_skb(params, xsk)) 101 return false; 102 103 if (order_base_2(linear_frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ) 104 return false; 105 106 if (MLX5_CAP_GEN(mdev, ext_stride_num_range)) 107 return true; 108 109 log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz); 110 signed_log_num_strides_param = 111 (s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE; 112 113 return signed_log_num_strides_param >= 0; 114 } 115 116 u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params, 117 struct mlx5e_xsk_param *xsk) 118 { 119 u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params, xsk); 120 121 /* Numbers are unsigned, don't subtract to avoid underflow. */ 122 if (params->log_rq_mtu_frames < 123 log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW) 124 return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW; 125 126 return params->log_rq_mtu_frames - log_pkts_per_wqe; 127 } 128 129 u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, 130 struct mlx5e_params *params, 131 struct mlx5e_xsk_param *xsk) 132 { 133 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) 134 return order_base_2(mlx5e_rx_get_linear_frag_sz(params, xsk)); 135 136 return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev); 137 } 138 139 u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, 140 struct mlx5e_params *params, 141 struct mlx5e_xsk_param *xsk) 142 { 143 return MLX5_MPWRQ_LOG_WQE_SZ - 144 mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); 145 } 146 147 u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, 148 struct mlx5e_params *params, 149 struct mlx5e_xsk_param *xsk) 150 { 151 bool is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ? 152 mlx5e_rx_is_linear_skb(params, xsk) : 153 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk); 154 155 return is_linear_skb ? mlx5e_get_linear_rq_headroom(params, xsk) : 0; 156 } 157 158 u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 159 { 160 bool is_mpwqe = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE); 161 u16 stop_room; 162 163 stop_room = mlx5e_tls_get_stop_room(mdev, params); 164 stop_room += mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS); 165 if (is_mpwqe) 166 /* A MPWQE can take up to the maximum-sized WQE + all the normal 167 * stop room can be taken if a new packet breaks the active 168 * MPWQE session and allocates its WQEs right away. 169 */ 170 stop_room += mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS); 171 172 return stop_room; 173 } 174 175 int mlx5e_validate_params(struct mlx5e_priv *priv, struct mlx5e_params *params) 176 { 177 size_t sq_size = 1 << params->log_sq_size; 178 u16 stop_room; 179 180 stop_room = mlx5e_calc_sq_stop_room(priv->mdev, params); 181 if (stop_room >= sq_size) { 182 netdev_err(priv->netdev, "Stop room %u is bigger than the SQ size %zu\n", 183 stop_room, sq_size); 184 return -EINVAL; 185 } 186 187 return 0; 188 } 189