1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ 3 4 #include "fs_core.h" 5 #include "lib/ipsec_fs_roce.h" 6 #include "mlx5_core.h" 7 8 struct mlx5_ipsec_miss { 9 struct mlx5_flow_group *group; 10 struct mlx5_flow_handle *rule; 11 }; 12 13 struct mlx5_ipsec_rx_roce { 14 struct mlx5_flow_group *g; 15 struct mlx5_flow_table *ft; 16 struct mlx5_flow_handle *rule; 17 struct mlx5_ipsec_miss roce_miss; 18 19 struct mlx5_flow_table *ft_rdma; 20 struct mlx5_flow_namespace *ns_rdma; 21 }; 22 23 struct mlx5_ipsec_tx_roce { 24 struct mlx5_flow_group *g; 25 struct mlx5_flow_table *ft; 26 struct mlx5_flow_handle *rule; 27 struct mlx5_flow_namespace *ns; 28 }; 29 30 struct mlx5_ipsec_fs { 31 struct mlx5_ipsec_rx_roce ipv4_rx; 32 struct mlx5_ipsec_rx_roce ipv6_rx; 33 struct mlx5_ipsec_tx_roce tx; 34 }; 35 36 static void ipsec_fs_roce_setup_udp_dport(struct mlx5_flow_spec *spec, 37 u16 dport) 38 { 39 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 40 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 41 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP); 42 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport); 43 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport); 44 } 45 46 static int 47 ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev *mdev, 48 struct mlx5_flow_destination *default_dst, 49 struct mlx5_ipsec_rx_roce *roce) 50 { 51 struct mlx5_flow_destination dst = {}; 52 MLX5_DECLARE_FLOW_ACT(flow_act); 53 struct mlx5_flow_handle *rule; 54 struct mlx5_flow_spec *spec; 55 int err = 0; 56 57 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 58 if (!spec) 59 return -ENOMEM; 60 61 ipsec_fs_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT); 62 63 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 64 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 65 dst.ft = roce->ft_rdma; 66 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1); 67 if (IS_ERR(rule)) { 68 err = PTR_ERR(rule); 69 mlx5_core_err(mdev, "Fail to add RX RoCE IPsec rule err=%d\n", 70 err); 71 goto fail_add_rule; 72 } 73 74 roce->rule = rule; 75 76 memset(spec, 0, sizeof(*spec)); 77 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, default_dst, 1); 78 if (IS_ERR(rule)) { 79 err = PTR_ERR(rule); 80 mlx5_core_err(mdev, "Fail to add RX RoCE IPsec miss rule err=%d\n", 81 err); 82 goto fail_add_default_rule; 83 } 84 85 roce->roce_miss.rule = rule; 86 87 kvfree(spec); 88 return 0; 89 90 fail_add_default_rule: 91 mlx5_del_flow_rules(roce->rule); 92 fail_add_rule: 93 kvfree(spec); 94 return err; 95 } 96 97 static int ipsec_fs_roce_tx_rule_setup(struct mlx5_core_dev *mdev, 98 struct mlx5_ipsec_tx_roce *roce, 99 struct mlx5_flow_table *pol_ft) 100 { 101 struct mlx5_flow_destination dst = {}; 102 MLX5_DECLARE_FLOW_ACT(flow_act); 103 struct mlx5_flow_handle *rule; 104 int err = 0; 105 106 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 107 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 108 dst.ft = pol_ft; 109 rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, &dst, 110 1); 111 if (IS_ERR(rule)) { 112 err = PTR_ERR(rule); 113 mlx5_core_err(mdev, "Fail to add TX RoCE IPsec rule err=%d\n", 114 err); 115 goto out; 116 } 117 roce->rule = rule; 118 119 out: 120 return err; 121 } 122 123 void mlx5_ipsec_fs_roce_tx_destroy(struct mlx5_ipsec_fs *ipsec_roce) 124 { 125 struct mlx5_ipsec_tx_roce *tx_roce; 126 127 if (!ipsec_roce) 128 return; 129 130 tx_roce = &ipsec_roce->tx; 131 132 mlx5_del_flow_rules(tx_roce->rule); 133 mlx5_destroy_flow_group(tx_roce->g); 134 mlx5_destroy_flow_table(tx_roce->ft); 135 } 136 137 #define MLX5_TX_ROCE_GROUP_SIZE BIT(0) 138 139 int mlx5_ipsec_fs_roce_tx_create(struct mlx5_core_dev *mdev, 140 struct mlx5_ipsec_fs *ipsec_roce, 141 struct mlx5_flow_table *pol_ft) 142 { 143 struct mlx5_flow_table_attr ft_attr = {}; 144 struct mlx5_ipsec_tx_roce *roce; 145 struct mlx5_flow_table *ft; 146 struct mlx5_flow_group *g; 147 int ix = 0; 148 int err; 149 u32 *in; 150 151 if (!ipsec_roce) 152 return 0; 153 154 roce = &ipsec_roce->tx; 155 156 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); 157 if (!in) 158 return -ENOMEM; 159 160 ft_attr.max_fte = 1; 161 ft = mlx5_create_flow_table(roce->ns, &ft_attr); 162 if (IS_ERR(ft)) { 163 err = PTR_ERR(ft); 164 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx ft err=%d\n", err); 165 goto free_in; 166 } 167 168 roce->ft = ft; 169 170 MLX5_SET_CFG(in, start_flow_index, ix); 171 ix += MLX5_TX_ROCE_GROUP_SIZE; 172 MLX5_SET_CFG(in, end_flow_index, ix - 1); 173 g = mlx5_create_flow_group(ft, in); 174 if (IS_ERR(g)) { 175 err = PTR_ERR(g); 176 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group err=%d\n", err); 177 goto destroy_table; 178 } 179 roce->g = g; 180 181 err = ipsec_fs_roce_tx_rule_setup(mdev, roce, pol_ft); 182 if (err) { 183 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err); 184 goto destroy_group; 185 } 186 187 kvfree(in); 188 return 0; 189 190 destroy_group: 191 mlx5_destroy_flow_group(roce->g); 192 destroy_table: 193 mlx5_destroy_flow_table(ft); 194 free_in: 195 kvfree(in); 196 return err; 197 } 198 199 struct mlx5_flow_table *mlx5_ipsec_fs_roce_ft_get(struct mlx5_ipsec_fs *ipsec_roce, u32 family) 200 { 201 struct mlx5_ipsec_rx_roce *rx_roce; 202 203 if (!ipsec_roce) 204 return NULL; 205 206 rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx : 207 &ipsec_roce->ipv6_rx; 208 209 return rx_roce->ft; 210 } 211 212 void mlx5_ipsec_fs_roce_rx_destroy(struct mlx5_ipsec_fs *ipsec_roce, u32 family) 213 { 214 struct mlx5_ipsec_rx_roce *rx_roce; 215 216 if (!ipsec_roce) 217 return; 218 219 rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx : 220 &ipsec_roce->ipv6_rx; 221 222 mlx5_del_flow_rules(rx_roce->roce_miss.rule); 223 mlx5_del_flow_rules(rx_roce->rule); 224 mlx5_destroy_flow_table(rx_roce->ft_rdma); 225 mlx5_destroy_flow_group(rx_roce->roce_miss.group); 226 mlx5_destroy_flow_group(rx_roce->g); 227 mlx5_destroy_flow_table(rx_roce->ft); 228 } 229 230 #define MLX5_RX_ROCE_GROUP_SIZE BIT(0) 231 232 int mlx5_ipsec_fs_roce_rx_create(struct mlx5_core_dev *mdev, 233 struct mlx5_ipsec_fs *ipsec_roce, 234 struct mlx5_flow_namespace *ns, 235 struct mlx5_flow_destination *default_dst, 236 u32 family, u32 level, u32 prio) 237 { 238 struct mlx5_flow_table_attr ft_attr = {}; 239 struct mlx5_ipsec_rx_roce *roce; 240 struct mlx5_flow_table *ft; 241 struct mlx5_flow_group *g; 242 void *outer_headers_c; 243 int ix = 0; 244 u32 *in; 245 int err; 246 u8 *mc; 247 248 if (!ipsec_roce) 249 return 0; 250 251 roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx : 252 &ipsec_roce->ipv6_rx; 253 254 ft_attr.max_fte = 2; 255 ft_attr.level = level; 256 ft_attr.prio = prio; 257 ft = mlx5_create_flow_table(ns, &ft_attr); 258 if (IS_ERR(ft)) { 259 err = PTR_ERR(ft); 260 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at nic err=%d\n", err); 261 return err; 262 } 263 264 roce->ft = ft; 265 266 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); 267 if (!in) { 268 err = -ENOMEM; 269 goto fail_nomem; 270 } 271 272 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 273 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 274 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); 275 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); 276 277 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 278 MLX5_SET_CFG(in, start_flow_index, ix); 279 ix += MLX5_RX_ROCE_GROUP_SIZE; 280 MLX5_SET_CFG(in, end_flow_index, ix - 1); 281 g = mlx5_create_flow_group(ft, in); 282 if (IS_ERR(g)) { 283 err = PTR_ERR(g); 284 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx group at nic err=%d\n", err); 285 goto fail_group; 286 } 287 roce->g = g; 288 289 memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in)); 290 MLX5_SET_CFG(in, start_flow_index, ix); 291 ix += MLX5_RX_ROCE_GROUP_SIZE; 292 MLX5_SET_CFG(in, end_flow_index, ix - 1); 293 g = mlx5_create_flow_group(ft, in); 294 if (IS_ERR(g)) { 295 err = PTR_ERR(g); 296 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx miss group at nic err=%d\n", err); 297 goto fail_mgroup; 298 } 299 roce->roce_miss.group = g; 300 301 memset(&ft_attr, 0, sizeof(ft_attr)); 302 if (family == AF_INET) 303 ft_attr.level = 1; 304 ft = mlx5_create_flow_table(roce->ns_rdma, &ft_attr); 305 if (IS_ERR(ft)) { 306 err = PTR_ERR(ft); 307 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at rdma err=%d\n", err); 308 goto fail_rdma_table; 309 } 310 311 roce->ft_rdma = ft; 312 313 err = ipsec_fs_roce_rx_rule_setup(mdev, default_dst, roce); 314 if (err) { 315 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx rules err=%d\n", err); 316 goto fail_setup_rule; 317 } 318 319 kvfree(in); 320 return 0; 321 322 fail_setup_rule: 323 mlx5_destroy_flow_table(roce->ft_rdma); 324 fail_rdma_table: 325 mlx5_destroy_flow_group(roce->roce_miss.group); 326 fail_mgroup: 327 mlx5_destroy_flow_group(roce->g); 328 fail_group: 329 kvfree(in); 330 fail_nomem: 331 mlx5_destroy_flow_table(roce->ft); 332 return err; 333 } 334 335 void mlx5_ipsec_fs_roce_cleanup(struct mlx5_ipsec_fs *ipsec_roce) 336 { 337 kfree(ipsec_roce); 338 } 339 340 struct mlx5_ipsec_fs *mlx5_ipsec_fs_roce_init(struct mlx5_core_dev *mdev) 341 { 342 struct mlx5_ipsec_fs *roce_ipsec; 343 struct mlx5_flow_namespace *ns; 344 345 ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC); 346 if (!ns) { 347 mlx5_core_err(mdev, "Failed to get RoCE rx ns\n"); 348 return NULL; 349 } 350 351 roce_ipsec = kzalloc(sizeof(*roce_ipsec), GFP_KERNEL); 352 if (!roce_ipsec) 353 return NULL; 354 355 roce_ipsec->ipv4_rx.ns_rdma = ns; 356 roce_ipsec->ipv6_rx.ns_rdma = ns; 357 358 ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC); 359 if (!ns) { 360 mlx5_core_err(mdev, "Failed to get RoCE tx ns\n"); 361 goto err_tx; 362 } 363 364 roce_ipsec->tx.ns = ns; 365 366 return roce_ipsec; 367 368 err_tx: 369 kfree(roce_ipsec); 370 return NULL; 371 } 372