1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <rdma/ib_user_verbs.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/uverbs_types.h> 9 #include <rdma/uverbs_ioctl.h> 10 #include <rdma/uverbs_std_types.h> 11 #include <rdma/mlx5_user_ioctl_cmds.h> 12 #include <rdma/mlx5_user_ioctl_verbs.h> 13 #include <rdma/ib_umem.h> 14 #include <linux/mlx5/driver.h> 15 #include <linux/mlx5/fs.h> 16 #include <linux/mlx5/fs_helpers.h> 17 #include <linux/mlx5/accel.h> 18 #include <linux/mlx5/eswitch.h> 19 #include "mlx5_ib.h" 20 #include "counters.h" 21 #include "devx.h" 22 #include "fs.h" 23 24 #define UVERBS_MODULE_NAME mlx5_ib 25 #include <rdma/uverbs_named_ioctl.h> 26 27 enum { 28 MATCH_CRITERIA_ENABLE_OUTER_BIT, 29 MATCH_CRITERIA_ENABLE_MISC_BIT, 30 MATCH_CRITERIA_ENABLE_INNER_BIT, 31 MATCH_CRITERIA_ENABLE_MISC2_BIT 32 }; 33 34 #define HEADER_IS_ZERO(match_criteria, headers) \ 35 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 36 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 37 38 static u8 get_match_criteria_enable(u32 *match_criteria) 39 { 40 u8 match_criteria_enable; 41 42 match_criteria_enable = 43 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 44 MATCH_CRITERIA_ENABLE_OUTER_BIT; 45 match_criteria_enable |= 46 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 47 MATCH_CRITERIA_ENABLE_MISC_BIT; 48 match_criteria_enable |= 49 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 50 MATCH_CRITERIA_ENABLE_INNER_BIT; 51 match_criteria_enable |= 52 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 53 MATCH_CRITERIA_ENABLE_MISC2_BIT; 54 55 return match_criteria_enable; 56 } 57 58 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) 59 { 60 u8 entry_mask; 61 u8 entry_val; 62 int err = 0; 63 64 if (!mask) 65 goto out; 66 67 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, 68 ip_protocol); 69 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, 70 ip_protocol); 71 if (!entry_mask) { 72 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); 73 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); 74 goto out; 75 } 76 /* Don't override existing ip protocol */ 77 if (mask != entry_mask || val != entry_val) 78 err = -EINVAL; 79 out: 80 return err; 81 } 82 83 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, 84 bool inner) 85 { 86 if (inner) { 87 MLX5_SET(fte_match_set_misc, 88 misc_c, inner_ipv6_flow_label, mask); 89 MLX5_SET(fte_match_set_misc, 90 misc_v, inner_ipv6_flow_label, val); 91 } else { 92 MLX5_SET(fte_match_set_misc, 93 misc_c, outer_ipv6_flow_label, mask); 94 MLX5_SET(fte_match_set_misc, 95 misc_v, outer_ipv6_flow_label, val); 96 } 97 } 98 99 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) 100 { 101 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); 102 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); 103 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); 104 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); 105 } 106 107 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) 108 { 109 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && 110 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) 111 return -EOPNOTSUPP; 112 113 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && 114 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) 115 return -EOPNOTSUPP; 116 117 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && 118 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) 119 return -EOPNOTSUPP; 120 121 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && 122 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) 123 return -EOPNOTSUPP; 124 125 return 0; 126 } 127 128 #define LAST_ETH_FIELD vlan_tag 129 #define LAST_IB_FIELD sl 130 #define LAST_IPV4_FIELD tos 131 #define LAST_IPV6_FIELD traffic_class 132 #define LAST_TCP_UDP_FIELD src_port 133 #define LAST_TUNNEL_FIELD tunnel_id 134 #define LAST_FLOW_TAG_FIELD tag_id 135 #define LAST_DROP_FIELD size 136 #define LAST_COUNTERS_FIELD counters 137 138 /* Field is the last supported field */ 139 #define FIELDS_NOT_SUPPORTED(filter, field)\ 140 memchr_inv((void *)&filter.field +\ 141 sizeof(filter.field), 0,\ 142 sizeof(filter) -\ 143 offsetof(typeof(filter), field) -\ 144 sizeof(filter.field)) 145 146 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, 147 bool is_egress, 148 struct mlx5_flow_act *action) 149 { 150 151 switch (maction->ib_action.type) { 152 case IB_FLOW_ACTION_ESP: 153 if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | 154 MLX5_FLOW_CONTEXT_ACTION_DECRYPT)) 155 return -EINVAL; 156 /* Currently only AES_GCM keymat is supported by the driver */ 157 action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx; 158 action->action |= is_egress ? 159 MLX5_FLOW_CONTEXT_ACTION_ENCRYPT : 160 MLX5_FLOW_CONTEXT_ACTION_DECRYPT; 161 return 0; 162 case IB_FLOW_ACTION_UNSPECIFIED: 163 if (maction->flow_action_raw.sub_type == 164 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { 165 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 166 return -EINVAL; 167 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 168 action->modify_hdr = 169 maction->flow_action_raw.modify_hdr; 170 return 0; 171 } 172 if (maction->flow_action_raw.sub_type == 173 MLX5_IB_FLOW_ACTION_DECAP) { 174 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) 175 return -EINVAL; 176 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 177 return 0; 178 } 179 if (maction->flow_action_raw.sub_type == 180 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { 181 if (action->action & 182 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) 183 return -EINVAL; 184 action->action |= 185 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 186 action->pkt_reformat = 187 maction->flow_action_raw.pkt_reformat; 188 return 0; 189 } 190 fallthrough; 191 default: 192 return -EOPNOTSUPP; 193 } 194 } 195 196 static int parse_flow_attr(struct mlx5_core_dev *mdev, 197 struct mlx5_flow_spec *spec, 198 const union ib_flow_spec *ib_spec, 199 const struct ib_flow_attr *flow_attr, 200 struct mlx5_flow_act *action, u32 prev_type) 201 { 202 struct mlx5_flow_context *flow_context = &spec->flow_context; 203 u32 *match_c = spec->match_criteria; 204 u32 *match_v = spec->match_value; 205 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, 206 misc_parameters); 207 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, 208 misc_parameters); 209 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, 210 misc_parameters_2); 211 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, 212 misc_parameters_2); 213 void *headers_c; 214 void *headers_v; 215 int match_ipv; 216 int ret; 217 218 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 219 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 220 inner_headers); 221 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 222 inner_headers); 223 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 224 ft_field_support.inner_ip_version); 225 } else { 226 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 227 outer_headers); 228 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 229 outer_headers); 230 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 231 ft_field_support.outer_ip_version); 232 } 233 234 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 235 case IB_FLOW_SPEC_ETH: 236 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) 237 return -EOPNOTSUPP; 238 239 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 240 dmac_47_16), 241 ib_spec->eth.mask.dst_mac); 242 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 243 dmac_47_16), 244 ib_spec->eth.val.dst_mac); 245 246 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 247 smac_47_16), 248 ib_spec->eth.mask.src_mac); 249 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 250 smac_47_16), 251 ib_spec->eth.val.src_mac); 252 253 if (ib_spec->eth.mask.vlan_tag) { 254 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 255 cvlan_tag, 1); 256 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 257 cvlan_tag, 1); 258 259 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 260 first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); 261 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 262 first_vid, ntohs(ib_spec->eth.val.vlan_tag)); 263 264 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 265 first_cfi, 266 ntohs(ib_spec->eth.mask.vlan_tag) >> 12); 267 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 268 first_cfi, 269 ntohs(ib_spec->eth.val.vlan_tag) >> 12); 270 271 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 272 first_prio, 273 ntohs(ib_spec->eth.mask.vlan_tag) >> 13); 274 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 275 first_prio, 276 ntohs(ib_spec->eth.val.vlan_tag) >> 13); 277 } 278 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 279 ethertype, ntohs(ib_spec->eth.mask.ether_type)); 280 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 281 ethertype, ntohs(ib_spec->eth.val.ether_type)); 282 break; 283 case IB_FLOW_SPEC_IPV4: 284 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) 285 return -EOPNOTSUPP; 286 287 if (match_ipv) { 288 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 289 ip_version, 0xf); 290 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 291 ip_version, MLX5_FS_IPV4_VERSION); 292 } else { 293 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 294 ethertype, 0xffff); 295 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 296 ethertype, ETH_P_IP); 297 } 298 299 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 300 src_ipv4_src_ipv6.ipv4_layout.ipv4), 301 &ib_spec->ipv4.mask.src_ip, 302 sizeof(ib_spec->ipv4.mask.src_ip)); 303 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 304 src_ipv4_src_ipv6.ipv4_layout.ipv4), 305 &ib_spec->ipv4.val.src_ip, 306 sizeof(ib_spec->ipv4.val.src_ip)); 307 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 308 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 309 &ib_spec->ipv4.mask.dst_ip, 310 sizeof(ib_spec->ipv4.mask.dst_ip)); 311 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 312 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 313 &ib_spec->ipv4.val.dst_ip, 314 sizeof(ib_spec->ipv4.val.dst_ip)); 315 316 set_tos(headers_c, headers_v, 317 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); 318 319 if (set_proto(headers_c, headers_v, 320 ib_spec->ipv4.mask.proto, 321 ib_spec->ipv4.val.proto)) 322 return -EINVAL; 323 break; 324 case IB_FLOW_SPEC_IPV6: 325 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) 326 return -EOPNOTSUPP; 327 328 if (match_ipv) { 329 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 330 ip_version, 0xf); 331 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 332 ip_version, MLX5_FS_IPV6_VERSION); 333 } else { 334 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 335 ethertype, 0xffff); 336 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 337 ethertype, ETH_P_IPV6); 338 } 339 340 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 341 src_ipv4_src_ipv6.ipv6_layout.ipv6), 342 &ib_spec->ipv6.mask.src_ip, 343 sizeof(ib_spec->ipv6.mask.src_ip)); 344 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 345 src_ipv4_src_ipv6.ipv6_layout.ipv6), 346 &ib_spec->ipv6.val.src_ip, 347 sizeof(ib_spec->ipv6.val.src_ip)); 348 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 349 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 350 &ib_spec->ipv6.mask.dst_ip, 351 sizeof(ib_spec->ipv6.mask.dst_ip)); 352 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 353 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 354 &ib_spec->ipv6.val.dst_ip, 355 sizeof(ib_spec->ipv6.val.dst_ip)); 356 357 set_tos(headers_c, headers_v, 358 ib_spec->ipv6.mask.traffic_class, 359 ib_spec->ipv6.val.traffic_class); 360 361 if (set_proto(headers_c, headers_v, 362 ib_spec->ipv6.mask.next_hdr, 363 ib_spec->ipv6.val.next_hdr)) 364 return -EINVAL; 365 366 set_flow_label(misc_params_c, misc_params_v, 367 ntohl(ib_spec->ipv6.mask.flow_label), 368 ntohl(ib_spec->ipv6.val.flow_label), 369 ib_spec->type & IB_FLOW_SPEC_INNER); 370 break; 371 case IB_FLOW_SPEC_ESP: 372 if (ib_spec->esp.mask.seq) 373 return -EOPNOTSUPP; 374 375 MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi, 376 ntohl(ib_spec->esp.mask.spi)); 377 MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi, 378 ntohl(ib_spec->esp.val.spi)); 379 break; 380 case IB_FLOW_SPEC_TCP: 381 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 382 LAST_TCP_UDP_FIELD)) 383 return -EOPNOTSUPP; 384 385 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) 386 return -EINVAL; 387 388 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, 389 ntohs(ib_spec->tcp_udp.mask.src_port)); 390 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 391 ntohs(ib_spec->tcp_udp.val.src_port)); 392 393 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, 394 ntohs(ib_spec->tcp_udp.mask.dst_port)); 395 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 396 ntohs(ib_spec->tcp_udp.val.dst_port)); 397 break; 398 case IB_FLOW_SPEC_UDP: 399 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 400 LAST_TCP_UDP_FIELD)) 401 return -EOPNOTSUPP; 402 403 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) 404 return -EINVAL; 405 406 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 407 ntohs(ib_spec->tcp_udp.mask.src_port)); 408 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 409 ntohs(ib_spec->tcp_udp.val.src_port)); 410 411 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 412 ntohs(ib_spec->tcp_udp.mask.dst_port)); 413 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 414 ntohs(ib_spec->tcp_udp.val.dst_port)); 415 break; 416 case IB_FLOW_SPEC_GRE: 417 if (ib_spec->gre.mask.c_ks_res0_ver) 418 return -EOPNOTSUPP; 419 420 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) 421 return -EINVAL; 422 423 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 424 0xff); 425 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 426 IPPROTO_GRE); 427 428 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, 429 ntohs(ib_spec->gre.mask.protocol)); 430 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, 431 ntohs(ib_spec->gre.val.protocol)); 432 433 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, 434 gre_key.nvgre.hi), 435 &ib_spec->gre.mask.key, 436 sizeof(ib_spec->gre.mask.key)); 437 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, 438 gre_key.nvgre.hi), 439 &ib_spec->gre.val.key, 440 sizeof(ib_spec->gre.val.key)); 441 break; 442 case IB_FLOW_SPEC_MPLS: 443 switch (prev_type) { 444 case IB_FLOW_SPEC_UDP: 445 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 446 ft_field_support.outer_first_mpls_over_udp), 447 &ib_spec->mpls.mask.tag)) 448 return -EOPNOTSUPP; 449 450 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 451 outer_first_mpls_over_udp), 452 &ib_spec->mpls.val.tag, 453 sizeof(ib_spec->mpls.val.tag)); 454 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 455 outer_first_mpls_over_udp), 456 &ib_spec->mpls.mask.tag, 457 sizeof(ib_spec->mpls.mask.tag)); 458 break; 459 case IB_FLOW_SPEC_GRE: 460 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 461 ft_field_support.outer_first_mpls_over_gre), 462 &ib_spec->mpls.mask.tag)) 463 return -EOPNOTSUPP; 464 465 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 466 outer_first_mpls_over_gre), 467 &ib_spec->mpls.val.tag, 468 sizeof(ib_spec->mpls.val.tag)); 469 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 470 outer_first_mpls_over_gre), 471 &ib_spec->mpls.mask.tag, 472 sizeof(ib_spec->mpls.mask.tag)); 473 break; 474 default: 475 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 476 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 477 ft_field_support.inner_first_mpls), 478 &ib_spec->mpls.mask.tag)) 479 return -EOPNOTSUPP; 480 481 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 482 inner_first_mpls), 483 &ib_spec->mpls.val.tag, 484 sizeof(ib_spec->mpls.val.tag)); 485 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 486 inner_first_mpls), 487 &ib_spec->mpls.mask.tag, 488 sizeof(ib_spec->mpls.mask.tag)); 489 } else { 490 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 491 ft_field_support.outer_first_mpls), 492 &ib_spec->mpls.mask.tag)) 493 return -EOPNOTSUPP; 494 495 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 496 outer_first_mpls), 497 &ib_spec->mpls.val.tag, 498 sizeof(ib_spec->mpls.val.tag)); 499 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 500 outer_first_mpls), 501 &ib_spec->mpls.mask.tag, 502 sizeof(ib_spec->mpls.mask.tag)); 503 } 504 } 505 break; 506 case IB_FLOW_SPEC_VXLAN_TUNNEL: 507 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, 508 LAST_TUNNEL_FIELD)) 509 return -EOPNOTSUPP; 510 511 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, 512 ntohl(ib_spec->tunnel.mask.tunnel_id)); 513 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, 514 ntohl(ib_spec->tunnel.val.tunnel_id)); 515 break; 516 case IB_FLOW_SPEC_ACTION_TAG: 517 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, 518 LAST_FLOW_TAG_FIELD)) 519 return -EOPNOTSUPP; 520 if (ib_spec->flow_tag.tag_id >= BIT(24)) 521 return -EINVAL; 522 523 flow_context->flow_tag = ib_spec->flow_tag.tag_id; 524 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 525 break; 526 case IB_FLOW_SPEC_ACTION_DROP: 527 if (FIELDS_NOT_SUPPORTED(ib_spec->drop, 528 LAST_DROP_FIELD)) 529 return -EOPNOTSUPP; 530 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 531 break; 532 case IB_FLOW_SPEC_ACTION_HANDLE: 533 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), 534 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); 535 if (ret) 536 return ret; 537 break; 538 case IB_FLOW_SPEC_ACTION_COUNT: 539 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, 540 LAST_COUNTERS_FIELD)) 541 return -EOPNOTSUPP; 542 543 /* for now support only one counters spec per flow */ 544 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) 545 return -EINVAL; 546 547 action->counters = ib_spec->flow_count.counters; 548 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 549 break; 550 default: 551 return -EINVAL; 552 } 553 554 return 0; 555 } 556 557 /* If a flow could catch both multicast and unicast packets, 558 * it won't fall into the multicast flow steering table and this rule 559 * could steal other multicast packets. 560 */ 561 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) 562 { 563 union ib_flow_spec *flow_spec; 564 565 if (ib_attr->type != IB_FLOW_ATTR_NORMAL || 566 ib_attr->num_of_specs < 1) 567 return false; 568 569 flow_spec = (union ib_flow_spec *)(ib_attr + 1); 570 if (flow_spec->type == IB_FLOW_SPEC_IPV4) { 571 struct ib_flow_spec_ipv4 *ipv4_spec; 572 573 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; 574 if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) 575 return true; 576 577 return false; 578 } 579 580 if (flow_spec->type == IB_FLOW_SPEC_ETH) { 581 struct ib_flow_spec_eth *eth_spec; 582 583 eth_spec = (struct ib_flow_spec_eth *)flow_spec; 584 return is_multicast_ether_addr(eth_spec->mask.dst_mac) && 585 is_multicast_ether_addr(eth_spec->val.dst_mac); 586 } 587 588 return false; 589 } 590 591 enum valid_spec { 592 VALID_SPEC_INVALID, 593 VALID_SPEC_VALID, 594 VALID_SPEC_NA, 595 }; 596 597 static enum valid_spec 598 is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev, 599 const struct mlx5_flow_spec *spec, 600 const struct mlx5_flow_act *flow_act, 601 bool egress) 602 { 603 const u32 *match_c = spec->match_criteria; 604 bool is_crypto = 605 (flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | 606 MLX5_FLOW_CONTEXT_ACTION_DECRYPT)); 607 bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c); 608 bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP; 609 610 /* 611 * Currently only crypto is supported in egress, when regular egress 612 * rules would be supported, always return VALID_SPEC_NA. 613 */ 614 if (!is_crypto) 615 return VALID_SPEC_NA; 616 617 return is_crypto && is_ipsec && 618 (!egress || (!is_drop && 619 !(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ? 620 VALID_SPEC_VALID : VALID_SPEC_INVALID; 621 } 622 623 static bool is_valid_spec(struct mlx5_core_dev *mdev, 624 const struct mlx5_flow_spec *spec, 625 const struct mlx5_flow_act *flow_act, 626 bool egress) 627 { 628 /* We curretly only support ipsec egress flow */ 629 return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID; 630 } 631 632 static bool is_valid_ethertype(struct mlx5_core_dev *mdev, 633 const struct ib_flow_attr *flow_attr, 634 bool check_inner) 635 { 636 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); 637 int match_ipv = check_inner ? 638 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 639 ft_field_support.inner_ip_version) : 640 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 641 ft_field_support.outer_ip_version); 642 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; 643 bool ipv4_spec_valid, ipv6_spec_valid; 644 unsigned int ip_spec_type = 0; 645 bool has_ethertype = false; 646 unsigned int spec_index; 647 bool mask_valid = true; 648 u16 eth_type = 0; 649 bool type_valid; 650 651 /* Validate that ethertype is correct */ 652 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 653 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && 654 ib_spec->eth.mask.ether_type) { 655 mask_valid = (ib_spec->eth.mask.ether_type == 656 htons(0xffff)); 657 has_ethertype = true; 658 eth_type = ntohs(ib_spec->eth.val.ether_type); 659 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || 660 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { 661 ip_spec_type = ib_spec->type; 662 } 663 ib_spec = (void *)ib_spec + ib_spec->size; 664 } 665 666 type_valid = (!has_ethertype) || (!ip_spec_type); 667 if (!type_valid && mask_valid) { 668 ipv4_spec_valid = (eth_type == ETH_P_IP) && 669 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); 670 ipv6_spec_valid = (eth_type == ETH_P_IPV6) && 671 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); 672 673 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || 674 (((eth_type == ETH_P_MPLS_UC) || 675 (eth_type == ETH_P_MPLS_MC)) && match_ipv); 676 } 677 678 return type_valid; 679 } 680 681 static bool is_valid_attr(struct mlx5_core_dev *mdev, 682 const struct ib_flow_attr *flow_attr) 683 { 684 return is_valid_ethertype(mdev, flow_attr, false) && 685 is_valid_ethertype(mdev, flow_attr, true); 686 } 687 688 static void put_flow_table(struct mlx5_ib_dev *dev, 689 struct mlx5_ib_flow_prio *prio, bool ft_added) 690 { 691 prio->refcount -= !!ft_added; 692 if (!prio->refcount) { 693 mlx5_destroy_flow_table(prio->flow_table); 694 prio->flow_table = NULL; 695 } 696 } 697 698 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) 699 { 700 struct mlx5_ib_flow_handler *handler = container_of(flow_id, 701 struct mlx5_ib_flow_handler, 702 ibflow); 703 struct mlx5_ib_flow_handler *iter, *tmp; 704 struct mlx5_ib_dev *dev = handler->dev; 705 706 mutex_lock(&dev->flow_db->lock); 707 708 list_for_each_entry_safe(iter, tmp, &handler->list, list) { 709 mlx5_del_flow_rules(iter->rule); 710 put_flow_table(dev, iter->prio, true); 711 list_del(&iter->list); 712 kfree(iter); 713 } 714 715 mlx5_del_flow_rules(handler->rule); 716 put_flow_table(dev, handler->prio, true); 717 mlx5_ib_counters_clear_description(handler->ibcounters); 718 mutex_unlock(&dev->flow_db->lock); 719 if (handler->flow_matcher) 720 atomic_dec(&handler->flow_matcher->usecnt); 721 kfree(handler); 722 723 return 0; 724 } 725 726 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) 727 { 728 priority *= 2; 729 if (!dont_trap) 730 priority++; 731 return priority; 732 } 733 734 enum flow_table_type { 735 MLX5_IB_FT_RX, 736 MLX5_IB_FT_TX 737 }; 738 739 #define MLX5_FS_MAX_TYPES 6 740 #define MLX5_FS_MAX_ENTRIES BIT(16) 741 742 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns, 743 struct mlx5_ib_flow_prio *prio, 744 int priority, 745 int num_entries, int num_groups, 746 u32 flags) 747 { 748 struct mlx5_flow_table_attr ft_attr = {}; 749 struct mlx5_flow_table *ft; 750 751 ft_attr.prio = priority; 752 ft_attr.max_fte = num_entries; 753 ft_attr.flags = flags; 754 ft_attr.autogroup.max_num_groups = num_groups; 755 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 756 if (IS_ERR(ft)) 757 return ERR_CAST(ft); 758 759 prio->flow_table = ft; 760 prio->refcount = 0; 761 return prio; 762 } 763 764 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, 765 struct ib_flow_attr *flow_attr, 766 enum flow_table_type ft_type) 767 { 768 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; 769 struct mlx5_flow_namespace *ns = NULL; 770 struct mlx5_ib_flow_prio *prio; 771 struct mlx5_flow_table *ft; 772 int max_table_size; 773 int num_entries; 774 int num_groups; 775 bool esw_encap; 776 u32 flags = 0; 777 int priority; 778 779 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 780 log_max_ft_size)); 781 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 782 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 783 if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { 784 enum mlx5_flow_namespace_type fn_type; 785 786 if (flow_is_multicast_only(flow_attr) && 787 !dont_trap) 788 priority = MLX5_IB_FLOW_MCAST_PRIO; 789 else 790 priority = ib_prio_to_core_prio(flow_attr->priority, 791 dont_trap); 792 if (ft_type == MLX5_IB_FT_RX) { 793 fn_type = MLX5_FLOW_NAMESPACE_BYPASS; 794 prio = &dev->flow_db->prios[priority]; 795 if (!dev->is_rep && !esw_encap && 796 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) 797 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 798 if (!dev->is_rep && !esw_encap && 799 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 800 reformat_l3_tunnel_to_l2)) 801 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 802 } else { 803 max_table_size = 804 BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 805 log_max_ft_size)); 806 fn_type = MLX5_FLOW_NAMESPACE_EGRESS; 807 prio = &dev->flow_db->egress_prios[priority]; 808 if (!dev->is_rep && !esw_encap && 809 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) 810 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 811 } 812 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 813 num_entries = MLX5_FS_MAX_ENTRIES; 814 num_groups = MLX5_FS_MAX_TYPES; 815 } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 816 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { 817 ns = mlx5_get_flow_namespace(dev->mdev, 818 MLX5_FLOW_NAMESPACE_LEFTOVERS); 819 build_leftovers_ft_param(&priority, 820 &num_entries, 821 &num_groups); 822 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; 823 } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 824 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 825 allow_sniffer_and_nic_rx_shared_tir)) 826 return ERR_PTR(-EOPNOTSUPP); 827 828 ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ? 829 MLX5_FLOW_NAMESPACE_SNIFFER_RX : 830 MLX5_FLOW_NAMESPACE_SNIFFER_TX); 831 832 prio = &dev->flow_db->sniffer[ft_type]; 833 priority = 0; 834 num_entries = 1; 835 num_groups = 1; 836 } 837 838 if (!ns) 839 return ERR_PTR(-EOPNOTSUPP); 840 841 max_table_size = min_t(int, num_entries, max_table_size); 842 843 ft = prio->flow_table; 844 if (!ft) 845 return _get_prio(ns, prio, priority, max_table_size, num_groups, 846 flags); 847 848 return prio; 849 } 850 851 static void set_underlay_qp(struct mlx5_ib_dev *dev, 852 struct mlx5_flow_spec *spec, 853 u32 underlay_qpn) 854 { 855 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, 856 spec->match_criteria, 857 misc_parameters); 858 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 859 misc_parameters); 860 861 if (underlay_qpn && 862 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 863 ft_field_support.bth_dst_qp)) { 864 MLX5_SET(fte_match_set_misc, 865 misc_params_v, bth_dst_qp, underlay_qpn); 866 MLX5_SET(fte_match_set_misc, 867 misc_params_c, bth_dst_qp, 0xffffff); 868 } 869 } 870 871 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, 872 struct mlx5_flow_spec *spec, 873 struct mlx5_eswitch_rep *rep) 874 { 875 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; 876 void *misc; 877 878 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 879 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 880 misc_parameters_2); 881 882 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 883 mlx5_eswitch_get_vport_metadata_for_match(esw, 884 rep->vport)); 885 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 886 misc_parameters_2); 887 888 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 889 mlx5_eswitch_get_vport_metadata_mask()); 890 } else { 891 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 892 misc_parameters); 893 894 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); 895 896 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 897 misc_parameters); 898 899 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 900 } 901 } 902 903 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, 904 struct mlx5_ib_flow_prio *ft_prio, 905 const struct ib_flow_attr *flow_attr, 906 struct mlx5_flow_destination *dst, 907 u32 underlay_qpn, 908 struct mlx5_ib_create_flow *ucmd) 909 { 910 struct mlx5_flow_table *ft = ft_prio->flow_table; 911 struct mlx5_ib_flow_handler *handler; 912 struct mlx5_flow_act flow_act = {}; 913 struct mlx5_flow_spec *spec; 914 struct mlx5_flow_destination dest_arr[2] = {}; 915 struct mlx5_flow_destination *rule_dst = dest_arr; 916 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); 917 unsigned int spec_index; 918 u32 prev_type = 0; 919 int err = 0; 920 int dest_num = 0; 921 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 922 923 if (!is_valid_attr(dev->mdev, flow_attr)) 924 return ERR_PTR(-EINVAL); 925 926 if (dev->is_rep && is_egress) 927 return ERR_PTR(-EINVAL); 928 929 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 930 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 931 if (!handler || !spec) { 932 err = -ENOMEM; 933 goto free; 934 } 935 936 INIT_LIST_HEAD(&handler->list); 937 938 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 939 err = parse_flow_attr(dev->mdev, spec, 940 ib_flow, flow_attr, &flow_act, 941 prev_type); 942 if (err < 0) 943 goto free; 944 945 prev_type = ((union ib_flow_spec *)ib_flow)->type; 946 ib_flow += ((union ib_flow_spec *)ib_flow)->size; 947 } 948 949 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { 950 memcpy(&dest_arr[0], dst, sizeof(*dst)); 951 dest_num++; 952 } 953 954 if (!flow_is_multicast_only(flow_attr)) 955 set_underlay_qp(dev, spec, underlay_qpn); 956 957 if (dev->is_rep) { 958 struct mlx5_eswitch_rep *rep; 959 960 rep = dev->port[flow_attr->port - 1].rep; 961 if (!rep) { 962 err = -EINVAL; 963 goto free; 964 } 965 966 mlx5_ib_set_rule_source_port(dev, spec, rep); 967 } 968 969 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); 970 971 if (is_egress && 972 !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) { 973 err = -EINVAL; 974 goto free; 975 } 976 977 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 978 struct mlx5_ib_mcounters *mcounters; 979 980 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); 981 if (err) 982 goto free; 983 984 mcounters = to_mcounters(flow_act.counters); 985 handler->ibcounters = flow_act.counters; 986 dest_arr[dest_num].type = 987 MLX5_FLOW_DESTINATION_TYPE_COUNTER; 988 dest_arr[dest_num].counter_id = 989 mlx5_fc_id(mcounters->hw_cntrs_hndl); 990 dest_num++; 991 } 992 993 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { 994 if (!dest_num) 995 rule_dst = NULL; 996 } else { 997 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) 998 flow_act.action |= 999 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1000 if (is_egress) 1001 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1002 else if (dest_num) 1003 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1004 } 1005 1006 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && 1007 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1008 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1009 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", 1010 spec->flow_context.flow_tag, flow_attr->type); 1011 err = -EINVAL; 1012 goto free; 1013 } 1014 handler->rule = mlx5_add_flow_rules(ft, spec, 1015 &flow_act, 1016 rule_dst, dest_num); 1017 1018 if (IS_ERR(handler->rule)) { 1019 err = PTR_ERR(handler->rule); 1020 goto free; 1021 } 1022 1023 ft_prio->refcount++; 1024 handler->prio = ft_prio; 1025 handler->dev = dev; 1026 1027 ft_prio->flow_table = ft; 1028 free: 1029 if (err && handler) { 1030 mlx5_ib_counters_clear_description(handler->ibcounters); 1031 kfree(handler); 1032 } 1033 kvfree(spec); 1034 return err ? ERR_PTR(err) : handler; 1035 } 1036 1037 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, 1038 struct mlx5_ib_flow_prio *ft_prio, 1039 const struct ib_flow_attr *flow_attr, 1040 struct mlx5_flow_destination *dst) 1041 { 1042 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); 1043 } 1044 1045 enum { 1046 LEFTOVERS_MC, 1047 LEFTOVERS_UC, 1048 }; 1049 1050 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, 1051 struct mlx5_ib_flow_prio *ft_prio, 1052 struct ib_flow_attr *flow_attr, 1053 struct mlx5_flow_destination *dst) 1054 { 1055 struct mlx5_ib_flow_handler *handler_ucast = NULL; 1056 struct mlx5_ib_flow_handler *handler = NULL; 1057 1058 static struct { 1059 struct ib_flow_attr flow_attr; 1060 struct ib_flow_spec_eth eth_flow; 1061 } leftovers_specs[] = { 1062 [LEFTOVERS_MC] = { 1063 .flow_attr = { 1064 .num_of_specs = 1, 1065 .size = sizeof(leftovers_specs[0]) 1066 }, 1067 .eth_flow = { 1068 .type = IB_FLOW_SPEC_ETH, 1069 .size = sizeof(struct ib_flow_spec_eth), 1070 .mask = {.dst_mac = {0x1} }, 1071 .val = {.dst_mac = {0x1} } 1072 } 1073 }, 1074 [LEFTOVERS_UC] = { 1075 .flow_attr = { 1076 .num_of_specs = 1, 1077 .size = sizeof(leftovers_specs[0]) 1078 }, 1079 .eth_flow = { 1080 .type = IB_FLOW_SPEC_ETH, 1081 .size = sizeof(struct ib_flow_spec_eth), 1082 .mask = {.dst_mac = {0x1} }, 1083 .val = {.dst_mac = {} } 1084 } 1085 } 1086 }; 1087 1088 handler = create_flow_rule(dev, ft_prio, 1089 &leftovers_specs[LEFTOVERS_MC].flow_attr, 1090 dst); 1091 if (!IS_ERR(handler) && 1092 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { 1093 handler_ucast = create_flow_rule(dev, ft_prio, 1094 &leftovers_specs[LEFTOVERS_UC].flow_attr, 1095 dst); 1096 if (IS_ERR(handler_ucast)) { 1097 mlx5_del_flow_rules(handler->rule); 1098 ft_prio->refcount--; 1099 kfree(handler); 1100 handler = handler_ucast; 1101 } else { 1102 list_add(&handler_ucast->list, &handler->list); 1103 } 1104 } 1105 1106 return handler; 1107 } 1108 1109 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, 1110 struct mlx5_ib_flow_prio *ft_rx, 1111 struct mlx5_ib_flow_prio *ft_tx, 1112 struct mlx5_flow_destination *dst) 1113 { 1114 struct mlx5_ib_flow_handler *handler_rx; 1115 struct mlx5_ib_flow_handler *handler_tx; 1116 int err; 1117 static const struct ib_flow_attr flow_attr = { 1118 .num_of_specs = 0, 1119 .size = sizeof(flow_attr) 1120 }; 1121 1122 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); 1123 if (IS_ERR(handler_rx)) { 1124 err = PTR_ERR(handler_rx); 1125 goto err; 1126 } 1127 1128 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); 1129 if (IS_ERR(handler_tx)) { 1130 err = PTR_ERR(handler_tx); 1131 goto err_tx; 1132 } 1133 1134 list_add(&handler_tx->list, &handler_rx->list); 1135 1136 return handler_rx; 1137 1138 err_tx: 1139 mlx5_del_flow_rules(handler_rx->rule); 1140 ft_rx->refcount--; 1141 kfree(handler_rx); 1142 err: 1143 return ERR_PTR(err); 1144 } 1145 1146 1147 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, 1148 struct ib_flow_attr *flow_attr, 1149 int domain, 1150 struct ib_udata *udata) 1151 { 1152 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1153 struct mlx5_ib_qp *mqp = to_mqp(qp); 1154 struct mlx5_ib_flow_handler *handler = NULL; 1155 struct mlx5_flow_destination *dst = NULL; 1156 struct mlx5_ib_flow_prio *ft_prio_tx = NULL; 1157 struct mlx5_ib_flow_prio *ft_prio; 1158 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1159 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; 1160 size_t min_ucmd_sz, required_ucmd_sz; 1161 int err; 1162 int underlay_qpn; 1163 1164 if (udata && udata->inlen) { 1165 min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) + 1166 sizeof(ucmd_hdr.reserved); 1167 if (udata->inlen < min_ucmd_sz) 1168 return ERR_PTR(-EOPNOTSUPP); 1169 1170 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); 1171 if (err) 1172 return ERR_PTR(err); 1173 1174 /* currently supports only one counters data */ 1175 if (ucmd_hdr.ncounters_data > 1) 1176 return ERR_PTR(-EINVAL); 1177 1178 required_ucmd_sz = min_ucmd_sz + 1179 sizeof(struct mlx5_ib_flow_counters_data) * 1180 ucmd_hdr.ncounters_data; 1181 if (udata->inlen > required_ucmd_sz && 1182 !ib_is_udata_cleared(udata, required_ucmd_sz, 1183 udata->inlen - required_ucmd_sz)) 1184 return ERR_PTR(-EOPNOTSUPP); 1185 1186 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); 1187 if (!ucmd) 1188 return ERR_PTR(-ENOMEM); 1189 1190 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); 1191 if (err) 1192 goto free_ucmd; 1193 } 1194 1195 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { 1196 err = -ENOMEM; 1197 goto free_ucmd; 1198 } 1199 1200 if (domain != IB_FLOW_DOMAIN_USER || 1201 flow_attr->port > dev->num_ports || 1202 (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | 1203 IB_FLOW_ATTR_FLAGS_EGRESS))) { 1204 err = -EINVAL; 1205 goto free_ucmd; 1206 } 1207 1208 if (is_egress && 1209 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1210 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1211 err = -EINVAL; 1212 goto free_ucmd; 1213 } 1214 1215 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1216 if (!dst) { 1217 err = -ENOMEM; 1218 goto free_ucmd; 1219 } 1220 1221 mutex_lock(&dev->flow_db->lock); 1222 1223 ft_prio = get_flow_table(dev, flow_attr, 1224 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); 1225 if (IS_ERR(ft_prio)) { 1226 err = PTR_ERR(ft_prio); 1227 goto unlock; 1228 } 1229 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 1230 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); 1231 if (IS_ERR(ft_prio_tx)) { 1232 err = PTR_ERR(ft_prio_tx); 1233 ft_prio_tx = NULL; 1234 goto destroy_ft; 1235 } 1236 } 1237 1238 if (is_egress) { 1239 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1240 } else { 1241 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1242 if (mqp->is_rss) 1243 dst->tir_num = mqp->rss_qp.tirn; 1244 else 1245 dst->tir_num = mqp->raw_packet_qp.rq.tirn; 1246 } 1247 1248 if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { 1249 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? 1250 mqp->underlay_qpn : 1251 0; 1252 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, 1253 underlay_qpn, ucmd); 1254 } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1255 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { 1256 handler = create_leftovers_rule(dev, ft_prio, flow_attr, 1257 dst); 1258 } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 1259 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); 1260 } else { 1261 err = -EINVAL; 1262 goto destroy_ft; 1263 } 1264 1265 if (IS_ERR(handler)) { 1266 err = PTR_ERR(handler); 1267 handler = NULL; 1268 goto destroy_ft; 1269 } 1270 1271 mutex_unlock(&dev->flow_db->lock); 1272 kfree(dst); 1273 kfree(ucmd); 1274 1275 return &handler->ibflow; 1276 1277 destroy_ft: 1278 put_flow_table(dev, ft_prio, false); 1279 if (ft_prio_tx) 1280 put_flow_table(dev, ft_prio_tx, false); 1281 unlock: 1282 mutex_unlock(&dev->flow_db->lock); 1283 kfree(dst); 1284 free_ucmd: 1285 kfree(ucmd); 1286 return ERR_PTR(err); 1287 } 1288 1289 static struct mlx5_ib_flow_prio * 1290 _get_flow_table(struct mlx5_ib_dev *dev, 1291 struct mlx5_ib_flow_matcher *fs_matcher, 1292 bool mcast) 1293 { 1294 struct mlx5_flow_namespace *ns = NULL; 1295 struct mlx5_ib_flow_prio *prio = NULL; 1296 int max_table_size = 0; 1297 bool esw_encap; 1298 u32 flags = 0; 1299 int priority; 1300 1301 if (mcast) 1302 priority = MLX5_IB_FLOW_MCAST_PRIO; 1303 else 1304 priority = ib_prio_to_core_prio(fs_matcher->priority, false); 1305 1306 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 1307 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 1308 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 1309 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1310 log_max_ft_size)); 1311 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) 1312 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1313 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1314 reformat_l3_tunnel_to_l2) && 1315 !esw_encap) 1316 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1317 } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { 1318 max_table_size = BIT( 1319 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); 1320 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap) 1321 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1322 } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) { 1323 max_table_size = BIT( 1324 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); 1325 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) 1326 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1327 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) && 1328 esw_encap) 1329 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1330 priority = FDB_BYPASS_PATH; 1331 } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) { 1332 max_table_size = 1333 BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 1334 log_max_ft_size)); 1335 priority = fs_matcher->priority; 1336 } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) { 1337 max_table_size = 1338 BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 1339 log_max_ft_size)); 1340 priority = fs_matcher->priority; 1341 } 1342 1343 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); 1344 1345 ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type); 1346 if (!ns) 1347 return ERR_PTR(-EOPNOTSUPP); 1348 1349 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) 1350 prio = &dev->flow_db->prios[priority]; 1351 else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) 1352 prio = &dev->flow_db->egress_prios[priority]; 1353 else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) 1354 prio = &dev->flow_db->fdb; 1355 else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) 1356 prio = &dev->flow_db->rdma_rx[priority]; 1357 else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) 1358 prio = &dev->flow_db->rdma_tx[priority]; 1359 1360 if (!prio) 1361 return ERR_PTR(-EINVAL); 1362 1363 if (prio->flow_table) 1364 return prio; 1365 1366 return _get_prio(ns, prio, priority, max_table_size, 1367 MLX5_FS_MAX_TYPES, flags); 1368 } 1369 1370 static struct mlx5_ib_flow_handler * 1371 _create_raw_flow_rule(struct mlx5_ib_dev *dev, 1372 struct mlx5_ib_flow_prio *ft_prio, 1373 struct mlx5_flow_destination *dst, 1374 struct mlx5_ib_flow_matcher *fs_matcher, 1375 struct mlx5_flow_context *flow_context, 1376 struct mlx5_flow_act *flow_act, 1377 void *cmd_in, int inlen, 1378 int dst_num) 1379 { 1380 struct mlx5_ib_flow_handler *handler; 1381 struct mlx5_flow_spec *spec; 1382 struct mlx5_flow_table *ft = ft_prio->flow_table; 1383 int err = 0; 1384 1385 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1386 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 1387 if (!handler || !spec) { 1388 err = -ENOMEM; 1389 goto free; 1390 } 1391 1392 INIT_LIST_HEAD(&handler->list); 1393 1394 memcpy(spec->match_value, cmd_in, inlen); 1395 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, 1396 fs_matcher->mask_len); 1397 spec->match_criteria_enable = fs_matcher->match_criteria_enable; 1398 spec->flow_context = *flow_context; 1399 1400 handler->rule = mlx5_add_flow_rules(ft, spec, 1401 flow_act, dst, dst_num); 1402 1403 if (IS_ERR(handler->rule)) { 1404 err = PTR_ERR(handler->rule); 1405 goto free; 1406 } 1407 1408 ft_prio->refcount++; 1409 handler->prio = ft_prio; 1410 handler->dev = dev; 1411 ft_prio->flow_table = ft; 1412 1413 free: 1414 if (err) 1415 kfree(handler); 1416 kvfree(spec); 1417 return err ? ERR_PTR(err) : handler; 1418 } 1419 1420 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, 1421 void *match_v) 1422 { 1423 void *match_c; 1424 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; 1425 void *dmac, *dmac_mask; 1426 void *ipv4, *ipv4_mask; 1427 1428 if (!(fs_matcher->match_criteria_enable & 1429 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) 1430 return false; 1431 1432 match_c = fs_matcher->matcher_mask.match_params; 1433 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, 1434 outer_headers); 1435 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, 1436 outer_headers); 1437 1438 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 1439 dmac_47_16); 1440 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 1441 dmac_47_16); 1442 1443 if (is_multicast_ether_addr(dmac) && 1444 is_multicast_ether_addr(dmac_mask)) 1445 return true; 1446 1447 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 1448 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1449 1450 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 1451 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1452 1453 if (ipv4_is_multicast(*(__be32 *)(ipv4)) && 1454 ipv4_is_multicast(*(__be32 *)(ipv4_mask))) 1455 return true; 1456 1457 return false; 1458 } 1459 1460 static struct mlx5_ib_flow_handler *raw_fs_rule_add( 1461 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, 1462 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, 1463 u32 counter_id, void *cmd_in, int inlen, int dest_id, int dest_type) 1464 { 1465 struct mlx5_flow_destination *dst; 1466 struct mlx5_ib_flow_prio *ft_prio; 1467 struct mlx5_ib_flow_handler *handler; 1468 int dst_num = 0; 1469 bool mcast; 1470 int err; 1471 1472 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) 1473 return ERR_PTR(-EOPNOTSUPP); 1474 1475 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) 1476 return ERR_PTR(-ENOMEM); 1477 1478 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); 1479 if (!dst) 1480 return ERR_PTR(-ENOMEM); 1481 1482 mcast = raw_fs_is_multicast(fs_matcher, cmd_in); 1483 mutex_lock(&dev->flow_db->lock); 1484 1485 ft_prio = _get_flow_table(dev, fs_matcher, mcast); 1486 if (IS_ERR(ft_prio)) { 1487 err = PTR_ERR(ft_prio); 1488 goto unlock; 1489 } 1490 1491 if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) { 1492 dst[dst_num].type = dest_type; 1493 dst[dst_num++].tir_num = dest_id; 1494 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1495 } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) { 1496 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; 1497 dst[dst_num++].ft_num = dest_id; 1498 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1499 } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) { 1500 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1501 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1502 } 1503 1504 1505 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 1506 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1507 dst[dst_num].counter_id = counter_id; 1508 dst_num++; 1509 } 1510 1511 handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher, 1512 flow_context, flow_act, 1513 cmd_in, inlen, dst_num); 1514 1515 if (IS_ERR(handler)) { 1516 err = PTR_ERR(handler); 1517 goto destroy_ft; 1518 } 1519 1520 mutex_unlock(&dev->flow_db->lock); 1521 atomic_inc(&fs_matcher->usecnt); 1522 handler->flow_matcher = fs_matcher; 1523 1524 kfree(dst); 1525 1526 return handler; 1527 1528 destroy_ft: 1529 put_flow_table(dev, ft_prio, false); 1530 unlock: 1531 mutex_unlock(&dev->flow_db->lock); 1532 kfree(dst); 1533 1534 return ERR_PTR(err); 1535 } 1536 1537 static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags) 1538 { 1539 u32 flags = 0; 1540 1541 if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA) 1542 flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA; 1543 1544 return flags; 1545 } 1546 1547 #define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED \ 1548 MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA 1549 static struct ib_flow_action * 1550 mlx5_ib_create_flow_action_esp(struct ib_device *device, 1551 const struct ib_flow_action_attrs_esp *attr, 1552 struct uverbs_attr_bundle *attrs) 1553 { 1554 struct mlx5_ib_dev *mdev = to_mdev(device); 1555 struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm; 1556 struct mlx5_accel_esp_xfrm_attrs accel_attrs = {}; 1557 struct mlx5_ib_flow_action *action; 1558 u64 action_flags; 1559 u64 flags; 1560 int err = 0; 1561 1562 err = uverbs_get_flags64( 1563 &action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS, 1564 ((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1)); 1565 if (err) 1566 return ERR_PTR(err); 1567 1568 flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags); 1569 1570 /* We current only support a subset of the standard features. Only a 1571 * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn 1572 * (with overlap). Full offload mode isn't supported. 1573 */ 1574 if (!attr->keymat || attr->replay || attr->encap || 1575 attr->spi || attr->seq || attr->tfc_pad || 1576 attr->hard_limit_pkts || 1577 (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | 1578 IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT))) 1579 return ERR_PTR(-EOPNOTSUPP); 1580 1581 if (attr->keymat->protocol != 1582 IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM) 1583 return ERR_PTR(-EOPNOTSUPP); 1584 1585 aes_gcm = &attr->keymat->keymat.aes_gcm; 1586 1587 if (aes_gcm->icv_len != 16 || 1588 aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ) 1589 return ERR_PTR(-EOPNOTSUPP); 1590 1591 action = kmalloc(sizeof(*action), GFP_KERNEL); 1592 if (!action) 1593 return ERR_PTR(-ENOMEM); 1594 1595 action->esp_aes_gcm.ib_flags = attr->flags; 1596 memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key, 1597 sizeof(accel_attrs.keymat.aes_gcm.aes_key)); 1598 accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8; 1599 memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt, 1600 sizeof(accel_attrs.keymat.aes_gcm.salt)); 1601 memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv, 1602 sizeof(accel_attrs.keymat.aes_gcm.seq_iv)); 1603 accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8; 1604 accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ; 1605 accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM; 1606 1607 accel_attrs.esn = attr->esn; 1608 if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) 1609 accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; 1610 if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) 1611 accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 1612 1613 if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT) 1614 accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT; 1615 1616 action->esp_aes_gcm.ctx = 1617 mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags); 1618 if (IS_ERR(action->esp_aes_gcm.ctx)) { 1619 err = PTR_ERR(action->esp_aes_gcm.ctx); 1620 goto err_parse; 1621 } 1622 1623 action->esp_aes_gcm.ib_flags = attr->flags; 1624 1625 return &action->ib_action; 1626 1627 err_parse: 1628 kfree(action); 1629 return ERR_PTR(err); 1630 } 1631 1632 static int 1633 mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action, 1634 const struct ib_flow_action_attrs_esp *attr, 1635 struct uverbs_attr_bundle *attrs) 1636 { 1637 struct mlx5_ib_flow_action *maction = to_mflow_act(action); 1638 struct mlx5_accel_esp_xfrm_attrs accel_attrs; 1639 int err = 0; 1640 1641 if (attr->keymat || attr->replay || attr->encap || 1642 attr->spi || attr->seq || attr->tfc_pad || 1643 attr->hard_limit_pkts || 1644 (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | 1645 IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS | 1646 IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))) 1647 return -EOPNOTSUPP; 1648 1649 /* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can 1650 * be modified. 1651 */ 1652 if (!(maction->esp_aes_gcm.ib_flags & 1653 IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) && 1654 attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | 1655 IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)) 1656 return -EINVAL; 1657 1658 memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs, 1659 sizeof(accel_attrs)); 1660 1661 accel_attrs.esn = attr->esn; 1662 if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) 1663 accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 1664 else 1665 accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 1666 1667 err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx, 1668 &accel_attrs); 1669 if (err) 1670 return err; 1671 1672 maction->esp_aes_gcm.ib_flags &= 1673 ~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; 1674 maction->esp_aes_gcm.ib_flags |= 1675 attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; 1676 1677 return 0; 1678 } 1679 1680 static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) 1681 { 1682 switch (maction->flow_action_raw.sub_type) { 1683 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: 1684 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, 1685 maction->flow_action_raw.modify_hdr); 1686 break; 1687 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: 1688 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, 1689 maction->flow_action_raw.pkt_reformat); 1690 break; 1691 case MLX5_IB_FLOW_ACTION_DECAP: 1692 break; 1693 default: 1694 break; 1695 } 1696 } 1697 1698 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) 1699 { 1700 struct mlx5_ib_flow_action *maction = to_mflow_act(action); 1701 1702 switch (action->type) { 1703 case IB_FLOW_ACTION_ESP: 1704 /* 1705 * We only support aes_gcm by now, so we implicitly know this is 1706 * the underline crypto. 1707 */ 1708 mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx); 1709 break; 1710 case IB_FLOW_ACTION_UNSPECIFIED: 1711 destroy_flow_action_raw(maction); 1712 break; 1713 default: 1714 WARN_ON(true); 1715 break; 1716 } 1717 1718 kfree(maction); 1719 return 0; 1720 } 1721 1722 static int 1723 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, 1724 enum mlx5_flow_namespace_type *namespace) 1725 { 1726 switch (table_type) { 1727 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: 1728 *namespace = MLX5_FLOW_NAMESPACE_BYPASS; 1729 break; 1730 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: 1731 *namespace = MLX5_FLOW_NAMESPACE_EGRESS; 1732 break; 1733 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: 1734 *namespace = MLX5_FLOW_NAMESPACE_FDB; 1735 break; 1736 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: 1737 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; 1738 break; 1739 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: 1740 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; 1741 break; 1742 default: 1743 return -EINVAL; 1744 } 1745 1746 return 0; 1747 } 1748 1749 static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { 1750 [MLX5_IB_FLOW_TYPE_NORMAL] = { 1751 .type = UVERBS_ATTR_TYPE_PTR_IN, 1752 .u.ptr = { 1753 .len = sizeof(u16), /* data is priority */ 1754 .min_len = sizeof(u16), 1755 } 1756 }, 1757 [MLX5_IB_FLOW_TYPE_SNIFFER] = { 1758 .type = UVERBS_ATTR_TYPE_PTR_IN, 1759 UVERBS_ATTR_NO_DATA(), 1760 }, 1761 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { 1762 .type = UVERBS_ATTR_TYPE_PTR_IN, 1763 UVERBS_ATTR_NO_DATA(), 1764 }, 1765 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { 1766 .type = UVERBS_ATTR_TYPE_PTR_IN, 1767 UVERBS_ATTR_NO_DATA(), 1768 }, 1769 }; 1770 1771 static bool is_flow_dest(void *obj, int *dest_id, int *dest_type) 1772 { 1773 struct devx_obj *devx_obj = obj; 1774 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 1775 1776 switch (opcode) { 1777 case MLX5_CMD_OP_DESTROY_TIR: 1778 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1779 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 1780 obj_id); 1781 return true; 1782 1783 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 1784 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1785 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 1786 table_id); 1787 return true; 1788 default: 1789 return false; 1790 } 1791 } 1792 1793 static int get_dests(struct uverbs_attr_bundle *attrs, 1794 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, 1795 int *dest_type, struct ib_qp **qp, u32 *flags) 1796 { 1797 bool dest_devx, dest_qp; 1798 void *devx_obj; 1799 int err; 1800 1801 dest_devx = uverbs_attr_is_valid(attrs, 1802 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 1803 dest_qp = uverbs_attr_is_valid(attrs, 1804 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 1805 1806 *flags = 0; 1807 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 1808 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | 1809 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); 1810 if (err) 1811 return err; 1812 1813 /* Both flags are not allowed */ 1814 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && 1815 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 1816 return -EINVAL; 1817 1818 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 1819 if (dest_devx && (dest_qp || *flags)) 1820 return -EINVAL; 1821 else if (dest_qp && *flags) 1822 return -EINVAL; 1823 } 1824 1825 /* Allow only DEVX object, drop as dest for FDB */ 1826 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx || 1827 (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) 1828 return -EINVAL; 1829 1830 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ 1831 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && 1832 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) 1833 return -EINVAL; 1834 1835 *qp = NULL; 1836 if (dest_devx) { 1837 devx_obj = 1838 uverbs_attr_get_obj(attrs, 1839 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 1840 1841 /* Verify that the given DEVX object is a flow 1842 * steering destination. 1843 */ 1844 if (!is_flow_dest(devx_obj, dest_id, dest_type)) 1845 return -EINVAL; 1846 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ 1847 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB || 1848 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && 1849 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) 1850 return -EINVAL; 1851 } else if (dest_qp) { 1852 struct mlx5_ib_qp *mqp; 1853 1854 *qp = uverbs_attr_get_obj(attrs, 1855 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 1856 if (IS_ERR(*qp)) 1857 return PTR_ERR(*qp); 1858 1859 if ((*qp)->qp_type != IB_QPT_RAW_PACKET) 1860 return -EINVAL; 1861 1862 mqp = to_mqp(*qp); 1863 if (mqp->is_rss) 1864 *dest_id = mqp->rss_qp.tirn; 1865 else 1866 *dest_id = mqp->raw_packet_qp.rq.tirn; 1867 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1868 } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 1869 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) { 1870 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1871 } 1872 1873 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && 1874 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 1875 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX)) 1876 return -EINVAL; 1877 1878 return 0; 1879 } 1880 1881 static bool is_flow_counter(void *obj, u32 offset, u32 *counter_id) 1882 { 1883 struct devx_obj *devx_obj = obj; 1884 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 1885 1886 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 1887 1888 if (offset && offset >= devx_obj->flow_counter_bulk_size) 1889 return false; 1890 1891 *counter_id = MLX5_GET(dealloc_flow_counter_in, 1892 devx_obj->dinbox, 1893 flow_counter_id); 1894 *counter_id += offset; 1895 return true; 1896 } 1897 1898 return false; 1899 } 1900 1901 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 1902 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( 1903 struct uverbs_attr_bundle *attrs) 1904 { 1905 struct mlx5_flow_context flow_context = {.flow_tag = 1906 MLX5_FS_DEFAULT_FLOW_TAG}; 1907 u32 *offset_attr, offset = 0, counter_id = 0; 1908 int dest_id, dest_type = -1, inlen, len, ret, i; 1909 struct mlx5_ib_flow_handler *flow_handler; 1910 struct mlx5_ib_flow_matcher *fs_matcher; 1911 struct ib_uobject **arr_flow_actions; 1912 struct ib_uflow_resources *uflow_res; 1913 struct mlx5_flow_act flow_act = {}; 1914 struct ib_qp *qp = NULL; 1915 void *devx_obj, *cmd_in; 1916 struct ib_uobject *uobj; 1917 struct mlx5_ib_dev *dev; 1918 u32 flags; 1919 1920 if (!capable(CAP_NET_RAW)) 1921 return -EPERM; 1922 1923 fs_matcher = uverbs_attr_get_obj(attrs, 1924 MLX5_IB_ATTR_CREATE_FLOW_MATCHER); 1925 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); 1926 dev = mlx5_udata_to_mdev(&attrs->driver_udata); 1927 1928 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) 1929 return -EINVAL; 1930 1931 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) 1932 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; 1933 1934 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 1935 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 1936 1937 len = uverbs_attr_get_uobjs_arr(attrs, 1938 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); 1939 if (len) { 1940 devx_obj = arr_flow_actions[0]->object; 1941 1942 if (uverbs_attr_is_valid(attrs, 1943 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { 1944 1945 int num_offsets = uverbs_attr_ptr_get_array_size( 1946 attrs, 1947 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 1948 sizeof(u32)); 1949 1950 if (num_offsets != 1) 1951 return -EINVAL; 1952 1953 offset_attr = uverbs_attr_get_alloced_ptr( 1954 attrs, 1955 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); 1956 offset = *offset_attr; 1957 } 1958 1959 if (!is_flow_counter(devx_obj, offset, &counter_id)) 1960 return -EINVAL; 1961 1962 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 1963 } 1964 1965 cmd_in = uverbs_attr_get_alloced_ptr( 1966 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 1967 inlen = uverbs_attr_get_len(attrs, 1968 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 1969 1970 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); 1971 if (!uflow_res) 1972 return -ENOMEM; 1973 1974 len = uverbs_attr_get_uobjs_arr(attrs, 1975 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); 1976 for (i = 0; i < len; i++) { 1977 struct mlx5_ib_flow_action *maction = 1978 to_mflow_act(arr_flow_actions[i]->object); 1979 1980 ret = parse_flow_flow_action(maction, false, &flow_act); 1981 if (ret) 1982 goto err_out; 1983 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, 1984 arr_flow_actions[i]->object); 1985 } 1986 1987 ret = uverbs_copy_from(&flow_context.flow_tag, attrs, 1988 MLX5_IB_ATTR_CREATE_FLOW_TAG); 1989 if (!ret) { 1990 if (flow_context.flow_tag >= BIT(24)) { 1991 ret = -EINVAL; 1992 goto err_out; 1993 } 1994 flow_context.flags |= FLOW_CONTEXT_HAS_TAG; 1995 } 1996 1997 flow_handler = 1998 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act, 1999 counter_id, cmd_in, inlen, dest_id, dest_type); 2000 if (IS_ERR(flow_handler)) { 2001 ret = PTR_ERR(flow_handler); 2002 goto err_out; 2003 } 2004 2005 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); 2006 2007 return 0; 2008 err_out: 2009 ib_uverbs_flow_resources_free(uflow_res); 2010 return ret; 2011 } 2012 2013 static int flow_matcher_cleanup(struct ib_uobject *uobject, 2014 enum rdma_remove_reason why, 2015 struct uverbs_attr_bundle *attrs) 2016 { 2017 struct mlx5_ib_flow_matcher *obj = uobject->object; 2018 int ret; 2019 2020 ret = ib_destroy_usecnt(&obj->usecnt, why, uobject); 2021 if (ret) 2022 return ret; 2023 2024 kfree(obj); 2025 return 0; 2026 } 2027 2028 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, 2029 struct mlx5_ib_flow_matcher *obj) 2030 { 2031 enum mlx5_ib_uapi_flow_table_type ft_type = 2032 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; 2033 u32 flags; 2034 int err; 2035 2036 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older 2037 * users should switch to it. We leave this to not break userspace 2038 */ 2039 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && 2040 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) 2041 return -EINVAL; 2042 2043 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { 2044 err = uverbs_get_const(&ft_type, attrs, 2045 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); 2046 if (err) 2047 return err; 2048 2049 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); 2050 if (err) 2051 return err; 2052 2053 return 0; 2054 } 2055 2056 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { 2057 err = uverbs_get_flags32(&flags, attrs, 2058 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2059 IB_FLOW_ATTR_FLAGS_EGRESS); 2060 if (err) 2061 return err; 2062 2063 if (flags) { 2064 mlx5_ib_ft_type_to_namespace( 2065 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, 2066 &obj->ns_type); 2067 return 0; 2068 } 2069 } 2070 2071 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; 2072 2073 return 0; 2074 } 2075 2076 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( 2077 struct uverbs_attr_bundle *attrs) 2078 { 2079 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2080 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); 2081 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2082 struct mlx5_ib_flow_matcher *obj; 2083 int err; 2084 2085 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); 2086 if (!obj) 2087 return -ENOMEM; 2088 2089 obj->mask_len = uverbs_attr_get_len( 2090 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2091 err = uverbs_copy_from(&obj->matcher_mask, 2092 attrs, 2093 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2094 if (err) 2095 goto end; 2096 2097 obj->flow_type = uverbs_attr_get_enum_id( 2098 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2099 2100 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { 2101 err = uverbs_copy_from(&obj->priority, 2102 attrs, 2103 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2104 if (err) 2105 goto end; 2106 } 2107 2108 err = uverbs_copy_from(&obj->match_criteria_enable, 2109 attrs, 2110 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); 2111 if (err) 2112 goto end; 2113 2114 err = mlx5_ib_matcher_ns(attrs, obj); 2115 if (err) 2116 goto end; 2117 2118 uobj->object = obj; 2119 obj->mdev = dev->mdev; 2120 atomic_set(&obj->usecnt, 0); 2121 return 0; 2122 2123 end: 2124 kfree(obj); 2125 return err; 2126 } 2127 2128 static struct ib_flow_action * 2129 mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, 2130 enum mlx5_ib_uapi_flow_table_type ft_type, 2131 u8 num_actions, void *in) 2132 { 2133 enum mlx5_flow_namespace_type namespace; 2134 struct mlx5_ib_flow_action *maction; 2135 int ret; 2136 2137 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 2138 if (ret) 2139 return ERR_PTR(-EINVAL); 2140 2141 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 2142 if (!maction) 2143 return ERR_PTR(-ENOMEM); 2144 2145 maction->flow_action_raw.modify_hdr = 2146 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); 2147 2148 if (IS_ERR(maction->flow_action_raw.modify_hdr)) { 2149 ret = PTR_ERR(maction->flow_action_raw.modify_hdr); 2150 kfree(maction); 2151 return ERR_PTR(ret); 2152 } 2153 maction->flow_action_raw.sub_type = 2154 MLX5_IB_FLOW_ACTION_MODIFY_HEADER; 2155 maction->flow_action_raw.dev = dev; 2156 2157 return &maction->ib_action; 2158 } 2159 2160 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) 2161 { 2162 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 2163 max_modify_header_actions) || 2164 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 2165 max_modify_header_actions) || 2166 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 2167 max_modify_header_actions); 2168 } 2169 2170 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( 2171 struct uverbs_attr_bundle *attrs) 2172 { 2173 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2174 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); 2175 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 2176 enum mlx5_ib_uapi_flow_table_type ft_type; 2177 struct ib_flow_action *action; 2178 int num_actions; 2179 void *in; 2180 int ret; 2181 2182 if (!mlx5_ib_modify_header_supported(mdev)) 2183 return -EOPNOTSUPP; 2184 2185 in = uverbs_attr_get_alloced_ptr(attrs, 2186 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); 2187 2188 num_actions = uverbs_attr_ptr_get_array_size( 2189 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 2190 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); 2191 if (num_actions < 0) 2192 return num_actions; 2193 2194 ret = uverbs_get_const(&ft_type, attrs, 2195 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); 2196 if (ret) 2197 return ret; 2198 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); 2199 if (IS_ERR(action)) 2200 return PTR_ERR(action); 2201 2202 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, 2203 IB_FLOW_ACTION_UNSPECIFIED); 2204 2205 return 0; 2206 } 2207 2208 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, 2209 u8 packet_reformat_type, 2210 u8 ft_type) 2211 { 2212 switch (packet_reformat_type) { 2213 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 2214 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 2215 return MLX5_CAP_FLOWTABLE(ibdev->mdev, 2216 encap_general_header); 2217 break; 2218 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 2219 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 2220 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, 2221 reformat_l2_to_l3_tunnel); 2222 break; 2223 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 2224 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 2225 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, 2226 reformat_l3_tunnel_to_l2); 2227 break; 2228 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: 2229 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 2230 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); 2231 break; 2232 default: 2233 break; 2234 } 2235 2236 return false; 2237 } 2238 2239 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) 2240 { 2241 switch (dv_prt) { 2242 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 2243 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; 2244 break; 2245 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 2246 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 2247 break; 2248 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 2249 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 2250 break; 2251 default: 2252 return -EINVAL; 2253 } 2254 2255 return 0; 2256 } 2257 2258 static int mlx5_ib_flow_action_create_packet_reformat_ctx( 2259 struct mlx5_ib_dev *dev, 2260 struct mlx5_ib_flow_action *maction, 2261 u8 ft_type, u8 dv_prt, 2262 void *in, size_t len) 2263 { 2264 enum mlx5_flow_namespace_type namespace; 2265 u8 prm_prt; 2266 int ret; 2267 2268 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 2269 if (ret) 2270 return ret; 2271 2272 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); 2273 if (ret) 2274 return ret; 2275 2276 maction->flow_action_raw.pkt_reformat = 2277 mlx5_packet_reformat_alloc(dev->mdev, prm_prt, len, 2278 in, namespace); 2279 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { 2280 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); 2281 return ret; 2282 } 2283 2284 maction->flow_action_raw.sub_type = 2285 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; 2286 maction->flow_action_raw.dev = dev; 2287 2288 return 0; 2289 } 2290 2291 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( 2292 struct uverbs_attr_bundle *attrs) 2293 { 2294 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 2295 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); 2296 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 2297 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; 2298 enum mlx5_ib_uapi_flow_table_type ft_type; 2299 struct mlx5_ib_flow_action *maction; 2300 int ret; 2301 2302 ret = uverbs_get_const(&ft_type, attrs, 2303 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); 2304 if (ret) 2305 return ret; 2306 2307 ret = uverbs_get_const(&dv_prt, attrs, 2308 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); 2309 if (ret) 2310 return ret; 2311 2312 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) 2313 return -EOPNOTSUPP; 2314 2315 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 2316 if (!maction) 2317 return -ENOMEM; 2318 2319 if (dv_prt == 2320 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { 2321 maction->flow_action_raw.sub_type = 2322 MLX5_IB_FLOW_ACTION_DECAP; 2323 maction->flow_action_raw.dev = mdev; 2324 } else { 2325 void *in; 2326 int len; 2327 2328 in = uverbs_attr_get_alloced_ptr(attrs, 2329 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 2330 if (IS_ERR(in)) { 2331 ret = PTR_ERR(in); 2332 goto free_maction; 2333 } 2334 2335 len = uverbs_attr_get_len(attrs, 2336 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 2337 2338 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, 2339 maction, ft_type, dv_prt, in, len); 2340 if (ret) 2341 goto free_maction; 2342 } 2343 2344 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, 2345 IB_FLOW_ACTION_UNSPECIFIED); 2346 return 0; 2347 2348 free_maction: 2349 kfree(maction); 2350 return ret; 2351 } 2352 2353 DECLARE_UVERBS_NAMED_METHOD( 2354 MLX5_IB_METHOD_CREATE_FLOW, 2355 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 2356 UVERBS_OBJECT_FLOW, 2357 UVERBS_ACCESS_NEW, 2358 UA_MANDATORY), 2359 UVERBS_ATTR_PTR_IN( 2360 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, 2361 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 2362 UA_MANDATORY, 2363 UA_ALLOC_AND_COPY), 2364 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, 2365 MLX5_IB_OBJECT_FLOW_MATCHER, 2366 UVERBS_ACCESS_READ, 2367 UA_MANDATORY), 2368 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, 2369 UVERBS_OBJECT_QP, 2370 UVERBS_ACCESS_READ), 2371 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, 2372 MLX5_IB_OBJECT_DEVX_OBJ, 2373 UVERBS_ACCESS_READ), 2374 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, 2375 UVERBS_OBJECT_FLOW_ACTION, 2376 UVERBS_ACCESS_READ, 1, 2377 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, 2378 UA_OPTIONAL), 2379 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, 2380 UVERBS_ATTR_TYPE(u32), 2381 UA_OPTIONAL), 2382 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, 2383 MLX5_IB_OBJECT_DEVX_OBJ, 2384 UVERBS_ACCESS_READ, 1, 1, 2385 UA_OPTIONAL), 2386 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 2387 UVERBS_ATTR_MIN_SIZE(sizeof(u32)), 2388 UA_OPTIONAL, 2389 UA_ALLOC_AND_COPY), 2390 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 2391 enum mlx5_ib_create_flow_flags, 2392 UA_OPTIONAL)); 2393 2394 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 2395 MLX5_IB_METHOD_DESTROY_FLOW, 2396 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 2397 UVERBS_OBJECT_FLOW, 2398 UVERBS_ACCESS_DESTROY, 2399 UA_MANDATORY)); 2400 2401 ADD_UVERBS_METHODS(mlx5_ib_fs, 2402 UVERBS_OBJECT_FLOW, 2403 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), 2404 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); 2405 2406 DECLARE_UVERBS_NAMED_METHOD( 2407 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, 2408 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, 2409 UVERBS_OBJECT_FLOW_ACTION, 2410 UVERBS_ACCESS_NEW, 2411 UA_MANDATORY), 2412 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 2413 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( 2414 set_add_copy_action_in_auto)), 2415 UA_MANDATORY, 2416 UA_ALLOC_AND_COPY), 2417 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, 2418 enum mlx5_ib_uapi_flow_table_type, 2419 UA_MANDATORY)); 2420 2421 DECLARE_UVERBS_NAMED_METHOD( 2422 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, 2423 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, 2424 UVERBS_OBJECT_FLOW_ACTION, 2425 UVERBS_ACCESS_NEW, 2426 UA_MANDATORY), 2427 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, 2428 UVERBS_ATTR_MIN_SIZE(1), 2429 UA_ALLOC_AND_COPY, 2430 UA_OPTIONAL), 2431 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, 2432 enum mlx5_ib_uapi_flow_action_packet_reformat_type, 2433 UA_MANDATORY), 2434 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, 2435 enum mlx5_ib_uapi_flow_table_type, 2436 UA_MANDATORY)); 2437 2438 ADD_UVERBS_METHODS( 2439 mlx5_ib_flow_actions, 2440 UVERBS_OBJECT_FLOW_ACTION, 2441 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), 2442 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); 2443 2444 DECLARE_UVERBS_NAMED_METHOD( 2445 MLX5_IB_METHOD_FLOW_MATCHER_CREATE, 2446 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, 2447 MLX5_IB_OBJECT_FLOW_MATCHER, 2448 UVERBS_ACCESS_NEW, 2449 UA_MANDATORY), 2450 UVERBS_ATTR_PTR_IN( 2451 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, 2452 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 2453 UA_MANDATORY), 2454 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, 2455 mlx5_ib_flow_type, 2456 UA_MANDATORY), 2457 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, 2458 UVERBS_ATTR_TYPE(u8), 2459 UA_MANDATORY), 2460 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2461 enum ib_flow_flags, 2462 UA_OPTIONAL), 2463 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, 2464 enum mlx5_ib_uapi_flow_table_type, 2465 UA_OPTIONAL)); 2466 2467 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 2468 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, 2469 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, 2470 MLX5_IB_OBJECT_FLOW_MATCHER, 2471 UVERBS_ACCESS_DESTROY, 2472 UA_MANDATORY)); 2473 2474 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, 2475 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), 2476 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), 2477 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); 2478 2479 const struct uapi_definition mlx5_ib_flow_defs[] = { 2480 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 2481 MLX5_IB_OBJECT_FLOW_MATCHER), 2482 UAPI_DEF_CHAIN_OBJ_TREE( 2483 UVERBS_OBJECT_FLOW, 2484 &mlx5_ib_fs), 2485 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, 2486 &mlx5_ib_flow_actions), 2487 {}, 2488 }; 2489 2490 static const struct ib_device_ops flow_ops = { 2491 .create_flow = mlx5_ib_create_flow, 2492 .destroy_flow = mlx5_ib_destroy_flow, 2493 .destroy_flow_action = mlx5_ib_destroy_flow_action, 2494 }; 2495 2496 static const struct ib_device_ops flow_ipsec_ops = { 2497 .create_flow_action_esp = mlx5_ib_create_flow_action_esp, 2498 .modify_flow_action_esp = mlx5_ib_modify_flow_action_esp, 2499 }; 2500 2501 int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) 2502 { 2503 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); 2504 2505 if (!dev->flow_db) 2506 return -ENOMEM; 2507 2508 mutex_init(&dev->flow_db->lock); 2509 2510 ib_set_device_ops(&dev->ib_dev, &flow_ops); 2511 if (mlx5_accel_ipsec_device_caps(dev->mdev) & 2512 MLX5_ACCEL_IPSEC_CAP_DEVICE) 2513 ib_set_device_ops(&dev->ib_dev, &flow_ipsec_ops); 2514 2515 return 0; 2516 } 2517