1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <rdma/ib_user_verbs.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/uverbs_types.h> 9 #include <rdma/uverbs_ioctl.h> 10 #include <rdma/uverbs_std_types.h> 11 #include <rdma/mlx5_user_ioctl_cmds.h> 12 #include <rdma/mlx5_user_ioctl_verbs.h> 13 #include <rdma/ib_hdrs.h> 14 #include <rdma/ib_umem.h> 15 #include <linux/mlx5/driver.h> 16 #include <linux/mlx5/fs.h> 17 #include <linux/mlx5/fs_helpers.h> 18 #include <linux/mlx5/eswitch.h> 19 #include <net/inet_ecn.h> 20 #include "mlx5_ib.h" 21 #include "counters.h" 22 #include "devx.h" 23 #include "fs.h" 24 25 #define UVERBS_MODULE_NAME mlx5_ib 26 #include <rdma/uverbs_named_ioctl.h> 27 28 enum { 29 MATCH_CRITERIA_ENABLE_OUTER_BIT, 30 MATCH_CRITERIA_ENABLE_MISC_BIT, 31 MATCH_CRITERIA_ENABLE_INNER_BIT, 32 MATCH_CRITERIA_ENABLE_MISC2_BIT 33 }; 34 35 #define HEADER_IS_ZERO(match_criteria, headers) \ 36 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 37 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 38 39 static u8 get_match_criteria_enable(u32 *match_criteria) 40 { 41 u8 match_criteria_enable; 42 43 match_criteria_enable = 44 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 45 MATCH_CRITERIA_ENABLE_OUTER_BIT; 46 match_criteria_enable |= 47 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 48 MATCH_CRITERIA_ENABLE_MISC_BIT; 49 match_criteria_enable |= 50 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 51 MATCH_CRITERIA_ENABLE_INNER_BIT; 52 match_criteria_enable |= 53 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 54 MATCH_CRITERIA_ENABLE_MISC2_BIT; 55 56 return match_criteria_enable; 57 } 58 59 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) 60 { 61 u8 entry_mask; 62 u8 entry_val; 63 int err = 0; 64 65 if (!mask) 66 goto out; 67 68 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, 69 ip_protocol); 70 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, 71 ip_protocol); 72 if (!entry_mask) { 73 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); 74 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); 75 goto out; 76 } 77 /* Don't override existing ip protocol */ 78 if (mask != entry_mask || val != entry_val) 79 err = -EINVAL; 80 out: 81 return err; 82 } 83 84 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, 85 bool inner) 86 { 87 if (inner) { 88 MLX5_SET(fte_match_set_misc, 89 misc_c, inner_ipv6_flow_label, mask); 90 MLX5_SET(fte_match_set_misc, 91 misc_v, inner_ipv6_flow_label, val); 92 } else { 93 MLX5_SET(fte_match_set_misc, 94 misc_c, outer_ipv6_flow_label, mask); 95 MLX5_SET(fte_match_set_misc, 96 misc_v, outer_ipv6_flow_label, val); 97 } 98 } 99 100 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) 101 { 102 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); 103 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); 104 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); 105 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); 106 } 107 108 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) 109 { 110 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && 111 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) 112 return -EOPNOTSUPP; 113 114 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && 115 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) 116 return -EOPNOTSUPP; 117 118 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && 119 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) 120 return -EOPNOTSUPP; 121 122 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && 123 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) 124 return -EOPNOTSUPP; 125 126 return 0; 127 } 128 129 #define LAST_ETH_FIELD vlan_tag 130 #define LAST_IPV4_FIELD tos 131 #define LAST_IPV6_FIELD traffic_class 132 #define LAST_TCP_UDP_FIELD src_port 133 #define LAST_TUNNEL_FIELD tunnel_id 134 #define LAST_FLOW_TAG_FIELD tag_id 135 #define LAST_DROP_FIELD size 136 #define LAST_COUNTERS_FIELD counters 137 138 /* Field is the last supported field */ 139 #define FIELDS_NOT_SUPPORTED(filter, field) \ 140 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \ 141 sizeof(filter) - offsetofend(typeof(filter), field)) 142 143 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, 144 bool is_egress, 145 struct mlx5_flow_act *action) 146 { 147 148 switch (maction->ib_action.type) { 149 case IB_FLOW_ACTION_UNSPECIFIED: 150 if (maction->flow_action_raw.sub_type == 151 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { 152 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 153 return -EINVAL; 154 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 155 action->modify_hdr = 156 maction->flow_action_raw.modify_hdr; 157 return 0; 158 } 159 if (maction->flow_action_raw.sub_type == 160 MLX5_IB_FLOW_ACTION_DECAP) { 161 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) 162 return -EINVAL; 163 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 164 return 0; 165 } 166 if (maction->flow_action_raw.sub_type == 167 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { 168 if (action->action & 169 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) 170 return -EINVAL; 171 action->action |= 172 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 173 action->pkt_reformat = 174 maction->flow_action_raw.pkt_reformat; 175 return 0; 176 } 177 fallthrough; 178 default: 179 return -EOPNOTSUPP; 180 } 181 } 182 183 static int parse_flow_attr(struct mlx5_core_dev *mdev, 184 struct mlx5_flow_spec *spec, 185 const union ib_flow_spec *ib_spec, 186 const struct ib_flow_attr *flow_attr, 187 struct mlx5_flow_act *action, u32 prev_type) 188 { 189 struct mlx5_flow_context *flow_context = &spec->flow_context; 190 u32 *match_c = spec->match_criteria; 191 u32 *match_v = spec->match_value; 192 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, 193 misc_parameters); 194 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, 195 misc_parameters); 196 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, 197 misc_parameters_2); 198 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, 199 misc_parameters_2); 200 void *headers_c; 201 void *headers_v; 202 int match_ipv; 203 int ret; 204 205 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 206 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 207 inner_headers); 208 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 209 inner_headers); 210 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 211 ft_field_support.inner_ip_version); 212 } else { 213 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 214 outer_headers); 215 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 216 outer_headers); 217 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 218 ft_field_support.outer_ip_version); 219 } 220 221 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 222 case IB_FLOW_SPEC_ETH: 223 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) 224 return -EOPNOTSUPP; 225 226 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 227 dmac_47_16), 228 ib_spec->eth.mask.dst_mac); 229 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 230 dmac_47_16), 231 ib_spec->eth.val.dst_mac); 232 233 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 234 smac_47_16), 235 ib_spec->eth.mask.src_mac); 236 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 237 smac_47_16), 238 ib_spec->eth.val.src_mac); 239 240 if (ib_spec->eth.mask.vlan_tag) { 241 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 242 cvlan_tag, 1); 243 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 244 cvlan_tag, 1); 245 246 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 247 first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); 248 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 249 first_vid, ntohs(ib_spec->eth.val.vlan_tag)); 250 251 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 252 first_cfi, 253 ntohs(ib_spec->eth.mask.vlan_tag) >> 12); 254 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 255 first_cfi, 256 ntohs(ib_spec->eth.val.vlan_tag) >> 12); 257 258 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 259 first_prio, 260 ntohs(ib_spec->eth.mask.vlan_tag) >> 13); 261 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 262 first_prio, 263 ntohs(ib_spec->eth.val.vlan_tag) >> 13); 264 } 265 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 266 ethertype, ntohs(ib_spec->eth.mask.ether_type)); 267 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 268 ethertype, ntohs(ib_spec->eth.val.ether_type)); 269 break; 270 case IB_FLOW_SPEC_IPV4: 271 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) 272 return -EOPNOTSUPP; 273 274 if (match_ipv) { 275 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 276 ip_version, 0xf); 277 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 278 ip_version, MLX5_FS_IPV4_VERSION); 279 } else { 280 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 281 ethertype, 0xffff); 282 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 283 ethertype, ETH_P_IP); 284 } 285 286 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 287 src_ipv4_src_ipv6.ipv4_layout.ipv4), 288 &ib_spec->ipv4.mask.src_ip, 289 sizeof(ib_spec->ipv4.mask.src_ip)); 290 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 291 src_ipv4_src_ipv6.ipv4_layout.ipv4), 292 &ib_spec->ipv4.val.src_ip, 293 sizeof(ib_spec->ipv4.val.src_ip)); 294 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 295 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 296 &ib_spec->ipv4.mask.dst_ip, 297 sizeof(ib_spec->ipv4.mask.dst_ip)); 298 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 299 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 300 &ib_spec->ipv4.val.dst_ip, 301 sizeof(ib_spec->ipv4.val.dst_ip)); 302 303 set_tos(headers_c, headers_v, 304 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); 305 306 if (set_proto(headers_c, headers_v, 307 ib_spec->ipv4.mask.proto, 308 ib_spec->ipv4.val.proto)) 309 return -EINVAL; 310 break; 311 case IB_FLOW_SPEC_IPV6: 312 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) 313 return -EOPNOTSUPP; 314 315 if (match_ipv) { 316 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 317 ip_version, 0xf); 318 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 319 ip_version, MLX5_FS_IPV6_VERSION); 320 } else { 321 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 322 ethertype, 0xffff); 323 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 324 ethertype, ETH_P_IPV6); 325 } 326 327 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 328 src_ipv4_src_ipv6.ipv6_layout.ipv6), 329 &ib_spec->ipv6.mask.src_ip, 330 sizeof(ib_spec->ipv6.mask.src_ip)); 331 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 332 src_ipv4_src_ipv6.ipv6_layout.ipv6), 333 &ib_spec->ipv6.val.src_ip, 334 sizeof(ib_spec->ipv6.val.src_ip)); 335 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 336 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 337 &ib_spec->ipv6.mask.dst_ip, 338 sizeof(ib_spec->ipv6.mask.dst_ip)); 339 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 340 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 341 &ib_spec->ipv6.val.dst_ip, 342 sizeof(ib_spec->ipv6.val.dst_ip)); 343 344 set_tos(headers_c, headers_v, 345 ib_spec->ipv6.mask.traffic_class, 346 ib_spec->ipv6.val.traffic_class); 347 348 if (set_proto(headers_c, headers_v, 349 ib_spec->ipv6.mask.next_hdr, 350 ib_spec->ipv6.val.next_hdr)) 351 return -EINVAL; 352 353 set_flow_label(misc_params_c, misc_params_v, 354 ntohl(ib_spec->ipv6.mask.flow_label), 355 ntohl(ib_spec->ipv6.val.flow_label), 356 ib_spec->type & IB_FLOW_SPEC_INNER); 357 break; 358 case IB_FLOW_SPEC_ESP: 359 return -EOPNOTSUPP; 360 case IB_FLOW_SPEC_TCP: 361 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 362 LAST_TCP_UDP_FIELD)) 363 return -EOPNOTSUPP; 364 365 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) 366 return -EINVAL; 367 368 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, 369 ntohs(ib_spec->tcp_udp.mask.src_port)); 370 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 371 ntohs(ib_spec->tcp_udp.val.src_port)); 372 373 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, 374 ntohs(ib_spec->tcp_udp.mask.dst_port)); 375 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 376 ntohs(ib_spec->tcp_udp.val.dst_port)); 377 break; 378 case IB_FLOW_SPEC_UDP: 379 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 380 LAST_TCP_UDP_FIELD)) 381 return -EOPNOTSUPP; 382 383 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) 384 return -EINVAL; 385 386 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 387 ntohs(ib_spec->tcp_udp.mask.src_port)); 388 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 389 ntohs(ib_spec->tcp_udp.val.src_port)); 390 391 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 392 ntohs(ib_spec->tcp_udp.mask.dst_port)); 393 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 394 ntohs(ib_spec->tcp_udp.val.dst_port)); 395 break; 396 case IB_FLOW_SPEC_GRE: 397 if (ib_spec->gre.mask.c_ks_res0_ver) 398 return -EOPNOTSUPP; 399 400 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) 401 return -EINVAL; 402 403 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 404 0xff); 405 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 406 IPPROTO_GRE); 407 408 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, 409 ntohs(ib_spec->gre.mask.protocol)); 410 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, 411 ntohs(ib_spec->gre.val.protocol)); 412 413 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, 414 gre_key.nvgre.hi), 415 &ib_spec->gre.mask.key, 416 sizeof(ib_spec->gre.mask.key)); 417 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, 418 gre_key.nvgre.hi), 419 &ib_spec->gre.val.key, 420 sizeof(ib_spec->gre.val.key)); 421 break; 422 case IB_FLOW_SPEC_MPLS: 423 switch (prev_type) { 424 case IB_FLOW_SPEC_UDP: 425 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 426 ft_field_support.outer_first_mpls_over_udp), 427 &ib_spec->mpls.mask.tag)) 428 return -EOPNOTSUPP; 429 430 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 431 outer_first_mpls_over_udp), 432 &ib_spec->mpls.val.tag, 433 sizeof(ib_spec->mpls.val.tag)); 434 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 435 outer_first_mpls_over_udp), 436 &ib_spec->mpls.mask.tag, 437 sizeof(ib_spec->mpls.mask.tag)); 438 break; 439 case IB_FLOW_SPEC_GRE: 440 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 441 ft_field_support.outer_first_mpls_over_gre), 442 &ib_spec->mpls.mask.tag)) 443 return -EOPNOTSUPP; 444 445 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 446 outer_first_mpls_over_gre), 447 &ib_spec->mpls.val.tag, 448 sizeof(ib_spec->mpls.val.tag)); 449 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 450 outer_first_mpls_over_gre), 451 &ib_spec->mpls.mask.tag, 452 sizeof(ib_spec->mpls.mask.tag)); 453 break; 454 default: 455 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 456 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 457 ft_field_support.inner_first_mpls), 458 &ib_spec->mpls.mask.tag)) 459 return -EOPNOTSUPP; 460 461 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 462 inner_first_mpls), 463 &ib_spec->mpls.val.tag, 464 sizeof(ib_spec->mpls.val.tag)); 465 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 466 inner_first_mpls), 467 &ib_spec->mpls.mask.tag, 468 sizeof(ib_spec->mpls.mask.tag)); 469 } else { 470 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 471 ft_field_support.outer_first_mpls), 472 &ib_spec->mpls.mask.tag)) 473 return -EOPNOTSUPP; 474 475 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 476 outer_first_mpls), 477 &ib_spec->mpls.val.tag, 478 sizeof(ib_spec->mpls.val.tag)); 479 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 480 outer_first_mpls), 481 &ib_spec->mpls.mask.tag, 482 sizeof(ib_spec->mpls.mask.tag)); 483 } 484 } 485 break; 486 case IB_FLOW_SPEC_VXLAN_TUNNEL: 487 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, 488 LAST_TUNNEL_FIELD)) 489 return -EOPNOTSUPP; 490 491 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, 492 ntohl(ib_spec->tunnel.mask.tunnel_id)); 493 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, 494 ntohl(ib_spec->tunnel.val.tunnel_id)); 495 break; 496 case IB_FLOW_SPEC_ACTION_TAG: 497 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, 498 LAST_FLOW_TAG_FIELD)) 499 return -EOPNOTSUPP; 500 if (ib_spec->flow_tag.tag_id >= BIT(24)) 501 return -EINVAL; 502 503 flow_context->flow_tag = ib_spec->flow_tag.tag_id; 504 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 505 break; 506 case IB_FLOW_SPEC_ACTION_DROP: 507 if (FIELDS_NOT_SUPPORTED(ib_spec->drop, 508 LAST_DROP_FIELD)) 509 return -EOPNOTSUPP; 510 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 511 break; 512 case IB_FLOW_SPEC_ACTION_HANDLE: 513 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), 514 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); 515 if (ret) 516 return ret; 517 break; 518 case IB_FLOW_SPEC_ACTION_COUNT: 519 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, 520 LAST_COUNTERS_FIELD)) 521 return -EOPNOTSUPP; 522 523 /* for now support only one counters spec per flow */ 524 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) 525 return -EINVAL; 526 527 action->counters = ib_spec->flow_count.counters; 528 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 529 break; 530 default: 531 return -EINVAL; 532 } 533 534 return 0; 535 } 536 537 /* If a flow could catch both multicast and unicast packets, 538 * it won't fall into the multicast flow steering table and this rule 539 * could steal other multicast packets. 540 */ 541 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) 542 { 543 union ib_flow_spec *flow_spec; 544 545 if (ib_attr->type != IB_FLOW_ATTR_NORMAL || 546 ib_attr->num_of_specs < 1) 547 return false; 548 549 flow_spec = (union ib_flow_spec *)(ib_attr + 1); 550 if (flow_spec->type == IB_FLOW_SPEC_IPV4) { 551 struct ib_flow_spec_ipv4 *ipv4_spec; 552 553 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; 554 if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) 555 return true; 556 557 return false; 558 } 559 560 if (flow_spec->type == IB_FLOW_SPEC_ETH) { 561 struct ib_flow_spec_eth *eth_spec; 562 563 eth_spec = (struct ib_flow_spec_eth *)flow_spec; 564 return is_multicast_ether_addr(eth_spec->mask.dst_mac) && 565 is_multicast_ether_addr(eth_spec->val.dst_mac); 566 } 567 568 return false; 569 } 570 571 static bool is_valid_ethertype(struct mlx5_core_dev *mdev, 572 const struct ib_flow_attr *flow_attr, 573 bool check_inner) 574 { 575 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); 576 int match_ipv = check_inner ? 577 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 578 ft_field_support.inner_ip_version) : 579 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 580 ft_field_support.outer_ip_version); 581 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; 582 bool ipv4_spec_valid, ipv6_spec_valid; 583 unsigned int ip_spec_type = 0; 584 bool has_ethertype = false; 585 unsigned int spec_index; 586 bool mask_valid = true; 587 u16 eth_type = 0; 588 bool type_valid; 589 590 /* Validate that ethertype is correct */ 591 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 592 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && 593 ib_spec->eth.mask.ether_type) { 594 mask_valid = (ib_spec->eth.mask.ether_type == 595 htons(0xffff)); 596 has_ethertype = true; 597 eth_type = ntohs(ib_spec->eth.val.ether_type); 598 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || 599 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { 600 ip_spec_type = ib_spec->type; 601 } 602 ib_spec = (void *)ib_spec + ib_spec->size; 603 } 604 605 type_valid = (!has_ethertype) || (!ip_spec_type); 606 if (!type_valid && mask_valid) { 607 ipv4_spec_valid = (eth_type == ETH_P_IP) && 608 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); 609 ipv6_spec_valid = (eth_type == ETH_P_IPV6) && 610 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); 611 612 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || 613 (((eth_type == ETH_P_MPLS_UC) || 614 (eth_type == ETH_P_MPLS_MC)) && match_ipv); 615 } 616 617 return type_valid; 618 } 619 620 static bool is_valid_attr(struct mlx5_core_dev *mdev, 621 const struct ib_flow_attr *flow_attr) 622 { 623 return is_valid_ethertype(mdev, flow_attr, false) && 624 is_valid_ethertype(mdev, flow_attr, true); 625 } 626 627 static void put_flow_table(struct mlx5_ib_dev *dev, 628 struct mlx5_ib_flow_prio *prio, bool ft_added) 629 { 630 prio->refcount -= !!ft_added; 631 if (!prio->refcount) { 632 mlx5_destroy_flow_table(prio->flow_table); 633 prio->flow_table = NULL; 634 } 635 } 636 637 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) 638 { 639 struct mlx5_ib_flow_handler *handler = container_of(flow_id, 640 struct mlx5_ib_flow_handler, 641 ibflow); 642 struct mlx5_ib_flow_handler *iter, *tmp; 643 struct mlx5_ib_dev *dev = handler->dev; 644 645 mutex_lock(&dev->flow_db->lock); 646 647 list_for_each_entry_safe(iter, tmp, &handler->list, list) { 648 mlx5_del_flow_rules(iter->rule); 649 put_flow_table(dev, iter->prio, true); 650 list_del(&iter->list); 651 kfree(iter); 652 } 653 654 mlx5_del_flow_rules(handler->rule); 655 put_flow_table(dev, handler->prio, true); 656 mlx5_ib_counters_clear_description(handler->ibcounters); 657 mutex_unlock(&dev->flow_db->lock); 658 if (handler->flow_matcher) 659 atomic_dec(&handler->flow_matcher->usecnt); 660 kfree(handler); 661 662 return 0; 663 } 664 665 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) 666 { 667 priority *= 2; 668 if (!dont_trap) 669 priority++; 670 return priority; 671 } 672 673 enum flow_table_type { 674 MLX5_IB_FT_RX, 675 MLX5_IB_FT_TX 676 }; 677 678 #define MLX5_FS_MAX_TYPES 6 679 #define MLX5_FS_MAX_ENTRIES BIT(16) 680 681 static bool mlx5_ib_shared_ft_allowed(struct ib_device *device) 682 { 683 struct mlx5_ib_dev *dev = to_mdev(device); 684 685 return MLX5_CAP_GEN(dev->mdev, shared_object_to_user_object_allowed); 686 } 687 688 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_ib_dev *dev, 689 struct mlx5_flow_namespace *ns, 690 struct mlx5_ib_flow_prio *prio, 691 int priority, 692 int num_entries, int num_groups, 693 u32 flags) 694 { 695 struct mlx5_flow_table_attr ft_attr = {}; 696 struct mlx5_flow_table *ft; 697 698 if (mlx5_ib_shared_ft_allowed(&dev->ib_dev)) 699 ft_attr.uid = MLX5_SHARED_RESOURCE_UID; 700 ft_attr.prio = priority; 701 ft_attr.max_fte = num_entries; 702 ft_attr.flags = flags; 703 ft_attr.autogroup.max_num_groups = num_groups; 704 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 705 if (IS_ERR(ft)) 706 return ERR_CAST(ft); 707 708 prio->flow_table = ft; 709 prio->refcount = 0; 710 return prio; 711 } 712 713 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, 714 struct ib_flow_attr *flow_attr, 715 enum flow_table_type ft_type) 716 { 717 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; 718 struct mlx5_flow_namespace *ns = NULL; 719 enum mlx5_flow_namespace_type fn_type; 720 struct mlx5_ib_flow_prio *prio; 721 struct mlx5_flow_table *ft; 722 int max_table_size; 723 int num_entries; 724 int num_groups; 725 bool esw_encap; 726 u32 flags = 0; 727 int priority; 728 729 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 730 log_max_ft_size)); 731 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 732 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 733 switch (flow_attr->type) { 734 case IB_FLOW_ATTR_NORMAL: 735 if (flow_is_multicast_only(flow_attr) && !dont_trap) 736 priority = MLX5_IB_FLOW_MCAST_PRIO; 737 else 738 priority = ib_prio_to_core_prio(flow_attr->priority, 739 dont_trap); 740 if (ft_type == MLX5_IB_FT_RX) { 741 fn_type = MLX5_FLOW_NAMESPACE_BYPASS; 742 prio = &dev->flow_db->prios[priority]; 743 if (!dev->is_rep && !esw_encap && 744 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) 745 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 746 if (!dev->is_rep && !esw_encap && 747 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 748 reformat_l3_tunnel_to_l2)) 749 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 750 } else { 751 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX( 752 dev->mdev, log_max_ft_size)); 753 fn_type = MLX5_FLOW_NAMESPACE_EGRESS; 754 prio = &dev->flow_db->egress_prios[priority]; 755 if (!dev->is_rep && !esw_encap && 756 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) 757 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 758 } 759 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 760 num_entries = MLX5_FS_MAX_ENTRIES; 761 num_groups = MLX5_FS_MAX_TYPES; 762 break; 763 case IB_FLOW_ATTR_ALL_DEFAULT: 764 case IB_FLOW_ATTR_MC_DEFAULT: 765 ns = mlx5_get_flow_namespace(dev->mdev, 766 MLX5_FLOW_NAMESPACE_LEFTOVERS); 767 build_leftovers_ft_param(&priority, &num_entries, &num_groups); 768 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; 769 break; 770 case IB_FLOW_ATTR_SNIFFER: 771 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 772 allow_sniffer_and_nic_rx_shared_tir)) 773 return ERR_PTR(-EOPNOTSUPP); 774 775 ns = mlx5_get_flow_namespace( 776 dev->mdev, ft_type == MLX5_IB_FT_RX ? 777 MLX5_FLOW_NAMESPACE_SNIFFER_RX : 778 MLX5_FLOW_NAMESPACE_SNIFFER_TX); 779 780 prio = &dev->flow_db->sniffer[ft_type]; 781 priority = 0; 782 num_entries = 1; 783 num_groups = 1; 784 break; 785 default: 786 break; 787 } 788 789 if (!ns) 790 return ERR_PTR(-EOPNOTSUPP); 791 792 max_table_size = min_t(int, num_entries, max_table_size); 793 794 ft = prio->flow_table; 795 if (!ft) 796 return _get_prio(dev, ns, prio, priority, max_table_size, 797 num_groups, flags); 798 799 return prio; 800 } 801 802 enum { 803 RDMA_RX_ECN_OPCOUNTER_PRIO, 804 RDMA_RX_CNP_OPCOUNTER_PRIO, 805 }; 806 807 enum { 808 RDMA_TX_CNP_OPCOUNTER_PRIO, 809 }; 810 811 static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num, 812 struct mlx5_flow_spec *spec) 813 { 814 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 815 ft_field_support.source_vhca_port) || 816 !MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 817 ft_field_support.source_vhca_port)) 818 return -EOPNOTSUPP; 819 820 MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria, 821 misc_parameters.source_vhca_port); 822 MLX5_SET(fte_match_param, &spec->match_value, 823 misc_parameters.source_vhca_port, port_num); 824 825 return 0; 826 } 827 828 static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num, 829 struct mlx5_flow_spec *spec, int ipv) 830 { 831 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 832 ft_field_support.outer_ip_version)) 833 return -EOPNOTSUPP; 834 835 if (mlx5_core_mp_enabled(dev->mdev) && 836 set_vhca_port_spec(dev, port_num, spec)) 837 return -EOPNOTSUPP; 838 839 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 840 outer_headers.ip_ecn); 841 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn, 842 INET_ECN_CE); 843 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 844 outer_headers.ip_version); 845 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 846 ipv); 847 848 spec->match_criteria_enable = 849 get_match_criteria_enable(spec->match_criteria); 850 851 return 0; 852 } 853 854 static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num, 855 struct mlx5_flow_spec *spec) 856 { 857 if (mlx5_core_mp_enabled(dev->mdev) && 858 set_vhca_port_spec(dev, port_num, spec)) 859 return -EOPNOTSUPP; 860 861 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 862 misc_parameters.bth_opcode); 863 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode, 864 IB_BTH_OPCODE_CNP); 865 866 spec->match_criteria_enable = 867 get_match_criteria_enable(spec->match_criteria); 868 869 return 0; 870 } 871 872 int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num, 873 struct mlx5_ib_op_fc *opfc, 874 enum mlx5_ib_optional_counter_type type) 875 { 876 enum mlx5_flow_namespace_type fn_type; 877 int priority, i, err, spec_num; 878 struct mlx5_flow_act flow_act = {}; 879 struct mlx5_flow_destination dst; 880 struct mlx5_flow_namespace *ns; 881 struct mlx5_ib_flow_prio *prio; 882 struct mlx5_flow_spec *spec; 883 884 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL); 885 if (!spec) 886 return -ENOMEM; 887 888 switch (type) { 889 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 890 if (set_ecn_ce_spec(dev, port_num, &spec[0], 891 MLX5_FS_IPV4_VERSION) || 892 set_ecn_ce_spec(dev, port_num, &spec[1], 893 MLX5_FS_IPV6_VERSION)) { 894 err = -EOPNOTSUPP; 895 goto free; 896 } 897 spec_num = 2; 898 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 899 priority = RDMA_RX_ECN_OPCOUNTER_PRIO; 900 break; 901 902 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 903 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 904 ft_field_support_2_nic_receive_rdma.bth_opcode) || 905 set_cnp_spec(dev, port_num, &spec[0])) { 906 err = -EOPNOTSUPP; 907 goto free; 908 } 909 spec_num = 1; 910 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 911 priority = RDMA_RX_CNP_OPCOUNTER_PRIO; 912 break; 913 914 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 915 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 916 ft_field_support_2_nic_transmit_rdma.bth_opcode) || 917 set_cnp_spec(dev, port_num, &spec[0])) { 918 err = -EOPNOTSUPP; 919 goto free; 920 } 921 spec_num = 1; 922 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 923 priority = RDMA_TX_CNP_OPCOUNTER_PRIO; 924 break; 925 926 default: 927 err = -EOPNOTSUPP; 928 goto free; 929 } 930 931 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 932 if (!ns) { 933 err = -EOPNOTSUPP; 934 goto free; 935 } 936 937 prio = &dev->flow_db->opfcs[type]; 938 if (!prio->flow_table) { 939 prio = _get_prio(dev, ns, prio, priority, 940 dev->num_ports * MAX_OPFC_RULES, 1, 0); 941 if (IS_ERR(prio)) { 942 err = PTR_ERR(prio); 943 goto free; 944 } 945 } 946 947 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 948 dst.counter_id = mlx5_fc_id(opfc->fc); 949 950 flow_act.action = 951 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW; 952 953 for (i = 0; i < spec_num; i++) { 954 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i], 955 &flow_act, &dst, 1); 956 if (IS_ERR(opfc->rule[i])) { 957 err = PTR_ERR(opfc->rule[i]); 958 goto del_rules; 959 } 960 } 961 prio->refcount += spec_num; 962 kfree(spec); 963 964 return 0; 965 966 del_rules: 967 for (i -= 1; i >= 0; i--) 968 mlx5_del_flow_rules(opfc->rule[i]); 969 put_flow_table(dev, prio, false); 970 free: 971 kfree(spec); 972 return err; 973 } 974 975 void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev, 976 struct mlx5_ib_op_fc *opfc, 977 enum mlx5_ib_optional_counter_type type) 978 { 979 int i; 980 981 for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) { 982 mlx5_del_flow_rules(opfc->rule[i]); 983 put_flow_table(dev, &dev->flow_db->opfcs[type], true); 984 } 985 } 986 987 static void set_underlay_qp(struct mlx5_ib_dev *dev, 988 struct mlx5_flow_spec *spec, 989 u32 underlay_qpn) 990 { 991 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, 992 spec->match_criteria, 993 misc_parameters); 994 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 995 misc_parameters); 996 997 if (underlay_qpn && 998 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 999 ft_field_support.bth_dst_qp)) { 1000 MLX5_SET(fte_match_set_misc, 1001 misc_params_v, bth_dst_qp, underlay_qpn); 1002 MLX5_SET(fte_match_set_misc, 1003 misc_params_c, bth_dst_qp, 0xffffff); 1004 } 1005 } 1006 1007 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, 1008 struct mlx5_flow_spec *spec, 1009 struct mlx5_eswitch_rep *rep) 1010 { 1011 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; 1012 void *misc; 1013 1014 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1015 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1016 misc_parameters_2); 1017 1018 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1019 mlx5_eswitch_get_vport_metadata_for_match(rep->esw, 1020 rep->vport)); 1021 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1022 misc_parameters_2); 1023 1024 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1025 mlx5_eswitch_get_vport_metadata_mask()); 1026 } else { 1027 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1028 misc_parameters); 1029 1030 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); 1031 1032 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1033 misc_parameters); 1034 1035 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1036 } 1037 } 1038 1039 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, 1040 struct mlx5_ib_flow_prio *ft_prio, 1041 const struct ib_flow_attr *flow_attr, 1042 struct mlx5_flow_destination *dst, 1043 u32 underlay_qpn, 1044 struct mlx5_ib_create_flow *ucmd) 1045 { 1046 struct mlx5_flow_table *ft = ft_prio->flow_table; 1047 struct mlx5_ib_flow_handler *handler; 1048 struct mlx5_flow_act flow_act = {}; 1049 struct mlx5_flow_spec *spec; 1050 struct mlx5_flow_destination dest_arr[2] = {}; 1051 struct mlx5_flow_destination *rule_dst = dest_arr; 1052 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); 1053 unsigned int spec_index; 1054 u32 prev_type = 0; 1055 int err = 0; 1056 int dest_num = 0; 1057 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1058 1059 if (!is_valid_attr(dev->mdev, flow_attr)) 1060 return ERR_PTR(-EINVAL); 1061 1062 if (dev->is_rep && is_egress) 1063 return ERR_PTR(-EINVAL); 1064 1065 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1066 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 1067 if (!handler || !spec) { 1068 err = -ENOMEM; 1069 goto free; 1070 } 1071 1072 INIT_LIST_HEAD(&handler->list); 1073 1074 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 1075 err = parse_flow_attr(dev->mdev, spec, 1076 ib_flow, flow_attr, &flow_act, 1077 prev_type); 1078 if (err < 0) 1079 goto free; 1080 1081 prev_type = ((union ib_flow_spec *)ib_flow)->type; 1082 ib_flow += ((union ib_flow_spec *)ib_flow)->size; 1083 } 1084 1085 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { 1086 memcpy(&dest_arr[0], dst, sizeof(*dst)); 1087 dest_num++; 1088 } 1089 1090 if (!flow_is_multicast_only(flow_attr)) 1091 set_underlay_qp(dev, spec, underlay_qpn); 1092 1093 if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) { 1094 struct mlx5_eswitch_rep *rep; 1095 1096 rep = dev->port[flow_attr->port - 1].rep; 1097 if (!rep) { 1098 err = -EINVAL; 1099 goto free; 1100 } 1101 1102 mlx5_ib_set_rule_source_port(dev, spec, rep); 1103 } 1104 1105 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); 1106 1107 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 1108 struct mlx5_ib_mcounters *mcounters; 1109 1110 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); 1111 if (err) 1112 goto free; 1113 1114 mcounters = to_mcounters(flow_act.counters); 1115 handler->ibcounters = flow_act.counters; 1116 dest_arr[dest_num].type = 1117 MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1118 dest_arr[dest_num].counter_id = 1119 mlx5_fc_id(mcounters->hw_cntrs_hndl); 1120 dest_num++; 1121 } 1122 1123 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { 1124 if (!dest_num) 1125 rule_dst = NULL; 1126 } else { 1127 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) 1128 flow_act.action |= 1129 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1130 if (is_egress) 1131 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1132 else if (dest_num) 1133 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1134 } 1135 1136 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && 1137 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1138 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1139 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", 1140 spec->flow_context.flow_tag, flow_attr->type); 1141 err = -EINVAL; 1142 goto free; 1143 } 1144 handler->rule = mlx5_add_flow_rules(ft, spec, 1145 &flow_act, 1146 rule_dst, dest_num); 1147 1148 if (IS_ERR(handler->rule)) { 1149 err = PTR_ERR(handler->rule); 1150 goto free; 1151 } 1152 1153 ft_prio->refcount++; 1154 handler->prio = ft_prio; 1155 handler->dev = dev; 1156 1157 ft_prio->flow_table = ft; 1158 free: 1159 if (err && handler) { 1160 mlx5_ib_counters_clear_description(handler->ibcounters); 1161 kfree(handler); 1162 } 1163 kvfree(spec); 1164 return err ? ERR_PTR(err) : handler; 1165 } 1166 1167 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, 1168 struct mlx5_ib_flow_prio *ft_prio, 1169 const struct ib_flow_attr *flow_attr, 1170 struct mlx5_flow_destination *dst) 1171 { 1172 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); 1173 } 1174 1175 enum { 1176 LEFTOVERS_MC, 1177 LEFTOVERS_UC, 1178 }; 1179 1180 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, 1181 struct mlx5_ib_flow_prio *ft_prio, 1182 struct ib_flow_attr *flow_attr, 1183 struct mlx5_flow_destination *dst) 1184 { 1185 struct mlx5_ib_flow_handler *handler_ucast = NULL; 1186 struct mlx5_ib_flow_handler *handler = NULL; 1187 1188 static struct { 1189 struct ib_flow_attr flow_attr; 1190 struct ib_flow_spec_eth eth_flow; 1191 } leftovers_specs[] = { 1192 [LEFTOVERS_MC] = { 1193 .flow_attr = { 1194 .num_of_specs = 1, 1195 .size = sizeof(leftovers_specs[0]) 1196 }, 1197 .eth_flow = { 1198 .type = IB_FLOW_SPEC_ETH, 1199 .size = sizeof(struct ib_flow_spec_eth), 1200 .mask = {.dst_mac = {0x1} }, 1201 .val = {.dst_mac = {0x1} } 1202 } 1203 }, 1204 [LEFTOVERS_UC] = { 1205 .flow_attr = { 1206 .num_of_specs = 1, 1207 .size = sizeof(leftovers_specs[0]) 1208 }, 1209 .eth_flow = { 1210 .type = IB_FLOW_SPEC_ETH, 1211 .size = sizeof(struct ib_flow_spec_eth), 1212 .mask = {.dst_mac = {0x1} }, 1213 .val = {.dst_mac = {} } 1214 } 1215 } 1216 }; 1217 1218 handler = create_flow_rule(dev, ft_prio, 1219 &leftovers_specs[LEFTOVERS_MC].flow_attr, 1220 dst); 1221 if (!IS_ERR(handler) && 1222 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { 1223 handler_ucast = create_flow_rule(dev, ft_prio, 1224 &leftovers_specs[LEFTOVERS_UC].flow_attr, 1225 dst); 1226 if (IS_ERR(handler_ucast)) { 1227 mlx5_del_flow_rules(handler->rule); 1228 ft_prio->refcount--; 1229 kfree(handler); 1230 handler = handler_ucast; 1231 } else { 1232 list_add(&handler_ucast->list, &handler->list); 1233 } 1234 } 1235 1236 return handler; 1237 } 1238 1239 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, 1240 struct mlx5_ib_flow_prio *ft_rx, 1241 struct mlx5_ib_flow_prio *ft_tx, 1242 struct mlx5_flow_destination *dst) 1243 { 1244 struct mlx5_ib_flow_handler *handler_rx; 1245 struct mlx5_ib_flow_handler *handler_tx; 1246 int err; 1247 static const struct ib_flow_attr flow_attr = { 1248 .num_of_specs = 0, 1249 .type = IB_FLOW_ATTR_SNIFFER, 1250 .size = sizeof(flow_attr) 1251 }; 1252 1253 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); 1254 if (IS_ERR(handler_rx)) { 1255 err = PTR_ERR(handler_rx); 1256 goto err; 1257 } 1258 1259 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); 1260 if (IS_ERR(handler_tx)) { 1261 err = PTR_ERR(handler_tx); 1262 goto err_tx; 1263 } 1264 1265 list_add(&handler_tx->list, &handler_rx->list); 1266 1267 return handler_rx; 1268 1269 err_tx: 1270 mlx5_del_flow_rules(handler_rx->rule); 1271 ft_rx->refcount--; 1272 kfree(handler_rx); 1273 err: 1274 return ERR_PTR(err); 1275 } 1276 1277 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, 1278 struct ib_flow_attr *flow_attr, 1279 struct ib_udata *udata) 1280 { 1281 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1282 struct mlx5_ib_qp *mqp = to_mqp(qp); 1283 struct mlx5_ib_flow_handler *handler = NULL; 1284 struct mlx5_flow_destination *dst = NULL; 1285 struct mlx5_ib_flow_prio *ft_prio_tx = NULL; 1286 struct mlx5_ib_flow_prio *ft_prio; 1287 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1288 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; 1289 size_t min_ucmd_sz, required_ucmd_sz; 1290 int err; 1291 int underlay_qpn; 1292 1293 if (udata && udata->inlen) { 1294 min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved); 1295 if (udata->inlen < min_ucmd_sz) 1296 return ERR_PTR(-EOPNOTSUPP); 1297 1298 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); 1299 if (err) 1300 return ERR_PTR(err); 1301 1302 /* currently supports only one counters data */ 1303 if (ucmd_hdr.ncounters_data > 1) 1304 return ERR_PTR(-EINVAL); 1305 1306 required_ucmd_sz = min_ucmd_sz + 1307 sizeof(struct mlx5_ib_flow_counters_data) * 1308 ucmd_hdr.ncounters_data; 1309 if (udata->inlen > required_ucmd_sz && 1310 !ib_is_udata_cleared(udata, required_ucmd_sz, 1311 udata->inlen - required_ucmd_sz)) 1312 return ERR_PTR(-EOPNOTSUPP); 1313 1314 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); 1315 if (!ucmd) 1316 return ERR_PTR(-ENOMEM); 1317 1318 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); 1319 if (err) 1320 goto free_ucmd; 1321 } 1322 1323 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { 1324 err = -ENOMEM; 1325 goto free_ucmd; 1326 } 1327 1328 if (flow_attr->flags & 1329 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) { 1330 err = -EINVAL; 1331 goto free_ucmd; 1332 } 1333 1334 if (is_egress && 1335 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1336 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1337 err = -EINVAL; 1338 goto free_ucmd; 1339 } 1340 1341 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1342 if (!dst) { 1343 err = -ENOMEM; 1344 goto free_ucmd; 1345 } 1346 1347 mutex_lock(&dev->flow_db->lock); 1348 1349 ft_prio = get_flow_table(dev, flow_attr, 1350 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); 1351 if (IS_ERR(ft_prio)) { 1352 err = PTR_ERR(ft_prio); 1353 goto unlock; 1354 } 1355 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 1356 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); 1357 if (IS_ERR(ft_prio_tx)) { 1358 err = PTR_ERR(ft_prio_tx); 1359 ft_prio_tx = NULL; 1360 goto destroy_ft; 1361 } 1362 } 1363 1364 if (is_egress) { 1365 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1366 } else { 1367 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1368 if (mqp->is_rss) 1369 dst->tir_num = mqp->rss_qp.tirn; 1370 else 1371 dst->tir_num = mqp->raw_packet_qp.rq.tirn; 1372 } 1373 1374 switch (flow_attr->type) { 1375 case IB_FLOW_ATTR_NORMAL: 1376 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? 1377 mqp->underlay_qpn : 1378 0; 1379 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, 1380 underlay_qpn, ucmd); 1381 break; 1382 case IB_FLOW_ATTR_ALL_DEFAULT: 1383 case IB_FLOW_ATTR_MC_DEFAULT: 1384 handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst); 1385 break; 1386 case IB_FLOW_ATTR_SNIFFER: 1387 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); 1388 break; 1389 default: 1390 err = -EINVAL; 1391 goto destroy_ft; 1392 } 1393 1394 if (IS_ERR(handler)) { 1395 err = PTR_ERR(handler); 1396 handler = NULL; 1397 goto destroy_ft; 1398 } 1399 1400 mutex_unlock(&dev->flow_db->lock); 1401 kfree(dst); 1402 kfree(ucmd); 1403 1404 return &handler->ibflow; 1405 1406 destroy_ft: 1407 put_flow_table(dev, ft_prio, false); 1408 if (ft_prio_tx) 1409 put_flow_table(dev, ft_prio_tx, false); 1410 unlock: 1411 mutex_unlock(&dev->flow_db->lock); 1412 kfree(dst); 1413 free_ucmd: 1414 kfree(ucmd); 1415 return ERR_PTR(err); 1416 } 1417 1418 static struct mlx5_ib_flow_prio * 1419 _get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority, 1420 enum mlx5_flow_namespace_type ns_type, 1421 bool mcast) 1422 { 1423 struct mlx5_flow_namespace *ns = NULL; 1424 struct mlx5_ib_flow_prio *prio = NULL; 1425 int max_table_size = 0; 1426 bool esw_encap; 1427 u32 flags = 0; 1428 int priority; 1429 1430 if (mcast) 1431 priority = MLX5_IB_FLOW_MCAST_PRIO; 1432 else 1433 priority = ib_prio_to_core_prio(user_priority, false); 1434 1435 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 1436 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 1437 switch (ns_type) { 1438 case MLX5_FLOW_NAMESPACE_BYPASS: 1439 max_table_size = BIT( 1440 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); 1441 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) 1442 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1443 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1444 reformat_l3_tunnel_to_l2) && 1445 !esw_encap) 1446 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1447 break; 1448 case MLX5_FLOW_NAMESPACE_EGRESS: 1449 max_table_size = BIT( 1450 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); 1451 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && 1452 !esw_encap) 1453 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1454 break; 1455 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 1456 max_table_size = BIT( 1457 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); 1458 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) 1459 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1460 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, 1461 reformat_l3_tunnel_to_l2) && 1462 esw_encap) 1463 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1464 priority = user_priority; 1465 break; 1466 case MLX5_FLOW_NAMESPACE_RDMA_RX: 1467 max_table_size = BIT( 1468 MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size)); 1469 priority = user_priority; 1470 break; 1471 case MLX5_FLOW_NAMESPACE_RDMA_TX: 1472 max_table_size = BIT( 1473 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size)); 1474 priority = user_priority; 1475 break; 1476 default: 1477 break; 1478 } 1479 1480 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); 1481 1482 ns = mlx5_get_flow_namespace(dev->mdev, ns_type); 1483 if (!ns) 1484 return ERR_PTR(-EOPNOTSUPP); 1485 1486 switch (ns_type) { 1487 case MLX5_FLOW_NAMESPACE_BYPASS: 1488 prio = &dev->flow_db->prios[priority]; 1489 break; 1490 case MLX5_FLOW_NAMESPACE_EGRESS: 1491 prio = &dev->flow_db->egress_prios[priority]; 1492 break; 1493 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 1494 prio = &dev->flow_db->fdb[priority]; 1495 break; 1496 case MLX5_FLOW_NAMESPACE_RDMA_RX: 1497 prio = &dev->flow_db->rdma_rx[priority]; 1498 break; 1499 case MLX5_FLOW_NAMESPACE_RDMA_TX: 1500 prio = &dev->flow_db->rdma_tx[priority]; 1501 break; 1502 default: return ERR_PTR(-EINVAL); 1503 } 1504 1505 if (!prio) 1506 return ERR_PTR(-EINVAL); 1507 1508 if (prio->flow_table) 1509 return prio; 1510 1511 return _get_prio(dev, ns, prio, priority, max_table_size, 1512 MLX5_FS_MAX_TYPES, flags); 1513 } 1514 1515 static struct mlx5_ib_flow_handler * 1516 _create_raw_flow_rule(struct mlx5_ib_dev *dev, 1517 struct mlx5_ib_flow_prio *ft_prio, 1518 struct mlx5_flow_destination *dst, 1519 struct mlx5_ib_flow_matcher *fs_matcher, 1520 struct mlx5_flow_context *flow_context, 1521 struct mlx5_flow_act *flow_act, 1522 void *cmd_in, int inlen, 1523 int dst_num) 1524 { 1525 struct mlx5_ib_flow_handler *handler; 1526 struct mlx5_flow_spec *spec; 1527 struct mlx5_flow_table *ft = ft_prio->flow_table; 1528 int err = 0; 1529 1530 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1531 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 1532 if (!handler || !spec) { 1533 err = -ENOMEM; 1534 goto free; 1535 } 1536 1537 INIT_LIST_HEAD(&handler->list); 1538 1539 memcpy(spec->match_value, cmd_in, inlen); 1540 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, 1541 fs_matcher->mask_len); 1542 spec->match_criteria_enable = fs_matcher->match_criteria_enable; 1543 spec->flow_context = *flow_context; 1544 1545 handler->rule = mlx5_add_flow_rules(ft, spec, 1546 flow_act, dst, dst_num); 1547 1548 if (IS_ERR(handler->rule)) { 1549 err = PTR_ERR(handler->rule); 1550 goto free; 1551 } 1552 1553 ft_prio->refcount++; 1554 handler->prio = ft_prio; 1555 handler->dev = dev; 1556 ft_prio->flow_table = ft; 1557 1558 free: 1559 if (err) 1560 kfree(handler); 1561 kvfree(spec); 1562 return err ? ERR_PTR(err) : handler; 1563 } 1564 1565 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, 1566 void *match_v) 1567 { 1568 void *match_c; 1569 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; 1570 void *dmac, *dmac_mask; 1571 void *ipv4, *ipv4_mask; 1572 1573 if (!(fs_matcher->match_criteria_enable & 1574 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) 1575 return false; 1576 1577 match_c = fs_matcher->matcher_mask.match_params; 1578 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, 1579 outer_headers); 1580 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, 1581 outer_headers); 1582 1583 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 1584 dmac_47_16); 1585 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 1586 dmac_47_16); 1587 1588 if (is_multicast_ether_addr(dmac) && 1589 is_multicast_ether_addr(dmac_mask)) 1590 return true; 1591 1592 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 1593 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1594 1595 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 1596 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1597 1598 if (ipv4_is_multicast(*(__be32 *)(ipv4)) && 1599 ipv4_is_multicast(*(__be32 *)(ipv4_mask))) 1600 return true; 1601 1602 return false; 1603 } 1604 1605 static struct mlx5_ib_flow_handler *raw_fs_rule_add( 1606 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, 1607 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, 1608 u32 counter_id, void *cmd_in, int inlen, int dest_id, int dest_type) 1609 { 1610 struct mlx5_flow_destination *dst; 1611 struct mlx5_ib_flow_prio *ft_prio; 1612 struct mlx5_ib_flow_handler *handler; 1613 int dst_num = 0; 1614 bool mcast; 1615 int err; 1616 1617 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) 1618 return ERR_PTR(-EOPNOTSUPP); 1619 1620 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) 1621 return ERR_PTR(-ENOMEM); 1622 1623 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); 1624 if (!dst) 1625 return ERR_PTR(-ENOMEM); 1626 1627 mcast = raw_fs_is_multicast(fs_matcher, cmd_in); 1628 mutex_lock(&dev->flow_db->lock); 1629 1630 ft_prio = _get_flow_table(dev, fs_matcher->priority, 1631 fs_matcher->ns_type, mcast); 1632 if (IS_ERR(ft_prio)) { 1633 err = PTR_ERR(ft_prio); 1634 goto unlock; 1635 } 1636 1637 switch (dest_type) { 1638 case MLX5_FLOW_DESTINATION_TYPE_TIR: 1639 dst[dst_num].type = dest_type; 1640 dst[dst_num++].tir_num = dest_id; 1641 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1642 break; 1643 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE: 1644 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; 1645 dst[dst_num++].ft_num = dest_id; 1646 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1647 break; 1648 case MLX5_FLOW_DESTINATION_TYPE_PORT: 1649 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1650 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1651 break; 1652 default: 1653 break; 1654 } 1655 1656 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 1657 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1658 dst[dst_num].counter_id = counter_id; 1659 dst_num++; 1660 } 1661 1662 handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL, 1663 fs_matcher, flow_context, flow_act, 1664 cmd_in, inlen, dst_num); 1665 1666 if (IS_ERR(handler)) { 1667 err = PTR_ERR(handler); 1668 goto destroy_ft; 1669 } 1670 1671 mutex_unlock(&dev->flow_db->lock); 1672 atomic_inc(&fs_matcher->usecnt); 1673 handler->flow_matcher = fs_matcher; 1674 1675 kfree(dst); 1676 1677 return handler; 1678 1679 destroy_ft: 1680 put_flow_table(dev, ft_prio, false); 1681 unlock: 1682 mutex_unlock(&dev->flow_db->lock); 1683 kfree(dst); 1684 1685 return ERR_PTR(err); 1686 } 1687 1688 static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) 1689 { 1690 switch (maction->flow_action_raw.sub_type) { 1691 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: 1692 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, 1693 maction->flow_action_raw.modify_hdr); 1694 break; 1695 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: 1696 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, 1697 maction->flow_action_raw.pkt_reformat); 1698 break; 1699 case MLX5_IB_FLOW_ACTION_DECAP: 1700 break; 1701 default: 1702 break; 1703 } 1704 } 1705 1706 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) 1707 { 1708 struct mlx5_ib_flow_action *maction = to_mflow_act(action); 1709 1710 switch (action->type) { 1711 case IB_FLOW_ACTION_UNSPECIFIED: 1712 destroy_flow_action_raw(maction); 1713 break; 1714 default: 1715 WARN_ON(true); 1716 break; 1717 } 1718 1719 kfree(maction); 1720 return 0; 1721 } 1722 1723 static int 1724 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, 1725 enum mlx5_flow_namespace_type *namespace) 1726 { 1727 switch (table_type) { 1728 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: 1729 *namespace = MLX5_FLOW_NAMESPACE_BYPASS; 1730 break; 1731 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: 1732 *namespace = MLX5_FLOW_NAMESPACE_EGRESS; 1733 break; 1734 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: 1735 *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS; 1736 break; 1737 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: 1738 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; 1739 break; 1740 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: 1741 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; 1742 break; 1743 default: 1744 return -EINVAL; 1745 } 1746 1747 return 0; 1748 } 1749 1750 static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { 1751 [MLX5_IB_FLOW_TYPE_NORMAL] = { 1752 .type = UVERBS_ATTR_TYPE_PTR_IN, 1753 .u.ptr = { 1754 .len = sizeof(u16), /* data is priority */ 1755 .min_len = sizeof(u16), 1756 } 1757 }, 1758 [MLX5_IB_FLOW_TYPE_SNIFFER] = { 1759 .type = UVERBS_ATTR_TYPE_PTR_IN, 1760 UVERBS_ATTR_NO_DATA(), 1761 }, 1762 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { 1763 .type = UVERBS_ATTR_TYPE_PTR_IN, 1764 UVERBS_ATTR_NO_DATA(), 1765 }, 1766 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { 1767 .type = UVERBS_ATTR_TYPE_PTR_IN, 1768 UVERBS_ATTR_NO_DATA(), 1769 }, 1770 }; 1771 1772 static bool is_flow_dest(void *obj, int *dest_id, int *dest_type) 1773 { 1774 struct devx_obj *devx_obj = obj; 1775 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 1776 1777 switch (opcode) { 1778 case MLX5_CMD_OP_DESTROY_TIR: 1779 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1780 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 1781 obj_id); 1782 return true; 1783 1784 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 1785 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1786 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 1787 table_id); 1788 return true; 1789 default: 1790 return false; 1791 } 1792 } 1793 1794 static int get_dests(struct uverbs_attr_bundle *attrs, 1795 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, 1796 int *dest_type, struct ib_qp **qp, u32 *flags) 1797 { 1798 bool dest_devx, dest_qp; 1799 void *devx_obj; 1800 int err; 1801 1802 dest_devx = uverbs_attr_is_valid(attrs, 1803 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 1804 dest_qp = uverbs_attr_is_valid(attrs, 1805 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 1806 1807 *flags = 0; 1808 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 1809 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | 1810 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); 1811 if (err) 1812 return err; 1813 1814 /* Both flags are not allowed */ 1815 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && 1816 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 1817 return -EINVAL; 1818 1819 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 1820 if (dest_devx && (dest_qp || *flags)) 1821 return -EINVAL; 1822 else if (dest_qp && *flags) 1823 return -EINVAL; 1824 } 1825 1826 /* Allow only DEVX object, drop as dest for FDB */ 1827 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 1828 !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) 1829 return -EINVAL; 1830 1831 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ 1832 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && 1833 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) 1834 return -EINVAL; 1835 1836 *qp = NULL; 1837 if (dest_devx) { 1838 devx_obj = 1839 uverbs_attr_get_obj(attrs, 1840 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 1841 1842 /* Verify that the given DEVX object is a flow 1843 * steering destination. 1844 */ 1845 if (!is_flow_dest(devx_obj, dest_id, dest_type)) 1846 return -EINVAL; 1847 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ 1848 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS || 1849 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && 1850 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) 1851 return -EINVAL; 1852 } else if (dest_qp) { 1853 struct mlx5_ib_qp *mqp; 1854 1855 *qp = uverbs_attr_get_obj(attrs, 1856 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 1857 if (IS_ERR(*qp)) 1858 return PTR_ERR(*qp); 1859 1860 if ((*qp)->qp_type != IB_QPT_RAW_PACKET) 1861 return -EINVAL; 1862 1863 mqp = to_mqp(*qp); 1864 if (mqp->is_rss) 1865 *dest_id = mqp->rss_qp.tirn; 1866 else 1867 *dest_id = mqp->raw_packet_qp.rq.tirn; 1868 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1869 } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 1870 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) && 1871 !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) { 1872 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1873 } 1874 1875 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && 1876 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 1877 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX)) 1878 return -EINVAL; 1879 1880 return 0; 1881 } 1882 1883 static bool is_flow_counter(void *obj, u32 offset, u32 *counter_id) 1884 { 1885 struct devx_obj *devx_obj = obj; 1886 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 1887 1888 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 1889 1890 if (offset && offset >= devx_obj->flow_counter_bulk_size) 1891 return false; 1892 1893 *counter_id = MLX5_GET(dealloc_flow_counter_in, 1894 devx_obj->dinbox, 1895 flow_counter_id); 1896 *counter_id += offset; 1897 return true; 1898 } 1899 1900 return false; 1901 } 1902 1903 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 1904 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( 1905 struct uverbs_attr_bundle *attrs) 1906 { 1907 struct mlx5_flow_context flow_context = {.flow_tag = 1908 MLX5_FS_DEFAULT_FLOW_TAG}; 1909 u32 *offset_attr, offset = 0, counter_id = 0; 1910 int dest_id, dest_type = -1, inlen, len, ret, i; 1911 struct mlx5_ib_flow_handler *flow_handler; 1912 struct mlx5_ib_flow_matcher *fs_matcher; 1913 struct ib_uobject **arr_flow_actions; 1914 struct ib_uflow_resources *uflow_res; 1915 struct mlx5_flow_act flow_act = {}; 1916 struct ib_qp *qp = NULL; 1917 void *devx_obj, *cmd_in; 1918 struct ib_uobject *uobj; 1919 struct mlx5_ib_dev *dev; 1920 u32 flags; 1921 1922 if (!capable(CAP_NET_RAW)) 1923 return -EPERM; 1924 1925 fs_matcher = uverbs_attr_get_obj(attrs, 1926 MLX5_IB_ATTR_CREATE_FLOW_MATCHER); 1927 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); 1928 dev = mlx5_udata_to_mdev(&attrs->driver_udata); 1929 1930 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) 1931 return -EINVAL; 1932 1933 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) 1934 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; 1935 1936 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 1937 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 1938 1939 len = uverbs_attr_get_uobjs_arr(attrs, 1940 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); 1941 if (len) { 1942 devx_obj = arr_flow_actions[0]->object; 1943 1944 if (uverbs_attr_is_valid(attrs, 1945 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { 1946 1947 int num_offsets = uverbs_attr_ptr_get_array_size( 1948 attrs, 1949 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 1950 sizeof(u32)); 1951 1952 if (num_offsets != 1) 1953 return -EINVAL; 1954 1955 offset_attr = uverbs_attr_get_alloced_ptr( 1956 attrs, 1957 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); 1958 offset = *offset_attr; 1959 } 1960 1961 if (!is_flow_counter(devx_obj, offset, &counter_id)) 1962 return -EINVAL; 1963 1964 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 1965 } 1966 1967 cmd_in = uverbs_attr_get_alloced_ptr( 1968 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 1969 inlen = uverbs_attr_get_len(attrs, 1970 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 1971 1972 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); 1973 if (!uflow_res) 1974 return -ENOMEM; 1975 1976 len = uverbs_attr_get_uobjs_arr(attrs, 1977 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); 1978 for (i = 0; i < len; i++) { 1979 struct mlx5_ib_flow_action *maction = 1980 to_mflow_act(arr_flow_actions[i]->object); 1981 1982 ret = parse_flow_flow_action(maction, false, &flow_act); 1983 if (ret) 1984 goto err_out; 1985 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, 1986 arr_flow_actions[i]->object); 1987 } 1988 1989 ret = uverbs_copy_from(&flow_context.flow_tag, attrs, 1990 MLX5_IB_ATTR_CREATE_FLOW_TAG); 1991 if (!ret) { 1992 if (flow_context.flow_tag >= BIT(24)) { 1993 ret = -EINVAL; 1994 goto err_out; 1995 } 1996 flow_context.flags |= FLOW_CONTEXT_HAS_TAG; 1997 } 1998 1999 flow_handler = 2000 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act, 2001 counter_id, cmd_in, inlen, dest_id, dest_type); 2002 if (IS_ERR(flow_handler)) { 2003 ret = PTR_ERR(flow_handler); 2004 goto err_out; 2005 } 2006 2007 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); 2008 2009 return 0; 2010 err_out: 2011 ib_uverbs_flow_resources_free(uflow_res); 2012 return ret; 2013 } 2014 2015 static int flow_matcher_cleanup(struct ib_uobject *uobject, 2016 enum rdma_remove_reason why, 2017 struct uverbs_attr_bundle *attrs) 2018 { 2019 struct mlx5_ib_flow_matcher *obj = uobject->object; 2020 2021 if (atomic_read(&obj->usecnt)) 2022 return -EBUSY; 2023 2024 kfree(obj); 2025 return 0; 2026 } 2027 2028 static int steering_anchor_cleanup(struct ib_uobject *uobject, 2029 enum rdma_remove_reason why, 2030 struct uverbs_attr_bundle *attrs) 2031 { 2032 struct mlx5_ib_steering_anchor *obj = uobject->object; 2033 2034 if (atomic_read(&obj->usecnt)) 2035 return -EBUSY; 2036 2037 mutex_lock(&obj->dev->flow_db->lock); 2038 put_flow_table(obj->dev, obj->ft_prio, true); 2039 mutex_unlock(&obj->dev->flow_db->lock); 2040 2041 kfree(obj); 2042 return 0; 2043 } 2044 2045 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, 2046 struct mlx5_ib_flow_matcher *obj) 2047 { 2048 enum mlx5_ib_uapi_flow_table_type ft_type = 2049 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; 2050 u32 flags; 2051 int err; 2052 2053 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older 2054 * users should switch to it. We leave this to not break userspace 2055 */ 2056 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && 2057 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) 2058 return -EINVAL; 2059 2060 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { 2061 err = uverbs_get_const(&ft_type, attrs, 2062 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); 2063 if (err) 2064 return err; 2065 2066 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); 2067 if (err) 2068 return err; 2069 2070 return 0; 2071 } 2072 2073 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { 2074 err = uverbs_get_flags32(&flags, attrs, 2075 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2076 IB_FLOW_ATTR_FLAGS_EGRESS); 2077 if (err) 2078 return err; 2079 2080 if (flags) 2081 return mlx5_ib_ft_type_to_namespace( 2082 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, 2083 &obj->ns_type); 2084 } 2085 2086 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; 2087 2088 return 0; 2089 } 2090 2091 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( 2092 struct uverbs_attr_bundle *attrs) 2093 { 2094 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2095 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); 2096 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2097 struct mlx5_ib_flow_matcher *obj; 2098 int err; 2099 2100 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); 2101 if (!obj) 2102 return -ENOMEM; 2103 2104 obj->mask_len = uverbs_attr_get_len( 2105 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2106 err = uverbs_copy_from(&obj->matcher_mask, 2107 attrs, 2108 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2109 if (err) 2110 goto end; 2111 2112 obj->flow_type = uverbs_attr_get_enum_id( 2113 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2114 2115 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { 2116 err = uverbs_copy_from(&obj->priority, 2117 attrs, 2118 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2119 if (err) 2120 goto end; 2121 } 2122 2123 err = uverbs_copy_from(&obj->match_criteria_enable, 2124 attrs, 2125 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); 2126 if (err) 2127 goto end; 2128 2129 err = mlx5_ib_matcher_ns(attrs, obj); 2130 if (err) 2131 goto end; 2132 2133 if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 2134 mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) { 2135 err = -EINVAL; 2136 goto end; 2137 } 2138 2139 uobj->object = obj; 2140 obj->mdev = dev->mdev; 2141 atomic_set(&obj->usecnt, 0); 2142 return 0; 2143 2144 end: 2145 kfree(obj); 2146 return err; 2147 } 2148 2149 static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)( 2150 struct uverbs_attr_bundle *attrs) 2151 { 2152 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2153 attrs, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE); 2154 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2155 enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type; 2156 enum mlx5_flow_namespace_type ns_type; 2157 struct mlx5_ib_steering_anchor *obj; 2158 struct mlx5_ib_flow_prio *ft_prio; 2159 u16 priority; 2160 u32 ft_id; 2161 int err; 2162 2163 if (!capable(CAP_NET_RAW)) 2164 return -EPERM; 2165 2166 err = uverbs_get_const(&ib_uapi_ft_type, attrs, 2167 MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE); 2168 if (err) 2169 return err; 2170 2171 err = mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type, &ns_type); 2172 if (err) 2173 return err; 2174 2175 err = uverbs_copy_from(&priority, attrs, 2176 MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY); 2177 if (err) 2178 return err; 2179 2180 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 2181 if (!obj) 2182 return -ENOMEM; 2183 2184 mutex_lock(&dev->flow_db->lock); 2185 ft_prio = _get_flow_table(dev, priority, ns_type, 0); 2186 if (IS_ERR(ft_prio)) { 2187 mutex_unlock(&dev->flow_db->lock); 2188 err = PTR_ERR(ft_prio); 2189 goto free_obj; 2190 } 2191 2192 ft_prio->refcount++; 2193 ft_id = mlx5_flow_table_id(ft_prio->flow_table); 2194 mutex_unlock(&dev->flow_db->lock); 2195 2196 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID, 2197 &ft_id, sizeof(ft_id)); 2198 if (err) 2199 goto put_flow_table; 2200 2201 uobj->object = obj; 2202 obj->dev = dev; 2203 obj->ft_prio = ft_prio; 2204 atomic_set(&obj->usecnt, 0); 2205 2206 return 0; 2207 2208 put_flow_table: 2209 mutex_lock(&dev->flow_db->lock); 2210 put_flow_table(dev, ft_prio, true); 2211 mutex_unlock(&dev->flow_db->lock); 2212 free_obj: 2213 kfree(obj); 2214 2215 return err; 2216 } 2217 2218 static struct ib_flow_action * 2219 mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, 2220 enum mlx5_ib_uapi_flow_table_type ft_type, 2221 u8 num_actions, void *in) 2222 { 2223 enum mlx5_flow_namespace_type namespace; 2224 struct mlx5_ib_flow_action *maction; 2225 int ret; 2226 2227 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 2228 if (ret) 2229 return ERR_PTR(-EINVAL); 2230 2231 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 2232 if (!maction) 2233 return ERR_PTR(-ENOMEM); 2234 2235 maction->flow_action_raw.modify_hdr = 2236 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); 2237 2238 if (IS_ERR(maction->flow_action_raw.modify_hdr)) { 2239 ret = PTR_ERR(maction->flow_action_raw.modify_hdr); 2240 kfree(maction); 2241 return ERR_PTR(ret); 2242 } 2243 maction->flow_action_raw.sub_type = 2244 MLX5_IB_FLOW_ACTION_MODIFY_HEADER; 2245 maction->flow_action_raw.dev = dev; 2246 2247 return &maction->ib_action; 2248 } 2249 2250 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) 2251 { 2252 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 2253 max_modify_header_actions) || 2254 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 2255 max_modify_header_actions) || 2256 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 2257 max_modify_header_actions); 2258 } 2259 2260 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( 2261 struct uverbs_attr_bundle *attrs) 2262 { 2263 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2264 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); 2265 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 2266 enum mlx5_ib_uapi_flow_table_type ft_type; 2267 struct ib_flow_action *action; 2268 int num_actions; 2269 void *in; 2270 int ret; 2271 2272 if (!mlx5_ib_modify_header_supported(mdev)) 2273 return -EOPNOTSUPP; 2274 2275 in = uverbs_attr_get_alloced_ptr(attrs, 2276 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); 2277 2278 num_actions = uverbs_attr_ptr_get_array_size( 2279 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 2280 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); 2281 if (num_actions < 0) 2282 return num_actions; 2283 2284 ret = uverbs_get_const(&ft_type, attrs, 2285 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); 2286 if (ret) 2287 return ret; 2288 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); 2289 if (IS_ERR(action)) 2290 return PTR_ERR(action); 2291 2292 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, 2293 IB_FLOW_ACTION_UNSPECIFIED); 2294 2295 return 0; 2296 } 2297 2298 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, 2299 u8 packet_reformat_type, 2300 u8 ft_type) 2301 { 2302 switch (packet_reformat_type) { 2303 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 2304 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 2305 return MLX5_CAP_FLOWTABLE(ibdev->mdev, 2306 encap_general_header); 2307 break; 2308 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 2309 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 2310 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, 2311 reformat_l2_to_l3_tunnel); 2312 break; 2313 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 2314 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 2315 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, 2316 reformat_l3_tunnel_to_l2); 2317 break; 2318 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: 2319 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 2320 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); 2321 break; 2322 default: 2323 break; 2324 } 2325 2326 return false; 2327 } 2328 2329 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) 2330 { 2331 switch (dv_prt) { 2332 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 2333 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; 2334 break; 2335 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 2336 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 2337 break; 2338 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 2339 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 2340 break; 2341 default: 2342 return -EINVAL; 2343 } 2344 2345 return 0; 2346 } 2347 2348 static int mlx5_ib_flow_action_create_packet_reformat_ctx( 2349 struct mlx5_ib_dev *dev, 2350 struct mlx5_ib_flow_action *maction, 2351 u8 ft_type, u8 dv_prt, 2352 void *in, size_t len) 2353 { 2354 struct mlx5_pkt_reformat_params reformat_params; 2355 enum mlx5_flow_namespace_type namespace; 2356 u8 prm_prt; 2357 int ret; 2358 2359 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 2360 if (ret) 2361 return ret; 2362 2363 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); 2364 if (ret) 2365 return ret; 2366 2367 memset(&reformat_params, 0, sizeof(reformat_params)); 2368 reformat_params.type = prm_prt; 2369 reformat_params.size = len; 2370 reformat_params.data = in; 2371 maction->flow_action_raw.pkt_reformat = 2372 mlx5_packet_reformat_alloc(dev->mdev, &reformat_params, 2373 namespace); 2374 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { 2375 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); 2376 return ret; 2377 } 2378 2379 maction->flow_action_raw.sub_type = 2380 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; 2381 maction->flow_action_raw.dev = dev; 2382 2383 return 0; 2384 } 2385 2386 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( 2387 struct uverbs_attr_bundle *attrs) 2388 { 2389 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 2390 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); 2391 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 2392 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; 2393 enum mlx5_ib_uapi_flow_table_type ft_type; 2394 struct mlx5_ib_flow_action *maction; 2395 int ret; 2396 2397 ret = uverbs_get_const(&ft_type, attrs, 2398 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); 2399 if (ret) 2400 return ret; 2401 2402 ret = uverbs_get_const(&dv_prt, attrs, 2403 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); 2404 if (ret) 2405 return ret; 2406 2407 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) 2408 return -EOPNOTSUPP; 2409 2410 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 2411 if (!maction) 2412 return -ENOMEM; 2413 2414 if (dv_prt == 2415 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { 2416 maction->flow_action_raw.sub_type = 2417 MLX5_IB_FLOW_ACTION_DECAP; 2418 maction->flow_action_raw.dev = mdev; 2419 } else { 2420 void *in; 2421 int len; 2422 2423 in = uverbs_attr_get_alloced_ptr(attrs, 2424 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 2425 if (IS_ERR(in)) { 2426 ret = PTR_ERR(in); 2427 goto free_maction; 2428 } 2429 2430 len = uverbs_attr_get_len(attrs, 2431 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 2432 2433 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, 2434 maction, ft_type, dv_prt, in, len); 2435 if (ret) 2436 goto free_maction; 2437 } 2438 2439 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, 2440 IB_FLOW_ACTION_UNSPECIFIED); 2441 return 0; 2442 2443 free_maction: 2444 kfree(maction); 2445 return ret; 2446 } 2447 2448 DECLARE_UVERBS_NAMED_METHOD( 2449 MLX5_IB_METHOD_CREATE_FLOW, 2450 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 2451 UVERBS_OBJECT_FLOW, 2452 UVERBS_ACCESS_NEW, 2453 UA_MANDATORY), 2454 UVERBS_ATTR_PTR_IN( 2455 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, 2456 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 2457 UA_MANDATORY, 2458 UA_ALLOC_AND_COPY), 2459 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, 2460 MLX5_IB_OBJECT_FLOW_MATCHER, 2461 UVERBS_ACCESS_READ, 2462 UA_MANDATORY), 2463 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, 2464 UVERBS_OBJECT_QP, 2465 UVERBS_ACCESS_READ), 2466 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, 2467 MLX5_IB_OBJECT_DEVX_OBJ, 2468 UVERBS_ACCESS_READ), 2469 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, 2470 UVERBS_OBJECT_FLOW_ACTION, 2471 UVERBS_ACCESS_READ, 1, 2472 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, 2473 UA_OPTIONAL), 2474 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, 2475 UVERBS_ATTR_TYPE(u32), 2476 UA_OPTIONAL), 2477 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, 2478 MLX5_IB_OBJECT_DEVX_OBJ, 2479 UVERBS_ACCESS_READ, 1, 1, 2480 UA_OPTIONAL), 2481 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 2482 UVERBS_ATTR_MIN_SIZE(sizeof(u32)), 2483 UA_OPTIONAL, 2484 UA_ALLOC_AND_COPY), 2485 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 2486 enum mlx5_ib_create_flow_flags, 2487 UA_OPTIONAL)); 2488 2489 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 2490 MLX5_IB_METHOD_DESTROY_FLOW, 2491 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 2492 UVERBS_OBJECT_FLOW, 2493 UVERBS_ACCESS_DESTROY, 2494 UA_MANDATORY)); 2495 2496 ADD_UVERBS_METHODS(mlx5_ib_fs, 2497 UVERBS_OBJECT_FLOW, 2498 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), 2499 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); 2500 2501 DECLARE_UVERBS_NAMED_METHOD( 2502 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, 2503 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, 2504 UVERBS_OBJECT_FLOW_ACTION, 2505 UVERBS_ACCESS_NEW, 2506 UA_MANDATORY), 2507 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 2508 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( 2509 set_add_copy_action_in_auto)), 2510 UA_MANDATORY, 2511 UA_ALLOC_AND_COPY), 2512 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, 2513 enum mlx5_ib_uapi_flow_table_type, 2514 UA_MANDATORY)); 2515 2516 DECLARE_UVERBS_NAMED_METHOD( 2517 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, 2518 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, 2519 UVERBS_OBJECT_FLOW_ACTION, 2520 UVERBS_ACCESS_NEW, 2521 UA_MANDATORY), 2522 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, 2523 UVERBS_ATTR_MIN_SIZE(1), 2524 UA_ALLOC_AND_COPY, 2525 UA_OPTIONAL), 2526 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, 2527 enum mlx5_ib_uapi_flow_action_packet_reformat_type, 2528 UA_MANDATORY), 2529 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, 2530 enum mlx5_ib_uapi_flow_table_type, 2531 UA_MANDATORY)); 2532 2533 ADD_UVERBS_METHODS( 2534 mlx5_ib_flow_actions, 2535 UVERBS_OBJECT_FLOW_ACTION, 2536 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), 2537 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); 2538 2539 DECLARE_UVERBS_NAMED_METHOD( 2540 MLX5_IB_METHOD_FLOW_MATCHER_CREATE, 2541 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, 2542 MLX5_IB_OBJECT_FLOW_MATCHER, 2543 UVERBS_ACCESS_NEW, 2544 UA_MANDATORY), 2545 UVERBS_ATTR_PTR_IN( 2546 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, 2547 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 2548 UA_MANDATORY), 2549 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, 2550 mlx5_ib_flow_type, 2551 UA_MANDATORY), 2552 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, 2553 UVERBS_ATTR_TYPE(u8), 2554 UA_MANDATORY), 2555 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2556 enum ib_flow_flags, 2557 UA_OPTIONAL), 2558 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, 2559 enum mlx5_ib_uapi_flow_table_type, 2560 UA_OPTIONAL)); 2561 2562 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 2563 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, 2564 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, 2565 MLX5_IB_OBJECT_FLOW_MATCHER, 2566 UVERBS_ACCESS_DESTROY, 2567 UA_MANDATORY)); 2568 2569 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, 2570 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), 2571 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), 2572 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); 2573 2574 DECLARE_UVERBS_NAMED_METHOD( 2575 MLX5_IB_METHOD_STEERING_ANCHOR_CREATE, 2576 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE, 2577 MLX5_IB_OBJECT_STEERING_ANCHOR, 2578 UVERBS_ACCESS_NEW, 2579 UA_MANDATORY), 2580 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE, 2581 enum mlx5_ib_uapi_flow_table_type, 2582 UA_MANDATORY), 2583 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY, 2584 UVERBS_ATTR_TYPE(u16), 2585 UA_MANDATORY), 2586 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID, 2587 UVERBS_ATTR_TYPE(u32), 2588 UA_MANDATORY)); 2589 2590 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 2591 MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY, 2592 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE, 2593 MLX5_IB_OBJECT_STEERING_ANCHOR, 2594 UVERBS_ACCESS_DESTROY, 2595 UA_MANDATORY)); 2596 2597 DECLARE_UVERBS_NAMED_OBJECT( 2598 MLX5_IB_OBJECT_STEERING_ANCHOR, 2599 UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup), 2600 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE), 2601 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY)); 2602 2603 const struct uapi_definition mlx5_ib_flow_defs[] = { 2604 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 2605 MLX5_IB_OBJECT_FLOW_MATCHER), 2606 UAPI_DEF_CHAIN_OBJ_TREE( 2607 UVERBS_OBJECT_FLOW, 2608 &mlx5_ib_fs), 2609 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, 2610 &mlx5_ib_flow_actions), 2611 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 2612 MLX5_IB_OBJECT_STEERING_ANCHOR, 2613 UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)), 2614 {}, 2615 }; 2616 2617 static const struct ib_device_ops flow_ops = { 2618 .create_flow = mlx5_ib_create_flow, 2619 .destroy_flow = mlx5_ib_destroy_flow, 2620 .destroy_flow_action = mlx5_ib_destroy_flow_action, 2621 }; 2622 2623 int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) 2624 { 2625 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); 2626 2627 if (!dev->flow_db) 2628 return -ENOMEM; 2629 2630 mutex_init(&dev->flow_db->lock); 2631 2632 ib_set_device_ops(&dev->ib_dev, &flow_ops); 2633 return 0; 2634 } 2635