1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <rdma/ib_user_verbs.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/uverbs_types.h> 9 #include <rdma/uverbs_ioctl.h> 10 #include <rdma/uverbs_std_types.h> 11 #include <rdma/mlx5_user_ioctl_cmds.h> 12 #include <rdma/mlx5_user_ioctl_verbs.h> 13 #include <rdma/ib_hdrs.h> 14 #include <rdma/ib_umem.h> 15 #include <linux/mlx5/driver.h> 16 #include <linux/mlx5/fs.h> 17 #include <linux/mlx5/fs_helpers.h> 18 #include <linux/mlx5/eswitch.h> 19 #include <net/inet_ecn.h> 20 #include "mlx5_ib.h" 21 #include "counters.h" 22 #include "devx.h" 23 #include "fs.h" 24 25 #define UVERBS_MODULE_NAME mlx5_ib 26 #include <rdma/uverbs_named_ioctl.h> 27 28 enum { 29 MATCH_CRITERIA_ENABLE_OUTER_BIT, 30 MATCH_CRITERIA_ENABLE_MISC_BIT, 31 MATCH_CRITERIA_ENABLE_INNER_BIT, 32 MATCH_CRITERIA_ENABLE_MISC2_BIT 33 }; 34 35 #define HEADER_IS_ZERO(match_criteria, headers) \ 36 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 37 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 38 39 static u8 get_match_criteria_enable(u32 *match_criteria) 40 { 41 u8 match_criteria_enable; 42 43 match_criteria_enable = 44 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 45 MATCH_CRITERIA_ENABLE_OUTER_BIT; 46 match_criteria_enable |= 47 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 48 MATCH_CRITERIA_ENABLE_MISC_BIT; 49 match_criteria_enable |= 50 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 51 MATCH_CRITERIA_ENABLE_INNER_BIT; 52 match_criteria_enable |= 53 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 54 MATCH_CRITERIA_ENABLE_MISC2_BIT; 55 56 return match_criteria_enable; 57 } 58 59 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) 60 { 61 u8 entry_mask; 62 u8 entry_val; 63 int err = 0; 64 65 if (!mask) 66 goto out; 67 68 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, 69 ip_protocol); 70 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, 71 ip_protocol); 72 if (!entry_mask) { 73 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); 74 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); 75 goto out; 76 } 77 /* Don't override existing ip protocol */ 78 if (mask != entry_mask || val != entry_val) 79 err = -EINVAL; 80 out: 81 return err; 82 } 83 84 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, 85 bool inner) 86 { 87 if (inner) { 88 MLX5_SET(fte_match_set_misc, 89 misc_c, inner_ipv6_flow_label, mask); 90 MLX5_SET(fte_match_set_misc, 91 misc_v, inner_ipv6_flow_label, val); 92 } else { 93 MLX5_SET(fte_match_set_misc, 94 misc_c, outer_ipv6_flow_label, mask); 95 MLX5_SET(fte_match_set_misc, 96 misc_v, outer_ipv6_flow_label, val); 97 } 98 } 99 100 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) 101 { 102 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); 103 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); 104 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); 105 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); 106 } 107 108 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) 109 { 110 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && 111 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) 112 return -EOPNOTSUPP; 113 114 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && 115 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) 116 return -EOPNOTSUPP; 117 118 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && 119 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) 120 return -EOPNOTSUPP; 121 122 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && 123 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) 124 return -EOPNOTSUPP; 125 126 return 0; 127 } 128 129 #define LAST_ETH_FIELD vlan_tag 130 #define LAST_IB_FIELD sl 131 #define LAST_IPV4_FIELD tos 132 #define LAST_IPV6_FIELD traffic_class 133 #define LAST_TCP_UDP_FIELD src_port 134 #define LAST_TUNNEL_FIELD tunnel_id 135 #define LAST_FLOW_TAG_FIELD tag_id 136 #define LAST_DROP_FIELD size 137 #define LAST_COUNTERS_FIELD counters 138 139 /* Field is the last supported field */ 140 #define FIELDS_NOT_SUPPORTED(filter, field) \ 141 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \ 142 sizeof(filter) - offsetofend(typeof(filter), field)) 143 144 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, 145 bool is_egress, 146 struct mlx5_flow_act *action) 147 { 148 149 switch (maction->ib_action.type) { 150 case IB_FLOW_ACTION_UNSPECIFIED: 151 if (maction->flow_action_raw.sub_type == 152 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { 153 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 154 return -EINVAL; 155 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 156 action->modify_hdr = 157 maction->flow_action_raw.modify_hdr; 158 return 0; 159 } 160 if (maction->flow_action_raw.sub_type == 161 MLX5_IB_FLOW_ACTION_DECAP) { 162 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) 163 return -EINVAL; 164 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 165 return 0; 166 } 167 if (maction->flow_action_raw.sub_type == 168 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { 169 if (action->action & 170 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) 171 return -EINVAL; 172 action->action |= 173 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 174 action->pkt_reformat = 175 maction->flow_action_raw.pkt_reformat; 176 return 0; 177 } 178 fallthrough; 179 default: 180 return -EOPNOTSUPP; 181 } 182 } 183 184 static int parse_flow_attr(struct mlx5_core_dev *mdev, 185 struct mlx5_flow_spec *spec, 186 const union ib_flow_spec *ib_spec, 187 const struct ib_flow_attr *flow_attr, 188 struct mlx5_flow_act *action, u32 prev_type) 189 { 190 struct mlx5_flow_context *flow_context = &spec->flow_context; 191 u32 *match_c = spec->match_criteria; 192 u32 *match_v = spec->match_value; 193 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, 194 misc_parameters); 195 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, 196 misc_parameters); 197 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, 198 misc_parameters_2); 199 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, 200 misc_parameters_2); 201 void *headers_c; 202 void *headers_v; 203 int match_ipv; 204 int ret; 205 206 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 207 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 208 inner_headers); 209 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 210 inner_headers); 211 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 212 ft_field_support.inner_ip_version); 213 } else { 214 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 215 outer_headers); 216 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 217 outer_headers); 218 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 219 ft_field_support.outer_ip_version); 220 } 221 222 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 223 case IB_FLOW_SPEC_ETH: 224 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) 225 return -EOPNOTSUPP; 226 227 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 228 dmac_47_16), 229 ib_spec->eth.mask.dst_mac); 230 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 231 dmac_47_16), 232 ib_spec->eth.val.dst_mac); 233 234 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 235 smac_47_16), 236 ib_spec->eth.mask.src_mac); 237 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 238 smac_47_16), 239 ib_spec->eth.val.src_mac); 240 241 if (ib_spec->eth.mask.vlan_tag) { 242 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 243 cvlan_tag, 1); 244 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 245 cvlan_tag, 1); 246 247 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 248 first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); 249 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 250 first_vid, ntohs(ib_spec->eth.val.vlan_tag)); 251 252 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 253 first_cfi, 254 ntohs(ib_spec->eth.mask.vlan_tag) >> 12); 255 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 256 first_cfi, 257 ntohs(ib_spec->eth.val.vlan_tag) >> 12); 258 259 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 260 first_prio, 261 ntohs(ib_spec->eth.mask.vlan_tag) >> 13); 262 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 263 first_prio, 264 ntohs(ib_spec->eth.val.vlan_tag) >> 13); 265 } 266 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 267 ethertype, ntohs(ib_spec->eth.mask.ether_type)); 268 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 269 ethertype, ntohs(ib_spec->eth.val.ether_type)); 270 break; 271 case IB_FLOW_SPEC_IPV4: 272 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) 273 return -EOPNOTSUPP; 274 275 if (match_ipv) { 276 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 277 ip_version, 0xf); 278 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 279 ip_version, MLX5_FS_IPV4_VERSION); 280 } else { 281 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 282 ethertype, 0xffff); 283 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 284 ethertype, ETH_P_IP); 285 } 286 287 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 288 src_ipv4_src_ipv6.ipv4_layout.ipv4), 289 &ib_spec->ipv4.mask.src_ip, 290 sizeof(ib_spec->ipv4.mask.src_ip)); 291 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 292 src_ipv4_src_ipv6.ipv4_layout.ipv4), 293 &ib_spec->ipv4.val.src_ip, 294 sizeof(ib_spec->ipv4.val.src_ip)); 295 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 296 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 297 &ib_spec->ipv4.mask.dst_ip, 298 sizeof(ib_spec->ipv4.mask.dst_ip)); 299 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 300 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 301 &ib_spec->ipv4.val.dst_ip, 302 sizeof(ib_spec->ipv4.val.dst_ip)); 303 304 set_tos(headers_c, headers_v, 305 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); 306 307 if (set_proto(headers_c, headers_v, 308 ib_spec->ipv4.mask.proto, 309 ib_spec->ipv4.val.proto)) 310 return -EINVAL; 311 break; 312 case IB_FLOW_SPEC_IPV6: 313 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) 314 return -EOPNOTSUPP; 315 316 if (match_ipv) { 317 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 318 ip_version, 0xf); 319 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 320 ip_version, MLX5_FS_IPV6_VERSION); 321 } else { 322 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 323 ethertype, 0xffff); 324 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 325 ethertype, ETH_P_IPV6); 326 } 327 328 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 329 src_ipv4_src_ipv6.ipv6_layout.ipv6), 330 &ib_spec->ipv6.mask.src_ip, 331 sizeof(ib_spec->ipv6.mask.src_ip)); 332 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 333 src_ipv4_src_ipv6.ipv6_layout.ipv6), 334 &ib_spec->ipv6.val.src_ip, 335 sizeof(ib_spec->ipv6.val.src_ip)); 336 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 337 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 338 &ib_spec->ipv6.mask.dst_ip, 339 sizeof(ib_spec->ipv6.mask.dst_ip)); 340 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 341 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 342 &ib_spec->ipv6.val.dst_ip, 343 sizeof(ib_spec->ipv6.val.dst_ip)); 344 345 set_tos(headers_c, headers_v, 346 ib_spec->ipv6.mask.traffic_class, 347 ib_spec->ipv6.val.traffic_class); 348 349 if (set_proto(headers_c, headers_v, 350 ib_spec->ipv6.mask.next_hdr, 351 ib_spec->ipv6.val.next_hdr)) 352 return -EINVAL; 353 354 set_flow_label(misc_params_c, misc_params_v, 355 ntohl(ib_spec->ipv6.mask.flow_label), 356 ntohl(ib_spec->ipv6.val.flow_label), 357 ib_spec->type & IB_FLOW_SPEC_INNER); 358 break; 359 case IB_FLOW_SPEC_ESP: 360 return -EOPNOTSUPP; 361 case IB_FLOW_SPEC_TCP: 362 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 363 LAST_TCP_UDP_FIELD)) 364 return -EOPNOTSUPP; 365 366 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) 367 return -EINVAL; 368 369 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, 370 ntohs(ib_spec->tcp_udp.mask.src_port)); 371 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 372 ntohs(ib_spec->tcp_udp.val.src_port)); 373 374 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, 375 ntohs(ib_spec->tcp_udp.mask.dst_port)); 376 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 377 ntohs(ib_spec->tcp_udp.val.dst_port)); 378 break; 379 case IB_FLOW_SPEC_UDP: 380 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 381 LAST_TCP_UDP_FIELD)) 382 return -EOPNOTSUPP; 383 384 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) 385 return -EINVAL; 386 387 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 388 ntohs(ib_spec->tcp_udp.mask.src_port)); 389 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 390 ntohs(ib_spec->tcp_udp.val.src_port)); 391 392 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 393 ntohs(ib_spec->tcp_udp.mask.dst_port)); 394 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 395 ntohs(ib_spec->tcp_udp.val.dst_port)); 396 break; 397 case IB_FLOW_SPEC_GRE: 398 if (ib_spec->gre.mask.c_ks_res0_ver) 399 return -EOPNOTSUPP; 400 401 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) 402 return -EINVAL; 403 404 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 405 0xff); 406 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 407 IPPROTO_GRE); 408 409 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, 410 ntohs(ib_spec->gre.mask.protocol)); 411 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, 412 ntohs(ib_spec->gre.val.protocol)); 413 414 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, 415 gre_key.nvgre.hi), 416 &ib_spec->gre.mask.key, 417 sizeof(ib_spec->gre.mask.key)); 418 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, 419 gre_key.nvgre.hi), 420 &ib_spec->gre.val.key, 421 sizeof(ib_spec->gre.val.key)); 422 break; 423 case IB_FLOW_SPEC_MPLS: 424 switch (prev_type) { 425 case IB_FLOW_SPEC_UDP: 426 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 427 ft_field_support.outer_first_mpls_over_udp), 428 &ib_spec->mpls.mask.tag)) 429 return -EOPNOTSUPP; 430 431 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 432 outer_first_mpls_over_udp), 433 &ib_spec->mpls.val.tag, 434 sizeof(ib_spec->mpls.val.tag)); 435 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 436 outer_first_mpls_over_udp), 437 &ib_spec->mpls.mask.tag, 438 sizeof(ib_spec->mpls.mask.tag)); 439 break; 440 case IB_FLOW_SPEC_GRE: 441 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 442 ft_field_support.outer_first_mpls_over_gre), 443 &ib_spec->mpls.mask.tag)) 444 return -EOPNOTSUPP; 445 446 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 447 outer_first_mpls_over_gre), 448 &ib_spec->mpls.val.tag, 449 sizeof(ib_spec->mpls.val.tag)); 450 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 451 outer_first_mpls_over_gre), 452 &ib_spec->mpls.mask.tag, 453 sizeof(ib_spec->mpls.mask.tag)); 454 break; 455 default: 456 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 457 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 458 ft_field_support.inner_first_mpls), 459 &ib_spec->mpls.mask.tag)) 460 return -EOPNOTSUPP; 461 462 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 463 inner_first_mpls), 464 &ib_spec->mpls.val.tag, 465 sizeof(ib_spec->mpls.val.tag)); 466 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 467 inner_first_mpls), 468 &ib_spec->mpls.mask.tag, 469 sizeof(ib_spec->mpls.mask.tag)); 470 } else { 471 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 472 ft_field_support.outer_first_mpls), 473 &ib_spec->mpls.mask.tag)) 474 return -EOPNOTSUPP; 475 476 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 477 outer_first_mpls), 478 &ib_spec->mpls.val.tag, 479 sizeof(ib_spec->mpls.val.tag)); 480 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 481 outer_first_mpls), 482 &ib_spec->mpls.mask.tag, 483 sizeof(ib_spec->mpls.mask.tag)); 484 } 485 } 486 break; 487 case IB_FLOW_SPEC_VXLAN_TUNNEL: 488 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, 489 LAST_TUNNEL_FIELD)) 490 return -EOPNOTSUPP; 491 492 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, 493 ntohl(ib_spec->tunnel.mask.tunnel_id)); 494 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, 495 ntohl(ib_spec->tunnel.val.tunnel_id)); 496 break; 497 case IB_FLOW_SPEC_ACTION_TAG: 498 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, 499 LAST_FLOW_TAG_FIELD)) 500 return -EOPNOTSUPP; 501 if (ib_spec->flow_tag.tag_id >= BIT(24)) 502 return -EINVAL; 503 504 flow_context->flow_tag = ib_spec->flow_tag.tag_id; 505 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 506 break; 507 case IB_FLOW_SPEC_ACTION_DROP: 508 if (FIELDS_NOT_SUPPORTED(ib_spec->drop, 509 LAST_DROP_FIELD)) 510 return -EOPNOTSUPP; 511 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 512 break; 513 case IB_FLOW_SPEC_ACTION_HANDLE: 514 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), 515 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); 516 if (ret) 517 return ret; 518 break; 519 case IB_FLOW_SPEC_ACTION_COUNT: 520 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, 521 LAST_COUNTERS_FIELD)) 522 return -EOPNOTSUPP; 523 524 /* for now support only one counters spec per flow */ 525 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) 526 return -EINVAL; 527 528 action->counters = ib_spec->flow_count.counters; 529 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 530 break; 531 default: 532 return -EINVAL; 533 } 534 535 return 0; 536 } 537 538 /* If a flow could catch both multicast and unicast packets, 539 * it won't fall into the multicast flow steering table and this rule 540 * could steal other multicast packets. 541 */ 542 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) 543 { 544 union ib_flow_spec *flow_spec; 545 546 if (ib_attr->type != IB_FLOW_ATTR_NORMAL || 547 ib_attr->num_of_specs < 1) 548 return false; 549 550 flow_spec = (union ib_flow_spec *)(ib_attr + 1); 551 if (flow_spec->type == IB_FLOW_SPEC_IPV4) { 552 struct ib_flow_spec_ipv4 *ipv4_spec; 553 554 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; 555 if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) 556 return true; 557 558 return false; 559 } 560 561 if (flow_spec->type == IB_FLOW_SPEC_ETH) { 562 struct ib_flow_spec_eth *eth_spec; 563 564 eth_spec = (struct ib_flow_spec_eth *)flow_spec; 565 return is_multicast_ether_addr(eth_spec->mask.dst_mac) && 566 is_multicast_ether_addr(eth_spec->val.dst_mac); 567 } 568 569 return false; 570 } 571 572 static bool is_valid_ethertype(struct mlx5_core_dev *mdev, 573 const struct ib_flow_attr *flow_attr, 574 bool check_inner) 575 { 576 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); 577 int match_ipv = check_inner ? 578 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 579 ft_field_support.inner_ip_version) : 580 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 581 ft_field_support.outer_ip_version); 582 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; 583 bool ipv4_spec_valid, ipv6_spec_valid; 584 unsigned int ip_spec_type = 0; 585 bool has_ethertype = false; 586 unsigned int spec_index; 587 bool mask_valid = true; 588 u16 eth_type = 0; 589 bool type_valid; 590 591 /* Validate that ethertype is correct */ 592 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 593 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && 594 ib_spec->eth.mask.ether_type) { 595 mask_valid = (ib_spec->eth.mask.ether_type == 596 htons(0xffff)); 597 has_ethertype = true; 598 eth_type = ntohs(ib_spec->eth.val.ether_type); 599 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || 600 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { 601 ip_spec_type = ib_spec->type; 602 } 603 ib_spec = (void *)ib_spec + ib_spec->size; 604 } 605 606 type_valid = (!has_ethertype) || (!ip_spec_type); 607 if (!type_valid && mask_valid) { 608 ipv4_spec_valid = (eth_type == ETH_P_IP) && 609 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); 610 ipv6_spec_valid = (eth_type == ETH_P_IPV6) && 611 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); 612 613 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || 614 (((eth_type == ETH_P_MPLS_UC) || 615 (eth_type == ETH_P_MPLS_MC)) && match_ipv); 616 } 617 618 return type_valid; 619 } 620 621 static bool is_valid_attr(struct mlx5_core_dev *mdev, 622 const struct ib_flow_attr *flow_attr) 623 { 624 return is_valid_ethertype(mdev, flow_attr, false) && 625 is_valid_ethertype(mdev, flow_attr, true); 626 } 627 628 static void put_flow_table(struct mlx5_ib_dev *dev, 629 struct mlx5_ib_flow_prio *prio, bool ft_added) 630 { 631 prio->refcount -= !!ft_added; 632 if (!prio->refcount) { 633 mlx5_destroy_flow_table(prio->flow_table); 634 prio->flow_table = NULL; 635 } 636 } 637 638 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) 639 { 640 struct mlx5_ib_flow_handler *handler = container_of(flow_id, 641 struct mlx5_ib_flow_handler, 642 ibflow); 643 struct mlx5_ib_flow_handler *iter, *tmp; 644 struct mlx5_ib_dev *dev = handler->dev; 645 646 mutex_lock(&dev->flow_db->lock); 647 648 list_for_each_entry_safe(iter, tmp, &handler->list, list) { 649 mlx5_del_flow_rules(iter->rule); 650 put_flow_table(dev, iter->prio, true); 651 list_del(&iter->list); 652 kfree(iter); 653 } 654 655 mlx5_del_flow_rules(handler->rule); 656 put_flow_table(dev, handler->prio, true); 657 mlx5_ib_counters_clear_description(handler->ibcounters); 658 mutex_unlock(&dev->flow_db->lock); 659 if (handler->flow_matcher) 660 atomic_dec(&handler->flow_matcher->usecnt); 661 kfree(handler); 662 663 return 0; 664 } 665 666 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) 667 { 668 priority *= 2; 669 if (!dont_trap) 670 priority++; 671 return priority; 672 } 673 674 enum flow_table_type { 675 MLX5_IB_FT_RX, 676 MLX5_IB_FT_TX 677 }; 678 679 #define MLX5_FS_MAX_TYPES 6 680 #define MLX5_FS_MAX_ENTRIES BIT(16) 681 682 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns, 683 struct mlx5_ib_flow_prio *prio, 684 int priority, 685 int num_entries, int num_groups, 686 u32 flags) 687 { 688 struct mlx5_flow_table_attr ft_attr = {}; 689 struct mlx5_flow_table *ft; 690 691 ft_attr.prio = priority; 692 ft_attr.max_fte = num_entries; 693 ft_attr.flags = flags; 694 ft_attr.autogroup.max_num_groups = num_groups; 695 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 696 if (IS_ERR(ft)) 697 return ERR_CAST(ft); 698 699 prio->flow_table = ft; 700 prio->refcount = 0; 701 return prio; 702 } 703 704 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, 705 struct ib_flow_attr *flow_attr, 706 enum flow_table_type ft_type) 707 { 708 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; 709 struct mlx5_flow_namespace *ns = NULL; 710 enum mlx5_flow_namespace_type fn_type; 711 struct mlx5_ib_flow_prio *prio; 712 struct mlx5_flow_table *ft; 713 int max_table_size; 714 int num_entries; 715 int num_groups; 716 bool esw_encap; 717 u32 flags = 0; 718 int priority; 719 720 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 721 log_max_ft_size)); 722 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 723 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 724 switch (flow_attr->type) { 725 case IB_FLOW_ATTR_NORMAL: 726 if (flow_is_multicast_only(flow_attr) && !dont_trap) 727 priority = MLX5_IB_FLOW_MCAST_PRIO; 728 else 729 priority = ib_prio_to_core_prio(flow_attr->priority, 730 dont_trap); 731 if (ft_type == MLX5_IB_FT_RX) { 732 fn_type = MLX5_FLOW_NAMESPACE_BYPASS; 733 prio = &dev->flow_db->prios[priority]; 734 if (!dev->is_rep && !esw_encap && 735 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) 736 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 737 if (!dev->is_rep && !esw_encap && 738 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 739 reformat_l3_tunnel_to_l2)) 740 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 741 } else { 742 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX( 743 dev->mdev, log_max_ft_size)); 744 fn_type = MLX5_FLOW_NAMESPACE_EGRESS; 745 prio = &dev->flow_db->egress_prios[priority]; 746 if (!dev->is_rep && !esw_encap && 747 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) 748 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 749 } 750 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 751 num_entries = MLX5_FS_MAX_ENTRIES; 752 num_groups = MLX5_FS_MAX_TYPES; 753 break; 754 case IB_FLOW_ATTR_ALL_DEFAULT: 755 case IB_FLOW_ATTR_MC_DEFAULT: 756 ns = mlx5_get_flow_namespace(dev->mdev, 757 MLX5_FLOW_NAMESPACE_LEFTOVERS); 758 build_leftovers_ft_param(&priority, &num_entries, &num_groups); 759 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; 760 break; 761 case IB_FLOW_ATTR_SNIFFER: 762 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 763 allow_sniffer_and_nic_rx_shared_tir)) 764 return ERR_PTR(-EOPNOTSUPP); 765 766 ns = mlx5_get_flow_namespace( 767 dev->mdev, ft_type == MLX5_IB_FT_RX ? 768 MLX5_FLOW_NAMESPACE_SNIFFER_RX : 769 MLX5_FLOW_NAMESPACE_SNIFFER_TX); 770 771 prio = &dev->flow_db->sniffer[ft_type]; 772 priority = 0; 773 num_entries = 1; 774 num_groups = 1; 775 break; 776 default: 777 break; 778 } 779 780 if (!ns) 781 return ERR_PTR(-EOPNOTSUPP); 782 783 max_table_size = min_t(int, num_entries, max_table_size); 784 785 ft = prio->flow_table; 786 if (!ft) 787 return _get_prio(ns, prio, priority, max_table_size, num_groups, 788 flags); 789 790 return prio; 791 } 792 793 enum { 794 RDMA_RX_ECN_OPCOUNTER_PRIO, 795 RDMA_RX_CNP_OPCOUNTER_PRIO, 796 }; 797 798 enum { 799 RDMA_TX_CNP_OPCOUNTER_PRIO, 800 }; 801 802 static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num, 803 struct mlx5_flow_spec *spec) 804 { 805 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 806 ft_field_support.source_vhca_port) || 807 !MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 808 ft_field_support.source_vhca_port)) 809 return -EOPNOTSUPP; 810 811 MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria, 812 misc_parameters.source_vhca_port); 813 MLX5_SET(fte_match_param, &spec->match_value, 814 misc_parameters.source_vhca_port, port_num); 815 816 return 0; 817 } 818 819 static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num, 820 struct mlx5_flow_spec *spec, int ipv) 821 { 822 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 823 ft_field_support.outer_ip_version)) 824 return -EOPNOTSUPP; 825 826 if (mlx5_core_mp_enabled(dev->mdev) && 827 set_vhca_port_spec(dev, port_num, spec)) 828 return -EOPNOTSUPP; 829 830 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 831 outer_headers.ip_ecn); 832 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn, 833 INET_ECN_CE); 834 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 835 outer_headers.ip_version); 836 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 837 ipv); 838 839 spec->match_criteria_enable = 840 get_match_criteria_enable(spec->match_criteria); 841 842 return 0; 843 } 844 845 static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num, 846 struct mlx5_flow_spec *spec) 847 { 848 if (mlx5_core_mp_enabled(dev->mdev) && 849 set_vhca_port_spec(dev, port_num, spec)) 850 return -EOPNOTSUPP; 851 852 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 853 misc_parameters.bth_opcode); 854 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode, 855 IB_BTH_OPCODE_CNP); 856 857 spec->match_criteria_enable = 858 get_match_criteria_enable(spec->match_criteria); 859 860 return 0; 861 } 862 863 int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num, 864 struct mlx5_ib_op_fc *opfc, 865 enum mlx5_ib_optional_counter_type type) 866 { 867 enum mlx5_flow_namespace_type fn_type; 868 int priority, i, err, spec_num; 869 struct mlx5_flow_act flow_act = {}; 870 struct mlx5_flow_destination dst; 871 struct mlx5_flow_namespace *ns; 872 struct mlx5_ib_flow_prio *prio; 873 struct mlx5_flow_spec *spec; 874 875 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL); 876 if (!spec) 877 return -ENOMEM; 878 879 switch (type) { 880 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 881 if (set_ecn_ce_spec(dev, port_num, &spec[0], 882 MLX5_FS_IPV4_VERSION) || 883 set_ecn_ce_spec(dev, port_num, &spec[1], 884 MLX5_FS_IPV6_VERSION)) { 885 err = -EOPNOTSUPP; 886 goto free; 887 } 888 spec_num = 2; 889 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 890 priority = RDMA_RX_ECN_OPCOUNTER_PRIO; 891 break; 892 893 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 894 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 895 ft_field_support_2_nic_receive_rdma.bth_opcode) || 896 set_cnp_spec(dev, port_num, &spec[0])) { 897 err = -EOPNOTSUPP; 898 goto free; 899 } 900 spec_num = 1; 901 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 902 priority = RDMA_RX_CNP_OPCOUNTER_PRIO; 903 break; 904 905 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 906 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 907 ft_field_support_2_nic_transmit_rdma.bth_opcode) || 908 set_cnp_spec(dev, port_num, &spec[0])) { 909 err = -EOPNOTSUPP; 910 goto free; 911 } 912 spec_num = 1; 913 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 914 priority = RDMA_TX_CNP_OPCOUNTER_PRIO; 915 break; 916 917 default: 918 err = -EOPNOTSUPP; 919 goto free; 920 } 921 922 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 923 if (!ns) { 924 err = -EOPNOTSUPP; 925 goto free; 926 } 927 928 prio = &dev->flow_db->opfcs[type]; 929 if (!prio->flow_table) { 930 prio = _get_prio(ns, prio, priority, 931 dev->num_ports * MAX_OPFC_RULES, 1, 0); 932 if (IS_ERR(prio)) { 933 err = PTR_ERR(prio); 934 goto free; 935 } 936 } 937 938 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 939 dst.counter_id = mlx5_fc_id(opfc->fc); 940 941 flow_act.action = 942 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW; 943 944 for (i = 0; i < spec_num; i++) { 945 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i], 946 &flow_act, &dst, 1); 947 if (IS_ERR(opfc->rule[i])) { 948 err = PTR_ERR(opfc->rule[i]); 949 goto del_rules; 950 } 951 } 952 prio->refcount += spec_num; 953 kfree(spec); 954 955 return 0; 956 957 del_rules: 958 for (i -= 1; i >= 0; i--) 959 mlx5_del_flow_rules(opfc->rule[i]); 960 put_flow_table(dev, prio, false); 961 free: 962 kfree(spec); 963 return err; 964 } 965 966 void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev, 967 struct mlx5_ib_op_fc *opfc, 968 enum mlx5_ib_optional_counter_type type) 969 { 970 int i; 971 972 for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) { 973 mlx5_del_flow_rules(opfc->rule[i]); 974 put_flow_table(dev, &dev->flow_db->opfcs[type], true); 975 } 976 } 977 978 static void set_underlay_qp(struct mlx5_ib_dev *dev, 979 struct mlx5_flow_spec *spec, 980 u32 underlay_qpn) 981 { 982 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, 983 spec->match_criteria, 984 misc_parameters); 985 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 986 misc_parameters); 987 988 if (underlay_qpn && 989 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 990 ft_field_support.bth_dst_qp)) { 991 MLX5_SET(fte_match_set_misc, 992 misc_params_v, bth_dst_qp, underlay_qpn); 993 MLX5_SET(fte_match_set_misc, 994 misc_params_c, bth_dst_qp, 0xffffff); 995 } 996 } 997 998 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, 999 struct mlx5_flow_spec *spec, 1000 struct mlx5_eswitch_rep *rep) 1001 { 1002 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; 1003 void *misc; 1004 1005 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1006 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1007 misc_parameters_2); 1008 1009 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1010 mlx5_eswitch_get_vport_metadata_for_match(rep->esw, 1011 rep->vport)); 1012 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1013 misc_parameters_2); 1014 1015 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1016 mlx5_eswitch_get_vport_metadata_mask()); 1017 } else { 1018 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1019 misc_parameters); 1020 1021 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); 1022 1023 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1024 misc_parameters); 1025 1026 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1027 } 1028 } 1029 1030 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, 1031 struct mlx5_ib_flow_prio *ft_prio, 1032 const struct ib_flow_attr *flow_attr, 1033 struct mlx5_flow_destination *dst, 1034 u32 underlay_qpn, 1035 struct mlx5_ib_create_flow *ucmd) 1036 { 1037 struct mlx5_flow_table *ft = ft_prio->flow_table; 1038 struct mlx5_ib_flow_handler *handler; 1039 struct mlx5_flow_act flow_act = {}; 1040 struct mlx5_flow_spec *spec; 1041 struct mlx5_flow_destination dest_arr[2] = {}; 1042 struct mlx5_flow_destination *rule_dst = dest_arr; 1043 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); 1044 unsigned int spec_index; 1045 u32 prev_type = 0; 1046 int err = 0; 1047 int dest_num = 0; 1048 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1049 1050 if (!is_valid_attr(dev->mdev, flow_attr)) 1051 return ERR_PTR(-EINVAL); 1052 1053 if (dev->is_rep && is_egress) 1054 return ERR_PTR(-EINVAL); 1055 1056 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1057 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 1058 if (!handler || !spec) { 1059 err = -ENOMEM; 1060 goto free; 1061 } 1062 1063 INIT_LIST_HEAD(&handler->list); 1064 1065 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 1066 err = parse_flow_attr(dev->mdev, spec, 1067 ib_flow, flow_attr, &flow_act, 1068 prev_type); 1069 if (err < 0) 1070 goto free; 1071 1072 prev_type = ((union ib_flow_spec *)ib_flow)->type; 1073 ib_flow += ((union ib_flow_spec *)ib_flow)->size; 1074 } 1075 1076 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { 1077 memcpy(&dest_arr[0], dst, sizeof(*dst)); 1078 dest_num++; 1079 } 1080 1081 if (!flow_is_multicast_only(flow_attr)) 1082 set_underlay_qp(dev, spec, underlay_qpn); 1083 1084 if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) { 1085 struct mlx5_eswitch_rep *rep; 1086 1087 rep = dev->port[flow_attr->port - 1].rep; 1088 if (!rep) { 1089 err = -EINVAL; 1090 goto free; 1091 } 1092 1093 mlx5_ib_set_rule_source_port(dev, spec, rep); 1094 } 1095 1096 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); 1097 1098 if (is_egress) { 1099 err = -EINVAL; 1100 goto free; 1101 } 1102 1103 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 1104 struct mlx5_ib_mcounters *mcounters; 1105 1106 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); 1107 if (err) 1108 goto free; 1109 1110 mcounters = to_mcounters(flow_act.counters); 1111 handler->ibcounters = flow_act.counters; 1112 dest_arr[dest_num].type = 1113 MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1114 dest_arr[dest_num].counter_id = 1115 mlx5_fc_id(mcounters->hw_cntrs_hndl); 1116 dest_num++; 1117 } 1118 1119 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { 1120 if (!dest_num) 1121 rule_dst = NULL; 1122 } else { 1123 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) 1124 flow_act.action |= 1125 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1126 if (is_egress) 1127 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1128 else if (dest_num) 1129 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1130 } 1131 1132 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && 1133 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1134 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1135 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", 1136 spec->flow_context.flow_tag, flow_attr->type); 1137 err = -EINVAL; 1138 goto free; 1139 } 1140 handler->rule = mlx5_add_flow_rules(ft, spec, 1141 &flow_act, 1142 rule_dst, dest_num); 1143 1144 if (IS_ERR(handler->rule)) { 1145 err = PTR_ERR(handler->rule); 1146 goto free; 1147 } 1148 1149 ft_prio->refcount++; 1150 handler->prio = ft_prio; 1151 handler->dev = dev; 1152 1153 ft_prio->flow_table = ft; 1154 free: 1155 if (err && handler) { 1156 mlx5_ib_counters_clear_description(handler->ibcounters); 1157 kfree(handler); 1158 } 1159 kvfree(spec); 1160 return err ? ERR_PTR(err) : handler; 1161 } 1162 1163 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, 1164 struct mlx5_ib_flow_prio *ft_prio, 1165 const struct ib_flow_attr *flow_attr, 1166 struct mlx5_flow_destination *dst) 1167 { 1168 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); 1169 } 1170 1171 enum { 1172 LEFTOVERS_MC, 1173 LEFTOVERS_UC, 1174 }; 1175 1176 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, 1177 struct mlx5_ib_flow_prio *ft_prio, 1178 struct ib_flow_attr *flow_attr, 1179 struct mlx5_flow_destination *dst) 1180 { 1181 struct mlx5_ib_flow_handler *handler_ucast = NULL; 1182 struct mlx5_ib_flow_handler *handler = NULL; 1183 1184 static struct { 1185 struct ib_flow_attr flow_attr; 1186 struct ib_flow_spec_eth eth_flow; 1187 } leftovers_specs[] = { 1188 [LEFTOVERS_MC] = { 1189 .flow_attr = { 1190 .num_of_specs = 1, 1191 .size = sizeof(leftovers_specs[0]) 1192 }, 1193 .eth_flow = { 1194 .type = IB_FLOW_SPEC_ETH, 1195 .size = sizeof(struct ib_flow_spec_eth), 1196 .mask = {.dst_mac = {0x1} }, 1197 .val = {.dst_mac = {0x1} } 1198 } 1199 }, 1200 [LEFTOVERS_UC] = { 1201 .flow_attr = { 1202 .num_of_specs = 1, 1203 .size = sizeof(leftovers_specs[0]) 1204 }, 1205 .eth_flow = { 1206 .type = IB_FLOW_SPEC_ETH, 1207 .size = sizeof(struct ib_flow_spec_eth), 1208 .mask = {.dst_mac = {0x1} }, 1209 .val = {.dst_mac = {} } 1210 } 1211 } 1212 }; 1213 1214 handler = create_flow_rule(dev, ft_prio, 1215 &leftovers_specs[LEFTOVERS_MC].flow_attr, 1216 dst); 1217 if (!IS_ERR(handler) && 1218 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { 1219 handler_ucast = create_flow_rule(dev, ft_prio, 1220 &leftovers_specs[LEFTOVERS_UC].flow_attr, 1221 dst); 1222 if (IS_ERR(handler_ucast)) { 1223 mlx5_del_flow_rules(handler->rule); 1224 ft_prio->refcount--; 1225 kfree(handler); 1226 handler = handler_ucast; 1227 } else { 1228 list_add(&handler_ucast->list, &handler->list); 1229 } 1230 } 1231 1232 return handler; 1233 } 1234 1235 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, 1236 struct mlx5_ib_flow_prio *ft_rx, 1237 struct mlx5_ib_flow_prio *ft_tx, 1238 struct mlx5_flow_destination *dst) 1239 { 1240 struct mlx5_ib_flow_handler *handler_rx; 1241 struct mlx5_ib_flow_handler *handler_tx; 1242 int err; 1243 static const struct ib_flow_attr flow_attr = { 1244 .num_of_specs = 0, 1245 .type = IB_FLOW_ATTR_SNIFFER, 1246 .size = sizeof(flow_attr) 1247 }; 1248 1249 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); 1250 if (IS_ERR(handler_rx)) { 1251 err = PTR_ERR(handler_rx); 1252 goto err; 1253 } 1254 1255 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); 1256 if (IS_ERR(handler_tx)) { 1257 err = PTR_ERR(handler_tx); 1258 goto err_tx; 1259 } 1260 1261 list_add(&handler_tx->list, &handler_rx->list); 1262 1263 return handler_rx; 1264 1265 err_tx: 1266 mlx5_del_flow_rules(handler_rx->rule); 1267 ft_rx->refcount--; 1268 kfree(handler_rx); 1269 err: 1270 return ERR_PTR(err); 1271 } 1272 1273 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, 1274 struct ib_flow_attr *flow_attr, 1275 struct ib_udata *udata) 1276 { 1277 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1278 struct mlx5_ib_qp *mqp = to_mqp(qp); 1279 struct mlx5_ib_flow_handler *handler = NULL; 1280 struct mlx5_flow_destination *dst = NULL; 1281 struct mlx5_ib_flow_prio *ft_prio_tx = NULL; 1282 struct mlx5_ib_flow_prio *ft_prio; 1283 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1284 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; 1285 size_t min_ucmd_sz, required_ucmd_sz; 1286 int err; 1287 int underlay_qpn; 1288 1289 if (udata && udata->inlen) { 1290 min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved); 1291 if (udata->inlen < min_ucmd_sz) 1292 return ERR_PTR(-EOPNOTSUPP); 1293 1294 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); 1295 if (err) 1296 return ERR_PTR(err); 1297 1298 /* currently supports only one counters data */ 1299 if (ucmd_hdr.ncounters_data > 1) 1300 return ERR_PTR(-EINVAL); 1301 1302 required_ucmd_sz = min_ucmd_sz + 1303 sizeof(struct mlx5_ib_flow_counters_data) * 1304 ucmd_hdr.ncounters_data; 1305 if (udata->inlen > required_ucmd_sz && 1306 !ib_is_udata_cleared(udata, required_ucmd_sz, 1307 udata->inlen - required_ucmd_sz)) 1308 return ERR_PTR(-EOPNOTSUPP); 1309 1310 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); 1311 if (!ucmd) 1312 return ERR_PTR(-ENOMEM); 1313 1314 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); 1315 if (err) 1316 goto free_ucmd; 1317 } 1318 1319 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { 1320 err = -ENOMEM; 1321 goto free_ucmd; 1322 } 1323 1324 if (flow_attr->flags & 1325 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) { 1326 err = -EINVAL; 1327 goto free_ucmd; 1328 } 1329 1330 if (is_egress && 1331 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1332 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1333 err = -EINVAL; 1334 goto free_ucmd; 1335 } 1336 1337 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1338 if (!dst) { 1339 err = -ENOMEM; 1340 goto free_ucmd; 1341 } 1342 1343 mutex_lock(&dev->flow_db->lock); 1344 1345 ft_prio = get_flow_table(dev, flow_attr, 1346 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); 1347 if (IS_ERR(ft_prio)) { 1348 err = PTR_ERR(ft_prio); 1349 goto unlock; 1350 } 1351 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 1352 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); 1353 if (IS_ERR(ft_prio_tx)) { 1354 err = PTR_ERR(ft_prio_tx); 1355 ft_prio_tx = NULL; 1356 goto destroy_ft; 1357 } 1358 } 1359 1360 if (is_egress) { 1361 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1362 } else { 1363 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1364 if (mqp->is_rss) 1365 dst->tir_num = mqp->rss_qp.tirn; 1366 else 1367 dst->tir_num = mqp->raw_packet_qp.rq.tirn; 1368 } 1369 1370 switch (flow_attr->type) { 1371 case IB_FLOW_ATTR_NORMAL: 1372 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? 1373 mqp->underlay_qpn : 1374 0; 1375 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, 1376 underlay_qpn, ucmd); 1377 break; 1378 case IB_FLOW_ATTR_ALL_DEFAULT: 1379 case IB_FLOW_ATTR_MC_DEFAULT: 1380 handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst); 1381 break; 1382 case IB_FLOW_ATTR_SNIFFER: 1383 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); 1384 break; 1385 default: 1386 err = -EINVAL; 1387 goto destroy_ft; 1388 } 1389 1390 if (IS_ERR(handler)) { 1391 err = PTR_ERR(handler); 1392 handler = NULL; 1393 goto destroy_ft; 1394 } 1395 1396 mutex_unlock(&dev->flow_db->lock); 1397 kfree(dst); 1398 kfree(ucmd); 1399 1400 return &handler->ibflow; 1401 1402 destroy_ft: 1403 put_flow_table(dev, ft_prio, false); 1404 if (ft_prio_tx) 1405 put_flow_table(dev, ft_prio_tx, false); 1406 unlock: 1407 mutex_unlock(&dev->flow_db->lock); 1408 kfree(dst); 1409 free_ucmd: 1410 kfree(ucmd); 1411 return ERR_PTR(err); 1412 } 1413 1414 static struct mlx5_ib_flow_prio * 1415 _get_flow_table(struct mlx5_ib_dev *dev, 1416 struct mlx5_ib_flow_matcher *fs_matcher, 1417 bool mcast) 1418 { 1419 struct mlx5_flow_namespace *ns = NULL; 1420 struct mlx5_ib_flow_prio *prio = NULL; 1421 int max_table_size = 0; 1422 bool esw_encap; 1423 u32 flags = 0; 1424 int priority; 1425 1426 if (mcast) 1427 priority = MLX5_IB_FLOW_MCAST_PRIO; 1428 else 1429 priority = ib_prio_to_core_prio(fs_matcher->priority, false); 1430 1431 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 1432 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 1433 switch (fs_matcher->ns_type) { 1434 case MLX5_FLOW_NAMESPACE_BYPASS: 1435 max_table_size = BIT( 1436 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); 1437 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) 1438 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1439 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1440 reformat_l3_tunnel_to_l2) && 1441 !esw_encap) 1442 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1443 break; 1444 case MLX5_FLOW_NAMESPACE_EGRESS: 1445 max_table_size = BIT( 1446 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); 1447 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && 1448 !esw_encap) 1449 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1450 break; 1451 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 1452 max_table_size = BIT( 1453 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); 1454 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) 1455 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1456 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, 1457 reformat_l3_tunnel_to_l2) && 1458 esw_encap) 1459 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1460 priority = fs_matcher->priority; 1461 break; 1462 case MLX5_FLOW_NAMESPACE_RDMA_RX: 1463 max_table_size = BIT( 1464 MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size)); 1465 priority = fs_matcher->priority; 1466 break; 1467 case MLX5_FLOW_NAMESPACE_RDMA_TX: 1468 max_table_size = BIT( 1469 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size)); 1470 priority = fs_matcher->priority; 1471 break; 1472 default: 1473 break; 1474 } 1475 1476 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); 1477 1478 ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type); 1479 if (!ns) 1480 return ERR_PTR(-EOPNOTSUPP); 1481 1482 switch (fs_matcher->ns_type) { 1483 case MLX5_FLOW_NAMESPACE_BYPASS: 1484 prio = &dev->flow_db->prios[priority]; 1485 break; 1486 case MLX5_FLOW_NAMESPACE_EGRESS: 1487 prio = &dev->flow_db->egress_prios[priority]; 1488 break; 1489 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 1490 prio = &dev->flow_db->fdb[priority]; 1491 break; 1492 case MLX5_FLOW_NAMESPACE_RDMA_RX: 1493 prio = &dev->flow_db->rdma_rx[priority]; 1494 break; 1495 case MLX5_FLOW_NAMESPACE_RDMA_TX: 1496 prio = &dev->flow_db->rdma_tx[priority]; 1497 break; 1498 default: return ERR_PTR(-EINVAL); 1499 } 1500 1501 if (!prio) 1502 return ERR_PTR(-EINVAL); 1503 1504 if (prio->flow_table) 1505 return prio; 1506 1507 return _get_prio(ns, prio, priority, max_table_size, 1508 MLX5_FS_MAX_TYPES, flags); 1509 } 1510 1511 static struct mlx5_ib_flow_handler * 1512 _create_raw_flow_rule(struct mlx5_ib_dev *dev, 1513 struct mlx5_ib_flow_prio *ft_prio, 1514 struct mlx5_flow_destination *dst, 1515 struct mlx5_ib_flow_matcher *fs_matcher, 1516 struct mlx5_flow_context *flow_context, 1517 struct mlx5_flow_act *flow_act, 1518 void *cmd_in, int inlen, 1519 int dst_num) 1520 { 1521 struct mlx5_ib_flow_handler *handler; 1522 struct mlx5_flow_spec *spec; 1523 struct mlx5_flow_table *ft = ft_prio->flow_table; 1524 int err = 0; 1525 1526 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1527 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 1528 if (!handler || !spec) { 1529 err = -ENOMEM; 1530 goto free; 1531 } 1532 1533 INIT_LIST_HEAD(&handler->list); 1534 1535 memcpy(spec->match_value, cmd_in, inlen); 1536 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, 1537 fs_matcher->mask_len); 1538 spec->match_criteria_enable = fs_matcher->match_criteria_enable; 1539 spec->flow_context = *flow_context; 1540 1541 handler->rule = mlx5_add_flow_rules(ft, spec, 1542 flow_act, dst, dst_num); 1543 1544 if (IS_ERR(handler->rule)) { 1545 err = PTR_ERR(handler->rule); 1546 goto free; 1547 } 1548 1549 ft_prio->refcount++; 1550 handler->prio = ft_prio; 1551 handler->dev = dev; 1552 ft_prio->flow_table = ft; 1553 1554 free: 1555 if (err) 1556 kfree(handler); 1557 kvfree(spec); 1558 return err ? ERR_PTR(err) : handler; 1559 } 1560 1561 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, 1562 void *match_v) 1563 { 1564 void *match_c; 1565 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; 1566 void *dmac, *dmac_mask; 1567 void *ipv4, *ipv4_mask; 1568 1569 if (!(fs_matcher->match_criteria_enable & 1570 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) 1571 return false; 1572 1573 match_c = fs_matcher->matcher_mask.match_params; 1574 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, 1575 outer_headers); 1576 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, 1577 outer_headers); 1578 1579 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 1580 dmac_47_16); 1581 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 1582 dmac_47_16); 1583 1584 if (is_multicast_ether_addr(dmac) && 1585 is_multicast_ether_addr(dmac_mask)) 1586 return true; 1587 1588 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 1589 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1590 1591 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 1592 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1593 1594 if (ipv4_is_multicast(*(__be32 *)(ipv4)) && 1595 ipv4_is_multicast(*(__be32 *)(ipv4_mask))) 1596 return true; 1597 1598 return false; 1599 } 1600 1601 static struct mlx5_ib_flow_handler *raw_fs_rule_add( 1602 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, 1603 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, 1604 u32 counter_id, void *cmd_in, int inlen, int dest_id, int dest_type) 1605 { 1606 struct mlx5_flow_destination *dst; 1607 struct mlx5_ib_flow_prio *ft_prio; 1608 struct mlx5_ib_flow_handler *handler; 1609 int dst_num = 0; 1610 bool mcast; 1611 int err; 1612 1613 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) 1614 return ERR_PTR(-EOPNOTSUPP); 1615 1616 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) 1617 return ERR_PTR(-ENOMEM); 1618 1619 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); 1620 if (!dst) 1621 return ERR_PTR(-ENOMEM); 1622 1623 mcast = raw_fs_is_multicast(fs_matcher, cmd_in); 1624 mutex_lock(&dev->flow_db->lock); 1625 1626 ft_prio = _get_flow_table(dev, fs_matcher, mcast); 1627 if (IS_ERR(ft_prio)) { 1628 err = PTR_ERR(ft_prio); 1629 goto unlock; 1630 } 1631 1632 switch (dest_type) { 1633 case MLX5_FLOW_DESTINATION_TYPE_TIR: 1634 dst[dst_num].type = dest_type; 1635 dst[dst_num++].tir_num = dest_id; 1636 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1637 break; 1638 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE: 1639 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; 1640 dst[dst_num++].ft_num = dest_id; 1641 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1642 break; 1643 case MLX5_FLOW_DESTINATION_TYPE_PORT: 1644 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1645 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1646 break; 1647 default: 1648 break; 1649 } 1650 1651 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 1652 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1653 dst[dst_num].counter_id = counter_id; 1654 dst_num++; 1655 } 1656 1657 handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL, 1658 fs_matcher, flow_context, flow_act, 1659 cmd_in, inlen, dst_num); 1660 1661 if (IS_ERR(handler)) { 1662 err = PTR_ERR(handler); 1663 goto destroy_ft; 1664 } 1665 1666 mutex_unlock(&dev->flow_db->lock); 1667 atomic_inc(&fs_matcher->usecnt); 1668 handler->flow_matcher = fs_matcher; 1669 1670 kfree(dst); 1671 1672 return handler; 1673 1674 destroy_ft: 1675 put_flow_table(dev, ft_prio, false); 1676 unlock: 1677 mutex_unlock(&dev->flow_db->lock); 1678 kfree(dst); 1679 1680 return ERR_PTR(err); 1681 } 1682 1683 static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) 1684 { 1685 switch (maction->flow_action_raw.sub_type) { 1686 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: 1687 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, 1688 maction->flow_action_raw.modify_hdr); 1689 break; 1690 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: 1691 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, 1692 maction->flow_action_raw.pkt_reformat); 1693 break; 1694 case MLX5_IB_FLOW_ACTION_DECAP: 1695 break; 1696 default: 1697 break; 1698 } 1699 } 1700 1701 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) 1702 { 1703 struct mlx5_ib_flow_action *maction = to_mflow_act(action); 1704 1705 switch (action->type) { 1706 case IB_FLOW_ACTION_UNSPECIFIED: 1707 destroy_flow_action_raw(maction); 1708 break; 1709 default: 1710 WARN_ON(true); 1711 break; 1712 } 1713 1714 kfree(maction); 1715 return 0; 1716 } 1717 1718 static int 1719 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, 1720 enum mlx5_flow_namespace_type *namespace) 1721 { 1722 switch (table_type) { 1723 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: 1724 *namespace = MLX5_FLOW_NAMESPACE_BYPASS; 1725 break; 1726 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: 1727 *namespace = MLX5_FLOW_NAMESPACE_EGRESS; 1728 break; 1729 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: 1730 *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS; 1731 break; 1732 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: 1733 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; 1734 break; 1735 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: 1736 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; 1737 break; 1738 default: 1739 return -EINVAL; 1740 } 1741 1742 return 0; 1743 } 1744 1745 static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { 1746 [MLX5_IB_FLOW_TYPE_NORMAL] = { 1747 .type = UVERBS_ATTR_TYPE_PTR_IN, 1748 .u.ptr = { 1749 .len = sizeof(u16), /* data is priority */ 1750 .min_len = sizeof(u16), 1751 } 1752 }, 1753 [MLX5_IB_FLOW_TYPE_SNIFFER] = { 1754 .type = UVERBS_ATTR_TYPE_PTR_IN, 1755 UVERBS_ATTR_NO_DATA(), 1756 }, 1757 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { 1758 .type = UVERBS_ATTR_TYPE_PTR_IN, 1759 UVERBS_ATTR_NO_DATA(), 1760 }, 1761 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { 1762 .type = UVERBS_ATTR_TYPE_PTR_IN, 1763 UVERBS_ATTR_NO_DATA(), 1764 }, 1765 }; 1766 1767 static bool is_flow_dest(void *obj, int *dest_id, int *dest_type) 1768 { 1769 struct devx_obj *devx_obj = obj; 1770 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 1771 1772 switch (opcode) { 1773 case MLX5_CMD_OP_DESTROY_TIR: 1774 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1775 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 1776 obj_id); 1777 return true; 1778 1779 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 1780 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1781 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 1782 table_id); 1783 return true; 1784 default: 1785 return false; 1786 } 1787 } 1788 1789 static int get_dests(struct uverbs_attr_bundle *attrs, 1790 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, 1791 int *dest_type, struct ib_qp **qp, u32 *flags) 1792 { 1793 bool dest_devx, dest_qp; 1794 void *devx_obj; 1795 int err; 1796 1797 dest_devx = uverbs_attr_is_valid(attrs, 1798 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 1799 dest_qp = uverbs_attr_is_valid(attrs, 1800 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 1801 1802 *flags = 0; 1803 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 1804 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | 1805 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); 1806 if (err) 1807 return err; 1808 1809 /* Both flags are not allowed */ 1810 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && 1811 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 1812 return -EINVAL; 1813 1814 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 1815 if (dest_devx && (dest_qp || *flags)) 1816 return -EINVAL; 1817 else if (dest_qp && *flags) 1818 return -EINVAL; 1819 } 1820 1821 /* Allow only DEVX object, drop as dest for FDB */ 1822 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 1823 !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) 1824 return -EINVAL; 1825 1826 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ 1827 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && 1828 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) 1829 return -EINVAL; 1830 1831 *qp = NULL; 1832 if (dest_devx) { 1833 devx_obj = 1834 uverbs_attr_get_obj(attrs, 1835 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 1836 1837 /* Verify that the given DEVX object is a flow 1838 * steering destination. 1839 */ 1840 if (!is_flow_dest(devx_obj, dest_id, dest_type)) 1841 return -EINVAL; 1842 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ 1843 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS || 1844 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && 1845 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) 1846 return -EINVAL; 1847 } else if (dest_qp) { 1848 struct mlx5_ib_qp *mqp; 1849 1850 *qp = uverbs_attr_get_obj(attrs, 1851 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 1852 if (IS_ERR(*qp)) 1853 return PTR_ERR(*qp); 1854 1855 if ((*qp)->qp_type != IB_QPT_RAW_PACKET) 1856 return -EINVAL; 1857 1858 mqp = to_mqp(*qp); 1859 if (mqp->is_rss) 1860 *dest_id = mqp->rss_qp.tirn; 1861 else 1862 *dest_id = mqp->raw_packet_qp.rq.tirn; 1863 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1864 } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 1865 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) && 1866 !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) { 1867 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1868 } 1869 1870 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && 1871 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 1872 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX)) 1873 return -EINVAL; 1874 1875 return 0; 1876 } 1877 1878 static bool is_flow_counter(void *obj, u32 offset, u32 *counter_id) 1879 { 1880 struct devx_obj *devx_obj = obj; 1881 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 1882 1883 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 1884 1885 if (offset && offset >= devx_obj->flow_counter_bulk_size) 1886 return false; 1887 1888 *counter_id = MLX5_GET(dealloc_flow_counter_in, 1889 devx_obj->dinbox, 1890 flow_counter_id); 1891 *counter_id += offset; 1892 return true; 1893 } 1894 1895 return false; 1896 } 1897 1898 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 1899 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( 1900 struct uverbs_attr_bundle *attrs) 1901 { 1902 struct mlx5_flow_context flow_context = {.flow_tag = 1903 MLX5_FS_DEFAULT_FLOW_TAG}; 1904 u32 *offset_attr, offset = 0, counter_id = 0; 1905 int dest_id, dest_type = -1, inlen, len, ret, i; 1906 struct mlx5_ib_flow_handler *flow_handler; 1907 struct mlx5_ib_flow_matcher *fs_matcher; 1908 struct ib_uobject **arr_flow_actions; 1909 struct ib_uflow_resources *uflow_res; 1910 struct mlx5_flow_act flow_act = {}; 1911 struct ib_qp *qp = NULL; 1912 void *devx_obj, *cmd_in; 1913 struct ib_uobject *uobj; 1914 struct mlx5_ib_dev *dev; 1915 u32 flags; 1916 1917 if (!capable(CAP_NET_RAW)) 1918 return -EPERM; 1919 1920 fs_matcher = uverbs_attr_get_obj(attrs, 1921 MLX5_IB_ATTR_CREATE_FLOW_MATCHER); 1922 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); 1923 dev = mlx5_udata_to_mdev(&attrs->driver_udata); 1924 1925 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) 1926 return -EINVAL; 1927 1928 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) 1929 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; 1930 1931 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 1932 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 1933 1934 len = uverbs_attr_get_uobjs_arr(attrs, 1935 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); 1936 if (len) { 1937 devx_obj = arr_flow_actions[0]->object; 1938 1939 if (uverbs_attr_is_valid(attrs, 1940 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { 1941 1942 int num_offsets = uverbs_attr_ptr_get_array_size( 1943 attrs, 1944 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 1945 sizeof(u32)); 1946 1947 if (num_offsets != 1) 1948 return -EINVAL; 1949 1950 offset_attr = uverbs_attr_get_alloced_ptr( 1951 attrs, 1952 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); 1953 offset = *offset_attr; 1954 } 1955 1956 if (!is_flow_counter(devx_obj, offset, &counter_id)) 1957 return -EINVAL; 1958 1959 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 1960 } 1961 1962 cmd_in = uverbs_attr_get_alloced_ptr( 1963 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 1964 inlen = uverbs_attr_get_len(attrs, 1965 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 1966 1967 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); 1968 if (!uflow_res) 1969 return -ENOMEM; 1970 1971 len = uverbs_attr_get_uobjs_arr(attrs, 1972 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); 1973 for (i = 0; i < len; i++) { 1974 struct mlx5_ib_flow_action *maction = 1975 to_mflow_act(arr_flow_actions[i]->object); 1976 1977 ret = parse_flow_flow_action(maction, false, &flow_act); 1978 if (ret) 1979 goto err_out; 1980 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, 1981 arr_flow_actions[i]->object); 1982 } 1983 1984 ret = uverbs_copy_from(&flow_context.flow_tag, attrs, 1985 MLX5_IB_ATTR_CREATE_FLOW_TAG); 1986 if (!ret) { 1987 if (flow_context.flow_tag >= BIT(24)) { 1988 ret = -EINVAL; 1989 goto err_out; 1990 } 1991 flow_context.flags |= FLOW_CONTEXT_HAS_TAG; 1992 } 1993 1994 flow_handler = 1995 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act, 1996 counter_id, cmd_in, inlen, dest_id, dest_type); 1997 if (IS_ERR(flow_handler)) { 1998 ret = PTR_ERR(flow_handler); 1999 goto err_out; 2000 } 2001 2002 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); 2003 2004 return 0; 2005 err_out: 2006 ib_uverbs_flow_resources_free(uflow_res); 2007 return ret; 2008 } 2009 2010 static int flow_matcher_cleanup(struct ib_uobject *uobject, 2011 enum rdma_remove_reason why, 2012 struct uverbs_attr_bundle *attrs) 2013 { 2014 struct mlx5_ib_flow_matcher *obj = uobject->object; 2015 2016 if (atomic_read(&obj->usecnt)) 2017 return -EBUSY; 2018 2019 kfree(obj); 2020 return 0; 2021 } 2022 2023 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, 2024 struct mlx5_ib_flow_matcher *obj) 2025 { 2026 enum mlx5_ib_uapi_flow_table_type ft_type = 2027 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; 2028 u32 flags; 2029 int err; 2030 2031 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older 2032 * users should switch to it. We leave this to not break userspace 2033 */ 2034 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && 2035 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) 2036 return -EINVAL; 2037 2038 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { 2039 err = uverbs_get_const(&ft_type, attrs, 2040 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); 2041 if (err) 2042 return err; 2043 2044 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); 2045 if (err) 2046 return err; 2047 2048 return 0; 2049 } 2050 2051 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { 2052 err = uverbs_get_flags32(&flags, attrs, 2053 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2054 IB_FLOW_ATTR_FLAGS_EGRESS); 2055 if (err) 2056 return err; 2057 2058 if (flags) { 2059 mlx5_ib_ft_type_to_namespace( 2060 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, 2061 &obj->ns_type); 2062 return 0; 2063 } 2064 } 2065 2066 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; 2067 2068 return 0; 2069 } 2070 2071 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( 2072 struct uverbs_attr_bundle *attrs) 2073 { 2074 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2075 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); 2076 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2077 struct mlx5_ib_flow_matcher *obj; 2078 int err; 2079 2080 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); 2081 if (!obj) 2082 return -ENOMEM; 2083 2084 obj->mask_len = uverbs_attr_get_len( 2085 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2086 err = uverbs_copy_from(&obj->matcher_mask, 2087 attrs, 2088 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2089 if (err) 2090 goto end; 2091 2092 obj->flow_type = uverbs_attr_get_enum_id( 2093 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2094 2095 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { 2096 err = uverbs_copy_from(&obj->priority, 2097 attrs, 2098 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2099 if (err) 2100 goto end; 2101 } 2102 2103 err = uverbs_copy_from(&obj->match_criteria_enable, 2104 attrs, 2105 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); 2106 if (err) 2107 goto end; 2108 2109 err = mlx5_ib_matcher_ns(attrs, obj); 2110 if (err) 2111 goto end; 2112 2113 if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 2114 mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) { 2115 err = -EINVAL; 2116 goto end; 2117 } 2118 2119 uobj->object = obj; 2120 obj->mdev = dev->mdev; 2121 atomic_set(&obj->usecnt, 0); 2122 return 0; 2123 2124 end: 2125 kfree(obj); 2126 return err; 2127 } 2128 2129 static struct ib_flow_action * 2130 mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, 2131 enum mlx5_ib_uapi_flow_table_type ft_type, 2132 u8 num_actions, void *in) 2133 { 2134 enum mlx5_flow_namespace_type namespace; 2135 struct mlx5_ib_flow_action *maction; 2136 int ret; 2137 2138 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 2139 if (ret) 2140 return ERR_PTR(-EINVAL); 2141 2142 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 2143 if (!maction) 2144 return ERR_PTR(-ENOMEM); 2145 2146 maction->flow_action_raw.modify_hdr = 2147 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); 2148 2149 if (IS_ERR(maction->flow_action_raw.modify_hdr)) { 2150 ret = PTR_ERR(maction->flow_action_raw.modify_hdr); 2151 kfree(maction); 2152 return ERR_PTR(ret); 2153 } 2154 maction->flow_action_raw.sub_type = 2155 MLX5_IB_FLOW_ACTION_MODIFY_HEADER; 2156 maction->flow_action_raw.dev = dev; 2157 2158 return &maction->ib_action; 2159 } 2160 2161 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) 2162 { 2163 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 2164 max_modify_header_actions) || 2165 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 2166 max_modify_header_actions) || 2167 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 2168 max_modify_header_actions); 2169 } 2170 2171 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( 2172 struct uverbs_attr_bundle *attrs) 2173 { 2174 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2175 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); 2176 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 2177 enum mlx5_ib_uapi_flow_table_type ft_type; 2178 struct ib_flow_action *action; 2179 int num_actions; 2180 void *in; 2181 int ret; 2182 2183 if (!mlx5_ib_modify_header_supported(mdev)) 2184 return -EOPNOTSUPP; 2185 2186 in = uverbs_attr_get_alloced_ptr(attrs, 2187 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); 2188 2189 num_actions = uverbs_attr_ptr_get_array_size( 2190 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 2191 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); 2192 if (num_actions < 0) 2193 return num_actions; 2194 2195 ret = uverbs_get_const(&ft_type, attrs, 2196 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); 2197 if (ret) 2198 return ret; 2199 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); 2200 if (IS_ERR(action)) 2201 return PTR_ERR(action); 2202 2203 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, 2204 IB_FLOW_ACTION_UNSPECIFIED); 2205 2206 return 0; 2207 } 2208 2209 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, 2210 u8 packet_reformat_type, 2211 u8 ft_type) 2212 { 2213 switch (packet_reformat_type) { 2214 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 2215 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 2216 return MLX5_CAP_FLOWTABLE(ibdev->mdev, 2217 encap_general_header); 2218 break; 2219 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 2220 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 2221 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, 2222 reformat_l2_to_l3_tunnel); 2223 break; 2224 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 2225 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 2226 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, 2227 reformat_l3_tunnel_to_l2); 2228 break; 2229 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: 2230 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 2231 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); 2232 break; 2233 default: 2234 break; 2235 } 2236 2237 return false; 2238 } 2239 2240 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) 2241 { 2242 switch (dv_prt) { 2243 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 2244 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; 2245 break; 2246 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 2247 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 2248 break; 2249 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 2250 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 2251 break; 2252 default: 2253 return -EINVAL; 2254 } 2255 2256 return 0; 2257 } 2258 2259 static int mlx5_ib_flow_action_create_packet_reformat_ctx( 2260 struct mlx5_ib_dev *dev, 2261 struct mlx5_ib_flow_action *maction, 2262 u8 ft_type, u8 dv_prt, 2263 void *in, size_t len) 2264 { 2265 struct mlx5_pkt_reformat_params reformat_params; 2266 enum mlx5_flow_namespace_type namespace; 2267 u8 prm_prt; 2268 int ret; 2269 2270 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 2271 if (ret) 2272 return ret; 2273 2274 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); 2275 if (ret) 2276 return ret; 2277 2278 memset(&reformat_params, 0, sizeof(reformat_params)); 2279 reformat_params.type = prm_prt; 2280 reformat_params.size = len; 2281 reformat_params.data = in; 2282 maction->flow_action_raw.pkt_reformat = 2283 mlx5_packet_reformat_alloc(dev->mdev, &reformat_params, 2284 namespace); 2285 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { 2286 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); 2287 return ret; 2288 } 2289 2290 maction->flow_action_raw.sub_type = 2291 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; 2292 maction->flow_action_raw.dev = dev; 2293 2294 return 0; 2295 } 2296 2297 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( 2298 struct uverbs_attr_bundle *attrs) 2299 { 2300 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 2301 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); 2302 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 2303 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; 2304 enum mlx5_ib_uapi_flow_table_type ft_type; 2305 struct mlx5_ib_flow_action *maction; 2306 int ret; 2307 2308 ret = uverbs_get_const(&ft_type, attrs, 2309 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); 2310 if (ret) 2311 return ret; 2312 2313 ret = uverbs_get_const(&dv_prt, attrs, 2314 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); 2315 if (ret) 2316 return ret; 2317 2318 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) 2319 return -EOPNOTSUPP; 2320 2321 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 2322 if (!maction) 2323 return -ENOMEM; 2324 2325 if (dv_prt == 2326 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { 2327 maction->flow_action_raw.sub_type = 2328 MLX5_IB_FLOW_ACTION_DECAP; 2329 maction->flow_action_raw.dev = mdev; 2330 } else { 2331 void *in; 2332 int len; 2333 2334 in = uverbs_attr_get_alloced_ptr(attrs, 2335 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 2336 if (IS_ERR(in)) { 2337 ret = PTR_ERR(in); 2338 goto free_maction; 2339 } 2340 2341 len = uverbs_attr_get_len(attrs, 2342 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 2343 2344 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, 2345 maction, ft_type, dv_prt, in, len); 2346 if (ret) 2347 goto free_maction; 2348 } 2349 2350 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, 2351 IB_FLOW_ACTION_UNSPECIFIED); 2352 return 0; 2353 2354 free_maction: 2355 kfree(maction); 2356 return ret; 2357 } 2358 2359 DECLARE_UVERBS_NAMED_METHOD( 2360 MLX5_IB_METHOD_CREATE_FLOW, 2361 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 2362 UVERBS_OBJECT_FLOW, 2363 UVERBS_ACCESS_NEW, 2364 UA_MANDATORY), 2365 UVERBS_ATTR_PTR_IN( 2366 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, 2367 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 2368 UA_MANDATORY, 2369 UA_ALLOC_AND_COPY), 2370 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, 2371 MLX5_IB_OBJECT_FLOW_MATCHER, 2372 UVERBS_ACCESS_READ, 2373 UA_MANDATORY), 2374 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, 2375 UVERBS_OBJECT_QP, 2376 UVERBS_ACCESS_READ), 2377 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, 2378 MLX5_IB_OBJECT_DEVX_OBJ, 2379 UVERBS_ACCESS_READ), 2380 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, 2381 UVERBS_OBJECT_FLOW_ACTION, 2382 UVERBS_ACCESS_READ, 1, 2383 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, 2384 UA_OPTIONAL), 2385 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, 2386 UVERBS_ATTR_TYPE(u32), 2387 UA_OPTIONAL), 2388 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, 2389 MLX5_IB_OBJECT_DEVX_OBJ, 2390 UVERBS_ACCESS_READ, 1, 1, 2391 UA_OPTIONAL), 2392 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 2393 UVERBS_ATTR_MIN_SIZE(sizeof(u32)), 2394 UA_OPTIONAL, 2395 UA_ALLOC_AND_COPY), 2396 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 2397 enum mlx5_ib_create_flow_flags, 2398 UA_OPTIONAL)); 2399 2400 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 2401 MLX5_IB_METHOD_DESTROY_FLOW, 2402 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 2403 UVERBS_OBJECT_FLOW, 2404 UVERBS_ACCESS_DESTROY, 2405 UA_MANDATORY)); 2406 2407 ADD_UVERBS_METHODS(mlx5_ib_fs, 2408 UVERBS_OBJECT_FLOW, 2409 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), 2410 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); 2411 2412 DECLARE_UVERBS_NAMED_METHOD( 2413 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, 2414 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, 2415 UVERBS_OBJECT_FLOW_ACTION, 2416 UVERBS_ACCESS_NEW, 2417 UA_MANDATORY), 2418 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 2419 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( 2420 set_add_copy_action_in_auto)), 2421 UA_MANDATORY, 2422 UA_ALLOC_AND_COPY), 2423 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, 2424 enum mlx5_ib_uapi_flow_table_type, 2425 UA_MANDATORY)); 2426 2427 DECLARE_UVERBS_NAMED_METHOD( 2428 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, 2429 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, 2430 UVERBS_OBJECT_FLOW_ACTION, 2431 UVERBS_ACCESS_NEW, 2432 UA_MANDATORY), 2433 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, 2434 UVERBS_ATTR_MIN_SIZE(1), 2435 UA_ALLOC_AND_COPY, 2436 UA_OPTIONAL), 2437 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, 2438 enum mlx5_ib_uapi_flow_action_packet_reformat_type, 2439 UA_MANDATORY), 2440 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, 2441 enum mlx5_ib_uapi_flow_table_type, 2442 UA_MANDATORY)); 2443 2444 ADD_UVERBS_METHODS( 2445 mlx5_ib_flow_actions, 2446 UVERBS_OBJECT_FLOW_ACTION, 2447 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), 2448 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); 2449 2450 DECLARE_UVERBS_NAMED_METHOD( 2451 MLX5_IB_METHOD_FLOW_MATCHER_CREATE, 2452 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, 2453 MLX5_IB_OBJECT_FLOW_MATCHER, 2454 UVERBS_ACCESS_NEW, 2455 UA_MANDATORY), 2456 UVERBS_ATTR_PTR_IN( 2457 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, 2458 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 2459 UA_MANDATORY), 2460 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, 2461 mlx5_ib_flow_type, 2462 UA_MANDATORY), 2463 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, 2464 UVERBS_ATTR_TYPE(u8), 2465 UA_MANDATORY), 2466 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2467 enum ib_flow_flags, 2468 UA_OPTIONAL), 2469 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, 2470 enum mlx5_ib_uapi_flow_table_type, 2471 UA_OPTIONAL)); 2472 2473 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 2474 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, 2475 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, 2476 MLX5_IB_OBJECT_FLOW_MATCHER, 2477 UVERBS_ACCESS_DESTROY, 2478 UA_MANDATORY)); 2479 2480 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, 2481 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), 2482 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), 2483 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); 2484 2485 const struct uapi_definition mlx5_ib_flow_defs[] = { 2486 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 2487 MLX5_IB_OBJECT_FLOW_MATCHER), 2488 UAPI_DEF_CHAIN_OBJ_TREE( 2489 UVERBS_OBJECT_FLOW, 2490 &mlx5_ib_fs), 2491 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, 2492 &mlx5_ib_flow_actions), 2493 {}, 2494 }; 2495 2496 static const struct ib_device_ops flow_ops = { 2497 .create_flow = mlx5_ib_create_flow, 2498 .destroy_flow = mlx5_ib_destroy_flow, 2499 .destroy_flow_action = mlx5_ib_destroy_flow_action, 2500 }; 2501 2502 int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) 2503 { 2504 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); 2505 2506 if (!dev->flow_db) 2507 return -ENOMEM; 2508 2509 mutex_init(&dev->flow_db->lock); 2510 2511 ib_set_device_ops(&dev->ib_dev, &flow_ops); 2512 return 0; 2513 } 2514