1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* Copyright (c) 2018 Mellanox Technologies. */ 3 4 #include <net/vxlan.h> 5 #include <net/gre.h> 6 #include <net/geneve.h> 7 #include <net/bareudp.h> 8 #include "en/tc_tun.h" 9 #include "en/tc_priv.h" 10 #include "en_tc.h" 11 #include "rep/tc.h" 12 #include "rep/neigh.h" 13 #include "lag/lag.h" 14 #include "lag/mp.h" 15 16 struct mlx5e_tc_tun_route_attr { 17 struct net_device *out_dev; 18 struct net_device *route_dev; 19 union { 20 struct flowi4 fl4; 21 struct flowi6 fl6; 22 } fl; 23 struct neighbour *n; 24 u8 ttl; 25 }; 26 27 #define TC_TUN_ROUTE_ATTR_INIT(name) struct mlx5e_tc_tun_route_attr name = {} 28 29 static void mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr *attr) 30 { 31 if (attr->n) 32 neigh_release(attr->n); 33 if (attr->route_dev) 34 dev_put(attr->route_dev); 35 } 36 37 struct mlx5e_tc_tunnel *mlx5e_get_tc_tun(struct net_device *tunnel_dev) 38 { 39 if (netif_is_vxlan(tunnel_dev)) 40 return &vxlan_tunnel; 41 else if (netif_is_geneve(tunnel_dev)) 42 return &geneve_tunnel; 43 else if (netif_is_gretap(tunnel_dev) || 44 netif_is_ip6gretap(tunnel_dev)) 45 return &gre_tunnel; 46 else if (netif_is_bareudp(tunnel_dev)) 47 return &mplsoudp_tunnel; 48 else 49 return NULL; 50 } 51 52 static int get_route_and_out_devs(struct mlx5e_priv *priv, 53 struct net_device *dev, 54 struct net_device **route_dev, 55 struct net_device **out_dev) 56 { 57 struct net_device *uplink_dev, *uplink_upper, *real_dev; 58 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 59 bool dst_is_lag_dev; 60 61 real_dev = is_vlan_dev(dev) ? vlan_dev_real_dev(dev) : dev; 62 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH); 63 64 rcu_read_lock(); 65 uplink_upper = netdev_master_upper_dev_get_rcu(uplink_dev); 66 /* mlx5_lag_is_sriov() is a blocking function which can't be called 67 * while holding rcu read lock. Take the net_device for correctness 68 * sake. 69 */ 70 if (uplink_upper) 71 dev_hold(uplink_upper); 72 rcu_read_unlock(); 73 74 dst_is_lag_dev = (uplink_upper && 75 netif_is_lag_master(uplink_upper) && 76 real_dev == uplink_upper && 77 mlx5_lag_is_sriov(priv->mdev)); 78 if (uplink_upper) 79 dev_put(uplink_upper); 80 81 /* if the egress device isn't on the same HW e-switch or 82 * it's a LAG device, use the uplink 83 */ 84 *route_dev = dev; 85 if (!netdev_port_same_parent_id(priv->netdev, real_dev) || 86 dst_is_lag_dev || is_vlan_dev(*route_dev) || 87 netif_is_ovs_master(*route_dev)) 88 *out_dev = uplink_dev; 89 else if (mlx5e_eswitch_rep(dev) && 90 mlx5e_is_valid_eswitch_fwd_dev(priv, dev)) 91 *out_dev = *route_dev; 92 else 93 return -EOPNOTSUPP; 94 95 if (!(mlx5e_eswitch_rep(*out_dev) && 96 mlx5e_is_uplink_rep(netdev_priv(*out_dev)))) 97 return -EOPNOTSUPP; 98 99 if (mlx5e_eswitch_uplink_rep(priv->netdev) && *out_dev != priv->netdev) 100 return -EOPNOTSUPP; 101 102 return 0; 103 } 104 105 static int mlx5e_route_lookup_ipv4_get(struct mlx5e_priv *priv, 106 struct net_device *mirred_dev, 107 struct mlx5e_tc_tun_route_attr *attr) 108 { 109 struct net_device *route_dev; 110 struct net_device *out_dev; 111 struct neighbour *n; 112 struct rtable *rt; 113 114 #if IS_ENABLED(CONFIG_INET) 115 struct mlx5_core_dev *mdev = priv->mdev; 116 struct net_device *uplink_dev; 117 int ret; 118 119 if (mlx5_lag_is_multipath(mdev)) { 120 struct mlx5_eswitch *esw = mdev->priv.eswitch; 121 122 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH); 123 attr->fl.fl4.flowi4_oif = uplink_dev->ifindex; 124 } else { 125 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(mirred_dev); 126 127 if (tunnel && tunnel->get_remote_ifindex) 128 attr->fl.fl4.flowi4_oif = tunnel->get_remote_ifindex(mirred_dev); 129 } 130 131 rt = ip_route_output_key(dev_net(mirred_dev), &attr->fl.fl4); 132 if (IS_ERR(rt)) 133 return PTR_ERR(rt); 134 135 if (rt->rt_type != RTN_UNICAST) { 136 ret = -ENETUNREACH; 137 goto err_rt_release; 138 } 139 140 if (mlx5_lag_is_multipath(mdev) && rt->rt_gw_family != AF_INET) { 141 ret = -ENETUNREACH; 142 goto err_rt_release; 143 } 144 #else 145 return -EOPNOTSUPP; 146 #endif 147 148 ret = get_route_and_out_devs(priv, rt->dst.dev, &route_dev, &out_dev); 149 if (ret < 0) 150 goto err_rt_release; 151 dev_hold(route_dev); 152 153 if (!attr->ttl) 154 attr->ttl = ip4_dst_hoplimit(&rt->dst); 155 n = dst_neigh_lookup(&rt->dst, &attr->fl.fl4.daddr); 156 if (!n) { 157 ret = -ENOMEM; 158 goto err_dev_release; 159 } 160 161 ip_rt_put(rt); 162 attr->route_dev = route_dev; 163 attr->out_dev = out_dev; 164 attr->n = n; 165 return 0; 166 167 err_dev_release: 168 dev_put(route_dev); 169 err_rt_release: 170 ip_rt_put(rt); 171 return ret; 172 } 173 174 static void mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr *attr) 175 { 176 mlx5e_tc_tun_route_attr_cleanup(attr); 177 } 178 179 static const char *mlx5e_netdev_kind(struct net_device *dev) 180 { 181 if (dev->rtnl_link_ops) 182 return dev->rtnl_link_ops->kind; 183 else 184 return "unknown"; 185 } 186 187 static int mlx5e_gen_ip_tunnel_header(char buf[], __u8 *ip_proto, 188 struct mlx5e_encap_entry *e) 189 { 190 if (!e->tunnel) { 191 pr_warn("mlx5: Cannot generate tunnel header for this tunnel\n"); 192 return -EOPNOTSUPP; 193 } 194 195 return e->tunnel->generate_ip_tun_hdr(buf, ip_proto, e); 196 } 197 198 static char *gen_eth_tnl_hdr(char *buf, struct net_device *dev, 199 struct mlx5e_encap_entry *e, 200 u16 proto) 201 { 202 struct ethhdr *eth = (struct ethhdr *)buf; 203 char *ip; 204 205 ether_addr_copy(eth->h_dest, e->h_dest); 206 ether_addr_copy(eth->h_source, dev->dev_addr); 207 if (is_vlan_dev(dev)) { 208 struct vlan_hdr *vlan = (struct vlan_hdr *) 209 ((char *)eth + ETH_HLEN); 210 ip = (char *)vlan + VLAN_HLEN; 211 eth->h_proto = vlan_dev_vlan_proto(dev); 212 vlan->h_vlan_TCI = htons(vlan_dev_vlan_id(dev)); 213 vlan->h_vlan_encapsulated_proto = htons(proto); 214 } else { 215 eth->h_proto = htons(proto); 216 ip = (char *)eth + ETH_HLEN; 217 } 218 219 return ip; 220 } 221 222 int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv, 223 struct net_device *mirred_dev, 224 struct mlx5e_encap_entry *e) 225 { 226 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 227 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 228 struct mlx5_pkt_reformat_params reformat_params; 229 struct mlx5e_neigh m_neigh = {}; 230 TC_TUN_ROUTE_ATTR_INIT(attr); 231 int ipv4_encap_size; 232 char *encap_header; 233 struct iphdr *ip; 234 u8 nud_state; 235 int err; 236 237 /* add the IP fields */ 238 attr.fl.fl4.flowi4_tos = tun_key->tos; 239 attr.fl.fl4.daddr = tun_key->u.ipv4.dst; 240 attr.fl.fl4.saddr = tun_key->u.ipv4.src; 241 attr.ttl = tun_key->ttl; 242 243 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr); 244 if (err) 245 return err; 246 247 ipv4_encap_size = 248 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 249 sizeof(struct iphdr) + 250 e->tunnel->calc_hlen(e); 251 252 if (max_encap_size < ipv4_encap_size) { 253 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 254 ipv4_encap_size, max_encap_size); 255 err = -EOPNOTSUPP; 256 goto release_neigh; 257 } 258 259 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL); 260 if (!encap_header) { 261 err = -ENOMEM; 262 goto release_neigh; 263 } 264 265 m_neigh.family = attr.n->ops->family; 266 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len); 267 e->out_dev = attr.out_dev; 268 e->route_dev_ifindex = attr.route_dev->ifindex; 269 270 /* It's important to add the neigh to the hash table before checking 271 * the neigh validity state. So if we'll get a notification, in case the 272 * neigh changes it's validity state, we would find the relevant neigh 273 * in the hash. 274 */ 275 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev); 276 if (err) 277 goto free_encap; 278 279 read_lock_bh(&attr.n->lock); 280 nud_state = attr.n->nud_state; 281 ether_addr_copy(e->h_dest, attr.n->ha); 282 read_unlock_bh(&attr.n->lock); 283 284 /* add ethernet header */ 285 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 286 ETH_P_IP); 287 288 /* add ip header */ 289 ip->tos = tun_key->tos; 290 ip->version = 0x4; 291 ip->ihl = 0x5; 292 ip->ttl = attr.ttl; 293 ip->daddr = attr.fl.fl4.daddr; 294 ip->saddr = attr.fl.fl4.saddr; 295 296 /* add tunneling protocol header */ 297 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr), 298 &ip->protocol, e); 299 if (err) 300 goto destroy_neigh_entry; 301 302 e->encap_size = ipv4_encap_size; 303 e->encap_header = encap_header; 304 305 if (!(nud_state & NUD_VALID)) { 306 neigh_event_send(attr.n, NULL); 307 /* the encap entry will be made valid on neigh update event 308 * and not used before that. 309 */ 310 goto release_neigh; 311 } 312 313 memset(&reformat_params, 0, sizeof(reformat_params)); 314 reformat_params.type = e->reformat_type; 315 reformat_params.size = ipv4_encap_size; 316 reformat_params.data = encap_header; 317 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, 318 MLX5_FLOW_NAMESPACE_FDB); 319 if (IS_ERR(e->pkt_reformat)) { 320 err = PTR_ERR(e->pkt_reformat); 321 goto destroy_neigh_entry; 322 } 323 324 e->flags |= MLX5_ENCAP_ENTRY_VALID; 325 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 326 mlx5e_route_lookup_ipv4_put(&attr); 327 return err; 328 329 destroy_neigh_entry: 330 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e); 331 free_encap: 332 kfree(encap_header); 333 release_neigh: 334 mlx5e_route_lookup_ipv4_put(&attr); 335 return err; 336 } 337 338 int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv, 339 struct net_device *mirred_dev, 340 struct mlx5e_encap_entry *e) 341 { 342 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 343 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 344 struct mlx5_pkt_reformat_params reformat_params; 345 TC_TUN_ROUTE_ATTR_INIT(attr); 346 int ipv4_encap_size; 347 char *encap_header; 348 struct iphdr *ip; 349 u8 nud_state; 350 int err; 351 352 /* add the IP fields */ 353 attr.fl.fl4.flowi4_tos = tun_key->tos; 354 attr.fl.fl4.daddr = tun_key->u.ipv4.dst; 355 attr.fl.fl4.saddr = tun_key->u.ipv4.src; 356 attr.ttl = tun_key->ttl; 357 358 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr); 359 if (err) 360 return err; 361 362 ipv4_encap_size = 363 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 364 sizeof(struct iphdr) + 365 e->tunnel->calc_hlen(e); 366 367 if (max_encap_size < ipv4_encap_size) { 368 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 369 ipv4_encap_size, max_encap_size); 370 err = -EOPNOTSUPP; 371 goto release_neigh; 372 } 373 374 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL); 375 if (!encap_header) { 376 err = -ENOMEM; 377 goto release_neigh; 378 } 379 380 e->route_dev_ifindex = attr.route_dev->ifindex; 381 382 read_lock_bh(&attr.n->lock); 383 nud_state = attr.n->nud_state; 384 ether_addr_copy(e->h_dest, attr.n->ha); 385 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev); 386 read_unlock_bh(&attr.n->lock); 387 388 /* add ethernet header */ 389 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 390 ETH_P_IP); 391 392 /* add ip header */ 393 ip->tos = tun_key->tos; 394 ip->version = 0x4; 395 ip->ihl = 0x5; 396 ip->ttl = attr.ttl; 397 ip->daddr = attr.fl.fl4.daddr; 398 ip->saddr = attr.fl.fl4.saddr; 399 400 /* add tunneling protocol header */ 401 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr), 402 &ip->protocol, e); 403 if (err) 404 goto free_encap; 405 406 e->encap_size = ipv4_encap_size; 407 kfree(e->encap_header); 408 e->encap_header = encap_header; 409 410 if (!(nud_state & NUD_VALID)) { 411 neigh_event_send(attr.n, NULL); 412 /* the encap entry will be made valid on neigh update event 413 * and not used before that. 414 */ 415 goto release_neigh; 416 } 417 418 memset(&reformat_params, 0, sizeof(reformat_params)); 419 reformat_params.type = e->reformat_type; 420 reformat_params.size = ipv4_encap_size; 421 reformat_params.data = encap_header; 422 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, 423 MLX5_FLOW_NAMESPACE_FDB); 424 if (IS_ERR(e->pkt_reformat)) { 425 err = PTR_ERR(e->pkt_reformat); 426 goto free_encap; 427 } 428 429 e->flags |= MLX5_ENCAP_ENTRY_VALID; 430 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 431 mlx5e_route_lookup_ipv4_put(&attr); 432 return err; 433 434 free_encap: 435 kfree(encap_header); 436 release_neigh: 437 mlx5e_route_lookup_ipv4_put(&attr); 438 return err; 439 } 440 441 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) 442 static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv, 443 struct net_device *mirred_dev, 444 struct mlx5e_tc_tun_route_attr *attr) 445 { 446 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(mirred_dev); 447 struct net_device *route_dev; 448 struct net_device *out_dev; 449 struct dst_entry *dst; 450 struct neighbour *n; 451 int ret; 452 453 if (tunnel && tunnel->get_remote_ifindex) 454 attr->fl.fl6.flowi6_oif = tunnel->get_remote_ifindex(mirred_dev); 455 dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(mirred_dev), NULL, &attr->fl.fl6, 456 NULL); 457 if (IS_ERR(dst)) 458 return PTR_ERR(dst); 459 460 if (!attr->ttl) 461 attr->ttl = ip6_dst_hoplimit(dst); 462 463 ret = get_route_and_out_devs(priv, dst->dev, &route_dev, &out_dev); 464 if (ret < 0) 465 goto err_dst_release; 466 467 dev_hold(route_dev); 468 n = dst_neigh_lookup(dst, &attr->fl.fl6.daddr); 469 if (!n) { 470 ret = -ENOMEM; 471 goto err_dev_release; 472 } 473 474 dst_release(dst); 475 attr->out_dev = out_dev; 476 attr->route_dev = route_dev; 477 attr->n = n; 478 return 0; 479 480 err_dev_release: 481 dev_put(route_dev); 482 err_dst_release: 483 dst_release(dst); 484 return ret; 485 } 486 487 static void mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr *attr) 488 { 489 mlx5e_tc_tun_route_attr_cleanup(attr); 490 } 491 492 int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv, 493 struct net_device *mirred_dev, 494 struct mlx5e_encap_entry *e) 495 { 496 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 497 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 498 struct mlx5_pkt_reformat_params reformat_params; 499 struct mlx5e_neigh m_neigh = {}; 500 TC_TUN_ROUTE_ATTR_INIT(attr); 501 struct ipv6hdr *ip6h; 502 int ipv6_encap_size; 503 char *encap_header; 504 u8 nud_state; 505 int err; 506 507 attr.ttl = tun_key->ttl; 508 attr.fl.fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label); 509 attr.fl.fl6.daddr = tun_key->u.ipv6.dst; 510 attr.fl.fl6.saddr = tun_key->u.ipv6.src; 511 512 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr); 513 if (err) 514 return err; 515 516 ipv6_encap_size = 517 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 518 sizeof(struct ipv6hdr) + 519 e->tunnel->calc_hlen(e); 520 521 if (max_encap_size < ipv6_encap_size) { 522 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 523 ipv6_encap_size, max_encap_size); 524 err = -EOPNOTSUPP; 525 goto release_neigh; 526 } 527 528 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL); 529 if (!encap_header) { 530 err = -ENOMEM; 531 goto release_neigh; 532 } 533 534 m_neigh.family = attr.n->ops->family; 535 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len); 536 e->out_dev = attr.out_dev; 537 e->route_dev_ifindex = attr.route_dev->ifindex; 538 539 /* It's important to add the neigh to the hash table before checking 540 * the neigh validity state. So if we'll get a notification, in case the 541 * neigh changes it's validity state, we would find the relevant neigh 542 * in the hash. 543 */ 544 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev); 545 if (err) 546 goto free_encap; 547 548 read_lock_bh(&attr.n->lock); 549 nud_state = attr.n->nud_state; 550 ether_addr_copy(e->h_dest, attr.n->ha); 551 read_unlock_bh(&attr.n->lock); 552 553 /* add ethernet header */ 554 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 555 ETH_P_IPV6); 556 557 /* add ip header */ 558 ip6_flow_hdr(ip6h, tun_key->tos, 0); 559 /* the HW fills up ipv6 payload len */ 560 ip6h->hop_limit = attr.ttl; 561 ip6h->daddr = attr.fl.fl6.daddr; 562 ip6h->saddr = attr.fl.fl6.saddr; 563 564 /* add tunneling protocol header */ 565 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr), 566 &ip6h->nexthdr, e); 567 if (err) 568 goto destroy_neigh_entry; 569 570 e->encap_size = ipv6_encap_size; 571 e->encap_header = encap_header; 572 573 if (!(nud_state & NUD_VALID)) { 574 neigh_event_send(attr.n, NULL); 575 /* the encap entry will be made valid on neigh update event 576 * and not used before that. 577 */ 578 goto release_neigh; 579 } 580 581 memset(&reformat_params, 0, sizeof(reformat_params)); 582 reformat_params.type = e->reformat_type; 583 reformat_params.size = ipv6_encap_size; 584 reformat_params.data = encap_header; 585 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, 586 MLX5_FLOW_NAMESPACE_FDB); 587 if (IS_ERR(e->pkt_reformat)) { 588 err = PTR_ERR(e->pkt_reformat); 589 goto destroy_neigh_entry; 590 } 591 592 e->flags |= MLX5_ENCAP_ENTRY_VALID; 593 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 594 mlx5e_route_lookup_ipv6_put(&attr); 595 return err; 596 597 destroy_neigh_entry: 598 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e); 599 free_encap: 600 kfree(encap_header); 601 release_neigh: 602 mlx5e_route_lookup_ipv6_put(&attr); 603 return err; 604 } 605 606 int mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv, 607 struct net_device *mirred_dev, 608 struct mlx5e_encap_entry *e) 609 { 610 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 611 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 612 struct mlx5_pkt_reformat_params reformat_params; 613 TC_TUN_ROUTE_ATTR_INIT(attr); 614 struct ipv6hdr *ip6h; 615 int ipv6_encap_size; 616 char *encap_header; 617 u8 nud_state; 618 int err; 619 620 attr.ttl = tun_key->ttl; 621 622 attr.fl.fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label); 623 attr.fl.fl6.daddr = tun_key->u.ipv6.dst; 624 attr.fl.fl6.saddr = tun_key->u.ipv6.src; 625 626 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr); 627 if (err) 628 return err; 629 630 ipv6_encap_size = 631 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 632 sizeof(struct ipv6hdr) + 633 e->tunnel->calc_hlen(e); 634 635 if (max_encap_size < ipv6_encap_size) { 636 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 637 ipv6_encap_size, max_encap_size); 638 err = -EOPNOTSUPP; 639 goto release_neigh; 640 } 641 642 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL); 643 if (!encap_header) { 644 err = -ENOMEM; 645 goto release_neigh; 646 } 647 648 e->route_dev_ifindex = attr.route_dev->ifindex; 649 650 read_lock_bh(&attr.n->lock); 651 nud_state = attr.n->nud_state; 652 ether_addr_copy(e->h_dest, attr.n->ha); 653 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev); 654 read_unlock_bh(&attr.n->lock); 655 656 /* add ethernet header */ 657 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 658 ETH_P_IPV6); 659 660 /* add ip header */ 661 ip6_flow_hdr(ip6h, tun_key->tos, 0); 662 /* the HW fills up ipv6 payload len */ 663 ip6h->hop_limit = attr.ttl; 664 ip6h->daddr = attr.fl.fl6.daddr; 665 ip6h->saddr = attr.fl.fl6.saddr; 666 667 /* add tunneling protocol header */ 668 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr), 669 &ip6h->nexthdr, e); 670 if (err) 671 goto free_encap; 672 673 e->encap_size = ipv6_encap_size; 674 kfree(e->encap_header); 675 e->encap_header = encap_header; 676 677 if (!(nud_state & NUD_VALID)) { 678 neigh_event_send(attr.n, NULL); 679 /* the encap entry will be made valid on neigh update event 680 * and not used before that. 681 */ 682 goto release_neigh; 683 } 684 685 memset(&reformat_params, 0, sizeof(reformat_params)); 686 reformat_params.type = e->reformat_type; 687 reformat_params.size = ipv6_encap_size; 688 reformat_params.data = encap_header; 689 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, 690 MLX5_FLOW_NAMESPACE_FDB); 691 if (IS_ERR(e->pkt_reformat)) { 692 err = PTR_ERR(e->pkt_reformat); 693 goto free_encap; 694 } 695 696 e->flags |= MLX5_ENCAP_ENTRY_VALID; 697 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 698 mlx5e_route_lookup_ipv6_put(&attr); 699 return err; 700 701 free_encap: 702 kfree(encap_header); 703 release_neigh: 704 mlx5e_route_lookup_ipv6_put(&attr); 705 return err; 706 } 707 #endif 708 709 int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv, 710 struct mlx5_flow_spec *spec, 711 struct mlx5_flow_attr *flow_attr) 712 { 713 struct mlx5_esw_flow_attr *esw_attr = flow_attr->esw_attr; 714 struct mlx5e_tc_int_port *int_port; 715 TC_TUN_ROUTE_ATTR_INIT(attr); 716 u16 vport_num; 717 int err = 0; 718 719 if (flow_attr->tun_ip_version == 4) { 720 /* Addresses are swapped for decap */ 721 attr.fl.fl4.saddr = esw_attr->rx_tun_attr->dst_ip.v4; 722 attr.fl.fl4.daddr = esw_attr->rx_tun_attr->src_ip.v4; 723 err = mlx5e_route_lookup_ipv4_get(priv, priv->netdev, &attr); 724 } 725 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) 726 else if (flow_attr->tun_ip_version == 6) { 727 /* Addresses are swapped for decap */ 728 attr.fl.fl6.saddr = esw_attr->rx_tun_attr->dst_ip.v6; 729 attr.fl.fl6.daddr = esw_attr->rx_tun_attr->src_ip.v6; 730 err = mlx5e_route_lookup_ipv6_get(priv, priv->netdev, &attr); 731 } 732 #endif 733 else 734 return 0; 735 736 if (err) 737 return err; 738 739 if (attr.route_dev->netdev_ops == &mlx5e_netdev_ops && 740 mlx5e_tc_is_vf_tunnel(attr.out_dev, attr.route_dev)) { 741 err = mlx5e_tc_query_route_vport(attr.out_dev, attr.route_dev, &vport_num); 742 if (err) 743 goto out; 744 745 esw_attr->rx_tun_attr->vni = MLX5_GET(fte_match_param, spec->match_value, 746 misc_parameters.vxlan_vni); 747 esw_attr->rx_tun_attr->decap_vport = vport_num; 748 } else if (netif_is_ovs_master(attr.route_dev)) { 749 int_port = mlx5e_tc_int_port_get(mlx5e_get_int_port_priv(priv), 750 attr.route_dev->ifindex, 751 MLX5E_TC_INT_PORT_INGRESS); 752 if (IS_ERR(int_port)) { 753 err = PTR_ERR(int_port); 754 goto out; 755 } 756 esw_attr->int_port = int_port; 757 } 758 759 out: 760 if (flow_attr->tun_ip_version == 4) 761 mlx5e_route_lookup_ipv4_put(&attr); 762 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) 763 else if (flow_attr->tun_ip_version == 6) 764 mlx5e_route_lookup_ipv6_put(&attr); 765 #endif 766 return err; 767 } 768 769 bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv, 770 struct net_device *netdev) 771 { 772 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(netdev); 773 774 if (tunnel && tunnel->can_offload(priv)) 775 return true; 776 else 777 return false; 778 } 779 780 int mlx5e_tc_tun_init_encap_attr(struct net_device *tunnel_dev, 781 struct mlx5e_priv *priv, 782 struct mlx5e_encap_entry *e, 783 struct netlink_ext_ack *extack) 784 { 785 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(tunnel_dev); 786 787 if (!tunnel) { 788 e->reformat_type = -1; 789 return -EOPNOTSUPP; 790 } 791 792 return tunnel->init_encap_attr(tunnel_dev, priv, e, extack); 793 } 794 795 int mlx5e_tc_tun_parse(struct net_device *filter_dev, 796 struct mlx5e_priv *priv, 797 struct mlx5_flow_spec *spec, 798 struct flow_cls_offload *f, 799 u8 *match_level) 800 { 801 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(filter_dev); 802 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 803 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 804 outer_headers); 805 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 806 outer_headers); 807 struct netlink_ext_ack *extack = f->common.extack; 808 int err = 0; 809 810 if (!tunnel) { 811 netdev_warn(priv->netdev, 812 "decapsulation offload is not supported for %s net device\n", 813 mlx5e_netdev_kind(filter_dev)); 814 err = -EOPNOTSUPP; 815 goto out; 816 } 817 818 *match_level = tunnel->match_level; 819 820 if (tunnel->parse_udp_ports) { 821 err = tunnel->parse_udp_ports(priv, spec, f, 822 headers_c, headers_v); 823 if (err) 824 goto out; 825 } 826 827 if (tunnel->parse_tunnel) { 828 err = tunnel->parse_tunnel(priv, spec, f, 829 headers_c, headers_v); 830 if (err) 831 goto out; 832 } 833 834 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) { 835 struct flow_dissector_key_basic key_basic = {}; 836 struct flow_dissector_key_basic mask_basic = { 837 .n_proto = htons(0xFFFF), 838 }; 839 struct flow_match_basic match_basic = { 840 .key = &key_basic, .mask = &mask_basic, 841 }; 842 struct flow_match_control match; 843 u16 addr_type; 844 845 flow_rule_match_enc_control(rule, &match); 846 addr_type = match.key->addr_type; 847 848 /* For tunnel addr_type used same key id`s as for non-tunnel */ 849 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 850 struct flow_match_ipv4_addrs match; 851 852 flow_rule_match_enc_ipv4_addrs(rule, &match); 853 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 854 src_ipv4_src_ipv6.ipv4_layout.ipv4, 855 ntohl(match.mask->src)); 856 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 857 src_ipv4_src_ipv6.ipv4_layout.ipv4, 858 ntohl(match.key->src)); 859 860 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 861 dst_ipv4_dst_ipv6.ipv4_layout.ipv4, 862 ntohl(match.mask->dst)); 863 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 864 dst_ipv4_dst_ipv6.ipv4_layout.ipv4, 865 ntohl(match.key->dst)); 866 867 key_basic.n_proto = htons(ETH_P_IP); 868 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true, 869 headers_c, headers_v); 870 } else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 871 struct flow_match_ipv6_addrs match; 872 873 flow_rule_match_enc_ipv6_addrs(rule, &match); 874 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 875 src_ipv4_src_ipv6.ipv6_layout.ipv6), 876 &match.mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout, 877 ipv6)); 878 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 879 src_ipv4_src_ipv6.ipv6_layout.ipv6), 880 &match.key->src, MLX5_FLD_SZ_BYTES(ipv6_layout, 881 ipv6)); 882 883 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 884 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 885 &match.mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, 886 ipv6)); 887 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 888 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 889 &match.key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, 890 ipv6)); 891 892 key_basic.n_proto = htons(ETH_P_IPV6); 893 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true, 894 headers_c, headers_v); 895 } 896 } 897 898 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_IP)) { 899 struct flow_match_ip match; 900 901 flow_rule_match_enc_ip(rule, &match); 902 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn, 903 match.mask->tos & 0x3); 904 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, 905 match.key->tos & 0x3); 906 907 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp, 908 match.mask->tos >> 2); 909 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, 910 match.key->tos >> 2); 911 912 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit, 913 match.mask->ttl); 914 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit, 915 match.key->ttl); 916 917 if (match.mask->ttl && 918 !MLX5_CAP_ESW_FLOWTABLE_FDB 919 (priv->mdev, 920 ft_field_support.outer_ipv4_ttl)) { 921 NL_SET_ERR_MSG_MOD(extack, 922 "Matching on TTL is not supported"); 923 err = -EOPNOTSUPP; 924 goto out; 925 } 926 } 927 928 /* let software handle IP fragments */ 929 MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1); 930 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); 931 932 return 0; 933 934 out: 935 return err; 936 } 937 938 int mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv *priv, 939 struct mlx5_flow_spec *spec, 940 struct flow_cls_offload *f, 941 void *headers_c, 942 void *headers_v) 943 { 944 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 945 struct netlink_ext_ack *extack = f->common.extack; 946 struct flow_match_ports enc_ports; 947 948 /* Full udp dst port must be given */ 949 950 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_PORTS)) { 951 NL_SET_ERR_MSG_MOD(extack, 952 "UDP tunnel decap filter must include enc_dst_port condition"); 953 netdev_warn(priv->netdev, 954 "UDP tunnel decap filter must include enc_dst_port condition\n"); 955 return -EOPNOTSUPP; 956 } 957 958 flow_rule_match_enc_ports(rule, &enc_ports); 959 960 if (memchr_inv(&enc_ports.mask->dst, 0xff, 961 sizeof(enc_ports.mask->dst))) { 962 NL_SET_ERR_MSG_MOD(extack, 963 "UDP tunnel decap filter must match enc_dst_port fully"); 964 netdev_warn(priv->netdev, 965 "UDP tunnel decap filter must match enc_dst_port fully\n"); 966 return -EOPNOTSUPP; 967 } 968 969 /* match on UDP protocol and dst port number */ 970 971 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol); 972 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP); 973 974 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 975 ntohs(enc_ports.mask->dst)); 976 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 977 ntohs(enc_ports.key->dst)); 978 979 /* UDP src port on outer header is generated by HW, 980 * so it is probably a bad idea to request matching it. 981 * Nonetheless, it is allowed. 982 */ 983 984 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 985 ntohs(enc_ports.mask->src)); 986 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 987 ntohs(enc_ports.key->src)); 988 989 return 0; 990 } 991