1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* Copyright (c) 2018 Mellanox Technologies. */ 3 4 #include <net/vxlan.h> 5 #include <net/gre.h> 6 #include <net/geneve.h> 7 #include <net/bareudp.h> 8 #include "en/tc_tun.h" 9 #include "en/tc_priv.h" 10 #include "en_tc.h" 11 #include "rep/tc.h" 12 #include "rep/neigh.h" 13 14 struct mlx5e_tc_tun_route_attr { 15 struct net_device *out_dev; 16 struct net_device *route_dev; 17 union { 18 struct flowi4 fl4; 19 struct flowi6 fl6; 20 } fl; 21 struct neighbour *n; 22 u8 ttl; 23 }; 24 25 #define TC_TUN_ROUTE_ATTR_INIT(name) struct mlx5e_tc_tun_route_attr name = {} 26 27 static void mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr *attr) 28 { 29 if (attr->n) 30 neigh_release(attr->n); 31 if (attr->route_dev) 32 dev_put(attr->route_dev); 33 } 34 35 struct mlx5e_tc_tunnel *mlx5e_get_tc_tun(struct net_device *tunnel_dev) 36 { 37 if (netif_is_vxlan(tunnel_dev)) 38 return &vxlan_tunnel; 39 else if (netif_is_geneve(tunnel_dev)) 40 return &geneve_tunnel; 41 else if (netif_is_gretap(tunnel_dev) || 42 netif_is_ip6gretap(tunnel_dev)) 43 return &gre_tunnel; 44 else if (netif_is_bareudp(tunnel_dev)) 45 return &mplsoudp_tunnel; 46 else 47 return NULL; 48 } 49 50 static int get_route_and_out_devs(struct mlx5e_priv *priv, 51 struct net_device *dev, 52 struct net_device **route_dev, 53 struct net_device **out_dev) 54 { 55 struct net_device *uplink_dev, *uplink_upper, *real_dev; 56 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 57 bool dst_is_lag_dev; 58 59 real_dev = is_vlan_dev(dev) ? vlan_dev_real_dev(dev) : dev; 60 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH); 61 62 rcu_read_lock(); 63 uplink_upper = netdev_master_upper_dev_get_rcu(uplink_dev); 64 /* mlx5_lag_is_sriov() is a blocking function which can't be called 65 * while holding rcu read lock. Take the net_device for correctness 66 * sake. 67 */ 68 if (uplink_upper) 69 dev_hold(uplink_upper); 70 rcu_read_unlock(); 71 72 dst_is_lag_dev = (uplink_upper && 73 netif_is_lag_master(uplink_upper) && 74 real_dev == uplink_upper && 75 mlx5_lag_is_sriov(priv->mdev)); 76 if (uplink_upper) 77 dev_put(uplink_upper); 78 79 /* if the egress device isn't on the same HW e-switch or 80 * it's a LAG device, use the uplink 81 */ 82 *route_dev = dev; 83 if (!netdev_port_same_parent_id(priv->netdev, real_dev) || 84 dst_is_lag_dev || is_vlan_dev(*route_dev)) 85 *out_dev = uplink_dev; 86 else if (mlx5e_eswitch_rep(dev) && 87 mlx5e_is_valid_eswitch_fwd_dev(priv, dev)) 88 *out_dev = *route_dev; 89 else 90 return -EOPNOTSUPP; 91 92 if (!(mlx5e_eswitch_rep(*out_dev) && 93 mlx5e_is_uplink_rep(netdev_priv(*out_dev)))) 94 return -EOPNOTSUPP; 95 96 if (mlx5e_eswitch_uplink_rep(priv->netdev) && *out_dev != priv->netdev) 97 return -EOPNOTSUPP; 98 99 return 0; 100 } 101 102 static int mlx5e_route_lookup_ipv4_get(struct mlx5e_priv *priv, 103 struct net_device *mirred_dev, 104 struct mlx5e_tc_tun_route_attr *attr) 105 { 106 struct net_device *route_dev; 107 struct net_device *out_dev; 108 struct neighbour *n; 109 struct rtable *rt; 110 111 #if IS_ENABLED(CONFIG_INET) 112 struct mlx5_core_dev *mdev = priv->mdev; 113 struct net_device *uplink_dev; 114 int ret; 115 116 if (mlx5_lag_is_multipath(mdev)) { 117 struct mlx5_eswitch *esw = mdev->priv.eswitch; 118 119 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH); 120 attr->fl.fl4.flowi4_oif = uplink_dev->ifindex; 121 } 122 123 rt = ip_route_output_key(dev_net(mirred_dev), &attr->fl.fl4); 124 if (IS_ERR(rt)) 125 return PTR_ERR(rt); 126 127 if (mlx5_lag_is_multipath(mdev) && rt->rt_gw_family != AF_INET) { 128 ret = -ENETUNREACH; 129 goto err_rt_release; 130 } 131 #else 132 return -EOPNOTSUPP; 133 #endif 134 135 ret = get_route_and_out_devs(priv, rt->dst.dev, &route_dev, &out_dev); 136 if (ret < 0) 137 goto err_rt_release; 138 dev_hold(route_dev); 139 140 if (!attr->ttl) 141 attr->ttl = ip4_dst_hoplimit(&rt->dst); 142 n = dst_neigh_lookup(&rt->dst, &attr->fl.fl4.daddr); 143 if (!n) { 144 ret = -ENOMEM; 145 goto err_dev_release; 146 } 147 148 ip_rt_put(rt); 149 attr->route_dev = route_dev; 150 attr->out_dev = out_dev; 151 attr->n = n; 152 return 0; 153 154 err_dev_release: 155 dev_put(route_dev); 156 err_rt_release: 157 ip_rt_put(rt); 158 return ret; 159 } 160 161 static void mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr *attr) 162 { 163 mlx5e_tc_tun_route_attr_cleanup(attr); 164 } 165 166 static const char *mlx5e_netdev_kind(struct net_device *dev) 167 { 168 if (dev->rtnl_link_ops) 169 return dev->rtnl_link_ops->kind; 170 else 171 return "unknown"; 172 } 173 174 static int mlx5e_gen_ip_tunnel_header(char buf[], __u8 *ip_proto, 175 struct mlx5e_encap_entry *e) 176 { 177 if (!e->tunnel) { 178 pr_warn("mlx5: Cannot generate tunnel header for this tunnel\n"); 179 return -EOPNOTSUPP; 180 } 181 182 return e->tunnel->generate_ip_tun_hdr(buf, ip_proto, e); 183 } 184 185 static char *gen_eth_tnl_hdr(char *buf, struct net_device *dev, 186 struct mlx5e_encap_entry *e, 187 u16 proto) 188 { 189 struct ethhdr *eth = (struct ethhdr *)buf; 190 char *ip; 191 192 ether_addr_copy(eth->h_dest, e->h_dest); 193 ether_addr_copy(eth->h_source, dev->dev_addr); 194 if (is_vlan_dev(dev)) { 195 struct vlan_hdr *vlan = (struct vlan_hdr *) 196 ((char *)eth + ETH_HLEN); 197 ip = (char *)vlan + VLAN_HLEN; 198 eth->h_proto = vlan_dev_vlan_proto(dev); 199 vlan->h_vlan_TCI = htons(vlan_dev_vlan_id(dev)); 200 vlan->h_vlan_encapsulated_proto = htons(proto); 201 } else { 202 eth->h_proto = htons(proto); 203 ip = (char *)eth + ETH_HLEN; 204 } 205 206 return ip; 207 } 208 209 int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv, 210 struct net_device *mirred_dev, 211 struct mlx5e_encap_entry *e) 212 { 213 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 214 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 215 struct mlx5e_neigh m_neigh = {}; 216 TC_TUN_ROUTE_ATTR_INIT(attr); 217 int ipv4_encap_size; 218 char *encap_header; 219 struct iphdr *ip; 220 u8 nud_state; 221 int err; 222 223 /* add the IP fields */ 224 attr.fl.fl4.flowi4_tos = tun_key->tos; 225 attr.fl.fl4.daddr = tun_key->u.ipv4.dst; 226 attr.fl.fl4.saddr = tun_key->u.ipv4.src; 227 attr.ttl = tun_key->ttl; 228 229 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr); 230 if (err) 231 return err; 232 233 ipv4_encap_size = 234 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 235 sizeof(struct iphdr) + 236 e->tunnel->calc_hlen(e); 237 238 if (max_encap_size < ipv4_encap_size) { 239 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 240 ipv4_encap_size, max_encap_size); 241 err = -EOPNOTSUPP; 242 goto release_neigh; 243 } 244 245 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL); 246 if (!encap_header) { 247 err = -ENOMEM; 248 goto release_neigh; 249 } 250 251 m_neigh.family = attr.n->ops->family; 252 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len); 253 e->out_dev = attr.out_dev; 254 e->route_dev_ifindex = attr.route_dev->ifindex; 255 256 /* It's important to add the neigh to the hash table before checking 257 * the neigh validity state. So if we'll get a notification, in case the 258 * neigh changes it's validity state, we would find the relevant neigh 259 * in the hash. 260 */ 261 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev); 262 if (err) 263 goto free_encap; 264 265 read_lock_bh(&attr.n->lock); 266 nud_state = attr.n->nud_state; 267 ether_addr_copy(e->h_dest, attr.n->ha); 268 read_unlock_bh(&attr.n->lock); 269 270 /* add ethernet header */ 271 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 272 ETH_P_IP); 273 274 /* add ip header */ 275 ip->tos = tun_key->tos; 276 ip->version = 0x4; 277 ip->ihl = 0x5; 278 ip->ttl = attr.ttl; 279 ip->daddr = attr.fl.fl4.daddr; 280 ip->saddr = attr.fl.fl4.saddr; 281 282 /* add tunneling protocol header */ 283 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr), 284 &ip->protocol, e); 285 if (err) 286 goto destroy_neigh_entry; 287 288 e->encap_size = ipv4_encap_size; 289 e->encap_header = encap_header; 290 291 if (!(nud_state & NUD_VALID)) { 292 neigh_event_send(attr.n, NULL); 293 /* the encap entry will be made valid on neigh update event 294 * and not used before that. 295 */ 296 goto release_neigh; 297 } 298 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, 299 e->reformat_type, 300 ipv4_encap_size, encap_header, 301 MLX5_FLOW_NAMESPACE_FDB); 302 if (IS_ERR(e->pkt_reformat)) { 303 err = PTR_ERR(e->pkt_reformat); 304 goto destroy_neigh_entry; 305 } 306 307 e->flags |= MLX5_ENCAP_ENTRY_VALID; 308 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 309 mlx5e_route_lookup_ipv4_put(&attr); 310 return err; 311 312 destroy_neigh_entry: 313 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e); 314 free_encap: 315 kfree(encap_header); 316 release_neigh: 317 mlx5e_route_lookup_ipv4_put(&attr); 318 return err; 319 } 320 321 int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv, 322 struct net_device *mirred_dev, 323 struct mlx5e_encap_entry *e) 324 { 325 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 326 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 327 TC_TUN_ROUTE_ATTR_INIT(attr); 328 int ipv4_encap_size; 329 char *encap_header; 330 struct iphdr *ip; 331 u8 nud_state; 332 int err; 333 334 /* add the IP fields */ 335 attr.fl.fl4.flowi4_tos = tun_key->tos; 336 attr.fl.fl4.daddr = tun_key->u.ipv4.dst; 337 attr.fl.fl4.saddr = tun_key->u.ipv4.src; 338 attr.ttl = tun_key->ttl; 339 340 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr); 341 if (err) 342 return err; 343 344 ipv4_encap_size = 345 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 346 sizeof(struct iphdr) + 347 e->tunnel->calc_hlen(e); 348 349 if (max_encap_size < ipv4_encap_size) { 350 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 351 ipv4_encap_size, max_encap_size); 352 err = -EOPNOTSUPP; 353 goto release_neigh; 354 } 355 356 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL); 357 if (!encap_header) { 358 err = -ENOMEM; 359 goto release_neigh; 360 } 361 362 e->route_dev_ifindex = attr.route_dev->ifindex; 363 364 read_lock_bh(&attr.n->lock); 365 nud_state = attr.n->nud_state; 366 ether_addr_copy(e->h_dest, attr.n->ha); 367 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev); 368 read_unlock_bh(&attr.n->lock); 369 370 /* add ethernet header */ 371 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 372 ETH_P_IP); 373 374 /* add ip header */ 375 ip->tos = tun_key->tos; 376 ip->version = 0x4; 377 ip->ihl = 0x5; 378 ip->ttl = attr.ttl; 379 ip->daddr = attr.fl.fl4.daddr; 380 ip->saddr = attr.fl.fl4.saddr; 381 382 /* add tunneling protocol header */ 383 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr), 384 &ip->protocol, e); 385 if (err) 386 goto free_encap; 387 388 e->encap_size = ipv4_encap_size; 389 kfree(e->encap_header); 390 e->encap_header = encap_header; 391 392 if (!(nud_state & NUD_VALID)) { 393 neigh_event_send(attr.n, NULL); 394 /* the encap entry will be made valid on neigh update event 395 * and not used before that. 396 */ 397 goto release_neigh; 398 } 399 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, 400 e->reformat_type, 401 ipv4_encap_size, encap_header, 402 MLX5_FLOW_NAMESPACE_FDB); 403 if (IS_ERR(e->pkt_reformat)) { 404 err = PTR_ERR(e->pkt_reformat); 405 goto free_encap; 406 } 407 408 e->flags |= MLX5_ENCAP_ENTRY_VALID; 409 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 410 mlx5e_route_lookup_ipv4_put(&attr); 411 return err; 412 413 free_encap: 414 kfree(encap_header); 415 release_neigh: 416 mlx5e_route_lookup_ipv4_put(&attr); 417 return err; 418 } 419 420 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) 421 static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv, 422 struct net_device *mirred_dev, 423 struct mlx5e_tc_tun_route_attr *attr) 424 { 425 struct net_device *route_dev; 426 struct net_device *out_dev; 427 struct dst_entry *dst; 428 struct neighbour *n; 429 int ret; 430 431 dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(mirred_dev), NULL, &attr->fl.fl6, 432 NULL); 433 if (IS_ERR(dst)) 434 return PTR_ERR(dst); 435 436 if (!attr->ttl) 437 attr->ttl = ip6_dst_hoplimit(dst); 438 439 ret = get_route_and_out_devs(priv, dst->dev, &route_dev, &out_dev); 440 if (ret < 0) 441 goto err_dst_release; 442 443 dev_hold(route_dev); 444 n = dst_neigh_lookup(dst, &attr->fl.fl6.daddr); 445 if (!n) { 446 ret = -ENOMEM; 447 goto err_dev_release; 448 } 449 450 dst_release(dst); 451 attr->out_dev = out_dev; 452 attr->route_dev = route_dev; 453 attr->n = n; 454 return 0; 455 456 err_dev_release: 457 dev_put(route_dev); 458 err_dst_release: 459 dst_release(dst); 460 return ret; 461 } 462 463 static void mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr *attr) 464 { 465 mlx5e_tc_tun_route_attr_cleanup(attr); 466 } 467 468 int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv, 469 struct net_device *mirred_dev, 470 struct mlx5e_encap_entry *e) 471 { 472 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 473 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 474 struct mlx5e_neigh m_neigh = {}; 475 TC_TUN_ROUTE_ATTR_INIT(attr); 476 struct ipv6hdr *ip6h; 477 int ipv6_encap_size; 478 char *encap_header; 479 u8 nud_state; 480 int err; 481 482 attr.ttl = tun_key->ttl; 483 attr.fl.fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label); 484 attr.fl.fl6.daddr = tun_key->u.ipv6.dst; 485 attr.fl.fl6.saddr = tun_key->u.ipv6.src; 486 487 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr); 488 if (err) 489 return err; 490 491 ipv6_encap_size = 492 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 493 sizeof(struct ipv6hdr) + 494 e->tunnel->calc_hlen(e); 495 496 if (max_encap_size < ipv6_encap_size) { 497 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 498 ipv6_encap_size, max_encap_size); 499 err = -EOPNOTSUPP; 500 goto release_neigh; 501 } 502 503 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL); 504 if (!encap_header) { 505 err = -ENOMEM; 506 goto release_neigh; 507 } 508 509 m_neigh.family = attr.n->ops->family; 510 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len); 511 e->out_dev = attr.out_dev; 512 e->route_dev_ifindex = attr.route_dev->ifindex; 513 514 /* It's importent to add the neigh to the hash table before checking 515 * the neigh validity state. So if we'll get a notification, in case the 516 * neigh changes it's validity state, we would find the relevant neigh 517 * in the hash. 518 */ 519 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev); 520 if (err) 521 goto free_encap; 522 523 read_lock_bh(&attr.n->lock); 524 nud_state = attr.n->nud_state; 525 ether_addr_copy(e->h_dest, attr.n->ha); 526 read_unlock_bh(&attr.n->lock); 527 528 /* add ethernet header */ 529 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 530 ETH_P_IPV6); 531 532 /* add ip header */ 533 ip6_flow_hdr(ip6h, tun_key->tos, 0); 534 /* the HW fills up ipv6 payload len */ 535 ip6h->hop_limit = attr.ttl; 536 ip6h->daddr = attr.fl.fl6.daddr; 537 ip6h->saddr = attr.fl.fl6.saddr; 538 539 /* add tunneling protocol header */ 540 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr), 541 &ip6h->nexthdr, e); 542 if (err) 543 goto destroy_neigh_entry; 544 545 e->encap_size = ipv6_encap_size; 546 e->encap_header = encap_header; 547 548 if (!(nud_state & NUD_VALID)) { 549 neigh_event_send(attr.n, NULL); 550 /* the encap entry will be made valid on neigh update event 551 * and not used before that. 552 */ 553 goto release_neigh; 554 } 555 556 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, 557 e->reformat_type, 558 ipv6_encap_size, encap_header, 559 MLX5_FLOW_NAMESPACE_FDB); 560 if (IS_ERR(e->pkt_reformat)) { 561 err = PTR_ERR(e->pkt_reformat); 562 goto destroy_neigh_entry; 563 } 564 565 e->flags |= MLX5_ENCAP_ENTRY_VALID; 566 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 567 mlx5e_route_lookup_ipv6_put(&attr); 568 return err; 569 570 destroy_neigh_entry: 571 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e); 572 free_encap: 573 kfree(encap_header); 574 release_neigh: 575 mlx5e_route_lookup_ipv6_put(&attr); 576 return err; 577 } 578 579 int mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv, 580 struct net_device *mirred_dev, 581 struct mlx5e_encap_entry *e) 582 { 583 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 584 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 585 TC_TUN_ROUTE_ATTR_INIT(attr); 586 struct ipv6hdr *ip6h; 587 int ipv6_encap_size; 588 char *encap_header; 589 u8 nud_state; 590 int err; 591 592 attr.ttl = tun_key->ttl; 593 594 attr.fl.fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label); 595 attr.fl.fl6.daddr = tun_key->u.ipv6.dst; 596 attr.fl.fl6.saddr = tun_key->u.ipv6.src; 597 598 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr); 599 if (err) 600 return err; 601 602 ipv6_encap_size = 603 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 604 sizeof(struct ipv6hdr) + 605 e->tunnel->calc_hlen(e); 606 607 if (max_encap_size < ipv6_encap_size) { 608 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 609 ipv6_encap_size, max_encap_size); 610 err = -EOPNOTSUPP; 611 goto release_neigh; 612 } 613 614 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL); 615 if (!encap_header) { 616 err = -ENOMEM; 617 goto release_neigh; 618 } 619 620 e->route_dev_ifindex = attr.route_dev->ifindex; 621 622 read_lock_bh(&attr.n->lock); 623 nud_state = attr.n->nud_state; 624 ether_addr_copy(e->h_dest, attr.n->ha); 625 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev); 626 read_unlock_bh(&attr.n->lock); 627 628 /* add ethernet header */ 629 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 630 ETH_P_IPV6); 631 632 /* add ip header */ 633 ip6_flow_hdr(ip6h, tun_key->tos, 0); 634 /* the HW fills up ipv6 payload len */ 635 ip6h->hop_limit = attr.ttl; 636 ip6h->daddr = attr.fl.fl6.daddr; 637 ip6h->saddr = attr.fl.fl6.saddr; 638 639 /* add tunneling protocol header */ 640 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr), 641 &ip6h->nexthdr, e); 642 if (err) 643 goto free_encap; 644 645 e->encap_size = ipv6_encap_size; 646 kfree(e->encap_header); 647 e->encap_header = encap_header; 648 649 if (!(nud_state & NUD_VALID)) { 650 neigh_event_send(attr.n, NULL); 651 /* the encap entry will be made valid on neigh update event 652 * and not used before that. 653 */ 654 goto release_neigh; 655 } 656 657 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, 658 e->reformat_type, 659 ipv6_encap_size, encap_header, 660 MLX5_FLOW_NAMESPACE_FDB); 661 if (IS_ERR(e->pkt_reformat)) { 662 err = PTR_ERR(e->pkt_reformat); 663 goto free_encap; 664 } 665 666 e->flags |= MLX5_ENCAP_ENTRY_VALID; 667 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 668 mlx5e_route_lookup_ipv6_put(&attr); 669 return err; 670 671 free_encap: 672 kfree(encap_header); 673 release_neigh: 674 mlx5e_route_lookup_ipv6_put(&attr); 675 return err; 676 } 677 #endif 678 679 int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv, 680 struct mlx5_flow_spec *spec, 681 struct mlx5_flow_attr *flow_attr) 682 { 683 struct mlx5_esw_flow_attr *esw_attr = flow_attr->esw_attr; 684 TC_TUN_ROUTE_ATTR_INIT(attr); 685 u16 vport_num; 686 int err = 0; 687 688 if (flow_attr->ip_version == 4) { 689 /* Addresses are swapped for decap */ 690 attr.fl.fl4.saddr = esw_attr->rx_tun_attr->dst_ip.v4; 691 attr.fl.fl4.daddr = esw_attr->rx_tun_attr->src_ip.v4; 692 err = mlx5e_route_lookup_ipv4_get(priv, priv->netdev, &attr); 693 } 694 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) 695 else if (flow_attr->ip_version == 6) { 696 /* Addresses are swapped for decap */ 697 attr.fl.fl6.saddr = esw_attr->rx_tun_attr->dst_ip.v6; 698 attr.fl.fl6.daddr = esw_attr->rx_tun_attr->src_ip.v6; 699 err = mlx5e_route_lookup_ipv6_get(priv, priv->netdev, &attr); 700 } 701 #endif 702 else 703 return 0; 704 705 if (err) 706 return err; 707 708 if (attr.route_dev->netdev_ops != &mlx5e_netdev_ops || 709 !mlx5e_tc_is_vf_tunnel(attr.out_dev, attr.route_dev)) 710 goto out; 711 712 err = mlx5e_tc_query_route_vport(attr.out_dev, attr.route_dev, &vport_num); 713 if (err) 714 goto out; 715 716 esw_attr->rx_tun_attr->vni = MLX5_GET(fte_match_param, spec->match_value, 717 misc_parameters.vxlan_vni); 718 esw_attr->rx_tun_attr->decap_vport = vport_num; 719 720 out: 721 if (flow_attr->ip_version == 4) 722 mlx5e_route_lookup_ipv4_put(&attr); 723 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) 724 else if (flow_attr->ip_version == 6) 725 mlx5e_route_lookup_ipv6_put(&attr); 726 #endif 727 return err; 728 } 729 730 bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv, 731 struct net_device *netdev) 732 { 733 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(netdev); 734 735 if (tunnel && tunnel->can_offload(priv)) 736 return true; 737 else 738 return false; 739 } 740 741 int mlx5e_tc_tun_init_encap_attr(struct net_device *tunnel_dev, 742 struct mlx5e_priv *priv, 743 struct mlx5e_encap_entry *e, 744 struct netlink_ext_ack *extack) 745 { 746 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(tunnel_dev); 747 748 if (!tunnel) { 749 e->reformat_type = -1; 750 return -EOPNOTSUPP; 751 } 752 753 return tunnel->init_encap_attr(tunnel_dev, priv, e, extack); 754 } 755 756 int mlx5e_tc_tun_parse(struct net_device *filter_dev, 757 struct mlx5e_priv *priv, 758 struct mlx5_flow_spec *spec, 759 struct flow_cls_offload *f, 760 u8 *match_level) 761 { 762 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(filter_dev); 763 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 764 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 765 outer_headers); 766 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 767 outer_headers); 768 struct netlink_ext_ack *extack = f->common.extack; 769 int err = 0; 770 771 if (!tunnel) { 772 netdev_warn(priv->netdev, 773 "decapsulation offload is not supported for %s net device\n", 774 mlx5e_netdev_kind(filter_dev)); 775 err = -EOPNOTSUPP; 776 goto out; 777 } 778 779 *match_level = tunnel->match_level; 780 781 if (tunnel->parse_udp_ports) { 782 err = tunnel->parse_udp_ports(priv, spec, f, 783 headers_c, headers_v); 784 if (err) 785 goto out; 786 } 787 788 if (tunnel->parse_tunnel) { 789 err = tunnel->parse_tunnel(priv, spec, f, 790 headers_c, headers_v); 791 if (err) 792 goto out; 793 } 794 795 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) { 796 struct flow_dissector_key_basic key_basic = {}; 797 struct flow_dissector_key_basic mask_basic = { 798 .n_proto = htons(0xFFFF), 799 }; 800 struct flow_match_basic match_basic = { 801 .key = &key_basic, .mask = &mask_basic, 802 }; 803 struct flow_match_control match; 804 u16 addr_type; 805 806 flow_rule_match_enc_control(rule, &match); 807 addr_type = match.key->addr_type; 808 809 /* For tunnel addr_type used same key id`s as for non-tunnel */ 810 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 811 struct flow_match_ipv4_addrs match; 812 813 flow_rule_match_enc_ipv4_addrs(rule, &match); 814 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 815 src_ipv4_src_ipv6.ipv4_layout.ipv4, 816 ntohl(match.mask->src)); 817 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 818 src_ipv4_src_ipv6.ipv4_layout.ipv4, 819 ntohl(match.key->src)); 820 821 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 822 dst_ipv4_dst_ipv6.ipv4_layout.ipv4, 823 ntohl(match.mask->dst)); 824 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 825 dst_ipv4_dst_ipv6.ipv4_layout.ipv4, 826 ntohl(match.key->dst)); 827 828 key_basic.n_proto = htons(ETH_P_IP); 829 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true, 830 headers_c, headers_v); 831 } else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 832 struct flow_match_ipv6_addrs match; 833 834 flow_rule_match_enc_ipv6_addrs(rule, &match); 835 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 836 src_ipv4_src_ipv6.ipv6_layout.ipv6), 837 &match.mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout, 838 ipv6)); 839 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 840 src_ipv4_src_ipv6.ipv6_layout.ipv6), 841 &match.key->src, MLX5_FLD_SZ_BYTES(ipv6_layout, 842 ipv6)); 843 844 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 845 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 846 &match.mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, 847 ipv6)); 848 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 849 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 850 &match.key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, 851 ipv6)); 852 853 key_basic.n_proto = htons(ETH_P_IPV6); 854 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true, 855 headers_c, headers_v); 856 } 857 } 858 859 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_IP)) { 860 struct flow_match_ip match; 861 862 flow_rule_match_enc_ip(rule, &match); 863 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn, 864 match.mask->tos & 0x3); 865 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, 866 match.key->tos & 0x3); 867 868 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp, 869 match.mask->tos >> 2); 870 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, 871 match.key->tos >> 2); 872 873 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit, 874 match.mask->ttl); 875 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit, 876 match.key->ttl); 877 878 if (match.mask->ttl && 879 !MLX5_CAP_ESW_FLOWTABLE_FDB 880 (priv->mdev, 881 ft_field_support.outer_ipv4_ttl)) { 882 NL_SET_ERR_MSG_MOD(extack, 883 "Matching on TTL is not supported"); 884 err = -EOPNOTSUPP; 885 goto out; 886 } 887 } 888 889 /* let software handle IP fragments */ 890 MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1); 891 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); 892 893 return 0; 894 895 out: 896 return err; 897 } 898 899 int mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv *priv, 900 struct mlx5_flow_spec *spec, 901 struct flow_cls_offload *f, 902 void *headers_c, 903 void *headers_v) 904 { 905 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 906 struct netlink_ext_ack *extack = f->common.extack; 907 struct flow_match_ports enc_ports; 908 909 /* Full udp dst port must be given */ 910 911 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_PORTS)) { 912 NL_SET_ERR_MSG_MOD(extack, 913 "UDP tunnel decap filter must include enc_dst_port condition"); 914 netdev_warn(priv->netdev, 915 "UDP tunnel decap filter must include enc_dst_port condition\n"); 916 return -EOPNOTSUPP; 917 } 918 919 flow_rule_match_enc_ports(rule, &enc_ports); 920 921 if (memchr_inv(&enc_ports.mask->dst, 0xff, 922 sizeof(enc_ports.mask->dst))) { 923 NL_SET_ERR_MSG_MOD(extack, 924 "UDP tunnel decap filter must match enc_dst_port fully"); 925 netdev_warn(priv->netdev, 926 "UDP tunnel decap filter must match enc_dst_port fully\n"); 927 return -EOPNOTSUPP; 928 } 929 930 /* match on UDP protocol and dst port number */ 931 932 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol); 933 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP); 934 935 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 936 ntohs(enc_ports.mask->dst)); 937 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 938 ntohs(enc_ports.key->dst)); 939 940 /* UDP src port on outer header is generated by HW, 941 * so it is probably a bad idea to request matching it. 942 * Nonetheless, it is allowed. 943 */ 944 945 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 946 ntohs(enc_ports.mask->src)); 947 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 948 ntohs(enc_ports.key->src)); 949 950 return 0; 951 } 952