1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* Copyright (c) 2018 Mellanox Technologies. */ 3 4 #include <net/vxlan.h> 5 #include <net/gre.h> 6 #include <net/geneve.h> 7 #include <net/bareudp.h> 8 #include "en/tc_tun.h" 9 #include "en/tc_priv.h" 10 #include "en_tc.h" 11 #include "rep/tc.h" 12 #include "rep/neigh.h" 13 14 struct mlx5e_tc_tun_route_attr { 15 struct net_device *out_dev; 16 struct net_device *route_dev; 17 union { 18 struct flowi4 fl4; 19 struct flowi6 fl6; 20 } fl; 21 struct neighbour *n; 22 u8 ttl; 23 }; 24 25 #define TC_TUN_ROUTE_ATTR_INIT(name) struct mlx5e_tc_tun_route_attr name = {} 26 27 static void mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr *attr) 28 { 29 if (attr->n) 30 neigh_release(attr->n); 31 if (attr->route_dev) 32 dev_put(attr->route_dev); 33 } 34 35 struct mlx5e_tc_tunnel *mlx5e_get_tc_tun(struct net_device *tunnel_dev) 36 { 37 if (netif_is_vxlan(tunnel_dev)) 38 return &vxlan_tunnel; 39 else if (netif_is_geneve(tunnel_dev)) 40 return &geneve_tunnel; 41 else if (netif_is_gretap(tunnel_dev) || 42 netif_is_ip6gretap(tunnel_dev)) 43 return &gre_tunnel; 44 else if (netif_is_bareudp(tunnel_dev)) 45 return &mplsoudp_tunnel; 46 else 47 return NULL; 48 } 49 50 static int get_route_and_out_devs(struct mlx5e_priv *priv, 51 struct net_device *dev, 52 struct net_device **route_dev, 53 struct net_device **out_dev) 54 { 55 struct net_device *uplink_dev, *uplink_upper, *real_dev; 56 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 57 bool dst_is_lag_dev; 58 59 real_dev = is_vlan_dev(dev) ? vlan_dev_real_dev(dev) : dev; 60 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH); 61 62 rcu_read_lock(); 63 uplink_upper = netdev_master_upper_dev_get_rcu(uplink_dev); 64 /* mlx5_lag_is_sriov() is a blocking function which can't be called 65 * while holding rcu read lock. Take the net_device for correctness 66 * sake. 67 */ 68 if (uplink_upper) 69 dev_hold(uplink_upper); 70 rcu_read_unlock(); 71 72 dst_is_lag_dev = (uplink_upper && 73 netif_is_lag_master(uplink_upper) && 74 real_dev == uplink_upper && 75 mlx5_lag_is_sriov(priv->mdev)); 76 if (uplink_upper) 77 dev_put(uplink_upper); 78 79 /* if the egress device isn't on the same HW e-switch or 80 * it's a LAG device, use the uplink 81 */ 82 *route_dev = dev; 83 if (!netdev_port_same_parent_id(priv->netdev, real_dev) || 84 dst_is_lag_dev || is_vlan_dev(*route_dev)) 85 *out_dev = uplink_dev; 86 else if (mlx5e_eswitch_rep(dev) && 87 mlx5e_is_valid_eswitch_fwd_dev(priv, dev)) 88 *out_dev = *route_dev; 89 else 90 return -EOPNOTSUPP; 91 92 if (!(mlx5e_eswitch_rep(*out_dev) && 93 mlx5e_is_uplink_rep(netdev_priv(*out_dev)))) 94 return -EOPNOTSUPP; 95 96 if (mlx5e_eswitch_uplink_rep(priv->netdev) && *out_dev != priv->netdev) 97 return -EOPNOTSUPP; 98 99 return 0; 100 } 101 102 static int mlx5e_route_lookup_ipv4_get(struct mlx5e_priv *priv, 103 struct net_device *mirred_dev, 104 struct mlx5e_tc_tun_route_attr *attr) 105 { 106 struct net_device *route_dev; 107 struct net_device *out_dev; 108 struct neighbour *n; 109 struct rtable *rt; 110 111 #if IS_ENABLED(CONFIG_INET) 112 struct mlx5_core_dev *mdev = priv->mdev; 113 struct net_device *uplink_dev; 114 int ret; 115 116 if (mlx5_lag_is_multipath(mdev)) { 117 struct mlx5_eswitch *esw = mdev->priv.eswitch; 118 119 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH); 120 attr->fl.fl4.flowi4_oif = uplink_dev->ifindex; 121 } 122 123 rt = ip_route_output_key(dev_net(mirred_dev), &attr->fl.fl4); 124 if (IS_ERR(rt)) 125 return PTR_ERR(rt); 126 127 if (mlx5_lag_is_multipath(mdev) && rt->rt_gw_family != AF_INET) { 128 ret = -ENETUNREACH; 129 goto err_rt_release; 130 } 131 #else 132 return -EOPNOTSUPP; 133 #endif 134 135 ret = get_route_and_out_devs(priv, rt->dst.dev, &route_dev, &out_dev); 136 if (ret < 0) 137 goto err_rt_release; 138 dev_hold(route_dev); 139 140 if (!attr->ttl) 141 attr->ttl = ip4_dst_hoplimit(&rt->dst); 142 n = dst_neigh_lookup(&rt->dst, &attr->fl.fl4.daddr); 143 if (!n) { 144 ret = -ENOMEM; 145 goto err_dev_release; 146 } 147 148 ip_rt_put(rt); 149 attr->route_dev = route_dev; 150 attr->out_dev = out_dev; 151 attr->n = n; 152 return 0; 153 154 err_dev_release: 155 dev_put(route_dev); 156 err_rt_release: 157 ip_rt_put(rt); 158 return ret; 159 } 160 161 static void mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr *attr) 162 { 163 mlx5e_tc_tun_route_attr_cleanup(attr); 164 } 165 166 static const char *mlx5e_netdev_kind(struct net_device *dev) 167 { 168 if (dev->rtnl_link_ops) 169 return dev->rtnl_link_ops->kind; 170 else 171 return "unknown"; 172 } 173 174 static int mlx5e_gen_ip_tunnel_header(char buf[], __u8 *ip_proto, 175 struct mlx5e_encap_entry *e) 176 { 177 if (!e->tunnel) { 178 pr_warn("mlx5: Cannot generate tunnel header for this tunnel\n"); 179 return -EOPNOTSUPP; 180 } 181 182 return e->tunnel->generate_ip_tun_hdr(buf, ip_proto, e); 183 } 184 185 static char *gen_eth_tnl_hdr(char *buf, struct net_device *dev, 186 struct mlx5e_encap_entry *e, 187 u16 proto) 188 { 189 struct ethhdr *eth = (struct ethhdr *)buf; 190 char *ip; 191 192 ether_addr_copy(eth->h_dest, e->h_dest); 193 ether_addr_copy(eth->h_source, dev->dev_addr); 194 if (is_vlan_dev(dev)) { 195 struct vlan_hdr *vlan = (struct vlan_hdr *) 196 ((char *)eth + ETH_HLEN); 197 ip = (char *)vlan + VLAN_HLEN; 198 eth->h_proto = vlan_dev_vlan_proto(dev); 199 vlan->h_vlan_TCI = htons(vlan_dev_vlan_id(dev)); 200 vlan->h_vlan_encapsulated_proto = htons(proto); 201 } else { 202 eth->h_proto = htons(proto); 203 ip = (char *)eth + ETH_HLEN; 204 } 205 206 return ip; 207 } 208 209 int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv, 210 struct net_device *mirred_dev, 211 struct mlx5e_encap_entry *e) 212 { 213 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 214 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 215 struct mlx5_pkt_reformat_params reformat_params; 216 struct mlx5e_neigh m_neigh = {}; 217 TC_TUN_ROUTE_ATTR_INIT(attr); 218 int ipv4_encap_size; 219 char *encap_header; 220 struct iphdr *ip; 221 u8 nud_state; 222 int err; 223 224 /* add the IP fields */ 225 attr.fl.fl4.flowi4_tos = tun_key->tos; 226 attr.fl.fl4.daddr = tun_key->u.ipv4.dst; 227 attr.fl.fl4.saddr = tun_key->u.ipv4.src; 228 attr.ttl = tun_key->ttl; 229 230 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr); 231 if (err) 232 return err; 233 234 ipv4_encap_size = 235 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 236 sizeof(struct iphdr) + 237 e->tunnel->calc_hlen(e); 238 239 if (max_encap_size < ipv4_encap_size) { 240 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 241 ipv4_encap_size, max_encap_size); 242 err = -EOPNOTSUPP; 243 goto release_neigh; 244 } 245 246 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL); 247 if (!encap_header) { 248 err = -ENOMEM; 249 goto release_neigh; 250 } 251 252 m_neigh.family = attr.n->ops->family; 253 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len); 254 e->out_dev = attr.out_dev; 255 e->route_dev_ifindex = attr.route_dev->ifindex; 256 257 /* It's important to add the neigh to the hash table before checking 258 * the neigh validity state. So if we'll get a notification, in case the 259 * neigh changes it's validity state, we would find the relevant neigh 260 * in the hash. 261 */ 262 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev); 263 if (err) 264 goto free_encap; 265 266 read_lock_bh(&attr.n->lock); 267 nud_state = attr.n->nud_state; 268 ether_addr_copy(e->h_dest, attr.n->ha); 269 read_unlock_bh(&attr.n->lock); 270 271 /* add ethernet header */ 272 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 273 ETH_P_IP); 274 275 /* add ip header */ 276 ip->tos = tun_key->tos; 277 ip->version = 0x4; 278 ip->ihl = 0x5; 279 ip->ttl = attr.ttl; 280 ip->daddr = attr.fl.fl4.daddr; 281 ip->saddr = attr.fl.fl4.saddr; 282 283 /* add tunneling protocol header */ 284 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr), 285 &ip->protocol, e); 286 if (err) 287 goto destroy_neigh_entry; 288 289 e->encap_size = ipv4_encap_size; 290 e->encap_header = encap_header; 291 292 if (!(nud_state & NUD_VALID)) { 293 neigh_event_send(attr.n, NULL); 294 /* the encap entry will be made valid on neigh update event 295 * and not used before that. 296 */ 297 goto release_neigh; 298 } 299 300 memset(&reformat_params, 0, sizeof(reformat_params)); 301 reformat_params.type = e->reformat_type; 302 reformat_params.size = ipv4_encap_size; 303 reformat_params.data = encap_header; 304 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, 305 MLX5_FLOW_NAMESPACE_FDB); 306 if (IS_ERR(e->pkt_reformat)) { 307 err = PTR_ERR(e->pkt_reformat); 308 goto destroy_neigh_entry; 309 } 310 311 e->flags |= MLX5_ENCAP_ENTRY_VALID; 312 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 313 mlx5e_route_lookup_ipv4_put(&attr); 314 return err; 315 316 destroy_neigh_entry: 317 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e); 318 free_encap: 319 kfree(encap_header); 320 release_neigh: 321 mlx5e_route_lookup_ipv4_put(&attr); 322 return err; 323 } 324 325 int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv, 326 struct net_device *mirred_dev, 327 struct mlx5e_encap_entry *e) 328 { 329 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 330 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 331 struct mlx5_pkt_reformat_params reformat_params; 332 TC_TUN_ROUTE_ATTR_INIT(attr); 333 int ipv4_encap_size; 334 char *encap_header; 335 struct iphdr *ip; 336 u8 nud_state; 337 int err; 338 339 /* add the IP fields */ 340 attr.fl.fl4.flowi4_tos = tun_key->tos; 341 attr.fl.fl4.daddr = tun_key->u.ipv4.dst; 342 attr.fl.fl4.saddr = tun_key->u.ipv4.src; 343 attr.ttl = tun_key->ttl; 344 345 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr); 346 if (err) 347 return err; 348 349 ipv4_encap_size = 350 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 351 sizeof(struct iphdr) + 352 e->tunnel->calc_hlen(e); 353 354 if (max_encap_size < ipv4_encap_size) { 355 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 356 ipv4_encap_size, max_encap_size); 357 err = -EOPNOTSUPP; 358 goto release_neigh; 359 } 360 361 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL); 362 if (!encap_header) { 363 err = -ENOMEM; 364 goto release_neigh; 365 } 366 367 e->route_dev_ifindex = attr.route_dev->ifindex; 368 369 read_lock_bh(&attr.n->lock); 370 nud_state = attr.n->nud_state; 371 ether_addr_copy(e->h_dest, attr.n->ha); 372 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev); 373 read_unlock_bh(&attr.n->lock); 374 375 /* add ethernet header */ 376 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 377 ETH_P_IP); 378 379 /* add ip header */ 380 ip->tos = tun_key->tos; 381 ip->version = 0x4; 382 ip->ihl = 0x5; 383 ip->ttl = attr.ttl; 384 ip->daddr = attr.fl.fl4.daddr; 385 ip->saddr = attr.fl.fl4.saddr; 386 387 /* add tunneling protocol header */ 388 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr), 389 &ip->protocol, e); 390 if (err) 391 goto free_encap; 392 393 e->encap_size = ipv4_encap_size; 394 kfree(e->encap_header); 395 e->encap_header = encap_header; 396 397 if (!(nud_state & NUD_VALID)) { 398 neigh_event_send(attr.n, NULL); 399 /* the encap entry will be made valid on neigh update event 400 * and not used before that. 401 */ 402 goto release_neigh; 403 } 404 405 memset(&reformat_params, 0, sizeof(reformat_params)); 406 reformat_params.type = e->reformat_type; 407 reformat_params.size = ipv4_encap_size; 408 reformat_params.data = encap_header; 409 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, 410 MLX5_FLOW_NAMESPACE_FDB); 411 if (IS_ERR(e->pkt_reformat)) { 412 err = PTR_ERR(e->pkt_reformat); 413 goto free_encap; 414 } 415 416 e->flags |= MLX5_ENCAP_ENTRY_VALID; 417 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 418 mlx5e_route_lookup_ipv4_put(&attr); 419 return err; 420 421 free_encap: 422 kfree(encap_header); 423 release_neigh: 424 mlx5e_route_lookup_ipv4_put(&attr); 425 return err; 426 } 427 428 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) 429 static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv, 430 struct net_device *mirred_dev, 431 struct mlx5e_tc_tun_route_attr *attr) 432 { 433 struct net_device *route_dev; 434 struct net_device *out_dev; 435 struct dst_entry *dst; 436 struct neighbour *n; 437 int ret; 438 439 dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(mirred_dev), NULL, &attr->fl.fl6, 440 NULL); 441 if (IS_ERR(dst)) 442 return PTR_ERR(dst); 443 444 if (!attr->ttl) 445 attr->ttl = ip6_dst_hoplimit(dst); 446 447 ret = get_route_and_out_devs(priv, dst->dev, &route_dev, &out_dev); 448 if (ret < 0) 449 goto err_dst_release; 450 451 dev_hold(route_dev); 452 n = dst_neigh_lookup(dst, &attr->fl.fl6.daddr); 453 if (!n) { 454 ret = -ENOMEM; 455 goto err_dev_release; 456 } 457 458 dst_release(dst); 459 attr->out_dev = out_dev; 460 attr->route_dev = route_dev; 461 attr->n = n; 462 return 0; 463 464 err_dev_release: 465 dev_put(route_dev); 466 err_dst_release: 467 dst_release(dst); 468 return ret; 469 } 470 471 static void mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr *attr) 472 { 473 mlx5e_tc_tun_route_attr_cleanup(attr); 474 } 475 476 int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv, 477 struct net_device *mirred_dev, 478 struct mlx5e_encap_entry *e) 479 { 480 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 481 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 482 struct mlx5_pkt_reformat_params reformat_params; 483 struct mlx5e_neigh m_neigh = {}; 484 TC_TUN_ROUTE_ATTR_INIT(attr); 485 struct ipv6hdr *ip6h; 486 int ipv6_encap_size; 487 char *encap_header; 488 u8 nud_state; 489 int err; 490 491 attr.ttl = tun_key->ttl; 492 attr.fl.fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label); 493 attr.fl.fl6.daddr = tun_key->u.ipv6.dst; 494 attr.fl.fl6.saddr = tun_key->u.ipv6.src; 495 496 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr); 497 if (err) 498 return err; 499 500 ipv6_encap_size = 501 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 502 sizeof(struct ipv6hdr) + 503 e->tunnel->calc_hlen(e); 504 505 if (max_encap_size < ipv6_encap_size) { 506 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 507 ipv6_encap_size, max_encap_size); 508 err = -EOPNOTSUPP; 509 goto release_neigh; 510 } 511 512 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL); 513 if (!encap_header) { 514 err = -ENOMEM; 515 goto release_neigh; 516 } 517 518 m_neigh.family = attr.n->ops->family; 519 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len); 520 e->out_dev = attr.out_dev; 521 e->route_dev_ifindex = attr.route_dev->ifindex; 522 523 /* It's importent to add the neigh to the hash table before checking 524 * the neigh validity state. So if we'll get a notification, in case the 525 * neigh changes it's validity state, we would find the relevant neigh 526 * in the hash. 527 */ 528 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev); 529 if (err) 530 goto free_encap; 531 532 read_lock_bh(&attr.n->lock); 533 nud_state = attr.n->nud_state; 534 ether_addr_copy(e->h_dest, attr.n->ha); 535 read_unlock_bh(&attr.n->lock); 536 537 /* add ethernet header */ 538 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 539 ETH_P_IPV6); 540 541 /* add ip header */ 542 ip6_flow_hdr(ip6h, tun_key->tos, 0); 543 /* the HW fills up ipv6 payload len */ 544 ip6h->hop_limit = attr.ttl; 545 ip6h->daddr = attr.fl.fl6.daddr; 546 ip6h->saddr = attr.fl.fl6.saddr; 547 548 /* add tunneling protocol header */ 549 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr), 550 &ip6h->nexthdr, e); 551 if (err) 552 goto destroy_neigh_entry; 553 554 e->encap_size = ipv6_encap_size; 555 e->encap_header = encap_header; 556 557 if (!(nud_state & NUD_VALID)) { 558 neigh_event_send(attr.n, NULL); 559 /* the encap entry will be made valid on neigh update event 560 * and not used before that. 561 */ 562 goto release_neigh; 563 } 564 565 memset(&reformat_params, 0, sizeof(reformat_params)); 566 reformat_params.type = e->reformat_type; 567 reformat_params.size = ipv6_encap_size; 568 reformat_params.data = encap_header; 569 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, 570 MLX5_FLOW_NAMESPACE_FDB); 571 if (IS_ERR(e->pkt_reformat)) { 572 err = PTR_ERR(e->pkt_reformat); 573 goto destroy_neigh_entry; 574 } 575 576 e->flags |= MLX5_ENCAP_ENTRY_VALID; 577 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 578 mlx5e_route_lookup_ipv6_put(&attr); 579 return err; 580 581 destroy_neigh_entry: 582 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e); 583 free_encap: 584 kfree(encap_header); 585 release_neigh: 586 mlx5e_route_lookup_ipv6_put(&attr); 587 return err; 588 } 589 590 int mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv, 591 struct net_device *mirred_dev, 592 struct mlx5e_encap_entry *e) 593 { 594 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); 595 const struct ip_tunnel_key *tun_key = &e->tun_info->key; 596 struct mlx5_pkt_reformat_params reformat_params; 597 TC_TUN_ROUTE_ATTR_INIT(attr); 598 struct ipv6hdr *ip6h; 599 int ipv6_encap_size; 600 char *encap_header; 601 u8 nud_state; 602 int err; 603 604 attr.ttl = tun_key->ttl; 605 606 attr.fl.fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label); 607 attr.fl.fl6.daddr = tun_key->u.ipv6.dst; 608 attr.fl.fl6.saddr = tun_key->u.ipv6.src; 609 610 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr); 611 if (err) 612 return err; 613 614 ipv6_encap_size = 615 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) + 616 sizeof(struct ipv6hdr) + 617 e->tunnel->calc_hlen(e); 618 619 if (max_encap_size < ipv6_encap_size) { 620 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n", 621 ipv6_encap_size, max_encap_size); 622 err = -EOPNOTSUPP; 623 goto release_neigh; 624 } 625 626 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL); 627 if (!encap_header) { 628 err = -ENOMEM; 629 goto release_neigh; 630 } 631 632 e->route_dev_ifindex = attr.route_dev->ifindex; 633 634 read_lock_bh(&attr.n->lock); 635 nud_state = attr.n->nud_state; 636 ether_addr_copy(e->h_dest, attr.n->ha); 637 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev); 638 read_unlock_bh(&attr.n->lock); 639 640 /* add ethernet header */ 641 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e, 642 ETH_P_IPV6); 643 644 /* add ip header */ 645 ip6_flow_hdr(ip6h, tun_key->tos, 0); 646 /* the HW fills up ipv6 payload len */ 647 ip6h->hop_limit = attr.ttl; 648 ip6h->daddr = attr.fl.fl6.daddr; 649 ip6h->saddr = attr.fl.fl6.saddr; 650 651 /* add tunneling protocol header */ 652 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr), 653 &ip6h->nexthdr, e); 654 if (err) 655 goto free_encap; 656 657 e->encap_size = ipv6_encap_size; 658 kfree(e->encap_header); 659 e->encap_header = encap_header; 660 661 if (!(nud_state & NUD_VALID)) { 662 neigh_event_send(attr.n, NULL); 663 /* the encap entry will be made valid on neigh update event 664 * and not used before that. 665 */ 666 goto release_neigh; 667 } 668 669 memset(&reformat_params, 0, sizeof(reformat_params)); 670 reformat_params.type = e->reformat_type; 671 reformat_params.size = ipv6_encap_size; 672 reformat_params.data = encap_header; 673 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, 674 MLX5_FLOW_NAMESPACE_FDB); 675 if (IS_ERR(e->pkt_reformat)) { 676 err = PTR_ERR(e->pkt_reformat); 677 goto free_encap; 678 } 679 680 e->flags |= MLX5_ENCAP_ENTRY_VALID; 681 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev)); 682 mlx5e_route_lookup_ipv6_put(&attr); 683 return err; 684 685 free_encap: 686 kfree(encap_header); 687 release_neigh: 688 mlx5e_route_lookup_ipv6_put(&attr); 689 return err; 690 } 691 #endif 692 693 int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv, 694 struct mlx5_flow_spec *spec, 695 struct mlx5_flow_attr *flow_attr) 696 { 697 struct mlx5_esw_flow_attr *esw_attr = flow_attr->esw_attr; 698 TC_TUN_ROUTE_ATTR_INIT(attr); 699 u16 vport_num; 700 int err = 0; 701 702 if (flow_attr->tun_ip_version == 4) { 703 /* Addresses are swapped for decap */ 704 attr.fl.fl4.saddr = esw_attr->rx_tun_attr->dst_ip.v4; 705 attr.fl.fl4.daddr = esw_attr->rx_tun_attr->src_ip.v4; 706 err = mlx5e_route_lookup_ipv4_get(priv, priv->netdev, &attr); 707 } 708 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) 709 else if (flow_attr->tun_ip_version == 6) { 710 /* Addresses are swapped for decap */ 711 attr.fl.fl6.saddr = esw_attr->rx_tun_attr->dst_ip.v6; 712 attr.fl.fl6.daddr = esw_attr->rx_tun_attr->src_ip.v6; 713 err = mlx5e_route_lookup_ipv6_get(priv, priv->netdev, &attr); 714 } 715 #endif 716 else 717 return 0; 718 719 if (err) 720 return err; 721 722 if (attr.route_dev->netdev_ops != &mlx5e_netdev_ops || 723 !mlx5e_tc_is_vf_tunnel(attr.out_dev, attr.route_dev)) 724 goto out; 725 726 err = mlx5e_tc_query_route_vport(attr.out_dev, attr.route_dev, &vport_num); 727 if (err) 728 goto out; 729 730 esw_attr->rx_tun_attr->vni = MLX5_GET(fte_match_param, spec->match_value, 731 misc_parameters.vxlan_vni); 732 esw_attr->rx_tun_attr->decap_vport = vport_num; 733 734 out: 735 if (flow_attr->tun_ip_version == 4) 736 mlx5e_route_lookup_ipv4_put(&attr); 737 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) 738 else if (flow_attr->tun_ip_version == 6) 739 mlx5e_route_lookup_ipv6_put(&attr); 740 #endif 741 return err; 742 } 743 744 bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv, 745 struct net_device *netdev) 746 { 747 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(netdev); 748 749 if (tunnel && tunnel->can_offload(priv)) 750 return true; 751 else 752 return false; 753 } 754 755 int mlx5e_tc_tun_init_encap_attr(struct net_device *tunnel_dev, 756 struct mlx5e_priv *priv, 757 struct mlx5e_encap_entry *e, 758 struct netlink_ext_ack *extack) 759 { 760 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(tunnel_dev); 761 762 if (!tunnel) { 763 e->reformat_type = -1; 764 return -EOPNOTSUPP; 765 } 766 767 return tunnel->init_encap_attr(tunnel_dev, priv, e, extack); 768 } 769 770 int mlx5e_tc_tun_parse(struct net_device *filter_dev, 771 struct mlx5e_priv *priv, 772 struct mlx5_flow_spec *spec, 773 struct flow_cls_offload *f, 774 u8 *match_level) 775 { 776 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(filter_dev); 777 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 778 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 779 outer_headers); 780 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 781 outer_headers); 782 struct netlink_ext_ack *extack = f->common.extack; 783 int err = 0; 784 785 if (!tunnel) { 786 netdev_warn(priv->netdev, 787 "decapsulation offload is not supported for %s net device\n", 788 mlx5e_netdev_kind(filter_dev)); 789 err = -EOPNOTSUPP; 790 goto out; 791 } 792 793 *match_level = tunnel->match_level; 794 795 if (tunnel->parse_udp_ports) { 796 err = tunnel->parse_udp_ports(priv, spec, f, 797 headers_c, headers_v); 798 if (err) 799 goto out; 800 } 801 802 if (tunnel->parse_tunnel) { 803 err = tunnel->parse_tunnel(priv, spec, f, 804 headers_c, headers_v); 805 if (err) 806 goto out; 807 } 808 809 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) { 810 struct flow_dissector_key_basic key_basic = {}; 811 struct flow_dissector_key_basic mask_basic = { 812 .n_proto = htons(0xFFFF), 813 }; 814 struct flow_match_basic match_basic = { 815 .key = &key_basic, .mask = &mask_basic, 816 }; 817 struct flow_match_control match; 818 u16 addr_type; 819 820 flow_rule_match_enc_control(rule, &match); 821 addr_type = match.key->addr_type; 822 823 /* For tunnel addr_type used same key id`s as for non-tunnel */ 824 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 825 struct flow_match_ipv4_addrs match; 826 827 flow_rule_match_enc_ipv4_addrs(rule, &match); 828 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 829 src_ipv4_src_ipv6.ipv4_layout.ipv4, 830 ntohl(match.mask->src)); 831 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 832 src_ipv4_src_ipv6.ipv4_layout.ipv4, 833 ntohl(match.key->src)); 834 835 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 836 dst_ipv4_dst_ipv6.ipv4_layout.ipv4, 837 ntohl(match.mask->dst)); 838 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 839 dst_ipv4_dst_ipv6.ipv4_layout.ipv4, 840 ntohl(match.key->dst)); 841 842 key_basic.n_proto = htons(ETH_P_IP); 843 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true, 844 headers_c, headers_v); 845 } else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 846 struct flow_match_ipv6_addrs match; 847 848 flow_rule_match_enc_ipv6_addrs(rule, &match); 849 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 850 src_ipv4_src_ipv6.ipv6_layout.ipv6), 851 &match.mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout, 852 ipv6)); 853 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 854 src_ipv4_src_ipv6.ipv6_layout.ipv6), 855 &match.key->src, MLX5_FLD_SZ_BYTES(ipv6_layout, 856 ipv6)); 857 858 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 859 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 860 &match.mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, 861 ipv6)); 862 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 863 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 864 &match.key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, 865 ipv6)); 866 867 key_basic.n_proto = htons(ETH_P_IPV6); 868 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true, 869 headers_c, headers_v); 870 } 871 } 872 873 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_IP)) { 874 struct flow_match_ip match; 875 876 flow_rule_match_enc_ip(rule, &match); 877 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn, 878 match.mask->tos & 0x3); 879 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, 880 match.key->tos & 0x3); 881 882 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp, 883 match.mask->tos >> 2); 884 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, 885 match.key->tos >> 2); 886 887 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit, 888 match.mask->ttl); 889 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit, 890 match.key->ttl); 891 892 if (match.mask->ttl && 893 !MLX5_CAP_ESW_FLOWTABLE_FDB 894 (priv->mdev, 895 ft_field_support.outer_ipv4_ttl)) { 896 NL_SET_ERR_MSG_MOD(extack, 897 "Matching on TTL is not supported"); 898 err = -EOPNOTSUPP; 899 goto out; 900 } 901 } 902 903 /* let software handle IP fragments */ 904 MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1); 905 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); 906 907 return 0; 908 909 out: 910 return err; 911 } 912 913 int mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv *priv, 914 struct mlx5_flow_spec *spec, 915 struct flow_cls_offload *f, 916 void *headers_c, 917 void *headers_v) 918 { 919 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 920 struct netlink_ext_ack *extack = f->common.extack; 921 struct flow_match_ports enc_ports; 922 923 /* Full udp dst port must be given */ 924 925 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_PORTS)) { 926 NL_SET_ERR_MSG_MOD(extack, 927 "UDP tunnel decap filter must include enc_dst_port condition"); 928 netdev_warn(priv->netdev, 929 "UDP tunnel decap filter must include enc_dst_port condition\n"); 930 return -EOPNOTSUPP; 931 } 932 933 flow_rule_match_enc_ports(rule, &enc_ports); 934 935 if (memchr_inv(&enc_ports.mask->dst, 0xff, 936 sizeof(enc_ports.mask->dst))) { 937 NL_SET_ERR_MSG_MOD(extack, 938 "UDP tunnel decap filter must match enc_dst_port fully"); 939 netdev_warn(priv->netdev, 940 "UDP tunnel decap filter must match enc_dst_port fully\n"); 941 return -EOPNOTSUPP; 942 } 943 944 /* match on UDP protocol and dst port number */ 945 946 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol); 947 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP); 948 949 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 950 ntohs(enc_ports.mask->dst)); 951 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 952 ntohs(enc_ports.key->dst)); 953 954 /* UDP src port on outer header is generated by HW, 955 * so it is probably a bad idea to request matching it. 956 * Nonetheless, it is allowed. 957 */ 958 959 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 960 ntohs(enc_ports.mask->src)); 961 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 962 ntohs(enc_ports.key->src)); 963 964 return 0; 965 } 966