1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * SR-IPv6 implementation 4 * 5 * Authors: 6 * David Lebrun <david.lebrun@uclouvain.be> 7 * eBPF support: Mathieu Xhonneux <m.xhonneux@gmail.com> 8 */ 9 10 #include <linux/filter.h> 11 #include <linux/types.h> 12 #include <linux/skbuff.h> 13 #include <linux/net.h> 14 #include <linux/module.h> 15 #include <net/ip.h> 16 #include <net/lwtunnel.h> 17 #include <net/netevent.h> 18 #include <net/netns/generic.h> 19 #include <net/ip6_fib.h> 20 #include <net/route.h> 21 #include <net/seg6.h> 22 #include <linux/seg6.h> 23 #include <linux/seg6_local.h> 24 #include <net/addrconf.h> 25 #include <net/ip6_route.h> 26 #include <net/dst_cache.h> 27 #include <net/ip_tunnels.h> 28 #ifdef CONFIG_IPV6_SEG6_HMAC 29 #include <net/seg6_hmac.h> 30 #endif 31 #include <net/seg6_local.h> 32 #include <linux/etherdevice.h> 33 #include <linux/bpf.h> 34 #include <linux/netfilter.h> 35 36 #define SEG6_F_ATTR(i) BIT(i) 37 38 struct seg6_local_lwt; 39 40 /* callbacks used for customizing the creation and destruction of a behavior */ 41 struct seg6_local_lwtunnel_ops { 42 int (*build_state)(struct seg6_local_lwt *slwt, const void *cfg, 43 struct netlink_ext_ack *extack); 44 void (*destroy_state)(struct seg6_local_lwt *slwt); 45 }; 46 47 struct seg6_action_desc { 48 int action; 49 unsigned long attrs; 50 51 /* The optattrs field is used for specifying all the optional 52 * attributes supported by a specific behavior. 53 * It means that if one of these attributes is not provided in the 54 * netlink message during the behavior creation, no errors will be 55 * returned to the userspace. 56 * 57 * Each attribute can be only of two types (mutually exclusive): 58 * 1) required or 2) optional. 59 * Every user MUST obey to this rule! If you set an attribute as 60 * required the same attribute CANNOT be set as optional and vice 61 * versa. 62 */ 63 unsigned long optattrs; 64 65 int (*input)(struct sk_buff *skb, struct seg6_local_lwt *slwt); 66 int static_headroom; 67 68 struct seg6_local_lwtunnel_ops slwt_ops; 69 }; 70 71 struct bpf_lwt_prog { 72 struct bpf_prog *prog; 73 char *name; 74 }; 75 76 enum seg6_end_dt_mode { 77 DT_INVALID_MODE = -EINVAL, 78 DT_LEGACY_MODE = 0, 79 DT_VRF_MODE = 1, 80 }; 81 82 struct seg6_end_dt_info { 83 enum seg6_end_dt_mode mode; 84 85 struct net *net; 86 /* VRF device associated to the routing table used by the SRv6 87 * End.DT4/DT6 behavior for routing IPv4/IPv6 packets. 88 */ 89 int vrf_ifindex; 90 int vrf_table; 91 92 /* tunneled packet family (IPv4 or IPv6). 93 * Protocol and header length are inferred from family. 94 */ 95 u16 family; 96 }; 97 98 struct pcpu_seg6_local_counters { 99 u64_stats_t packets; 100 u64_stats_t bytes; 101 u64_stats_t errors; 102 103 struct u64_stats_sync syncp; 104 }; 105 106 /* This struct groups all the SRv6 Behavior counters supported so far. 107 * 108 * put_nla_counters() makes use of this data structure to collect all counter 109 * values after the per-CPU counter evaluation has been performed. 110 * Finally, each counter value (in seg6_local_counters) is stored in the 111 * corresponding netlink attribute and sent to user space. 112 * 113 * NB: we don't want to expose this structure to user space! 114 */ 115 struct seg6_local_counters { 116 __u64 packets; 117 __u64 bytes; 118 __u64 errors; 119 }; 120 121 #define seg6_local_alloc_pcpu_counters(__gfp) \ 122 __netdev_alloc_pcpu_stats(struct pcpu_seg6_local_counters, \ 123 ((__gfp) | __GFP_ZERO)) 124 125 #define SEG6_F_LOCAL_COUNTERS SEG6_F_ATTR(SEG6_LOCAL_COUNTERS) 126 127 struct seg6_local_lwt { 128 int action; 129 struct ipv6_sr_hdr *srh; 130 int table; 131 struct in_addr nh4; 132 struct in6_addr nh6; 133 int iif; 134 int oif; 135 struct bpf_lwt_prog bpf; 136 #ifdef CONFIG_NET_L3_MASTER_DEV 137 struct seg6_end_dt_info dt_info; 138 #endif 139 struct pcpu_seg6_local_counters __percpu *pcpu_counters; 140 141 int headroom; 142 struct seg6_action_desc *desc; 143 /* unlike the required attrs, we have to track the optional attributes 144 * that have been effectively parsed. 145 */ 146 unsigned long parsed_optattrs; 147 }; 148 149 static struct seg6_local_lwt *seg6_local_lwtunnel(struct lwtunnel_state *lwt) 150 { 151 return (struct seg6_local_lwt *)lwt->data; 152 } 153 154 static struct ipv6_sr_hdr *get_and_validate_srh(struct sk_buff *skb) 155 { 156 struct ipv6_sr_hdr *srh; 157 158 srh = seg6_get_srh(skb, IP6_FH_F_SKIP_RH); 159 if (!srh) 160 return NULL; 161 162 #ifdef CONFIG_IPV6_SEG6_HMAC 163 if (!seg6_hmac_validate_skb(skb)) 164 return NULL; 165 #endif 166 167 return srh; 168 } 169 170 static bool decap_and_validate(struct sk_buff *skb, int proto) 171 { 172 struct ipv6_sr_hdr *srh; 173 unsigned int off = 0; 174 175 srh = seg6_get_srh(skb, 0); 176 if (srh && srh->segments_left > 0) 177 return false; 178 179 #ifdef CONFIG_IPV6_SEG6_HMAC 180 if (srh && !seg6_hmac_validate_skb(skb)) 181 return false; 182 #endif 183 184 if (ipv6_find_hdr(skb, &off, proto, NULL, NULL) < 0) 185 return false; 186 187 if (!pskb_pull(skb, off)) 188 return false; 189 190 skb_postpull_rcsum(skb, skb_network_header(skb), off); 191 192 skb_reset_network_header(skb); 193 skb_reset_transport_header(skb); 194 if (iptunnel_pull_offloads(skb)) 195 return false; 196 197 return true; 198 } 199 200 static void advance_nextseg(struct ipv6_sr_hdr *srh, struct in6_addr *daddr) 201 { 202 struct in6_addr *addr; 203 204 srh->segments_left--; 205 addr = srh->segments + srh->segments_left; 206 *daddr = *addr; 207 } 208 209 static int 210 seg6_lookup_any_nexthop(struct sk_buff *skb, struct in6_addr *nhaddr, 211 u32 tbl_id, bool local_delivery) 212 { 213 struct net *net = dev_net(skb->dev); 214 struct ipv6hdr *hdr = ipv6_hdr(skb); 215 int flags = RT6_LOOKUP_F_HAS_SADDR; 216 struct dst_entry *dst = NULL; 217 struct rt6_info *rt; 218 struct flowi6 fl6; 219 int dev_flags = 0; 220 221 fl6.flowi6_iif = skb->dev->ifindex; 222 fl6.daddr = nhaddr ? *nhaddr : hdr->daddr; 223 fl6.saddr = hdr->saddr; 224 fl6.flowlabel = ip6_flowinfo(hdr); 225 fl6.flowi6_mark = skb->mark; 226 fl6.flowi6_proto = hdr->nexthdr; 227 228 if (nhaddr) 229 fl6.flowi6_flags = FLOWI_FLAG_KNOWN_NH; 230 231 if (!tbl_id) { 232 dst = ip6_route_input_lookup(net, skb->dev, &fl6, skb, flags); 233 } else { 234 struct fib6_table *table; 235 236 table = fib6_get_table(net, tbl_id); 237 if (!table) 238 goto out; 239 240 rt = ip6_pol_route(net, table, 0, &fl6, skb, flags); 241 dst = &rt->dst; 242 } 243 244 /* we want to discard traffic destined for local packet processing, 245 * if @local_delivery is set to false. 246 */ 247 if (!local_delivery) 248 dev_flags |= IFF_LOOPBACK; 249 250 if (dst && (dst->dev->flags & dev_flags) && !dst->error) { 251 dst_release(dst); 252 dst = NULL; 253 } 254 255 out: 256 if (!dst) { 257 rt = net->ipv6.ip6_blk_hole_entry; 258 dst = &rt->dst; 259 dst_hold(dst); 260 } 261 262 skb_dst_drop(skb); 263 skb_dst_set(skb, dst); 264 return dst->error; 265 } 266 267 int seg6_lookup_nexthop(struct sk_buff *skb, 268 struct in6_addr *nhaddr, u32 tbl_id) 269 { 270 return seg6_lookup_any_nexthop(skb, nhaddr, tbl_id, false); 271 } 272 273 /* regular endpoint function */ 274 static int input_action_end(struct sk_buff *skb, struct seg6_local_lwt *slwt) 275 { 276 struct ipv6_sr_hdr *srh; 277 278 srh = get_and_validate_srh(skb); 279 if (!srh) 280 goto drop; 281 282 advance_nextseg(srh, &ipv6_hdr(skb)->daddr); 283 284 seg6_lookup_nexthop(skb, NULL, 0); 285 286 return dst_input(skb); 287 288 drop: 289 kfree_skb(skb); 290 return -EINVAL; 291 } 292 293 /* regular endpoint, and forward to specified nexthop */ 294 static int input_action_end_x(struct sk_buff *skb, struct seg6_local_lwt *slwt) 295 { 296 struct ipv6_sr_hdr *srh; 297 298 srh = get_and_validate_srh(skb); 299 if (!srh) 300 goto drop; 301 302 advance_nextseg(srh, &ipv6_hdr(skb)->daddr); 303 304 seg6_lookup_nexthop(skb, &slwt->nh6, 0); 305 306 return dst_input(skb); 307 308 drop: 309 kfree_skb(skb); 310 return -EINVAL; 311 } 312 313 static int input_action_end_t(struct sk_buff *skb, struct seg6_local_lwt *slwt) 314 { 315 struct ipv6_sr_hdr *srh; 316 317 srh = get_and_validate_srh(skb); 318 if (!srh) 319 goto drop; 320 321 advance_nextseg(srh, &ipv6_hdr(skb)->daddr); 322 323 seg6_lookup_nexthop(skb, NULL, slwt->table); 324 325 return dst_input(skb); 326 327 drop: 328 kfree_skb(skb); 329 return -EINVAL; 330 } 331 332 /* decapsulate and forward inner L2 frame on specified interface */ 333 static int input_action_end_dx2(struct sk_buff *skb, 334 struct seg6_local_lwt *slwt) 335 { 336 struct net *net = dev_net(skb->dev); 337 struct net_device *odev; 338 struct ethhdr *eth; 339 340 if (!decap_and_validate(skb, IPPROTO_ETHERNET)) 341 goto drop; 342 343 if (!pskb_may_pull(skb, ETH_HLEN)) 344 goto drop; 345 346 skb_reset_mac_header(skb); 347 eth = (struct ethhdr *)skb->data; 348 349 /* To determine the frame's protocol, we assume it is 802.3. This avoids 350 * a call to eth_type_trans(), which is not really relevant for our 351 * use case. 352 */ 353 if (!eth_proto_is_802_3(eth->h_proto)) 354 goto drop; 355 356 odev = dev_get_by_index_rcu(net, slwt->oif); 357 if (!odev) 358 goto drop; 359 360 /* As we accept Ethernet frames, make sure the egress device is of 361 * the correct type. 362 */ 363 if (odev->type != ARPHRD_ETHER) 364 goto drop; 365 366 if (!(odev->flags & IFF_UP) || !netif_carrier_ok(odev)) 367 goto drop; 368 369 skb_orphan(skb); 370 371 if (skb_warn_if_lro(skb)) 372 goto drop; 373 374 skb_forward_csum(skb); 375 376 if (skb->len - ETH_HLEN > odev->mtu) 377 goto drop; 378 379 skb->dev = odev; 380 skb->protocol = eth->h_proto; 381 382 return dev_queue_xmit(skb); 383 384 drop: 385 kfree_skb(skb); 386 return -EINVAL; 387 } 388 389 static int input_action_end_dx6_finish(struct net *net, struct sock *sk, 390 struct sk_buff *skb) 391 { 392 struct dst_entry *orig_dst = skb_dst(skb); 393 struct in6_addr *nhaddr = NULL; 394 struct seg6_local_lwt *slwt; 395 396 slwt = seg6_local_lwtunnel(orig_dst->lwtstate); 397 398 /* The inner packet is not associated to any local interface, 399 * so we do not call netif_rx(). 400 * 401 * If slwt->nh6 is set to ::, then lookup the nexthop for the 402 * inner packet's DA. Otherwise, use the specified nexthop. 403 */ 404 if (!ipv6_addr_any(&slwt->nh6)) 405 nhaddr = &slwt->nh6; 406 407 seg6_lookup_nexthop(skb, nhaddr, 0); 408 409 return dst_input(skb); 410 } 411 412 /* decapsulate and forward to specified nexthop */ 413 static int input_action_end_dx6(struct sk_buff *skb, 414 struct seg6_local_lwt *slwt) 415 { 416 /* this function accepts IPv6 encapsulated packets, with either 417 * an SRH with SL=0, or no SRH. 418 */ 419 420 if (!decap_and_validate(skb, IPPROTO_IPV6)) 421 goto drop; 422 423 if (!pskb_may_pull(skb, sizeof(struct ipv6hdr))) 424 goto drop; 425 426 skb_set_transport_header(skb, sizeof(struct ipv6hdr)); 427 nf_reset_ct(skb); 428 429 if (static_branch_unlikely(&nf_hooks_lwtunnel_enabled)) 430 return NF_HOOK(NFPROTO_IPV6, NF_INET_PRE_ROUTING, 431 dev_net(skb->dev), NULL, skb, NULL, 432 skb_dst(skb)->dev, input_action_end_dx6_finish); 433 434 return input_action_end_dx6_finish(dev_net(skb->dev), NULL, skb); 435 drop: 436 kfree_skb(skb); 437 return -EINVAL; 438 } 439 440 static int input_action_end_dx4_finish(struct net *net, struct sock *sk, 441 struct sk_buff *skb) 442 { 443 struct dst_entry *orig_dst = skb_dst(skb); 444 struct seg6_local_lwt *slwt; 445 struct iphdr *iph; 446 __be32 nhaddr; 447 int err; 448 449 slwt = seg6_local_lwtunnel(orig_dst->lwtstate); 450 451 iph = ip_hdr(skb); 452 453 nhaddr = slwt->nh4.s_addr ?: iph->daddr; 454 455 skb_dst_drop(skb); 456 457 err = ip_route_input(skb, nhaddr, iph->saddr, 0, skb->dev); 458 if (err) { 459 kfree_skb(skb); 460 return -EINVAL; 461 } 462 463 return dst_input(skb); 464 } 465 466 static int input_action_end_dx4(struct sk_buff *skb, 467 struct seg6_local_lwt *slwt) 468 { 469 if (!decap_and_validate(skb, IPPROTO_IPIP)) 470 goto drop; 471 472 if (!pskb_may_pull(skb, sizeof(struct iphdr))) 473 goto drop; 474 475 skb->protocol = htons(ETH_P_IP); 476 skb_set_transport_header(skb, sizeof(struct iphdr)); 477 nf_reset_ct(skb); 478 479 if (static_branch_unlikely(&nf_hooks_lwtunnel_enabled)) 480 return NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING, 481 dev_net(skb->dev), NULL, skb, NULL, 482 skb_dst(skb)->dev, input_action_end_dx4_finish); 483 484 return input_action_end_dx4_finish(dev_net(skb->dev), NULL, skb); 485 drop: 486 kfree_skb(skb); 487 return -EINVAL; 488 } 489 490 #ifdef CONFIG_NET_L3_MASTER_DEV 491 static struct net *fib6_config_get_net(const struct fib6_config *fib6_cfg) 492 { 493 const struct nl_info *nli = &fib6_cfg->fc_nlinfo; 494 495 return nli->nl_net; 496 } 497 498 static int __seg6_end_dt_vrf_build(struct seg6_local_lwt *slwt, const void *cfg, 499 u16 family, struct netlink_ext_ack *extack) 500 { 501 struct seg6_end_dt_info *info = &slwt->dt_info; 502 int vrf_ifindex; 503 struct net *net; 504 505 net = fib6_config_get_net(cfg); 506 507 /* note that vrf_table was already set by parse_nla_vrftable() */ 508 vrf_ifindex = l3mdev_ifindex_lookup_by_table_id(L3MDEV_TYPE_VRF, net, 509 info->vrf_table); 510 if (vrf_ifindex < 0) { 511 if (vrf_ifindex == -EPERM) { 512 NL_SET_ERR_MSG(extack, 513 "Strict mode for VRF is disabled"); 514 } else if (vrf_ifindex == -ENODEV) { 515 NL_SET_ERR_MSG(extack, 516 "Table has no associated VRF device"); 517 } else { 518 pr_debug("seg6local: SRv6 End.DT* creation error=%d\n", 519 vrf_ifindex); 520 } 521 522 return vrf_ifindex; 523 } 524 525 info->net = net; 526 info->vrf_ifindex = vrf_ifindex; 527 528 info->family = family; 529 info->mode = DT_VRF_MODE; 530 531 return 0; 532 } 533 534 /* The SRv6 End.DT4/DT6 behavior extracts the inner (IPv4/IPv6) packet and 535 * routes the IPv4/IPv6 packet by looking at the configured routing table. 536 * 537 * In the SRv6 End.DT4/DT6 use case, we can receive traffic (IPv6+Segment 538 * Routing Header packets) from several interfaces and the outer IPv6 539 * destination address (DA) is used for retrieving the specific instance of the 540 * End.DT4/DT6 behavior that should process the packets. 541 * 542 * However, the inner IPv4/IPv6 packet is not really bound to any receiving 543 * interface and thus the End.DT4/DT6 sets the VRF (associated with the 544 * corresponding routing table) as the *receiving* interface. 545 * In other words, the End.DT4/DT6 processes a packet as if it has been received 546 * directly by the VRF (and not by one of its slave devices, if any). 547 * In this way, the VRF interface is used for routing the IPv4/IPv6 packet in 548 * according to the routing table configured by the End.DT4/DT6 instance. 549 * 550 * This design allows you to get some interesting features like: 551 * 1) the statistics on rx packets; 552 * 2) the possibility to install a packet sniffer on the receiving interface 553 * (the VRF one) for looking at the incoming packets; 554 * 3) the possibility to leverage the netfilter prerouting hook for the inner 555 * IPv4 packet. 556 * 557 * This function returns: 558 * - the sk_buff* when the VRF rcv handler has processed the packet correctly; 559 * - NULL when the skb is consumed by the VRF rcv handler; 560 * - a pointer which encodes a negative error number in case of error. 561 * Note that in this case, the function takes care of freeing the skb. 562 */ 563 static struct sk_buff *end_dt_vrf_rcv(struct sk_buff *skb, u16 family, 564 struct net_device *dev) 565 { 566 /* based on l3mdev_ip_rcv; we are only interested in the master */ 567 if (unlikely(!netif_is_l3_master(dev) && !netif_has_l3_rx_handler(dev))) 568 goto drop; 569 570 if (unlikely(!dev->l3mdev_ops->l3mdev_l3_rcv)) 571 goto drop; 572 573 /* the decap packet IPv4/IPv6 does not come with any mac header info. 574 * We must unset the mac header to allow the VRF device to rebuild it, 575 * just in case there is a sniffer attached on the device. 576 */ 577 skb_unset_mac_header(skb); 578 579 skb = dev->l3mdev_ops->l3mdev_l3_rcv(dev, skb, family); 580 if (!skb) 581 /* the skb buffer was consumed by the handler */ 582 return NULL; 583 584 /* when a packet is received by a VRF or by one of its slaves, the 585 * master device reference is set into the skb. 586 */ 587 if (unlikely(skb->dev != dev || skb->skb_iif != dev->ifindex)) 588 goto drop; 589 590 return skb; 591 592 drop: 593 kfree_skb(skb); 594 return ERR_PTR(-EINVAL); 595 } 596 597 static struct net_device *end_dt_get_vrf_rcu(struct sk_buff *skb, 598 struct seg6_end_dt_info *info) 599 { 600 int vrf_ifindex = info->vrf_ifindex; 601 struct net *net = info->net; 602 603 if (unlikely(vrf_ifindex < 0)) 604 goto error; 605 606 if (unlikely(!net_eq(dev_net(skb->dev), net))) 607 goto error; 608 609 return dev_get_by_index_rcu(net, vrf_ifindex); 610 611 error: 612 return NULL; 613 } 614 615 static struct sk_buff *end_dt_vrf_core(struct sk_buff *skb, 616 struct seg6_local_lwt *slwt, u16 family) 617 { 618 struct seg6_end_dt_info *info = &slwt->dt_info; 619 struct net_device *vrf; 620 __be16 protocol; 621 int hdrlen; 622 623 vrf = end_dt_get_vrf_rcu(skb, info); 624 if (unlikely(!vrf)) 625 goto drop; 626 627 switch (family) { 628 case AF_INET: 629 protocol = htons(ETH_P_IP); 630 hdrlen = sizeof(struct iphdr); 631 break; 632 case AF_INET6: 633 protocol = htons(ETH_P_IPV6); 634 hdrlen = sizeof(struct ipv6hdr); 635 break; 636 case AF_UNSPEC: 637 fallthrough; 638 default: 639 goto drop; 640 } 641 642 if (unlikely(info->family != AF_UNSPEC && info->family != family)) { 643 pr_warn_once("seg6local: SRv6 End.DT* family mismatch"); 644 goto drop; 645 } 646 647 skb->protocol = protocol; 648 649 skb_dst_drop(skb); 650 651 skb_set_transport_header(skb, hdrlen); 652 nf_reset_ct(skb); 653 654 return end_dt_vrf_rcv(skb, family, vrf); 655 656 drop: 657 kfree_skb(skb); 658 return ERR_PTR(-EINVAL); 659 } 660 661 static int input_action_end_dt4(struct sk_buff *skb, 662 struct seg6_local_lwt *slwt) 663 { 664 struct iphdr *iph; 665 int err; 666 667 if (!decap_and_validate(skb, IPPROTO_IPIP)) 668 goto drop; 669 670 if (!pskb_may_pull(skb, sizeof(struct iphdr))) 671 goto drop; 672 673 skb = end_dt_vrf_core(skb, slwt, AF_INET); 674 if (!skb) 675 /* packet has been processed and consumed by the VRF */ 676 return 0; 677 678 if (IS_ERR(skb)) 679 return PTR_ERR(skb); 680 681 iph = ip_hdr(skb); 682 683 err = ip_route_input(skb, iph->daddr, iph->saddr, 0, skb->dev); 684 if (unlikely(err)) 685 goto drop; 686 687 return dst_input(skb); 688 689 drop: 690 kfree_skb(skb); 691 return -EINVAL; 692 } 693 694 static int seg6_end_dt4_build(struct seg6_local_lwt *slwt, const void *cfg, 695 struct netlink_ext_ack *extack) 696 { 697 return __seg6_end_dt_vrf_build(slwt, cfg, AF_INET, extack); 698 } 699 700 static enum 701 seg6_end_dt_mode seg6_end_dt6_parse_mode(struct seg6_local_lwt *slwt) 702 { 703 unsigned long parsed_optattrs = slwt->parsed_optattrs; 704 bool legacy, vrfmode; 705 706 legacy = !!(parsed_optattrs & SEG6_F_ATTR(SEG6_LOCAL_TABLE)); 707 vrfmode = !!(parsed_optattrs & SEG6_F_ATTR(SEG6_LOCAL_VRFTABLE)); 708 709 if (!(legacy ^ vrfmode)) 710 /* both are absent or present: invalid DT6 mode */ 711 return DT_INVALID_MODE; 712 713 return legacy ? DT_LEGACY_MODE : DT_VRF_MODE; 714 } 715 716 static enum seg6_end_dt_mode seg6_end_dt6_get_mode(struct seg6_local_lwt *slwt) 717 { 718 struct seg6_end_dt_info *info = &slwt->dt_info; 719 720 return info->mode; 721 } 722 723 static int seg6_end_dt6_build(struct seg6_local_lwt *slwt, const void *cfg, 724 struct netlink_ext_ack *extack) 725 { 726 enum seg6_end_dt_mode mode = seg6_end_dt6_parse_mode(slwt); 727 struct seg6_end_dt_info *info = &slwt->dt_info; 728 729 switch (mode) { 730 case DT_LEGACY_MODE: 731 info->mode = DT_LEGACY_MODE; 732 return 0; 733 case DT_VRF_MODE: 734 return __seg6_end_dt_vrf_build(slwt, cfg, AF_INET6, extack); 735 default: 736 NL_SET_ERR_MSG(extack, "table or vrftable must be specified"); 737 return -EINVAL; 738 } 739 } 740 #endif 741 742 static int input_action_end_dt6(struct sk_buff *skb, 743 struct seg6_local_lwt *slwt) 744 { 745 if (!decap_and_validate(skb, IPPROTO_IPV6)) 746 goto drop; 747 748 if (!pskb_may_pull(skb, sizeof(struct ipv6hdr))) 749 goto drop; 750 751 #ifdef CONFIG_NET_L3_MASTER_DEV 752 if (seg6_end_dt6_get_mode(slwt) == DT_LEGACY_MODE) 753 goto legacy_mode; 754 755 /* DT6_VRF_MODE */ 756 skb = end_dt_vrf_core(skb, slwt, AF_INET6); 757 if (!skb) 758 /* packet has been processed and consumed by the VRF */ 759 return 0; 760 761 if (IS_ERR(skb)) 762 return PTR_ERR(skb); 763 764 /* note: this time we do not need to specify the table because the VRF 765 * takes care of selecting the correct table. 766 */ 767 seg6_lookup_any_nexthop(skb, NULL, 0, true); 768 769 return dst_input(skb); 770 771 legacy_mode: 772 #endif 773 skb_set_transport_header(skb, sizeof(struct ipv6hdr)); 774 775 seg6_lookup_any_nexthop(skb, NULL, slwt->table, true); 776 777 return dst_input(skb); 778 779 drop: 780 kfree_skb(skb); 781 return -EINVAL; 782 } 783 784 #ifdef CONFIG_NET_L3_MASTER_DEV 785 static int seg6_end_dt46_build(struct seg6_local_lwt *slwt, const void *cfg, 786 struct netlink_ext_ack *extack) 787 { 788 return __seg6_end_dt_vrf_build(slwt, cfg, AF_UNSPEC, extack); 789 } 790 791 static int input_action_end_dt46(struct sk_buff *skb, 792 struct seg6_local_lwt *slwt) 793 { 794 unsigned int off = 0; 795 int nexthdr; 796 797 nexthdr = ipv6_find_hdr(skb, &off, -1, NULL, NULL); 798 if (unlikely(nexthdr < 0)) 799 goto drop; 800 801 switch (nexthdr) { 802 case IPPROTO_IPIP: 803 return input_action_end_dt4(skb, slwt); 804 case IPPROTO_IPV6: 805 return input_action_end_dt6(skb, slwt); 806 } 807 808 drop: 809 kfree_skb(skb); 810 return -EINVAL; 811 } 812 #endif 813 814 /* push an SRH on top of the current one */ 815 static int input_action_end_b6(struct sk_buff *skb, struct seg6_local_lwt *slwt) 816 { 817 struct ipv6_sr_hdr *srh; 818 int err = -EINVAL; 819 820 srh = get_and_validate_srh(skb); 821 if (!srh) 822 goto drop; 823 824 err = seg6_do_srh_inline(skb, slwt->srh); 825 if (err) 826 goto drop; 827 828 ipv6_hdr(skb)->payload_len = htons(skb->len - sizeof(struct ipv6hdr)); 829 skb_set_transport_header(skb, sizeof(struct ipv6hdr)); 830 831 seg6_lookup_nexthop(skb, NULL, 0); 832 833 return dst_input(skb); 834 835 drop: 836 kfree_skb(skb); 837 return err; 838 } 839 840 /* encapsulate within an outer IPv6 header and a specified SRH */ 841 static int input_action_end_b6_encap(struct sk_buff *skb, 842 struct seg6_local_lwt *slwt) 843 { 844 struct ipv6_sr_hdr *srh; 845 int err = -EINVAL; 846 847 srh = get_and_validate_srh(skb); 848 if (!srh) 849 goto drop; 850 851 advance_nextseg(srh, &ipv6_hdr(skb)->daddr); 852 853 skb_reset_inner_headers(skb); 854 skb->encapsulation = 1; 855 856 err = seg6_do_srh_encap(skb, slwt->srh, IPPROTO_IPV6); 857 if (err) 858 goto drop; 859 860 ipv6_hdr(skb)->payload_len = htons(skb->len - sizeof(struct ipv6hdr)); 861 skb_set_transport_header(skb, sizeof(struct ipv6hdr)); 862 863 seg6_lookup_nexthop(skb, NULL, 0); 864 865 return dst_input(skb); 866 867 drop: 868 kfree_skb(skb); 869 return err; 870 } 871 872 DEFINE_PER_CPU(struct seg6_bpf_srh_state, seg6_bpf_srh_states); 873 874 bool seg6_bpf_has_valid_srh(struct sk_buff *skb) 875 { 876 struct seg6_bpf_srh_state *srh_state = 877 this_cpu_ptr(&seg6_bpf_srh_states); 878 struct ipv6_sr_hdr *srh = srh_state->srh; 879 880 if (unlikely(srh == NULL)) 881 return false; 882 883 if (unlikely(!srh_state->valid)) { 884 if ((srh_state->hdrlen & 7) != 0) 885 return false; 886 887 srh->hdrlen = (u8)(srh_state->hdrlen >> 3); 888 if (!seg6_validate_srh(srh, (srh->hdrlen + 1) << 3, true)) 889 return false; 890 891 srh_state->valid = true; 892 } 893 894 return true; 895 } 896 897 static int input_action_end_bpf(struct sk_buff *skb, 898 struct seg6_local_lwt *slwt) 899 { 900 struct seg6_bpf_srh_state *srh_state = 901 this_cpu_ptr(&seg6_bpf_srh_states); 902 struct ipv6_sr_hdr *srh; 903 int ret; 904 905 srh = get_and_validate_srh(skb); 906 if (!srh) { 907 kfree_skb(skb); 908 return -EINVAL; 909 } 910 advance_nextseg(srh, &ipv6_hdr(skb)->daddr); 911 912 /* preempt_disable is needed to protect the per-CPU buffer srh_state, 913 * which is also accessed by the bpf_lwt_seg6_* helpers 914 */ 915 preempt_disable(); 916 srh_state->srh = srh; 917 srh_state->hdrlen = srh->hdrlen << 3; 918 srh_state->valid = true; 919 920 rcu_read_lock(); 921 bpf_compute_data_pointers(skb); 922 ret = bpf_prog_run_save_cb(slwt->bpf.prog, skb); 923 rcu_read_unlock(); 924 925 switch (ret) { 926 case BPF_OK: 927 case BPF_REDIRECT: 928 break; 929 case BPF_DROP: 930 goto drop; 931 default: 932 pr_warn_once("bpf-seg6local: Illegal return value %u\n", ret); 933 goto drop; 934 } 935 936 if (srh_state->srh && !seg6_bpf_has_valid_srh(skb)) 937 goto drop; 938 939 preempt_enable(); 940 if (ret != BPF_REDIRECT) 941 seg6_lookup_nexthop(skb, NULL, 0); 942 943 return dst_input(skb); 944 945 drop: 946 preempt_enable(); 947 kfree_skb(skb); 948 return -EINVAL; 949 } 950 951 static struct seg6_action_desc seg6_action_table[] = { 952 { 953 .action = SEG6_LOCAL_ACTION_END, 954 .attrs = 0, 955 .optattrs = SEG6_F_LOCAL_COUNTERS, 956 .input = input_action_end, 957 }, 958 { 959 .action = SEG6_LOCAL_ACTION_END_X, 960 .attrs = SEG6_F_ATTR(SEG6_LOCAL_NH6), 961 .optattrs = SEG6_F_LOCAL_COUNTERS, 962 .input = input_action_end_x, 963 }, 964 { 965 .action = SEG6_LOCAL_ACTION_END_T, 966 .attrs = SEG6_F_ATTR(SEG6_LOCAL_TABLE), 967 .optattrs = SEG6_F_LOCAL_COUNTERS, 968 .input = input_action_end_t, 969 }, 970 { 971 .action = SEG6_LOCAL_ACTION_END_DX2, 972 .attrs = SEG6_F_ATTR(SEG6_LOCAL_OIF), 973 .optattrs = SEG6_F_LOCAL_COUNTERS, 974 .input = input_action_end_dx2, 975 }, 976 { 977 .action = SEG6_LOCAL_ACTION_END_DX6, 978 .attrs = SEG6_F_ATTR(SEG6_LOCAL_NH6), 979 .optattrs = SEG6_F_LOCAL_COUNTERS, 980 .input = input_action_end_dx6, 981 }, 982 { 983 .action = SEG6_LOCAL_ACTION_END_DX4, 984 .attrs = SEG6_F_ATTR(SEG6_LOCAL_NH4), 985 .optattrs = SEG6_F_LOCAL_COUNTERS, 986 .input = input_action_end_dx4, 987 }, 988 { 989 .action = SEG6_LOCAL_ACTION_END_DT4, 990 .attrs = SEG6_F_ATTR(SEG6_LOCAL_VRFTABLE), 991 .optattrs = SEG6_F_LOCAL_COUNTERS, 992 #ifdef CONFIG_NET_L3_MASTER_DEV 993 .input = input_action_end_dt4, 994 .slwt_ops = { 995 .build_state = seg6_end_dt4_build, 996 }, 997 #endif 998 }, 999 { 1000 .action = SEG6_LOCAL_ACTION_END_DT6, 1001 #ifdef CONFIG_NET_L3_MASTER_DEV 1002 .attrs = 0, 1003 .optattrs = SEG6_F_LOCAL_COUNTERS | 1004 SEG6_F_ATTR(SEG6_LOCAL_TABLE) | 1005 SEG6_F_ATTR(SEG6_LOCAL_VRFTABLE), 1006 .slwt_ops = { 1007 .build_state = seg6_end_dt6_build, 1008 }, 1009 #else 1010 .attrs = SEG6_F_ATTR(SEG6_LOCAL_TABLE), 1011 .optattrs = SEG6_F_LOCAL_COUNTERS, 1012 #endif 1013 .input = input_action_end_dt6, 1014 }, 1015 { 1016 .action = SEG6_LOCAL_ACTION_END_DT46, 1017 .attrs = SEG6_F_ATTR(SEG6_LOCAL_VRFTABLE), 1018 .optattrs = SEG6_F_LOCAL_COUNTERS, 1019 #ifdef CONFIG_NET_L3_MASTER_DEV 1020 .input = input_action_end_dt46, 1021 .slwt_ops = { 1022 .build_state = seg6_end_dt46_build, 1023 }, 1024 #endif 1025 }, 1026 { 1027 .action = SEG6_LOCAL_ACTION_END_B6, 1028 .attrs = SEG6_F_ATTR(SEG6_LOCAL_SRH), 1029 .optattrs = SEG6_F_LOCAL_COUNTERS, 1030 .input = input_action_end_b6, 1031 }, 1032 { 1033 .action = SEG6_LOCAL_ACTION_END_B6_ENCAP, 1034 .attrs = SEG6_F_ATTR(SEG6_LOCAL_SRH), 1035 .optattrs = SEG6_F_LOCAL_COUNTERS, 1036 .input = input_action_end_b6_encap, 1037 .static_headroom = sizeof(struct ipv6hdr), 1038 }, 1039 { 1040 .action = SEG6_LOCAL_ACTION_END_BPF, 1041 .attrs = SEG6_F_ATTR(SEG6_LOCAL_BPF), 1042 .optattrs = SEG6_F_LOCAL_COUNTERS, 1043 .input = input_action_end_bpf, 1044 }, 1045 1046 }; 1047 1048 static struct seg6_action_desc *__get_action_desc(int action) 1049 { 1050 struct seg6_action_desc *desc; 1051 int i, count; 1052 1053 count = ARRAY_SIZE(seg6_action_table); 1054 for (i = 0; i < count; i++) { 1055 desc = &seg6_action_table[i]; 1056 if (desc->action == action) 1057 return desc; 1058 } 1059 1060 return NULL; 1061 } 1062 1063 static bool seg6_lwtunnel_counters_enabled(struct seg6_local_lwt *slwt) 1064 { 1065 return slwt->parsed_optattrs & SEG6_F_LOCAL_COUNTERS; 1066 } 1067 1068 static void seg6_local_update_counters(struct seg6_local_lwt *slwt, 1069 unsigned int len, int err) 1070 { 1071 struct pcpu_seg6_local_counters *pcounters; 1072 1073 pcounters = this_cpu_ptr(slwt->pcpu_counters); 1074 u64_stats_update_begin(&pcounters->syncp); 1075 1076 if (likely(!err)) { 1077 u64_stats_inc(&pcounters->packets); 1078 u64_stats_add(&pcounters->bytes, len); 1079 } else { 1080 u64_stats_inc(&pcounters->errors); 1081 } 1082 1083 u64_stats_update_end(&pcounters->syncp); 1084 } 1085 1086 static int seg6_local_input_core(struct net *net, struct sock *sk, 1087 struct sk_buff *skb) 1088 { 1089 struct dst_entry *orig_dst = skb_dst(skb); 1090 struct seg6_action_desc *desc; 1091 struct seg6_local_lwt *slwt; 1092 unsigned int len = skb->len; 1093 int rc; 1094 1095 slwt = seg6_local_lwtunnel(orig_dst->lwtstate); 1096 desc = slwt->desc; 1097 1098 rc = desc->input(skb, slwt); 1099 1100 if (!seg6_lwtunnel_counters_enabled(slwt)) 1101 return rc; 1102 1103 seg6_local_update_counters(slwt, len, rc); 1104 1105 return rc; 1106 } 1107 1108 static int seg6_local_input(struct sk_buff *skb) 1109 { 1110 if (skb->protocol != htons(ETH_P_IPV6)) { 1111 kfree_skb(skb); 1112 return -EINVAL; 1113 } 1114 1115 if (static_branch_unlikely(&nf_hooks_lwtunnel_enabled)) 1116 return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_IN, 1117 dev_net(skb->dev), NULL, skb, skb->dev, NULL, 1118 seg6_local_input_core); 1119 1120 return seg6_local_input_core(dev_net(skb->dev), NULL, skb); 1121 } 1122 1123 static const struct nla_policy seg6_local_policy[SEG6_LOCAL_MAX + 1] = { 1124 [SEG6_LOCAL_ACTION] = { .type = NLA_U32 }, 1125 [SEG6_LOCAL_SRH] = { .type = NLA_BINARY }, 1126 [SEG6_LOCAL_TABLE] = { .type = NLA_U32 }, 1127 [SEG6_LOCAL_VRFTABLE] = { .type = NLA_U32 }, 1128 [SEG6_LOCAL_NH4] = { .type = NLA_BINARY, 1129 .len = sizeof(struct in_addr) }, 1130 [SEG6_LOCAL_NH6] = { .type = NLA_BINARY, 1131 .len = sizeof(struct in6_addr) }, 1132 [SEG6_LOCAL_IIF] = { .type = NLA_U32 }, 1133 [SEG6_LOCAL_OIF] = { .type = NLA_U32 }, 1134 [SEG6_LOCAL_BPF] = { .type = NLA_NESTED }, 1135 [SEG6_LOCAL_COUNTERS] = { .type = NLA_NESTED }, 1136 }; 1137 1138 static int parse_nla_srh(struct nlattr **attrs, struct seg6_local_lwt *slwt) 1139 { 1140 struct ipv6_sr_hdr *srh; 1141 int len; 1142 1143 srh = nla_data(attrs[SEG6_LOCAL_SRH]); 1144 len = nla_len(attrs[SEG6_LOCAL_SRH]); 1145 1146 /* SRH must contain at least one segment */ 1147 if (len < sizeof(*srh) + sizeof(struct in6_addr)) 1148 return -EINVAL; 1149 1150 if (!seg6_validate_srh(srh, len, false)) 1151 return -EINVAL; 1152 1153 slwt->srh = kmemdup(srh, len, GFP_KERNEL); 1154 if (!slwt->srh) 1155 return -ENOMEM; 1156 1157 slwt->headroom += len; 1158 1159 return 0; 1160 } 1161 1162 static int put_nla_srh(struct sk_buff *skb, struct seg6_local_lwt *slwt) 1163 { 1164 struct ipv6_sr_hdr *srh; 1165 struct nlattr *nla; 1166 int len; 1167 1168 srh = slwt->srh; 1169 len = (srh->hdrlen + 1) << 3; 1170 1171 nla = nla_reserve(skb, SEG6_LOCAL_SRH, len); 1172 if (!nla) 1173 return -EMSGSIZE; 1174 1175 memcpy(nla_data(nla), srh, len); 1176 1177 return 0; 1178 } 1179 1180 static int cmp_nla_srh(struct seg6_local_lwt *a, struct seg6_local_lwt *b) 1181 { 1182 int len = (a->srh->hdrlen + 1) << 3; 1183 1184 if (len != ((b->srh->hdrlen + 1) << 3)) 1185 return 1; 1186 1187 return memcmp(a->srh, b->srh, len); 1188 } 1189 1190 static void destroy_attr_srh(struct seg6_local_lwt *slwt) 1191 { 1192 kfree(slwt->srh); 1193 } 1194 1195 static int parse_nla_table(struct nlattr **attrs, struct seg6_local_lwt *slwt) 1196 { 1197 slwt->table = nla_get_u32(attrs[SEG6_LOCAL_TABLE]); 1198 1199 return 0; 1200 } 1201 1202 static int put_nla_table(struct sk_buff *skb, struct seg6_local_lwt *slwt) 1203 { 1204 if (nla_put_u32(skb, SEG6_LOCAL_TABLE, slwt->table)) 1205 return -EMSGSIZE; 1206 1207 return 0; 1208 } 1209 1210 static int cmp_nla_table(struct seg6_local_lwt *a, struct seg6_local_lwt *b) 1211 { 1212 if (a->table != b->table) 1213 return 1; 1214 1215 return 0; 1216 } 1217 1218 static struct 1219 seg6_end_dt_info *seg6_possible_end_dt_info(struct seg6_local_lwt *slwt) 1220 { 1221 #ifdef CONFIG_NET_L3_MASTER_DEV 1222 return &slwt->dt_info; 1223 #else 1224 return ERR_PTR(-EOPNOTSUPP); 1225 #endif 1226 } 1227 1228 static int parse_nla_vrftable(struct nlattr **attrs, 1229 struct seg6_local_lwt *slwt) 1230 { 1231 struct seg6_end_dt_info *info = seg6_possible_end_dt_info(slwt); 1232 1233 if (IS_ERR(info)) 1234 return PTR_ERR(info); 1235 1236 info->vrf_table = nla_get_u32(attrs[SEG6_LOCAL_VRFTABLE]); 1237 1238 return 0; 1239 } 1240 1241 static int put_nla_vrftable(struct sk_buff *skb, struct seg6_local_lwt *slwt) 1242 { 1243 struct seg6_end_dt_info *info = seg6_possible_end_dt_info(slwt); 1244 1245 if (IS_ERR(info)) 1246 return PTR_ERR(info); 1247 1248 if (nla_put_u32(skb, SEG6_LOCAL_VRFTABLE, info->vrf_table)) 1249 return -EMSGSIZE; 1250 1251 return 0; 1252 } 1253 1254 static int cmp_nla_vrftable(struct seg6_local_lwt *a, struct seg6_local_lwt *b) 1255 { 1256 struct seg6_end_dt_info *info_a = seg6_possible_end_dt_info(a); 1257 struct seg6_end_dt_info *info_b = seg6_possible_end_dt_info(b); 1258 1259 if (info_a->vrf_table != info_b->vrf_table) 1260 return 1; 1261 1262 return 0; 1263 } 1264 1265 static int parse_nla_nh4(struct nlattr **attrs, struct seg6_local_lwt *slwt) 1266 { 1267 memcpy(&slwt->nh4, nla_data(attrs[SEG6_LOCAL_NH4]), 1268 sizeof(struct in_addr)); 1269 1270 return 0; 1271 } 1272 1273 static int put_nla_nh4(struct sk_buff *skb, struct seg6_local_lwt *slwt) 1274 { 1275 struct nlattr *nla; 1276 1277 nla = nla_reserve(skb, SEG6_LOCAL_NH4, sizeof(struct in_addr)); 1278 if (!nla) 1279 return -EMSGSIZE; 1280 1281 memcpy(nla_data(nla), &slwt->nh4, sizeof(struct in_addr)); 1282 1283 return 0; 1284 } 1285 1286 static int cmp_nla_nh4(struct seg6_local_lwt *a, struct seg6_local_lwt *b) 1287 { 1288 return memcmp(&a->nh4, &b->nh4, sizeof(struct in_addr)); 1289 } 1290 1291 static int parse_nla_nh6(struct nlattr **attrs, struct seg6_local_lwt *slwt) 1292 { 1293 memcpy(&slwt->nh6, nla_data(attrs[SEG6_LOCAL_NH6]), 1294 sizeof(struct in6_addr)); 1295 1296 return 0; 1297 } 1298 1299 static int put_nla_nh6(struct sk_buff *skb, struct seg6_local_lwt *slwt) 1300 { 1301 struct nlattr *nla; 1302 1303 nla = nla_reserve(skb, SEG6_LOCAL_NH6, sizeof(struct in6_addr)); 1304 if (!nla) 1305 return -EMSGSIZE; 1306 1307 memcpy(nla_data(nla), &slwt->nh6, sizeof(struct in6_addr)); 1308 1309 return 0; 1310 } 1311 1312 static int cmp_nla_nh6(struct seg6_local_lwt *a, struct seg6_local_lwt *b) 1313 { 1314 return memcmp(&a->nh6, &b->nh6, sizeof(struct in6_addr)); 1315 } 1316 1317 static int parse_nla_iif(struct nlattr **attrs, struct seg6_local_lwt *slwt) 1318 { 1319 slwt->iif = nla_get_u32(attrs[SEG6_LOCAL_IIF]); 1320 1321 return 0; 1322 } 1323 1324 static int put_nla_iif(struct sk_buff *skb, struct seg6_local_lwt *slwt) 1325 { 1326 if (nla_put_u32(skb, SEG6_LOCAL_IIF, slwt->iif)) 1327 return -EMSGSIZE; 1328 1329 return 0; 1330 } 1331 1332 static int cmp_nla_iif(struct seg6_local_lwt *a, struct seg6_local_lwt *b) 1333 { 1334 if (a->iif != b->iif) 1335 return 1; 1336 1337 return 0; 1338 } 1339 1340 static int parse_nla_oif(struct nlattr **attrs, struct seg6_local_lwt *slwt) 1341 { 1342 slwt->oif = nla_get_u32(attrs[SEG6_LOCAL_OIF]); 1343 1344 return 0; 1345 } 1346 1347 static int put_nla_oif(struct sk_buff *skb, struct seg6_local_lwt *slwt) 1348 { 1349 if (nla_put_u32(skb, SEG6_LOCAL_OIF, slwt->oif)) 1350 return -EMSGSIZE; 1351 1352 return 0; 1353 } 1354 1355 static int cmp_nla_oif(struct seg6_local_lwt *a, struct seg6_local_lwt *b) 1356 { 1357 if (a->oif != b->oif) 1358 return 1; 1359 1360 return 0; 1361 } 1362 1363 #define MAX_PROG_NAME 256 1364 static const struct nla_policy bpf_prog_policy[SEG6_LOCAL_BPF_PROG_MAX + 1] = { 1365 [SEG6_LOCAL_BPF_PROG] = { .type = NLA_U32, }, 1366 [SEG6_LOCAL_BPF_PROG_NAME] = { .type = NLA_NUL_STRING, 1367 .len = MAX_PROG_NAME }, 1368 }; 1369 1370 static int parse_nla_bpf(struct nlattr **attrs, struct seg6_local_lwt *slwt) 1371 { 1372 struct nlattr *tb[SEG6_LOCAL_BPF_PROG_MAX + 1]; 1373 struct bpf_prog *p; 1374 int ret; 1375 u32 fd; 1376 1377 ret = nla_parse_nested_deprecated(tb, SEG6_LOCAL_BPF_PROG_MAX, 1378 attrs[SEG6_LOCAL_BPF], 1379 bpf_prog_policy, NULL); 1380 if (ret < 0) 1381 return ret; 1382 1383 if (!tb[SEG6_LOCAL_BPF_PROG] || !tb[SEG6_LOCAL_BPF_PROG_NAME]) 1384 return -EINVAL; 1385 1386 slwt->bpf.name = nla_memdup(tb[SEG6_LOCAL_BPF_PROG_NAME], GFP_KERNEL); 1387 if (!slwt->bpf.name) 1388 return -ENOMEM; 1389 1390 fd = nla_get_u32(tb[SEG6_LOCAL_BPF_PROG]); 1391 p = bpf_prog_get_type(fd, BPF_PROG_TYPE_LWT_SEG6LOCAL); 1392 if (IS_ERR(p)) { 1393 kfree(slwt->bpf.name); 1394 return PTR_ERR(p); 1395 } 1396 1397 slwt->bpf.prog = p; 1398 return 0; 1399 } 1400 1401 static int put_nla_bpf(struct sk_buff *skb, struct seg6_local_lwt *slwt) 1402 { 1403 struct nlattr *nest; 1404 1405 if (!slwt->bpf.prog) 1406 return 0; 1407 1408 nest = nla_nest_start_noflag(skb, SEG6_LOCAL_BPF); 1409 if (!nest) 1410 return -EMSGSIZE; 1411 1412 if (nla_put_u32(skb, SEG6_LOCAL_BPF_PROG, slwt->bpf.prog->aux->id)) 1413 return -EMSGSIZE; 1414 1415 if (slwt->bpf.name && 1416 nla_put_string(skb, SEG6_LOCAL_BPF_PROG_NAME, slwt->bpf.name)) 1417 return -EMSGSIZE; 1418 1419 return nla_nest_end(skb, nest); 1420 } 1421 1422 static int cmp_nla_bpf(struct seg6_local_lwt *a, struct seg6_local_lwt *b) 1423 { 1424 if (!a->bpf.name && !b->bpf.name) 1425 return 0; 1426 1427 if (!a->bpf.name || !b->bpf.name) 1428 return 1; 1429 1430 return strcmp(a->bpf.name, b->bpf.name); 1431 } 1432 1433 static void destroy_attr_bpf(struct seg6_local_lwt *slwt) 1434 { 1435 kfree(slwt->bpf.name); 1436 if (slwt->bpf.prog) 1437 bpf_prog_put(slwt->bpf.prog); 1438 } 1439 1440 static const struct 1441 nla_policy seg6_local_counters_policy[SEG6_LOCAL_CNT_MAX + 1] = { 1442 [SEG6_LOCAL_CNT_PACKETS] = { .type = NLA_U64 }, 1443 [SEG6_LOCAL_CNT_BYTES] = { .type = NLA_U64 }, 1444 [SEG6_LOCAL_CNT_ERRORS] = { .type = NLA_U64 }, 1445 }; 1446 1447 static int parse_nla_counters(struct nlattr **attrs, 1448 struct seg6_local_lwt *slwt) 1449 { 1450 struct pcpu_seg6_local_counters __percpu *pcounters; 1451 struct nlattr *tb[SEG6_LOCAL_CNT_MAX + 1]; 1452 int ret; 1453 1454 ret = nla_parse_nested_deprecated(tb, SEG6_LOCAL_CNT_MAX, 1455 attrs[SEG6_LOCAL_COUNTERS], 1456 seg6_local_counters_policy, NULL); 1457 if (ret < 0) 1458 return ret; 1459 1460 /* basic support for SRv6 Behavior counters requires at least: 1461 * packets, bytes and errors. 1462 */ 1463 if (!tb[SEG6_LOCAL_CNT_PACKETS] || !tb[SEG6_LOCAL_CNT_BYTES] || 1464 !tb[SEG6_LOCAL_CNT_ERRORS]) 1465 return -EINVAL; 1466 1467 /* counters are always zero initialized */ 1468 pcounters = seg6_local_alloc_pcpu_counters(GFP_KERNEL); 1469 if (!pcounters) 1470 return -ENOMEM; 1471 1472 slwt->pcpu_counters = pcounters; 1473 1474 return 0; 1475 } 1476 1477 static int seg6_local_fill_nla_counters(struct sk_buff *skb, 1478 struct seg6_local_counters *counters) 1479 { 1480 if (nla_put_u64_64bit(skb, SEG6_LOCAL_CNT_PACKETS, counters->packets, 1481 SEG6_LOCAL_CNT_PAD)) 1482 return -EMSGSIZE; 1483 1484 if (nla_put_u64_64bit(skb, SEG6_LOCAL_CNT_BYTES, counters->bytes, 1485 SEG6_LOCAL_CNT_PAD)) 1486 return -EMSGSIZE; 1487 1488 if (nla_put_u64_64bit(skb, SEG6_LOCAL_CNT_ERRORS, counters->errors, 1489 SEG6_LOCAL_CNT_PAD)) 1490 return -EMSGSIZE; 1491 1492 return 0; 1493 } 1494 1495 static int put_nla_counters(struct sk_buff *skb, struct seg6_local_lwt *slwt) 1496 { 1497 struct seg6_local_counters counters = { 0, 0, 0 }; 1498 struct nlattr *nest; 1499 int rc, i; 1500 1501 nest = nla_nest_start(skb, SEG6_LOCAL_COUNTERS); 1502 if (!nest) 1503 return -EMSGSIZE; 1504 1505 for_each_possible_cpu(i) { 1506 struct pcpu_seg6_local_counters *pcounters; 1507 u64 packets, bytes, errors; 1508 unsigned int start; 1509 1510 pcounters = per_cpu_ptr(slwt->pcpu_counters, i); 1511 do { 1512 start = u64_stats_fetch_begin_irq(&pcounters->syncp); 1513 1514 packets = u64_stats_read(&pcounters->packets); 1515 bytes = u64_stats_read(&pcounters->bytes); 1516 errors = u64_stats_read(&pcounters->errors); 1517 1518 } while (u64_stats_fetch_retry_irq(&pcounters->syncp, start)); 1519 1520 counters.packets += packets; 1521 counters.bytes += bytes; 1522 counters.errors += errors; 1523 } 1524 1525 rc = seg6_local_fill_nla_counters(skb, &counters); 1526 if (rc < 0) { 1527 nla_nest_cancel(skb, nest); 1528 return rc; 1529 } 1530 1531 return nla_nest_end(skb, nest); 1532 } 1533 1534 static int cmp_nla_counters(struct seg6_local_lwt *a, struct seg6_local_lwt *b) 1535 { 1536 /* a and b are equal if both have pcpu_counters set or not */ 1537 return (!!((unsigned long)a->pcpu_counters)) ^ 1538 (!!((unsigned long)b->pcpu_counters)); 1539 } 1540 1541 static void destroy_attr_counters(struct seg6_local_lwt *slwt) 1542 { 1543 free_percpu(slwt->pcpu_counters); 1544 } 1545 1546 struct seg6_action_param { 1547 int (*parse)(struct nlattr **attrs, struct seg6_local_lwt *slwt); 1548 int (*put)(struct sk_buff *skb, struct seg6_local_lwt *slwt); 1549 int (*cmp)(struct seg6_local_lwt *a, struct seg6_local_lwt *b); 1550 1551 /* optional destroy() callback useful for releasing resources which 1552 * have been previously acquired in the corresponding parse() 1553 * function. 1554 */ 1555 void (*destroy)(struct seg6_local_lwt *slwt); 1556 }; 1557 1558 static struct seg6_action_param seg6_action_params[SEG6_LOCAL_MAX + 1] = { 1559 [SEG6_LOCAL_SRH] = { .parse = parse_nla_srh, 1560 .put = put_nla_srh, 1561 .cmp = cmp_nla_srh, 1562 .destroy = destroy_attr_srh }, 1563 1564 [SEG6_LOCAL_TABLE] = { .parse = parse_nla_table, 1565 .put = put_nla_table, 1566 .cmp = cmp_nla_table }, 1567 1568 [SEG6_LOCAL_NH4] = { .parse = parse_nla_nh4, 1569 .put = put_nla_nh4, 1570 .cmp = cmp_nla_nh4 }, 1571 1572 [SEG6_LOCAL_NH6] = { .parse = parse_nla_nh6, 1573 .put = put_nla_nh6, 1574 .cmp = cmp_nla_nh6 }, 1575 1576 [SEG6_LOCAL_IIF] = { .parse = parse_nla_iif, 1577 .put = put_nla_iif, 1578 .cmp = cmp_nla_iif }, 1579 1580 [SEG6_LOCAL_OIF] = { .parse = parse_nla_oif, 1581 .put = put_nla_oif, 1582 .cmp = cmp_nla_oif }, 1583 1584 [SEG6_LOCAL_BPF] = { .parse = parse_nla_bpf, 1585 .put = put_nla_bpf, 1586 .cmp = cmp_nla_bpf, 1587 .destroy = destroy_attr_bpf }, 1588 1589 [SEG6_LOCAL_VRFTABLE] = { .parse = parse_nla_vrftable, 1590 .put = put_nla_vrftable, 1591 .cmp = cmp_nla_vrftable }, 1592 1593 [SEG6_LOCAL_COUNTERS] = { .parse = parse_nla_counters, 1594 .put = put_nla_counters, 1595 .cmp = cmp_nla_counters, 1596 .destroy = destroy_attr_counters }, 1597 }; 1598 1599 /* call the destroy() callback (if available) for each set attribute in 1600 * @parsed_attrs, starting from the first attribute up to the @max_parsed 1601 * (excluded) attribute. 1602 */ 1603 static void __destroy_attrs(unsigned long parsed_attrs, int max_parsed, 1604 struct seg6_local_lwt *slwt) 1605 { 1606 struct seg6_action_param *param; 1607 int i; 1608 1609 /* Every required seg6local attribute is identified by an ID which is 1610 * encoded as a flag (i.e: 1 << ID) in the 'attrs' bitmask; 1611 * 1612 * We scan the 'parsed_attrs' bitmask, starting from the first attribute 1613 * up to the @max_parsed (excluded) attribute. 1614 * For each set attribute, we retrieve the corresponding destroy() 1615 * callback. If the callback is not available, then we skip to the next 1616 * attribute; otherwise, we call the destroy() callback. 1617 */ 1618 for (i = 0; i < max_parsed; ++i) { 1619 if (!(parsed_attrs & SEG6_F_ATTR(i))) 1620 continue; 1621 1622 param = &seg6_action_params[i]; 1623 1624 if (param->destroy) 1625 param->destroy(slwt); 1626 } 1627 } 1628 1629 /* release all the resources that may have been acquired during parsing 1630 * operations. 1631 */ 1632 static void destroy_attrs(struct seg6_local_lwt *slwt) 1633 { 1634 unsigned long attrs = slwt->desc->attrs | slwt->parsed_optattrs; 1635 1636 __destroy_attrs(attrs, SEG6_LOCAL_MAX + 1, slwt); 1637 } 1638 1639 static int parse_nla_optional_attrs(struct nlattr **attrs, 1640 struct seg6_local_lwt *slwt) 1641 { 1642 struct seg6_action_desc *desc = slwt->desc; 1643 unsigned long parsed_optattrs = 0; 1644 struct seg6_action_param *param; 1645 int err, i; 1646 1647 for (i = 0; i < SEG6_LOCAL_MAX + 1; ++i) { 1648 if (!(desc->optattrs & SEG6_F_ATTR(i)) || !attrs[i]) 1649 continue; 1650 1651 /* once here, the i-th attribute is provided by the 1652 * userspace AND it is identified optional as well. 1653 */ 1654 param = &seg6_action_params[i]; 1655 1656 err = param->parse(attrs, slwt); 1657 if (err < 0) 1658 goto parse_optattrs_err; 1659 1660 /* current attribute has been correctly parsed */ 1661 parsed_optattrs |= SEG6_F_ATTR(i); 1662 } 1663 1664 /* store in the tunnel state all the optional attributed successfully 1665 * parsed. 1666 */ 1667 slwt->parsed_optattrs = parsed_optattrs; 1668 1669 return 0; 1670 1671 parse_optattrs_err: 1672 __destroy_attrs(parsed_optattrs, i, slwt); 1673 1674 return err; 1675 } 1676 1677 /* call the custom constructor of the behavior during its initialization phase 1678 * and after that all its attributes have been parsed successfully. 1679 */ 1680 static int 1681 seg6_local_lwtunnel_build_state(struct seg6_local_lwt *slwt, const void *cfg, 1682 struct netlink_ext_ack *extack) 1683 { 1684 struct seg6_action_desc *desc = slwt->desc; 1685 struct seg6_local_lwtunnel_ops *ops; 1686 1687 ops = &desc->slwt_ops; 1688 if (!ops->build_state) 1689 return 0; 1690 1691 return ops->build_state(slwt, cfg, extack); 1692 } 1693 1694 /* call the custom destructor of the behavior which is invoked before the 1695 * tunnel is going to be destroyed. 1696 */ 1697 static void seg6_local_lwtunnel_destroy_state(struct seg6_local_lwt *slwt) 1698 { 1699 struct seg6_action_desc *desc = slwt->desc; 1700 struct seg6_local_lwtunnel_ops *ops; 1701 1702 ops = &desc->slwt_ops; 1703 if (!ops->destroy_state) 1704 return; 1705 1706 ops->destroy_state(slwt); 1707 } 1708 1709 static int parse_nla_action(struct nlattr **attrs, struct seg6_local_lwt *slwt) 1710 { 1711 struct seg6_action_param *param; 1712 struct seg6_action_desc *desc; 1713 unsigned long invalid_attrs; 1714 int i, err; 1715 1716 desc = __get_action_desc(slwt->action); 1717 if (!desc) 1718 return -EINVAL; 1719 1720 if (!desc->input) 1721 return -EOPNOTSUPP; 1722 1723 slwt->desc = desc; 1724 slwt->headroom += desc->static_headroom; 1725 1726 /* Forcing the desc->optattrs *set* and the desc->attrs *set* to be 1727 * disjoined, this allow us to release acquired resources by optional 1728 * attributes and by required attributes independently from each other 1729 * without any interference. 1730 * In other terms, we are sure that we do not release some the acquired 1731 * resources twice. 1732 * 1733 * Note that if an attribute is configured both as required and as 1734 * optional, it means that the user has messed something up in the 1735 * seg6_action_table. Therefore, this check is required for SRv6 1736 * behaviors to work properly. 1737 */ 1738 invalid_attrs = desc->attrs & desc->optattrs; 1739 if (invalid_attrs) { 1740 WARN_ONCE(1, 1741 "An attribute cannot be both required AND optional"); 1742 return -EINVAL; 1743 } 1744 1745 /* parse the required attributes */ 1746 for (i = 0; i < SEG6_LOCAL_MAX + 1; i++) { 1747 if (desc->attrs & SEG6_F_ATTR(i)) { 1748 if (!attrs[i]) 1749 return -EINVAL; 1750 1751 param = &seg6_action_params[i]; 1752 1753 err = param->parse(attrs, slwt); 1754 if (err < 0) 1755 goto parse_attrs_err; 1756 } 1757 } 1758 1759 /* parse the optional attributes, if any */ 1760 err = parse_nla_optional_attrs(attrs, slwt); 1761 if (err < 0) 1762 goto parse_attrs_err; 1763 1764 return 0; 1765 1766 parse_attrs_err: 1767 /* release any resource that may have been acquired during the i-1 1768 * parse() operations. 1769 */ 1770 __destroy_attrs(desc->attrs, i, slwt); 1771 1772 return err; 1773 } 1774 1775 static int seg6_local_build_state(struct net *net, struct nlattr *nla, 1776 unsigned int family, const void *cfg, 1777 struct lwtunnel_state **ts, 1778 struct netlink_ext_ack *extack) 1779 { 1780 struct nlattr *tb[SEG6_LOCAL_MAX + 1]; 1781 struct lwtunnel_state *newts; 1782 struct seg6_local_lwt *slwt; 1783 int err; 1784 1785 if (family != AF_INET6) 1786 return -EINVAL; 1787 1788 err = nla_parse_nested_deprecated(tb, SEG6_LOCAL_MAX, nla, 1789 seg6_local_policy, extack); 1790 1791 if (err < 0) 1792 return err; 1793 1794 if (!tb[SEG6_LOCAL_ACTION]) 1795 return -EINVAL; 1796 1797 newts = lwtunnel_state_alloc(sizeof(*slwt)); 1798 if (!newts) 1799 return -ENOMEM; 1800 1801 slwt = seg6_local_lwtunnel(newts); 1802 slwt->action = nla_get_u32(tb[SEG6_LOCAL_ACTION]); 1803 1804 err = parse_nla_action(tb, slwt); 1805 if (err < 0) 1806 goto out_free; 1807 1808 err = seg6_local_lwtunnel_build_state(slwt, cfg, extack); 1809 if (err < 0) 1810 goto out_destroy_attrs; 1811 1812 newts->type = LWTUNNEL_ENCAP_SEG6_LOCAL; 1813 newts->flags = LWTUNNEL_STATE_INPUT_REDIRECT; 1814 newts->headroom = slwt->headroom; 1815 1816 *ts = newts; 1817 1818 return 0; 1819 1820 out_destroy_attrs: 1821 destroy_attrs(slwt); 1822 out_free: 1823 kfree(newts); 1824 return err; 1825 } 1826 1827 static void seg6_local_destroy_state(struct lwtunnel_state *lwt) 1828 { 1829 struct seg6_local_lwt *slwt = seg6_local_lwtunnel(lwt); 1830 1831 seg6_local_lwtunnel_destroy_state(slwt); 1832 1833 destroy_attrs(slwt); 1834 1835 return; 1836 } 1837 1838 static int seg6_local_fill_encap(struct sk_buff *skb, 1839 struct lwtunnel_state *lwt) 1840 { 1841 struct seg6_local_lwt *slwt = seg6_local_lwtunnel(lwt); 1842 struct seg6_action_param *param; 1843 unsigned long attrs; 1844 int i, err; 1845 1846 if (nla_put_u32(skb, SEG6_LOCAL_ACTION, slwt->action)) 1847 return -EMSGSIZE; 1848 1849 attrs = slwt->desc->attrs | slwt->parsed_optattrs; 1850 1851 for (i = 0; i < SEG6_LOCAL_MAX + 1; i++) { 1852 if (attrs & SEG6_F_ATTR(i)) { 1853 param = &seg6_action_params[i]; 1854 err = param->put(skb, slwt); 1855 if (err < 0) 1856 return err; 1857 } 1858 } 1859 1860 return 0; 1861 } 1862 1863 static int seg6_local_get_encap_size(struct lwtunnel_state *lwt) 1864 { 1865 struct seg6_local_lwt *slwt = seg6_local_lwtunnel(lwt); 1866 unsigned long attrs; 1867 int nlsize; 1868 1869 nlsize = nla_total_size(4); /* action */ 1870 1871 attrs = slwt->desc->attrs | slwt->parsed_optattrs; 1872 1873 if (attrs & SEG6_F_ATTR(SEG6_LOCAL_SRH)) 1874 nlsize += nla_total_size((slwt->srh->hdrlen + 1) << 3); 1875 1876 if (attrs & SEG6_F_ATTR(SEG6_LOCAL_TABLE)) 1877 nlsize += nla_total_size(4); 1878 1879 if (attrs & SEG6_F_ATTR(SEG6_LOCAL_NH4)) 1880 nlsize += nla_total_size(4); 1881 1882 if (attrs & SEG6_F_ATTR(SEG6_LOCAL_NH6)) 1883 nlsize += nla_total_size(16); 1884 1885 if (attrs & SEG6_F_ATTR(SEG6_LOCAL_IIF)) 1886 nlsize += nla_total_size(4); 1887 1888 if (attrs & SEG6_F_ATTR(SEG6_LOCAL_OIF)) 1889 nlsize += nla_total_size(4); 1890 1891 if (attrs & SEG6_F_ATTR(SEG6_LOCAL_BPF)) 1892 nlsize += nla_total_size(sizeof(struct nlattr)) + 1893 nla_total_size(MAX_PROG_NAME) + 1894 nla_total_size(4); 1895 1896 if (attrs & SEG6_F_ATTR(SEG6_LOCAL_VRFTABLE)) 1897 nlsize += nla_total_size(4); 1898 1899 if (attrs & SEG6_F_LOCAL_COUNTERS) 1900 nlsize += nla_total_size(0) + /* nest SEG6_LOCAL_COUNTERS */ 1901 /* SEG6_LOCAL_CNT_PACKETS */ 1902 nla_total_size_64bit(sizeof(__u64)) + 1903 /* SEG6_LOCAL_CNT_BYTES */ 1904 nla_total_size_64bit(sizeof(__u64)) + 1905 /* SEG6_LOCAL_CNT_ERRORS */ 1906 nla_total_size_64bit(sizeof(__u64)); 1907 1908 return nlsize; 1909 } 1910 1911 static int seg6_local_cmp_encap(struct lwtunnel_state *a, 1912 struct lwtunnel_state *b) 1913 { 1914 struct seg6_local_lwt *slwt_a, *slwt_b; 1915 struct seg6_action_param *param; 1916 unsigned long attrs_a, attrs_b; 1917 int i; 1918 1919 slwt_a = seg6_local_lwtunnel(a); 1920 slwt_b = seg6_local_lwtunnel(b); 1921 1922 if (slwt_a->action != slwt_b->action) 1923 return 1; 1924 1925 attrs_a = slwt_a->desc->attrs | slwt_a->parsed_optattrs; 1926 attrs_b = slwt_b->desc->attrs | slwt_b->parsed_optattrs; 1927 1928 if (attrs_a != attrs_b) 1929 return 1; 1930 1931 for (i = 0; i < SEG6_LOCAL_MAX + 1; i++) { 1932 if (attrs_a & SEG6_F_ATTR(i)) { 1933 param = &seg6_action_params[i]; 1934 if (param->cmp(slwt_a, slwt_b)) 1935 return 1; 1936 } 1937 } 1938 1939 return 0; 1940 } 1941 1942 static const struct lwtunnel_encap_ops seg6_local_ops = { 1943 .build_state = seg6_local_build_state, 1944 .destroy_state = seg6_local_destroy_state, 1945 .input = seg6_local_input, 1946 .fill_encap = seg6_local_fill_encap, 1947 .get_encap_size = seg6_local_get_encap_size, 1948 .cmp_encap = seg6_local_cmp_encap, 1949 .owner = THIS_MODULE, 1950 }; 1951 1952 int __init seg6_local_init(void) 1953 { 1954 /* If the max total number of defined attributes is reached, then your 1955 * kernel build stops here. 1956 * 1957 * This check is required to avoid arithmetic overflows when processing 1958 * behavior attributes and the maximum number of defined attributes 1959 * exceeds the allowed value. 1960 */ 1961 BUILD_BUG_ON(SEG6_LOCAL_MAX + 1 > BITS_PER_TYPE(unsigned long)); 1962 1963 return lwtunnel_encap_add_ops(&seg6_local_ops, 1964 LWTUNNEL_ENCAP_SEG6_LOCAL); 1965 } 1966 1967 void seg6_local_exit(void) 1968 { 1969 lwtunnel_encap_del_ops(&seg6_local_ops, LWTUNNEL_ENCAP_SEG6_LOCAL); 1970 } 1971