1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * XFRM virtual interface 4 * 5 * Copyright (C) 2018 secunet Security Networks AG 6 * 7 * Author: 8 * Steffen Klassert <steffen.klassert@secunet.com> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/capability.h> 13 #include <linux/errno.h> 14 #include <linux/types.h> 15 #include <linux/sockios.h> 16 #include <linux/icmp.h> 17 #include <linux/if.h> 18 #include <linux/in.h> 19 #include <linux/ip.h> 20 #include <linux/net.h> 21 #include <linux/in6.h> 22 #include <linux/netdevice.h> 23 #include <linux/if_link.h> 24 #include <linux/if_arp.h> 25 #include <linux/icmpv6.h> 26 #include <linux/init.h> 27 #include <linux/route.h> 28 #include <linux/rtnetlink.h> 29 #include <linux/netfilter_ipv6.h> 30 #include <linux/slab.h> 31 #include <linux/hash.h> 32 33 #include <linux/uaccess.h> 34 #include <linux/atomic.h> 35 36 #include <net/gso.h> 37 #include <net/icmp.h> 38 #include <net/ip.h> 39 #include <net/ipv6.h> 40 #include <net/ip6_route.h> 41 #include <net/ip_tunnels.h> 42 #include <net/addrconf.h> 43 #include <net/xfrm.h> 44 #include <net/net_namespace.h> 45 #include <net/dst_metadata.h> 46 #include <net/netns/generic.h> 47 #include <linux/etherdevice.h> 48 49 static int xfrmi_dev_init(struct net_device *dev); 50 static void xfrmi_dev_setup(struct net_device *dev); 51 static struct rtnl_link_ops xfrmi_link_ops __read_mostly; 52 static unsigned int xfrmi_net_id __read_mostly; 53 static const struct net_device_ops xfrmi_netdev_ops; 54 55 #define XFRMI_HASH_BITS 8 56 #define XFRMI_HASH_SIZE BIT(XFRMI_HASH_BITS) 57 58 struct xfrmi_net { 59 /* lists for storing interfaces in use */ 60 struct xfrm_if __rcu *xfrmi[XFRMI_HASH_SIZE]; 61 struct xfrm_if __rcu *collect_md_xfrmi; 62 }; 63 64 static const struct nla_policy xfrm_lwt_policy[LWT_XFRM_MAX + 1] = { 65 [LWT_XFRM_IF_ID] = NLA_POLICY_MIN(NLA_U32, 1), 66 [LWT_XFRM_LINK] = NLA_POLICY_MIN(NLA_U32, 1), 67 }; 68 69 static void xfrmi_destroy_state(struct lwtunnel_state *lwt) 70 { 71 } 72 73 static int xfrmi_build_state(struct net *net, struct nlattr *nla, 74 unsigned int family, const void *cfg, 75 struct lwtunnel_state **ts, 76 struct netlink_ext_ack *extack) 77 { 78 struct nlattr *tb[LWT_XFRM_MAX + 1]; 79 struct lwtunnel_state *new_state; 80 struct xfrm_md_info *info; 81 int ret; 82 83 ret = nla_parse_nested(tb, LWT_XFRM_MAX, nla, xfrm_lwt_policy, extack); 84 if (ret < 0) 85 return ret; 86 87 if (!tb[LWT_XFRM_IF_ID]) { 88 NL_SET_ERR_MSG(extack, "if_id must be set"); 89 return -EINVAL; 90 } 91 92 new_state = lwtunnel_state_alloc(sizeof(*info)); 93 if (!new_state) { 94 NL_SET_ERR_MSG(extack, "failed to create encap info"); 95 return -ENOMEM; 96 } 97 98 new_state->type = LWTUNNEL_ENCAP_XFRM; 99 100 info = lwt_xfrm_info(new_state); 101 102 info->if_id = nla_get_u32(tb[LWT_XFRM_IF_ID]); 103 104 if (tb[LWT_XFRM_LINK]) 105 info->link = nla_get_u32(tb[LWT_XFRM_LINK]); 106 107 *ts = new_state; 108 return 0; 109 } 110 111 static int xfrmi_fill_encap_info(struct sk_buff *skb, 112 struct lwtunnel_state *lwt) 113 { 114 struct xfrm_md_info *info = lwt_xfrm_info(lwt); 115 116 if (nla_put_u32(skb, LWT_XFRM_IF_ID, info->if_id) || 117 (info->link && nla_put_u32(skb, LWT_XFRM_LINK, info->link))) 118 return -EMSGSIZE; 119 120 return 0; 121 } 122 123 static int xfrmi_encap_nlsize(struct lwtunnel_state *lwtstate) 124 { 125 return nla_total_size(sizeof(u32)) + /* LWT_XFRM_IF_ID */ 126 nla_total_size(sizeof(u32)); /* LWT_XFRM_LINK */ 127 } 128 129 static int xfrmi_encap_cmp(struct lwtunnel_state *a, struct lwtunnel_state *b) 130 { 131 struct xfrm_md_info *a_info = lwt_xfrm_info(a); 132 struct xfrm_md_info *b_info = lwt_xfrm_info(b); 133 134 return memcmp(a_info, b_info, sizeof(*a_info)); 135 } 136 137 static const struct lwtunnel_encap_ops xfrmi_encap_ops = { 138 .build_state = xfrmi_build_state, 139 .destroy_state = xfrmi_destroy_state, 140 .fill_encap = xfrmi_fill_encap_info, 141 .get_encap_size = xfrmi_encap_nlsize, 142 .cmp_encap = xfrmi_encap_cmp, 143 .owner = THIS_MODULE, 144 }; 145 146 #define for_each_xfrmi_rcu(start, xi) \ 147 for (xi = rcu_dereference(start); xi; xi = rcu_dereference(xi->next)) 148 149 static u32 xfrmi_hash(u32 if_id) 150 { 151 return hash_32(if_id, XFRMI_HASH_BITS); 152 } 153 154 static struct xfrm_if *xfrmi_lookup(struct net *net, struct xfrm_state *x) 155 { 156 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 157 struct xfrm_if *xi; 158 159 for_each_xfrmi_rcu(xfrmn->xfrmi[xfrmi_hash(x->if_id)], xi) { 160 if (x->if_id == xi->p.if_id && 161 (xi->dev->flags & IFF_UP)) 162 return xi; 163 } 164 165 xi = rcu_dereference(xfrmn->collect_md_xfrmi); 166 if (xi && (xi->dev->flags & IFF_UP)) 167 return xi; 168 169 return NULL; 170 } 171 172 static bool xfrmi_decode_session(struct sk_buff *skb, 173 unsigned short family, 174 struct xfrm_if_decode_session_result *res) 175 { 176 struct net_device *dev; 177 struct xfrm_if *xi; 178 int ifindex = 0; 179 180 if (!secpath_exists(skb) || !skb->dev) 181 return false; 182 183 switch (family) { 184 case AF_INET6: 185 ifindex = inet6_sdif(skb); 186 break; 187 case AF_INET: 188 ifindex = inet_sdif(skb); 189 break; 190 } 191 192 if (ifindex) { 193 struct net *net = xs_net(xfrm_input_state(skb)); 194 195 dev = dev_get_by_index_rcu(net, ifindex); 196 } else { 197 dev = skb->dev; 198 } 199 200 if (!dev || !(dev->flags & IFF_UP)) 201 return false; 202 if (dev->netdev_ops != &xfrmi_netdev_ops) 203 return false; 204 205 xi = netdev_priv(dev); 206 res->net = xi->net; 207 208 if (xi->p.collect_md) 209 res->if_id = xfrm_input_state(skb)->if_id; 210 else 211 res->if_id = xi->p.if_id; 212 return true; 213 } 214 215 static void xfrmi_link(struct xfrmi_net *xfrmn, struct xfrm_if *xi) 216 { 217 struct xfrm_if __rcu **xip = &xfrmn->xfrmi[xfrmi_hash(xi->p.if_id)]; 218 219 rcu_assign_pointer(xi->next , rtnl_dereference(*xip)); 220 rcu_assign_pointer(*xip, xi); 221 } 222 223 static void xfrmi_unlink(struct xfrmi_net *xfrmn, struct xfrm_if *xi) 224 { 225 struct xfrm_if __rcu **xip; 226 struct xfrm_if *iter; 227 228 for (xip = &xfrmn->xfrmi[xfrmi_hash(xi->p.if_id)]; 229 (iter = rtnl_dereference(*xip)) != NULL; 230 xip = &iter->next) { 231 if (xi == iter) { 232 rcu_assign_pointer(*xip, xi->next); 233 break; 234 } 235 } 236 } 237 238 static void xfrmi_dev_free(struct net_device *dev) 239 { 240 struct xfrm_if *xi = netdev_priv(dev); 241 242 gro_cells_destroy(&xi->gro_cells); 243 free_percpu(dev->tstats); 244 } 245 246 static int xfrmi_create(struct net_device *dev) 247 { 248 struct xfrm_if *xi = netdev_priv(dev); 249 struct net *net = dev_net(dev); 250 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 251 int err; 252 253 dev->rtnl_link_ops = &xfrmi_link_ops; 254 err = register_netdevice(dev); 255 if (err < 0) 256 goto out; 257 258 if (xi->p.collect_md) 259 rcu_assign_pointer(xfrmn->collect_md_xfrmi, xi); 260 else 261 xfrmi_link(xfrmn, xi); 262 263 return 0; 264 265 out: 266 return err; 267 } 268 269 static struct xfrm_if *xfrmi_locate(struct net *net, struct xfrm_if_parms *p) 270 { 271 struct xfrm_if __rcu **xip; 272 struct xfrm_if *xi; 273 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 274 275 for (xip = &xfrmn->xfrmi[xfrmi_hash(p->if_id)]; 276 (xi = rtnl_dereference(*xip)) != NULL; 277 xip = &xi->next) 278 if (xi->p.if_id == p->if_id) 279 return xi; 280 281 return NULL; 282 } 283 284 static void xfrmi_dev_uninit(struct net_device *dev) 285 { 286 struct xfrm_if *xi = netdev_priv(dev); 287 struct xfrmi_net *xfrmn = net_generic(xi->net, xfrmi_net_id); 288 289 if (xi->p.collect_md) 290 RCU_INIT_POINTER(xfrmn->collect_md_xfrmi, NULL); 291 else 292 xfrmi_unlink(xfrmn, xi); 293 } 294 295 static void xfrmi_scrub_packet(struct sk_buff *skb, bool xnet) 296 { 297 skb_clear_tstamp(skb); 298 skb->pkt_type = PACKET_HOST; 299 skb->skb_iif = 0; 300 skb->ignore_df = 0; 301 skb_dst_drop(skb); 302 nf_reset_ct(skb); 303 nf_reset_trace(skb); 304 305 if (!xnet) 306 return; 307 308 ipvs_reset(skb); 309 secpath_reset(skb); 310 skb_orphan(skb); 311 skb->mark = 0; 312 } 313 314 static int xfrmi_input(struct sk_buff *skb, int nexthdr, __be32 spi, 315 int encap_type, unsigned short family) 316 { 317 struct sec_path *sp; 318 319 sp = skb_sec_path(skb); 320 if (sp && (sp->len || sp->olen) && 321 !xfrm_policy_check(NULL, XFRM_POLICY_IN, skb, family)) 322 goto discard; 323 324 XFRM_SPI_SKB_CB(skb)->family = family; 325 if (family == AF_INET) { 326 XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr); 327 XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL; 328 } else { 329 XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr); 330 XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL; 331 } 332 333 return xfrm_input(skb, nexthdr, spi, encap_type); 334 discard: 335 kfree_skb(skb); 336 return 0; 337 } 338 339 static int xfrmi4_rcv(struct sk_buff *skb) 340 { 341 return xfrmi_input(skb, ip_hdr(skb)->protocol, 0, 0, AF_INET); 342 } 343 344 static int xfrmi6_rcv(struct sk_buff *skb) 345 { 346 return xfrmi_input(skb, skb_network_header(skb)[IP6CB(skb)->nhoff], 347 0, 0, AF_INET6); 348 } 349 350 static int xfrmi4_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 351 { 352 return xfrmi_input(skb, nexthdr, spi, encap_type, AF_INET); 353 } 354 355 static int xfrmi6_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 356 { 357 return xfrmi_input(skb, nexthdr, spi, encap_type, AF_INET6); 358 } 359 360 static int xfrmi_rcv_cb(struct sk_buff *skb, int err) 361 { 362 const struct xfrm_mode *inner_mode; 363 struct net_device *dev; 364 struct xfrm_state *x; 365 struct xfrm_if *xi; 366 bool xnet; 367 int link; 368 369 if (err && !secpath_exists(skb)) 370 return 0; 371 372 x = xfrm_input_state(skb); 373 374 xi = xfrmi_lookup(xs_net(x), x); 375 if (!xi) 376 return 1; 377 378 link = skb->dev->ifindex; 379 dev = xi->dev; 380 skb->dev = dev; 381 382 if (err) { 383 dev->stats.rx_errors++; 384 dev->stats.rx_dropped++; 385 386 return 0; 387 } 388 389 xnet = !net_eq(xi->net, dev_net(skb->dev)); 390 391 if (xnet) { 392 inner_mode = &x->inner_mode; 393 394 if (x->sel.family == AF_UNSPEC) { 395 inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol); 396 if (inner_mode == NULL) { 397 XFRM_INC_STATS(dev_net(skb->dev), 398 LINUX_MIB_XFRMINSTATEMODEERROR); 399 return -EINVAL; 400 } 401 } 402 403 if (!xfrm_policy_check(NULL, XFRM_POLICY_IN, skb, 404 inner_mode->family)) 405 return -EPERM; 406 } 407 408 xfrmi_scrub_packet(skb, xnet); 409 if (xi->p.collect_md) { 410 struct metadata_dst *md_dst; 411 412 md_dst = metadata_dst_alloc(0, METADATA_XFRM, GFP_ATOMIC); 413 if (!md_dst) 414 return -ENOMEM; 415 416 md_dst->u.xfrm_info.if_id = x->if_id; 417 md_dst->u.xfrm_info.link = link; 418 skb_dst_set(skb, (struct dst_entry *)md_dst); 419 } 420 dev_sw_netstats_rx_add(dev, skb->len); 421 422 return 0; 423 } 424 425 static int 426 xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl) 427 { 428 struct xfrm_if *xi = netdev_priv(dev); 429 struct net_device_stats *stats = &xi->dev->stats; 430 struct dst_entry *dst = skb_dst(skb); 431 unsigned int length = skb->len; 432 struct net_device *tdev; 433 struct xfrm_state *x; 434 int err = -1; 435 u32 if_id; 436 int mtu; 437 438 if (xi->p.collect_md) { 439 struct xfrm_md_info *md_info = skb_xfrm_md_info(skb); 440 441 if (unlikely(!md_info)) 442 return -EINVAL; 443 444 if_id = md_info->if_id; 445 fl->flowi_oif = md_info->link; 446 if (md_info->dst_orig) { 447 struct dst_entry *tmp_dst = dst; 448 449 dst = md_info->dst_orig; 450 skb_dst_set(skb, dst); 451 md_info->dst_orig = NULL; 452 dst_release(tmp_dst); 453 } 454 } else { 455 if_id = xi->p.if_id; 456 } 457 458 dst_hold(dst); 459 dst = xfrm_lookup_with_ifid(xi->net, dst, fl, NULL, 0, if_id); 460 if (IS_ERR(dst)) { 461 err = PTR_ERR(dst); 462 dst = NULL; 463 goto tx_err_link_failure; 464 } 465 466 x = dst->xfrm; 467 if (!x) 468 goto tx_err_link_failure; 469 470 if (x->if_id != if_id) 471 goto tx_err_link_failure; 472 473 tdev = dst->dev; 474 475 if (tdev == dev) { 476 stats->collisions++; 477 net_warn_ratelimited("%s: Local routing loop detected!\n", 478 dev->name); 479 goto tx_err_dst_release; 480 } 481 482 mtu = dst_mtu(dst); 483 if ((!skb_is_gso(skb) && skb->len > mtu) || 484 (skb_is_gso(skb) && !skb_gso_validate_network_len(skb, mtu))) { 485 skb_dst_update_pmtu_no_confirm(skb, mtu); 486 487 if (skb->protocol == htons(ETH_P_IPV6)) { 488 if (mtu < IPV6_MIN_MTU) 489 mtu = IPV6_MIN_MTU; 490 491 if (skb->len > 1280) 492 icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 493 else 494 goto xmit; 495 } else { 496 if (!(ip_hdr(skb)->frag_off & htons(IP_DF))) 497 goto xmit; 498 icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, 499 htonl(mtu)); 500 } 501 502 dst_release(dst); 503 return -EMSGSIZE; 504 } 505 506 xmit: 507 xfrmi_scrub_packet(skb, !net_eq(xi->net, dev_net(dev))); 508 skb_dst_set(skb, dst); 509 skb->dev = tdev; 510 511 err = dst_output(xi->net, skb->sk, skb); 512 if (net_xmit_eval(err) == 0) { 513 dev_sw_netstats_tx_add(dev, 1, length); 514 } else { 515 stats->tx_errors++; 516 stats->tx_aborted_errors++; 517 } 518 519 return 0; 520 tx_err_link_failure: 521 stats->tx_carrier_errors++; 522 dst_link_failure(skb); 523 tx_err_dst_release: 524 dst_release(dst); 525 return err; 526 } 527 528 static netdev_tx_t xfrmi_xmit(struct sk_buff *skb, struct net_device *dev) 529 { 530 struct xfrm_if *xi = netdev_priv(dev); 531 struct net_device_stats *stats = &xi->dev->stats; 532 struct dst_entry *dst = skb_dst(skb); 533 struct flowi fl; 534 int ret; 535 536 memset(&fl, 0, sizeof(fl)); 537 538 switch (skb->protocol) { 539 case htons(ETH_P_IPV6): 540 xfrm_decode_session(skb, &fl, AF_INET6); 541 memset(IP6CB(skb), 0, sizeof(*IP6CB(skb))); 542 if (!dst) { 543 fl.u.ip6.flowi6_oif = dev->ifindex; 544 fl.u.ip6.flowi6_flags |= FLOWI_FLAG_ANYSRC; 545 dst = ip6_route_output(dev_net(dev), NULL, &fl.u.ip6); 546 if (dst->error) { 547 dst_release(dst); 548 stats->tx_carrier_errors++; 549 goto tx_err; 550 } 551 skb_dst_set(skb, dst); 552 } 553 break; 554 case htons(ETH_P_IP): 555 xfrm_decode_session(skb, &fl, AF_INET); 556 memset(IPCB(skb), 0, sizeof(*IPCB(skb))); 557 if (!dst) { 558 struct rtable *rt; 559 560 fl.u.ip4.flowi4_oif = dev->ifindex; 561 fl.u.ip4.flowi4_flags |= FLOWI_FLAG_ANYSRC; 562 rt = __ip_route_output_key(dev_net(dev), &fl.u.ip4); 563 if (IS_ERR(rt)) { 564 stats->tx_carrier_errors++; 565 goto tx_err; 566 } 567 skb_dst_set(skb, &rt->dst); 568 } 569 break; 570 default: 571 goto tx_err; 572 } 573 574 fl.flowi_oif = xi->p.link; 575 576 ret = xfrmi_xmit2(skb, dev, &fl); 577 if (ret < 0) 578 goto tx_err; 579 580 return NETDEV_TX_OK; 581 582 tx_err: 583 stats->tx_errors++; 584 stats->tx_dropped++; 585 kfree_skb(skb); 586 return NETDEV_TX_OK; 587 } 588 589 static int xfrmi4_err(struct sk_buff *skb, u32 info) 590 { 591 const struct iphdr *iph = (const struct iphdr *)skb->data; 592 struct net *net = dev_net(skb->dev); 593 int protocol = iph->protocol; 594 struct ip_comp_hdr *ipch; 595 struct ip_esp_hdr *esph; 596 struct ip_auth_hdr *ah ; 597 struct xfrm_state *x; 598 struct xfrm_if *xi; 599 __be32 spi; 600 601 switch (protocol) { 602 case IPPROTO_ESP: 603 esph = (struct ip_esp_hdr *)(skb->data+(iph->ihl<<2)); 604 spi = esph->spi; 605 break; 606 case IPPROTO_AH: 607 ah = (struct ip_auth_hdr *)(skb->data+(iph->ihl<<2)); 608 spi = ah->spi; 609 break; 610 case IPPROTO_COMP: 611 ipch = (struct ip_comp_hdr *)(skb->data+(iph->ihl<<2)); 612 spi = htonl(ntohs(ipch->cpi)); 613 break; 614 default: 615 return 0; 616 } 617 618 switch (icmp_hdr(skb)->type) { 619 case ICMP_DEST_UNREACH: 620 if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED) 621 return 0; 622 break; 623 case ICMP_REDIRECT: 624 break; 625 default: 626 return 0; 627 } 628 629 x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr, 630 spi, protocol, AF_INET); 631 if (!x) 632 return 0; 633 634 xi = xfrmi_lookup(net, x); 635 if (!xi) { 636 xfrm_state_put(x); 637 return -1; 638 } 639 640 if (icmp_hdr(skb)->type == ICMP_DEST_UNREACH) 641 ipv4_update_pmtu(skb, net, info, 0, protocol); 642 else 643 ipv4_redirect(skb, net, 0, protocol); 644 xfrm_state_put(x); 645 646 return 0; 647 } 648 649 static int xfrmi6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, 650 u8 type, u8 code, int offset, __be32 info) 651 { 652 const struct ipv6hdr *iph = (const struct ipv6hdr *)skb->data; 653 struct net *net = dev_net(skb->dev); 654 int protocol = iph->nexthdr; 655 struct ip_comp_hdr *ipch; 656 struct ip_esp_hdr *esph; 657 struct ip_auth_hdr *ah; 658 struct xfrm_state *x; 659 struct xfrm_if *xi; 660 __be32 spi; 661 662 switch (protocol) { 663 case IPPROTO_ESP: 664 esph = (struct ip_esp_hdr *)(skb->data + offset); 665 spi = esph->spi; 666 break; 667 case IPPROTO_AH: 668 ah = (struct ip_auth_hdr *)(skb->data + offset); 669 spi = ah->spi; 670 break; 671 case IPPROTO_COMP: 672 ipch = (struct ip_comp_hdr *)(skb->data + offset); 673 spi = htonl(ntohs(ipch->cpi)); 674 break; 675 default: 676 return 0; 677 } 678 679 if (type != ICMPV6_PKT_TOOBIG && 680 type != NDISC_REDIRECT) 681 return 0; 682 683 x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr, 684 spi, protocol, AF_INET6); 685 if (!x) 686 return 0; 687 688 xi = xfrmi_lookup(net, x); 689 if (!xi) { 690 xfrm_state_put(x); 691 return -1; 692 } 693 694 if (type == NDISC_REDIRECT) 695 ip6_redirect(skb, net, skb->dev->ifindex, 0, 696 sock_net_uid(net, NULL)); 697 else 698 ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL)); 699 xfrm_state_put(x); 700 701 return 0; 702 } 703 704 static int xfrmi_change(struct xfrm_if *xi, const struct xfrm_if_parms *p) 705 { 706 if (xi->p.link != p->link) 707 return -EINVAL; 708 709 xi->p.if_id = p->if_id; 710 711 return 0; 712 } 713 714 static int xfrmi_update(struct xfrm_if *xi, struct xfrm_if_parms *p) 715 { 716 struct net *net = xi->net; 717 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 718 int err; 719 720 xfrmi_unlink(xfrmn, xi); 721 synchronize_net(); 722 err = xfrmi_change(xi, p); 723 xfrmi_link(xfrmn, xi); 724 netdev_state_change(xi->dev); 725 return err; 726 } 727 728 static int xfrmi_get_iflink(const struct net_device *dev) 729 { 730 struct xfrm_if *xi = netdev_priv(dev); 731 732 return xi->p.link; 733 } 734 735 static const struct net_device_ops xfrmi_netdev_ops = { 736 .ndo_init = xfrmi_dev_init, 737 .ndo_uninit = xfrmi_dev_uninit, 738 .ndo_start_xmit = xfrmi_xmit, 739 .ndo_get_stats64 = dev_get_tstats64, 740 .ndo_get_iflink = xfrmi_get_iflink, 741 }; 742 743 static void xfrmi_dev_setup(struct net_device *dev) 744 { 745 dev->netdev_ops = &xfrmi_netdev_ops; 746 dev->header_ops = &ip_tunnel_header_ops; 747 dev->type = ARPHRD_NONE; 748 dev->mtu = ETH_DATA_LEN; 749 dev->min_mtu = ETH_MIN_MTU; 750 dev->max_mtu = IP_MAX_MTU; 751 dev->flags = IFF_NOARP; 752 dev->needs_free_netdev = true; 753 dev->priv_destructor = xfrmi_dev_free; 754 netif_keep_dst(dev); 755 756 eth_broadcast_addr(dev->broadcast); 757 } 758 759 #define XFRMI_FEATURES (NETIF_F_SG | \ 760 NETIF_F_FRAGLIST | \ 761 NETIF_F_GSO_SOFTWARE | \ 762 NETIF_F_HW_CSUM) 763 764 static int xfrmi_dev_init(struct net_device *dev) 765 { 766 struct xfrm_if *xi = netdev_priv(dev); 767 struct net_device *phydev = __dev_get_by_index(xi->net, xi->p.link); 768 int err; 769 770 dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); 771 if (!dev->tstats) 772 return -ENOMEM; 773 774 err = gro_cells_init(&xi->gro_cells, dev); 775 if (err) { 776 free_percpu(dev->tstats); 777 return err; 778 } 779 780 dev->features |= NETIF_F_LLTX; 781 dev->features |= XFRMI_FEATURES; 782 dev->hw_features |= XFRMI_FEATURES; 783 784 if (phydev) { 785 dev->needed_headroom = phydev->needed_headroom; 786 dev->needed_tailroom = phydev->needed_tailroom; 787 788 if (is_zero_ether_addr(dev->dev_addr)) 789 eth_hw_addr_inherit(dev, phydev); 790 if (is_zero_ether_addr(dev->broadcast)) 791 memcpy(dev->broadcast, phydev->broadcast, 792 dev->addr_len); 793 } else { 794 eth_hw_addr_random(dev); 795 eth_broadcast_addr(dev->broadcast); 796 } 797 798 return 0; 799 } 800 801 static int xfrmi_validate(struct nlattr *tb[], struct nlattr *data[], 802 struct netlink_ext_ack *extack) 803 { 804 return 0; 805 } 806 807 static void xfrmi_netlink_parms(struct nlattr *data[], 808 struct xfrm_if_parms *parms) 809 { 810 memset(parms, 0, sizeof(*parms)); 811 812 if (!data) 813 return; 814 815 if (data[IFLA_XFRM_LINK]) 816 parms->link = nla_get_u32(data[IFLA_XFRM_LINK]); 817 818 if (data[IFLA_XFRM_IF_ID]) 819 parms->if_id = nla_get_u32(data[IFLA_XFRM_IF_ID]); 820 821 if (data[IFLA_XFRM_COLLECT_METADATA]) 822 parms->collect_md = true; 823 } 824 825 static int xfrmi_newlink(struct net *src_net, struct net_device *dev, 826 struct nlattr *tb[], struct nlattr *data[], 827 struct netlink_ext_ack *extack) 828 { 829 struct net *net = dev_net(dev); 830 struct xfrm_if_parms p = {}; 831 struct xfrm_if *xi; 832 int err; 833 834 xfrmi_netlink_parms(data, &p); 835 if (p.collect_md) { 836 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 837 838 if (p.link || p.if_id) { 839 NL_SET_ERR_MSG(extack, "link and if_id must be zero"); 840 return -EINVAL; 841 } 842 843 if (rtnl_dereference(xfrmn->collect_md_xfrmi)) 844 return -EEXIST; 845 846 } else { 847 if (!p.if_id) { 848 NL_SET_ERR_MSG(extack, "if_id must be non zero"); 849 return -EINVAL; 850 } 851 852 xi = xfrmi_locate(net, &p); 853 if (xi) 854 return -EEXIST; 855 } 856 857 xi = netdev_priv(dev); 858 xi->p = p; 859 xi->net = net; 860 xi->dev = dev; 861 862 err = xfrmi_create(dev); 863 return err; 864 } 865 866 static void xfrmi_dellink(struct net_device *dev, struct list_head *head) 867 { 868 unregister_netdevice_queue(dev, head); 869 } 870 871 static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[], 872 struct nlattr *data[], 873 struct netlink_ext_ack *extack) 874 { 875 struct xfrm_if *xi = netdev_priv(dev); 876 struct net *net = xi->net; 877 struct xfrm_if_parms p = {}; 878 879 xfrmi_netlink_parms(data, &p); 880 if (!p.if_id) { 881 NL_SET_ERR_MSG(extack, "if_id must be non zero"); 882 return -EINVAL; 883 } 884 885 if (p.collect_md) { 886 NL_SET_ERR_MSG(extack, "collect_md can't be changed"); 887 return -EINVAL; 888 } 889 890 xi = xfrmi_locate(net, &p); 891 if (!xi) { 892 xi = netdev_priv(dev); 893 } else { 894 if (xi->dev != dev) 895 return -EEXIST; 896 if (xi->p.collect_md) { 897 NL_SET_ERR_MSG(extack, 898 "device can't be changed to collect_md"); 899 return -EINVAL; 900 } 901 } 902 903 return xfrmi_update(xi, &p); 904 } 905 906 static size_t xfrmi_get_size(const struct net_device *dev) 907 { 908 return 909 /* IFLA_XFRM_LINK */ 910 nla_total_size(4) + 911 /* IFLA_XFRM_IF_ID */ 912 nla_total_size(4) + 913 /* IFLA_XFRM_COLLECT_METADATA */ 914 nla_total_size(0) + 915 0; 916 } 917 918 static int xfrmi_fill_info(struct sk_buff *skb, const struct net_device *dev) 919 { 920 struct xfrm_if *xi = netdev_priv(dev); 921 struct xfrm_if_parms *parm = &xi->p; 922 923 if (nla_put_u32(skb, IFLA_XFRM_LINK, parm->link) || 924 nla_put_u32(skb, IFLA_XFRM_IF_ID, parm->if_id) || 925 (xi->p.collect_md && nla_put_flag(skb, IFLA_XFRM_COLLECT_METADATA))) 926 goto nla_put_failure; 927 return 0; 928 929 nla_put_failure: 930 return -EMSGSIZE; 931 } 932 933 static struct net *xfrmi_get_link_net(const struct net_device *dev) 934 { 935 struct xfrm_if *xi = netdev_priv(dev); 936 937 return xi->net; 938 } 939 940 static const struct nla_policy xfrmi_policy[IFLA_XFRM_MAX + 1] = { 941 [IFLA_XFRM_UNSPEC] = { .strict_start_type = IFLA_XFRM_COLLECT_METADATA }, 942 [IFLA_XFRM_LINK] = { .type = NLA_U32 }, 943 [IFLA_XFRM_IF_ID] = { .type = NLA_U32 }, 944 [IFLA_XFRM_COLLECT_METADATA] = { .type = NLA_FLAG }, 945 }; 946 947 static struct rtnl_link_ops xfrmi_link_ops __read_mostly = { 948 .kind = "xfrm", 949 .maxtype = IFLA_XFRM_MAX, 950 .policy = xfrmi_policy, 951 .priv_size = sizeof(struct xfrm_if), 952 .setup = xfrmi_dev_setup, 953 .validate = xfrmi_validate, 954 .newlink = xfrmi_newlink, 955 .dellink = xfrmi_dellink, 956 .changelink = xfrmi_changelink, 957 .get_size = xfrmi_get_size, 958 .fill_info = xfrmi_fill_info, 959 .get_link_net = xfrmi_get_link_net, 960 }; 961 962 static void __net_exit xfrmi_exit_batch_net(struct list_head *net_exit_list) 963 { 964 struct net *net; 965 LIST_HEAD(list); 966 967 rtnl_lock(); 968 list_for_each_entry(net, net_exit_list, exit_list) { 969 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 970 struct xfrm_if __rcu **xip; 971 struct xfrm_if *xi; 972 int i; 973 974 for (i = 0; i < XFRMI_HASH_SIZE; i++) { 975 for (xip = &xfrmn->xfrmi[i]; 976 (xi = rtnl_dereference(*xip)) != NULL; 977 xip = &xi->next) 978 unregister_netdevice_queue(xi->dev, &list); 979 } 980 xi = rtnl_dereference(xfrmn->collect_md_xfrmi); 981 if (xi) 982 unregister_netdevice_queue(xi->dev, &list); 983 } 984 unregister_netdevice_many(&list); 985 rtnl_unlock(); 986 } 987 988 static struct pernet_operations xfrmi_net_ops = { 989 .exit_batch = xfrmi_exit_batch_net, 990 .id = &xfrmi_net_id, 991 .size = sizeof(struct xfrmi_net), 992 }; 993 994 static struct xfrm6_protocol xfrmi_esp6_protocol __read_mostly = { 995 .handler = xfrmi6_rcv, 996 .input_handler = xfrmi6_input, 997 .cb_handler = xfrmi_rcv_cb, 998 .err_handler = xfrmi6_err, 999 .priority = 10, 1000 }; 1001 1002 static struct xfrm6_protocol xfrmi_ah6_protocol __read_mostly = { 1003 .handler = xfrm6_rcv, 1004 .input_handler = xfrm_input, 1005 .cb_handler = xfrmi_rcv_cb, 1006 .err_handler = xfrmi6_err, 1007 .priority = 10, 1008 }; 1009 1010 static struct xfrm6_protocol xfrmi_ipcomp6_protocol __read_mostly = { 1011 .handler = xfrm6_rcv, 1012 .input_handler = xfrm_input, 1013 .cb_handler = xfrmi_rcv_cb, 1014 .err_handler = xfrmi6_err, 1015 .priority = 10, 1016 }; 1017 1018 #if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL) 1019 static int xfrmi6_rcv_tunnel(struct sk_buff *skb) 1020 { 1021 const xfrm_address_t *saddr; 1022 __be32 spi; 1023 1024 saddr = (const xfrm_address_t *)&ipv6_hdr(skb)->saddr; 1025 spi = xfrm6_tunnel_spi_lookup(dev_net(skb->dev), saddr); 1026 1027 return xfrm6_rcv_spi(skb, IPPROTO_IPV6, spi, NULL); 1028 } 1029 1030 static struct xfrm6_tunnel xfrmi_ipv6_handler __read_mostly = { 1031 .handler = xfrmi6_rcv_tunnel, 1032 .cb_handler = xfrmi_rcv_cb, 1033 .err_handler = xfrmi6_err, 1034 .priority = 2, 1035 }; 1036 1037 static struct xfrm6_tunnel xfrmi_ip6ip_handler __read_mostly = { 1038 .handler = xfrmi6_rcv_tunnel, 1039 .cb_handler = xfrmi_rcv_cb, 1040 .err_handler = xfrmi6_err, 1041 .priority = 2, 1042 }; 1043 #endif 1044 1045 static struct xfrm4_protocol xfrmi_esp4_protocol __read_mostly = { 1046 .handler = xfrmi4_rcv, 1047 .input_handler = xfrmi4_input, 1048 .cb_handler = xfrmi_rcv_cb, 1049 .err_handler = xfrmi4_err, 1050 .priority = 10, 1051 }; 1052 1053 static struct xfrm4_protocol xfrmi_ah4_protocol __read_mostly = { 1054 .handler = xfrm4_rcv, 1055 .input_handler = xfrm_input, 1056 .cb_handler = xfrmi_rcv_cb, 1057 .err_handler = xfrmi4_err, 1058 .priority = 10, 1059 }; 1060 1061 static struct xfrm4_protocol xfrmi_ipcomp4_protocol __read_mostly = { 1062 .handler = xfrm4_rcv, 1063 .input_handler = xfrm_input, 1064 .cb_handler = xfrmi_rcv_cb, 1065 .err_handler = xfrmi4_err, 1066 .priority = 10, 1067 }; 1068 1069 #if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL) 1070 static int xfrmi4_rcv_tunnel(struct sk_buff *skb) 1071 { 1072 return xfrm4_rcv_spi(skb, IPPROTO_IPIP, ip_hdr(skb)->saddr); 1073 } 1074 1075 static struct xfrm_tunnel xfrmi_ipip_handler __read_mostly = { 1076 .handler = xfrmi4_rcv_tunnel, 1077 .cb_handler = xfrmi_rcv_cb, 1078 .err_handler = xfrmi4_err, 1079 .priority = 3, 1080 }; 1081 1082 static struct xfrm_tunnel xfrmi_ipip6_handler __read_mostly = { 1083 .handler = xfrmi4_rcv_tunnel, 1084 .cb_handler = xfrmi_rcv_cb, 1085 .err_handler = xfrmi4_err, 1086 .priority = 2, 1087 }; 1088 #endif 1089 1090 static int __init xfrmi4_init(void) 1091 { 1092 int err; 1093 1094 err = xfrm4_protocol_register(&xfrmi_esp4_protocol, IPPROTO_ESP); 1095 if (err < 0) 1096 goto xfrm_proto_esp_failed; 1097 err = xfrm4_protocol_register(&xfrmi_ah4_protocol, IPPROTO_AH); 1098 if (err < 0) 1099 goto xfrm_proto_ah_failed; 1100 err = xfrm4_protocol_register(&xfrmi_ipcomp4_protocol, IPPROTO_COMP); 1101 if (err < 0) 1102 goto xfrm_proto_comp_failed; 1103 #if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL) 1104 err = xfrm4_tunnel_register(&xfrmi_ipip_handler, AF_INET); 1105 if (err < 0) 1106 goto xfrm_tunnel_ipip_failed; 1107 err = xfrm4_tunnel_register(&xfrmi_ipip6_handler, AF_INET6); 1108 if (err < 0) 1109 goto xfrm_tunnel_ipip6_failed; 1110 #endif 1111 1112 return 0; 1113 1114 #if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL) 1115 xfrm_tunnel_ipip6_failed: 1116 xfrm4_tunnel_deregister(&xfrmi_ipip_handler, AF_INET); 1117 xfrm_tunnel_ipip_failed: 1118 xfrm4_protocol_deregister(&xfrmi_ipcomp4_protocol, IPPROTO_COMP); 1119 #endif 1120 xfrm_proto_comp_failed: 1121 xfrm4_protocol_deregister(&xfrmi_ah4_protocol, IPPROTO_AH); 1122 xfrm_proto_ah_failed: 1123 xfrm4_protocol_deregister(&xfrmi_esp4_protocol, IPPROTO_ESP); 1124 xfrm_proto_esp_failed: 1125 return err; 1126 } 1127 1128 static void xfrmi4_fini(void) 1129 { 1130 #if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL) 1131 xfrm4_tunnel_deregister(&xfrmi_ipip6_handler, AF_INET6); 1132 xfrm4_tunnel_deregister(&xfrmi_ipip_handler, AF_INET); 1133 #endif 1134 xfrm4_protocol_deregister(&xfrmi_ipcomp4_protocol, IPPROTO_COMP); 1135 xfrm4_protocol_deregister(&xfrmi_ah4_protocol, IPPROTO_AH); 1136 xfrm4_protocol_deregister(&xfrmi_esp4_protocol, IPPROTO_ESP); 1137 } 1138 1139 static int __init xfrmi6_init(void) 1140 { 1141 int err; 1142 1143 err = xfrm6_protocol_register(&xfrmi_esp6_protocol, IPPROTO_ESP); 1144 if (err < 0) 1145 goto xfrm_proto_esp_failed; 1146 err = xfrm6_protocol_register(&xfrmi_ah6_protocol, IPPROTO_AH); 1147 if (err < 0) 1148 goto xfrm_proto_ah_failed; 1149 err = xfrm6_protocol_register(&xfrmi_ipcomp6_protocol, IPPROTO_COMP); 1150 if (err < 0) 1151 goto xfrm_proto_comp_failed; 1152 #if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL) 1153 err = xfrm6_tunnel_register(&xfrmi_ipv6_handler, AF_INET6); 1154 if (err < 0) 1155 goto xfrm_tunnel_ipv6_failed; 1156 err = xfrm6_tunnel_register(&xfrmi_ip6ip_handler, AF_INET); 1157 if (err < 0) 1158 goto xfrm_tunnel_ip6ip_failed; 1159 #endif 1160 1161 return 0; 1162 1163 #if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL) 1164 xfrm_tunnel_ip6ip_failed: 1165 xfrm6_tunnel_deregister(&xfrmi_ipv6_handler, AF_INET6); 1166 xfrm_tunnel_ipv6_failed: 1167 xfrm6_protocol_deregister(&xfrmi_ipcomp6_protocol, IPPROTO_COMP); 1168 #endif 1169 xfrm_proto_comp_failed: 1170 xfrm6_protocol_deregister(&xfrmi_ah6_protocol, IPPROTO_AH); 1171 xfrm_proto_ah_failed: 1172 xfrm6_protocol_deregister(&xfrmi_esp6_protocol, IPPROTO_ESP); 1173 xfrm_proto_esp_failed: 1174 return err; 1175 } 1176 1177 static void xfrmi6_fini(void) 1178 { 1179 #if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL) 1180 xfrm6_tunnel_deregister(&xfrmi_ip6ip_handler, AF_INET); 1181 xfrm6_tunnel_deregister(&xfrmi_ipv6_handler, AF_INET6); 1182 #endif 1183 xfrm6_protocol_deregister(&xfrmi_ipcomp6_protocol, IPPROTO_COMP); 1184 xfrm6_protocol_deregister(&xfrmi_ah6_protocol, IPPROTO_AH); 1185 xfrm6_protocol_deregister(&xfrmi_esp6_protocol, IPPROTO_ESP); 1186 } 1187 1188 static const struct xfrm_if_cb xfrm_if_cb = { 1189 .decode_session = xfrmi_decode_session, 1190 }; 1191 1192 static int __init xfrmi_init(void) 1193 { 1194 const char *msg; 1195 int err; 1196 1197 pr_info("IPsec XFRM device driver\n"); 1198 1199 msg = "tunnel device"; 1200 err = register_pernet_device(&xfrmi_net_ops); 1201 if (err < 0) 1202 goto pernet_dev_failed; 1203 1204 msg = "xfrm4 protocols"; 1205 err = xfrmi4_init(); 1206 if (err < 0) 1207 goto xfrmi4_failed; 1208 1209 msg = "xfrm6 protocols"; 1210 err = xfrmi6_init(); 1211 if (err < 0) 1212 goto xfrmi6_failed; 1213 1214 1215 msg = "netlink interface"; 1216 err = rtnl_link_register(&xfrmi_link_ops); 1217 if (err < 0) 1218 goto rtnl_link_failed; 1219 1220 err = register_xfrm_interface_bpf(); 1221 if (err < 0) 1222 goto kfunc_failed; 1223 1224 lwtunnel_encap_add_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM); 1225 1226 xfrm_if_register_cb(&xfrm_if_cb); 1227 1228 return err; 1229 1230 kfunc_failed: 1231 rtnl_link_unregister(&xfrmi_link_ops); 1232 rtnl_link_failed: 1233 xfrmi6_fini(); 1234 xfrmi6_failed: 1235 xfrmi4_fini(); 1236 xfrmi4_failed: 1237 unregister_pernet_device(&xfrmi_net_ops); 1238 pernet_dev_failed: 1239 pr_err("xfrmi init: failed to register %s\n", msg); 1240 return err; 1241 } 1242 1243 static void __exit xfrmi_fini(void) 1244 { 1245 xfrm_if_unregister_cb(); 1246 lwtunnel_encap_del_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM); 1247 rtnl_link_unregister(&xfrmi_link_ops); 1248 xfrmi4_fini(); 1249 xfrmi6_fini(); 1250 unregister_pernet_device(&xfrmi_net_ops); 1251 } 1252 1253 module_init(xfrmi_init); 1254 module_exit(xfrmi_fini); 1255 MODULE_LICENSE("GPL"); 1256 MODULE_ALIAS_RTNL_LINK("xfrm"); 1257 MODULE_ALIAS_NETDEV("xfrm0"); 1258 MODULE_AUTHOR("Steffen Klassert"); 1259 MODULE_DESCRIPTION("XFRM virtual interface"); 1260