1 /* 2 * vrf.c: device driver to encapsulate a VRF space 3 * 4 * Copyright (c) 2015 Cumulus Networks. All rights reserved. 5 * Copyright (c) 2015 Shrijeet Mukherjee <shm@cumulusnetworks.com> 6 * Copyright (c) 2015 David Ahern <dsa@cumulusnetworks.com> 7 * 8 * Based on dummy, team and ipvlan drivers 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published by 12 * the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 */ 15 16 #include <linux/module.h> 17 #include <linux/kernel.h> 18 #include <linux/netdevice.h> 19 #include <linux/etherdevice.h> 20 #include <linux/ip.h> 21 #include <linux/init.h> 22 #include <linux/moduleparam.h> 23 #include <linux/netfilter.h> 24 #include <linux/rtnetlink.h> 25 #include <net/rtnetlink.h> 26 #include <linux/u64_stats_sync.h> 27 #include <linux/hashtable.h> 28 29 #include <linux/inetdevice.h> 30 #include <net/arp.h> 31 #include <net/ip.h> 32 #include <net/ip_fib.h> 33 #include <net/ip6_route.h> 34 #include <net/rtnetlink.h> 35 #include <net/route.h> 36 #include <net/addrconf.h> 37 #include <net/l3mdev.h> 38 39 #define DRV_NAME "vrf" 40 #define DRV_VERSION "1.0" 41 42 #define vrf_master_get_rcu(dev) \ 43 ((struct net_device *)rcu_dereference(dev->rx_handler_data)) 44 45 struct slave { 46 struct list_head list; 47 struct net_device *dev; 48 }; 49 50 struct slave_queue { 51 struct list_head all_slaves; 52 }; 53 54 struct net_vrf { 55 struct slave_queue queue; 56 struct rtable *rth; 57 u32 tb_id; 58 }; 59 60 struct pcpu_dstats { 61 u64 tx_pkts; 62 u64 tx_bytes; 63 u64 tx_drps; 64 u64 rx_pkts; 65 u64 rx_bytes; 66 struct u64_stats_sync syncp; 67 }; 68 69 static struct dst_entry *vrf_ip_check(struct dst_entry *dst, u32 cookie) 70 { 71 return dst; 72 } 73 74 static int vrf_ip_local_out(struct sk_buff *skb) 75 { 76 return ip_local_out(skb); 77 } 78 79 static unsigned int vrf_v4_mtu(const struct dst_entry *dst) 80 { 81 /* TO-DO: return max ethernet size? */ 82 return dst->dev->mtu; 83 } 84 85 static void vrf_dst_destroy(struct dst_entry *dst) 86 { 87 /* our dst lives forever - or until the device is closed */ 88 } 89 90 static unsigned int vrf_default_advmss(const struct dst_entry *dst) 91 { 92 return 65535 - 40; 93 } 94 95 static struct dst_ops vrf_dst_ops = { 96 .family = AF_INET, 97 .local_out = vrf_ip_local_out, 98 .check = vrf_ip_check, 99 .mtu = vrf_v4_mtu, 100 .destroy = vrf_dst_destroy, 101 .default_advmss = vrf_default_advmss, 102 }; 103 104 static bool is_ip_rx_frame(struct sk_buff *skb) 105 { 106 switch (skb->protocol) { 107 case htons(ETH_P_IP): 108 case htons(ETH_P_IPV6): 109 return true; 110 } 111 return false; 112 } 113 114 static void vrf_tx_error(struct net_device *vrf_dev, struct sk_buff *skb) 115 { 116 vrf_dev->stats.tx_errors++; 117 kfree_skb(skb); 118 } 119 120 /* note: already called with rcu_read_lock */ 121 static rx_handler_result_t vrf_handle_frame(struct sk_buff **pskb) 122 { 123 struct sk_buff *skb = *pskb; 124 125 if (is_ip_rx_frame(skb)) { 126 struct net_device *dev = vrf_master_get_rcu(skb->dev); 127 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats); 128 129 u64_stats_update_begin(&dstats->syncp); 130 dstats->rx_pkts++; 131 dstats->rx_bytes += skb->len; 132 u64_stats_update_end(&dstats->syncp); 133 134 skb->dev = dev; 135 136 return RX_HANDLER_ANOTHER; 137 } 138 return RX_HANDLER_PASS; 139 } 140 141 static struct rtnl_link_stats64 *vrf_get_stats64(struct net_device *dev, 142 struct rtnl_link_stats64 *stats) 143 { 144 int i; 145 146 for_each_possible_cpu(i) { 147 const struct pcpu_dstats *dstats; 148 u64 tbytes, tpkts, tdrops, rbytes, rpkts; 149 unsigned int start; 150 151 dstats = per_cpu_ptr(dev->dstats, i); 152 do { 153 start = u64_stats_fetch_begin_irq(&dstats->syncp); 154 tbytes = dstats->tx_bytes; 155 tpkts = dstats->tx_pkts; 156 tdrops = dstats->tx_drps; 157 rbytes = dstats->rx_bytes; 158 rpkts = dstats->rx_pkts; 159 } while (u64_stats_fetch_retry_irq(&dstats->syncp, start)); 160 stats->tx_bytes += tbytes; 161 stats->tx_packets += tpkts; 162 stats->tx_dropped += tdrops; 163 stats->rx_bytes += rbytes; 164 stats->rx_packets += rpkts; 165 } 166 return stats; 167 } 168 169 static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb, 170 struct net_device *dev) 171 { 172 vrf_tx_error(dev, skb); 173 return NET_XMIT_DROP; 174 } 175 176 static int vrf_send_v4_prep(struct sk_buff *skb, struct flowi4 *fl4, 177 struct net_device *vrf_dev) 178 { 179 struct rtable *rt; 180 int err = 1; 181 182 rt = ip_route_output_flow(dev_net(vrf_dev), fl4, NULL); 183 if (IS_ERR(rt)) 184 goto out; 185 186 /* TO-DO: what about broadcast ? */ 187 if (rt->rt_type != RTN_UNICAST && rt->rt_type != RTN_LOCAL) { 188 ip_rt_put(rt); 189 goto out; 190 } 191 192 skb_dst_drop(skb); 193 skb_dst_set(skb, &rt->dst); 194 err = 0; 195 out: 196 return err; 197 } 198 199 static netdev_tx_t vrf_process_v4_outbound(struct sk_buff *skb, 200 struct net_device *vrf_dev) 201 { 202 struct iphdr *ip4h = ip_hdr(skb); 203 int ret = NET_XMIT_DROP; 204 struct flowi4 fl4 = { 205 /* needed to match OIF rule */ 206 .flowi4_oif = vrf_dev->ifindex, 207 .flowi4_iif = LOOPBACK_IFINDEX, 208 .flowi4_tos = RT_TOS(ip4h->tos), 209 .flowi4_flags = FLOWI_FLAG_ANYSRC | FLOWI_FLAG_L3MDEV_SRC | 210 FLOWI_FLAG_SKIP_NH_OIF, 211 .daddr = ip4h->daddr, 212 }; 213 214 if (vrf_send_v4_prep(skb, &fl4, vrf_dev)) 215 goto err; 216 217 if (!ip4h->saddr) { 218 ip4h->saddr = inet_select_addr(skb_dst(skb)->dev, 0, 219 RT_SCOPE_LINK); 220 } 221 222 ret = ip_local_out(skb); 223 if (unlikely(net_xmit_eval(ret))) 224 vrf_dev->stats.tx_errors++; 225 else 226 ret = NET_XMIT_SUCCESS; 227 228 out: 229 return ret; 230 err: 231 vrf_tx_error(vrf_dev, skb); 232 goto out; 233 } 234 235 static netdev_tx_t is_ip_tx_frame(struct sk_buff *skb, struct net_device *dev) 236 { 237 /* strip the ethernet header added for pass through VRF device */ 238 __skb_pull(skb, skb_network_offset(skb)); 239 240 switch (skb->protocol) { 241 case htons(ETH_P_IP): 242 return vrf_process_v4_outbound(skb, dev); 243 case htons(ETH_P_IPV6): 244 return vrf_process_v6_outbound(skb, dev); 245 default: 246 vrf_tx_error(dev, skb); 247 return NET_XMIT_DROP; 248 } 249 } 250 251 static netdev_tx_t vrf_xmit(struct sk_buff *skb, struct net_device *dev) 252 { 253 netdev_tx_t ret = is_ip_tx_frame(skb, dev); 254 255 if (likely(ret == NET_XMIT_SUCCESS || ret == NET_XMIT_CN)) { 256 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats); 257 258 u64_stats_update_begin(&dstats->syncp); 259 dstats->tx_pkts++; 260 dstats->tx_bytes += skb->len; 261 u64_stats_update_end(&dstats->syncp); 262 } else { 263 this_cpu_inc(dev->dstats->tx_drps); 264 } 265 266 return ret; 267 } 268 269 /* modelled after ip_finish_output2 */ 270 static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 271 { 272 struct dst_entry *dst = skb_dst(skb); 273 struct rtable *rt = (struct rtable *)dst; 274 struct net_device *dev = dst->dev; 275 unsigned int hh_len = LL_RESERVED_SPACE(dev); 276 struct neighbour *neigh; 277 u32 nexthop; 278 int ret = -EINVAL; 279 280 /* Be paranoid, rather than too clever. */ 281 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { 282 struct sk_buff *skb2; 283 284 skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev)); 285 if (!skb2) { 286 ret = -ENOMEM; 287 goto err; 288 } 289 if (skb->sk) 290 skb_set_owner_w(skb2, skb->sk); 291 292 consume_skb(skb); 293 skb = skb2; 294 } 295 296 rcu_read_lock_bh(); 297 298 nexthop = (__force u32)rt_nexthop(rt, ip_hdr(skb)->daddr); 299 neigh = __ipv4_neigh_lookup_noref(dev, nexthop); 300 if (unlikely(!neigh)) 301 neigh = __neigh_create(&arp_tbl, &nexthop, dev, false); 302 if (!IS_ERR(neigh)) 303 ret = dst_neigh_output(dst, neigh, skb); 304 305 rcu_read_unlock_bh(); 306 err: 307 if (unlikely(ret < 0)) 308 vrf_tx_error(skb->dev, skb); 309 return ret; 310 } 311 312 static int vrf_output(struct sock *sk, struct sk_buff *skb) 313 { 314 struct net_device *dev = skb_dst(skb)->dev; 315 struct net *net = dev_net(dev); 316 317 IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len); 318 319 skb->dev = dev; 320 skb->protocol = htons(ETH_P_IP); 321 322 return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, 323 net, sk, skb, NULL, dev, 324 vrf_finish_output, 325 !(IPCB(skb)->flags & IPSKB_REROUTED)); 326 } 327 328 static void vrf_rtable_destroy(struct net_vrf *vrf) 329 { 330 struct dst_entry *dst = (struct dst_entry *)vrf->rth; 331 332 dst_destroy(dst); 333 vrf->rth = NULL; 334 } 335 336 static struct rtable *vrf_rtable_create(struct net_device *dev) 337 { 338 struct net_vrf *vrf = netdev_priv(dev); 339 struct rtable *rth; 340 341 rth = dst_alloc(&vrf_dst_ops, dev, 2, 342 DST_OBSOLETE_NONE, 343 (DST_HOST | DST_NOPOLICY | DST_NOXFRM)); 344 if (rth) { 345 rth->dst.output = vrf_output; 346 rth->rt_genid = rt_genid_ipv4(dev_net(dev)); 347 rth->rt_flags = 0; 348 rth->rt_type = RTN_UNICAST; 349 rth->rt_is_input = 0; 350 rth->rt_iif = 0; 351 rth->rt_pmtu = 0; 352 rth->rt_gateway = 0; 353 rth->rt_uses_gateway = 0; 354 rth->rt_table_id = vrf->tb_id; 355 INIT_LIST_HEAD(&rth->rt_uncached); 356 rth->rt_uncached_list = NULL; 357 } 358 359 return rth; 360 } 361 362 /**************************** device handling ********************/ 363 364 /* cycle interface to flush neighbor cache and move routes across tables */ 365 static void cycle_netdev(struct net_device *dev) 366 { 367 unsigned int flags = dev->flags; 368 int ret; 369 370 if (!netif_running(dev)) 371 return; 372 373 ret = dev_change_flags(dev, flags & ~IFF_UP); 374 if (ret >= 0) 375 ret = dev_change_flags(dev, flags); 376 377 if (ret < 0) { 378 netdev_err(dev, 379 "Failed to cycle device %s; route tables might be wrong!\n", 380 dev->name); 381 } 382 } 383 384 static struct slave *__vrf_find_slave_dev(struct slave_queue *queue, 385 struct net_device *dev) 386 { 387 struct list_head *head = &queue->all_slaves; 388 struct slave *slave; 389 390 list_for_each_entry(slave, head, list) { 391 if (slave->dev == dev) 392 return slave; 393 } 394 395 return NULL; 396 } 397 398 /* inverse of __vrf_insert_slave */ 399 static void __vrf_remove_slave(struct slave_queue *queue, struct slave *slave) 400 { 401 list_del(&slave->list); 402 } 403 404 static void __vrf_insert_slave(struct slave_queue *queue, struct slave *slave) 405 { 406 list_add(&slave->list, &queue->all_slaves); 407 } 408 409 static int do_vrf_add_slave(struct net_device *dev, struct net_device *port_dev) 410 { 411 struct slave *slave = kzalloc(sizeof(*slave), GFP_KERNEL); 412 struct net_vrf *vrf = netdev_priv(dev); 413 struct slave_queue *queue = &vrf->queue; 414 int ret = -ENOMEM; 415 416 if (!slave) 417 goto out_fail; 418 419 slave->dev = port_dev; 420 421 /* register the packet handler for slave ports */ 422 ret = netdev_rx_handler_register(port_dev, vrf_handle_frame, dev); 423 if (ret) { 424 netdev_err(port_dev, 425 "Device %s failed to register rx_handler\n", 426 port_dev->name); 427 goto out_fail; 428 } 429 430 ret = netdev_master_upper_dev_link(port_dev, dev); 431 if (ret < 0) 432 goto out_unregister; 433 434 port_dev->priv_flags |= IFF_L3MDEV_SLAVE; 435 __vrf_insert_slave(queue, slave); 436 cycle_netdev(port_dev); 437 438 return 0; 439 440 out_unregister: 441 netdev_rx_handler_unregister(port_dev); 442 out_fail: 443 kfree(slave); 444 return ret; 445 } 446 447 static int vrf_add_slave(struct net_device *dev, struct net_device *port_dev) 448 { 449 if (netif_is_l3_master(port_dev) || netif_is_l3_slave(port_dev)) 450 return -EINVAL; 451 452 return do_vrf_add_slave(dev, port_dev); 453 } 454 455 /* inverse of do_vrf_add_slave */ 456 static int do_vrf_del_slave(struct net_device *dev, struct net_device *port_dev) 457 { 458 struct net_vrf *vrf = netdev_priv(dev); 459 struct slave_queue *queue = &vrf->queue; 460 struct slave *slave; 461 462 netdev_upper_dev_unlink(port_dev, dev); 463 port_dev->priv_flags &= ~IFF_L3MDEV_SLAVE; 464 465 netdev_rx_handler_unregister(port_dev); 466 467 cycle_netdev(port_dev); 468 469 slave = __vrf_find_slave_dev(queue, port_dev); 470 if (slave) 471 __vrf_remove_slave(queue, slave); 472 473 kfree(slave); 474 475 return 0; 476 } 477 478 static int vrf_del_slave(struct net_device *dev, struct net_device *port_dev) 479 { 480 return do_vrf_del_slave(dev, port_dev); 481 } 482 483 static void vrf_dev_uninit(struct net_device *dev) 484 { 485 struct net_vrf *vrf = netdev_priv(dev); 486 struct slave_queue *queue = &vrf->queue; 487 struct list_head *head = &queue->all_slaves; 488 struct slave *slave, *next; 489 490 vrf_rtable_destroy(vrf); 491 492 list_for_each_entry_safe(slave, next, head, list) 493 vrf_del_slave(dev, slave->dev); 494 495 free_percpu(dev->dstats); 496 dev->dstats = NULL; 497 } 498 499 static int vrf_dev_init(struct net_device *dev) 500 { 501 struct net_vrf *vrf = netdev_priv(dev); 502 503 INIT_LIST_HEAD(&vrf->queue.all_slaves); 504 505 dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats); 506 if (!dev->dstats) 507 goto out_nomem; 508 509 /* create the default dst which points back to us */ 510 vrf->rth = vrf_rtable_create(dev); 511 if (!vrf->rth) 512 goto out_stats; 513 514 dev->flags = IFF_MASTER | IFF_NOARP; 515 516 return 0; 517 518 out_stats: 519 free_percpu(dev->dstats); 520 dev->dstats = NULL; 521 out_nomem: 522 return -ENOMEM; 523 } 524 525 static const struct net_device_ops vrf_netdev_ops = { 526 .ndo_init = vrf_dev_init, 527 .ndo_uninit = vrf_dev_uninit, 528 .ndo_start_xmit = vrf_xmit, 529 .ndo_get_stats64 = vrf_get_stats64, 530 .ndo_add_slave = vrf_add_slave, 531 .ndo_del_slave = vrf_del_slave, 532 }; 533 534 static u32 vrf_fib_table(const struct net_device *dev) 535 { 536 struct net_vrf *vrf = netdev_priv(dev); 537 538 return vrf->tb_id; 539 } 540 541 static struct rtable *vrf_get_rtable(const struct net_device *dev, 542 const struct flowi4 *fl4) 543 { 544 struct rtable *rth = NULL; 545 546 if (!(fl4->flowi4_flags & FLOWI_FLAG_L3MDEV_SRC)) { 547 struct net_vrf *vrf = netdev_priv(dev); 548 549 rth = vrf->rth; 550 atomic_inc(&rth->dst.__refcnt); 551 } 552 553 return rth; 554 } 555 556 static const struct l3mdev_ops vrf_l3mdev_ops = { 557 .l3mdev_fib_table = vrf_fib_table, 558 .l3mdev_get_rtable = vrf_get_rtable, 559 }; 560 561 static void vrf_get_drvinfo(struct net_device *dev, 562 struct ethtool_drvinfo *info) 563 { 564 strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); 565 strlcpy(info->version, DRV_VERSION, sizeof(info->version)); 566 } 567 568 static const struct ethtool_ops vrf_ethtool_ops = { 569 .get_drvinfo = vrf_get_drvinfo, 570 }; 571 572 static void vrf_setup(struct net_device *dev) 573 { 574 ether_setup(dev); 575 576 /* Initialize the device structure. */ 577 dev->netdev_ops = &vrf_netdev_ops; 578 dev->l3mdev_ops = &vrf_l3mdev_ops; 579 dev->ethtool_ops = &vrf_ethtool_ops; 580 dev->destructor = free_netdev; 581 582 /* Fill in device structure with ethernet-generic values. */ 583 eth_hw_addr_random(dev); 584 585 /* don't acquire vrf device's netif_tx_lock when transmitting */ 586 dev->features |= NETIF_F_LLTX; 587 588 /* don't allow vrf devices to change network namespaces. */ 589 dev->features |= NETIF_F_NETNS_LOCAL; 590 } 591 592 static int vrf_validate(struct nlattr *tb[], struct nlattr *data[]) 593 { 594 if (tb[IFLA_ADDRESS]) { 595 if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) 596 return -EINVAL; 597 if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) 598 return -EADDRNOTAVAIL; 599 } 600 return 0; 601 } 602 603 static void vrf_dellink(struct net_device *dev, struct list_head *head) 604 { 605 unregister_netdevice_queue(dev, head); 606 } 607 608 static int vrf_newlink(struct net *src_net, struct net_device *dev, 609 struct nlattr *tb[], struct nlattr *data[]) 610 { 611 struct net_vrf *vrf = netdev_priv(dev); 612 int err; 613 614 if (!data || !data[IFLA_VRF_TABLE]) 615 return -EINVAL; 616 617 vrf->tb_id = nla_get_u32(data[IFLA_VRF_TABLE]); 618 619 dev->priv_flags |= IFF_L3MDEV_MASTER; 620 621 err = register_netdevice(dev); 622 if (err < 0) 623 goto out_fail; 624 625 return 0; 626 627 out_fail: 628 free_netdev(dev); 629 return err; 630 } 631 632 static size_t vrf_nl_getsize(const struct net_device *dev) 633 { 634 return nla_total_size(sizeof(u32)); /* IFLA_VRF_TABLE */ 635 } 636 637 static int vrf_fillinfo(struct sk_buff *skb, 638 const struct net_device *dev) 639 { 640 struct net_vrf *vrf = netdev_priv(dev); 641 642 return nla_put_u32(skb, IFLA_VRF_TABLE, vrf->tb_id); 643 } 644 645 static const struct nla_policy vrf_nl_policy[IFLA_VRF_MAX + 1] = { 646 [IFLA_VRF_TABLE] = { .type = NLA_U32 }, 647 }; 648 649 static struct rtnl_link_ops vrf_link_ops __read_mostly = { 650 .kind = DRV_NAME, 651 .priv_size = sizeof(struct net_vrf), 652 653 .get_size = vrf_nl_getsize, 654 .policy = vrf_nl_policy, 655 .validate = vrf_validate, 656 .fill_info = vrf_fillinfo, 657 658 .newlink = vrf_newlink, 659 .dellink = vrf_dellink, 660 .setup = vrf_setup, 661 .maxtype = IFLA_VRF_MAX, 662 }; 663 664 static int vrf_device_event(struct notifier_block *unused, 665 unsigned long event, void *ptr) 666 { 667 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 668 669 /* only care about unregister events to drop slave references */ 670 if (event == NETDEV_UNREGISTER) { 671 struct net_device *vrf_dev; 672 673 if (!netif_is_l3_slave(dev)) 674 goto out; 675 676 vrf_dev = netdev_master_upper_dev_get(dev); 677 vrf_del_slave(vrf_dev, dev); 678 } 679 out: 680 return NOTIFY_DONE; 681 } 682 683 static struct notifier_block vrf_notifier_block __read_mostly = { 684 .notifier_call = vrf_device_event, 685 }; 686 687 static int __init vrf_init_module(void) 688 { 689 int rc; 690 691 vrf_dst_ops.kmem_cachep = 692 kmem_cache_create("vrf_ip_dst_cache", 693 sizeof(struct rtable), 0, 694 SLAB_HWCACHE_ALIGN, 695 NULL); 696 697 if (!vrf_dst_ops.kmem_cachep) 698 return -ENOMEM; 699 700 register_netdevice_notifier(&vrf_notifier_block); 701 702 rc = rtnl_link_register(&vrf_link_ops); 703 if (rc < 0) 704 goto error; 705 706 return 0; 707 708 error: 709 unregister_netdevice_notifier(&vrf_notifier_block); 710 kmem_cache_destroy(vrf_dst_ops.kmem_cachep); 711 return rc; 712 } 713 714 static void __exit vrf_cleanup_module(void) 715 { 716 rtnl_link_unregister(&vrf_link_ops); 717 unregister_netdevice_notifier(&vrf_notifier_block); 718 kmem_cache_destroy(vrf_dst_ops.kmem_cachep); 719 } 720 721 module_init(vrf_init_module); 722 module_exit(vrf_cleanup_module); 723 MODULE_AUTHOR("Shrijeet Mukherjee, David Ahern"); 724 MODULE_DESCRIPTION("Device driver to instantiate VRF domains"); 725 MODULE_LICENSE("GPL"); 726 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 727 MODULE_VERSION(DRV_VERSION); 728