1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * TUN - Universal TUN/TAP device driver. 4 * Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com> 5 * 6 * $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $ 7 */ 8 9 /* 10 * Changes: 11 * 12 * Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14 13 * Add TUNSETLINK ioctl to set the link encapsulation 14 * 15 * Mark Smith <markzzzsmith@yahoo.com.au> 16 * Use eth_random_addr() for tap MAC address. 17 * 18 * Harald Roelle <harald.roelle@ifi.lmu.de> 2004/04/20 19 * Fixes in packet dropping, queue length setting and queue wakeup. 20 * Increased default tx queue length. 21 * Added ethtool API. 22 * Minor cleanups 23 * 24 * Daniel Podlejski <underley@underley.eu.org> 25 * Modifications for 2.3.99-pre5 kernel. 26 */ 27 28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30 #define DRV_NAME "tun" 31 #define DRV_VERSION "1.6" 32 #define DRV_DESCRIPTION "Universal TUN/TAP device driver" 33 #define DRV_COPYRIGHT "(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>" 34 35 #include <linux/module.h> 36 #include <linux/errno.h> 37 #include <linux/kernel.h> 38 #include <linux/sched/signal.h> 39 #include <linux/major.h> 40 #include <linux/slab.h> 41 #include <linux/poll.h> 42 #include <linux/fcntl.h> 43 #include <linux/init.h> 44 #include <linux/skbuff.h> 45 #include <linux/netdevice.h> 46 #include <linux/etherdevice.h> 47 #include <linux/miscdevice.h> 48 #include <linux/ethtool.h> 49 #include <linux/rtnetlink.h> 50 #include <linux/compat.h> 51 #include <linux/if.h> 52 #include <linux/if_arp.h> 53 #include <linux/if_ether.h> 54 #include <linux/if_tun.h> 55 #include <linux/if_vlan.h> 56 #include <linux/crc32.h> 57 #include <linux/nsproxy.h> 58 #include <linux/virtio_net.h> 59 #include <linux/rcupdate.h> 60 #include <net/net_namespace.h> 61 #include <net/netns/generic.h> 62 #include <net/rtnetlink.h> 63 #include <net/sock.h> 64 #include <net/xdp.h> 65 #include <net/ip_tunnels.h> 66 #include <linux/seq_file.h> 67 #include <linux/uio.h> 68 #include <linux/skb_array.h> 69 #include <linux/bpf.h> 70 #include <linux/bpf_trace.h> 71 #include <linux/mutex.h> 72 #include <linux/ieee802154.h> 73 #include <linux/if_ltalk.h> 74 #include <uapi/linux/if_fddi.h> 75 #include <uapi/linux/if_hippi.h> 76 #include <uapi/linux/if_fc.h> 77 #include <net/ax25.h> 78 #include <net/rose.h> 79 #include <net/6lowpan.h> 80 81 #include <linux/uaccess.h> 82 #include <linux/proc_fs.h> 83 84 static void tun_default_link_ksettings(struct net_device *dev, 85 struct ethtool_link_ksettings *cmd); 86 87 #define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 88 89 /* TUN device flags */ 90 91 /* IFF_ATTACH_QUEUE is never stored in device flags, 92 * overload it to mean fasync when stored there. 93 */ 94 #define TUN_FASYNC IFF_ATTACH_QUEUE 95 /* High bits in flags field are unused. */ 96 #define TUN_VNET_LE 0x80000000 97 #define TUN_VNET_BE 0x40000000 98 99 #define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \ 100 IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS) 101 102 #define GOODCOPY_LEN 128 103 104 #define FLT_EXACT_COUNT 8 105 struct tap_filter { 106 unsigned int count; /* Number of addrs. Zero means disabled */ 107 u32 mask[2]; /* Mask of the hashed addrs */ 108 unsigned char addr[FLT_EXACT_COUNT][ETH_ALEN]; 109 }; 110 111 /* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal 112 * to max number of VCPUs in guest. */ 113 #define MAX_TAP_QUEUES 256 114 #define MAX_TAP_FLOWS 4096 115 116 #define TUN_FLOW_EXPIRE (3 * HZ) 117 118 /* A tun_file connects an open character device to a tuntap netdevice. It 119 * also contains all socket related structures (except sock_fprog and tap_filter) 120 * to serve as one transmit queue for tuntap device. The sock_fprog and 121 * tap_filter were kept in tun_struct since they were used for filtering for the 122 * netdevice not for a specific queue (at least I didn't see the requirement for 123 * this). 124 * 125 * RCU usage: 126 * The tun_file and tun_struct are loosely coupled, the pointer from one to the 127 * other can only be read while rcu_read_lock or rtnl_lock is held. 128 */ 129 struct tun_file { 130 struct sock sk; 131 struct socket socket; 132 struct tun_struct __rcu *tun; 133 struct fasync_struct *fasync; 134 /* only used for fasnyc */ 135 unsigned int flags; 136 union { 137 u16 queue_index; 138 unsigned int ifindex; 139 }; 140 struct napi_struct napi; 141 bool napi_enabled; 142 bool napi_frags_enabled; 143 struct mutex napi_mutex; /* Protects access to the above napi */ 144 struct list_head next; 145 struct tun_struct *detached; 146 struct ptr_ring tx_ring; 147 struct xdp_rxq_info xdp_rxq; 148 }; 149 150 struct tun_page { 151 struct page *page; 152 int count; 153 }; 154 155 struct tun_flow_entry { 156 struct hlist_node hash_link; 157 struct rcu_head rcu; 158 struct tun_struct *tun; 159 160 u32 rxhash; 161 u32 rps_rxhash; 162 int queue_index; 163 unsigned long updated ____cacheline_aligned_in_smp; 164 }; 165 166 #define TUN_NUM_FLOW_ENTRIES 1024 167 #define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1) 168 169 struct tun_prog { 170 struct rcu_head rcu; 171 struct bpf_prog *prog; 172 }; 173 174 /* Since the socket were moved to tun_file, to preserve the behavior of persist 175 * device, socket filter, sndbuf and vnet header size were restore when the 176 * file were attached to a persist device. 177 */ 178 struct tun_struct { 179 struct tun_file __rcu *tfiles[MAX_TAP_QUEUES]; 180 unsigned int numqueues; 181 unsigned int flags; 182 kuid_t owner; 183 kgid_t group; 184 185 struct net_device *dev; 186 netdev_features_t set_features; 187 #define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \ 188 NETIF_F_TSO6) 189 190 int align; 191 int vnet_hdr_sz; 192 int sndbuf; 193 struct tap_filter txflt; 194 struct sock_fprog fprog; 195 /* protected by rtnl lock */ 196 bool filter_attached; 197 u32 msg_enable; 198 spinlock_t lock; 199 struct hlist_head flows[TUN_NUM_FLOW_ENTRIES]; 200 struct timer_list flow_gc_timer; 201 unsigned long ageing_time; 202 unsigned int numdisabled; 203 struct list_head disabled; 204 void *security; 205 u32 flow_count; 206 u32 rx_batched; 207 atomic_long_t rx_frame_errors; 208 struct bpf_prog __rcu *xdp_prog; 209 struct tun_prog __rcu *steering_prog; 210 struct tun_prog __rcu *filter_prog; 211 struct ethtool_link_ksettings link_ksettings; 212 /* init args */ 213 struct file *file; 214 struct ifreq *ifr; 215 }; 216 217 struct veth { 218 __be16 h_vlan_proto; 219 __be16 h_vlan_TCI; 220 }; 221 222 static void tun_flow_init(struct tun_struct *tun); 223 static void tun_flow_uninit(struct tun_struct *tun); 224 225 static int tun_napi_receive(struct napi_struct *napi, int budget) 226 { 227 struct tun_file *tfile = container_of(napi, struct tun_file, napi); 228 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 229 struct sk_buff_head process_queue; 230 struct sk_buff *skb; 231 int received = 0; 232 233 __skb_queue_head_init(&process_queue); 234 235 spin_lock(&queue->lock); 236 skb_queue_splice_tail_init(queue, &process_queue); 237 spin_unlock(&queue->lock); 238 239 while (received < budget && (skb = __skb_dequeue(&process_queue))) { 240 napi_gro_receive(napi, skb); 241 ++received; 242 } 243 244 if (!skb_queue_empty(&process_queue)) { 245 spin_lock(&queue->lock); 246 skb_queue_splice(&process_queue, queue); 247 spin_unlock(&queue->lock); 248 } 249 250 return received; 251 } 252 253 static int tun_napi_poll(struct napi_struct *napi, int budget) 254 { 255 unsigned int received; 256 257 received = tun_napi_receive(napi, budget); 258 259 if (received < budget) 260 napi_complete_done(napi, received); 261 262 return received; 263 } 264 265 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile, 266 bool napi_en, bool napi_frags) 267 { 268 tfile->napi_enabled = napi_en; 269 tfile->napi_frags_enabled = napi_en && napi_frags; 270 if (napi_en) { 271 netif_napi_add_tx(tun->dev, &tfile->napi, tun_napi_poll); 272 napi_enable(&tfile->napi); 273 } 274 } 275 276 static void tun_napi_enable(struct tun_file *tfile) 277 { 278 if (tfile->napi_enabled) 279 napi_enable(&tfile->napi); 280 } 281 282 static void tun_napi_disable(struct tun_file *tfile) 283 { 284 if (tfile->napi_enabled) 285 napi_disable(&tfile->napi); 286 } 287 288 static void tun_napi_del(struct tun_file *tfile) 289 { 290 if (tfile->napi_enabled) 291 netif_napi_del(&tfile->napi); 292 } 293 294 static bool tun_napi_frags_enabled(const struct tun_file *tfile) 295 { 296 return tfile->napi_frags_enabled; 297 } 298 299 #ifdef CONFIG_TUN_VNET_CROSS_LE 300 static inline bool tun_legacy_is_little_endian(struct tun_struct *tun) 301 { 302 return tun->flags & TUN_VNET_BE ? false : 303 virtio_legacy_is_little_endian(); 304 } 305 306 static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp) 307 { 308 int be = !!(tun->flags & TUN_VNET_BE); 309 310 if (put_user(be, argp)) 311 return -EFAULT; 312 313 return 0; 314 } 315 316 static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp) 317 { 318 int be; 319 320 if (get_user(be, argp)) 321 return -EFAULT; 322 323 if (be) 324 tun->flags |= TUN_VNET_BE; 325 else 326 tun->flags &= ~TUN_VNET_BE; 327 328 return 0; 329 } 330 #else 331 static inline bool tun_legacy_is_little_endian(struct tun_struct *tun) 332 { 333 return virtio_legacy_is_little_endian(); 334 } 335 336 static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp) 337 { 338 return -EINVAL; 339 } 340 341 static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp) 342 { 343 return -EINVAL; 344 } 345 #endif /* CONFIG_TUN_VNET_CROSS_LE */ 346 347 static inline bool tun_is_little_endian(struct tun_struct *tun) 348 { 349 return tun->flags & TUN_VNET_LE || 350 tun_legacy_is_little_endian(tun); 351 } 352 353 static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val) 354 { 355 return __virtio16_to_cpu(tun_is_little_endian(tun), val); 356 } 357 358 static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val) 359 { 360 return __cpu_to_virtio16(tun_is_little_endian(tun), val); 361 } 362 363 static inline u32 tun_hashfn(u32 rxhash) 364 { 365 return rxhash & TUN_MASK_FLOW_ENTRIES; 366 } 367 368 static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash) 369 { 370 struct tun_flow_entry *e; 371 372 hlist_for_each_entry_rcu(e, head, hash_link) { 373 if (e->rxhash == rxhash) 374 return e; 375 } 376 return NULL; 377 } 378 379 static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun, 380 struct hlist_head *head, 381 u32 rxhash, u16 queue_index) 382 { 383 struct tun_flow_entry *e = kmalloc(sizeof(*e), GFP_ATOMIC); 384 385 if (e) { 386 netif_info(tun, tx_queued, tun->dev, 387 "create flow: hash %u index %u\n", 388 rxhash, queue_index); 389 e->updated = jiffies; 390 e->rxhash = rxhash; 391 e->rps_rxhash = 0; 392 e->queue_index = queue_index; 393 e->tun = tun; 394 hlist_add_head_rcu(&e->hash_link, head); 395 ++tun->flow_count; 396 } 397 return e; 398 } 399 400 static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e) 401 { 402 netif_info(tun, tx_queued, tun->dev, "delete flow: hash %u index %u\n", 403 e->rxhash, e->queue_index); 404 hlist_del_rcu(&e->hash_link); 405 kfree_rcu(e, rcu); 406 --tun->flow_count; 407 } 408 409 static void tun_flow_flush(struct tun_struct *tun) 410 { 411 int i; 412 413 spin_lock_bh(&tun->lock); 414 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 415 struct tun_flow_entry *e; 416 struct hlist_node *n; 417 418 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) 419 tun_flow_delete(tun, e); 420 } 421 spin_unlock_bh(&tun->lock); 422 } 423 424 static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index) 425 { 426 int i; 427 428 spin_lock_bh(&tun->lock); 429 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 430 struct tun_flow_entry *e; 431 struct hlist_node *n; 432 433 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) { 434 if (e->queue_index == queue_index) 435 tun_flow_delete(tun, e); 436 } 437 } 438 spin_unlock_bh(&tun->lock); 439 } 440 441 static void tun_flow_cleanup(struct timer_list *t) 442 { 443 struct tun_struct *tun = from_timer(tun, t, flow_gc_timer); 444 unsigned long delay = tun->ageing_time; 445 unsigned long next_timer = jiffies + delay; 446 unsigned long count = 0; 447 int i; 448 449 spin_lock(&tun->lock); 450 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 451 struct tun_flow_entry *e; 452 struct hlist_node *n; 453 454 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) { 455 unsigned long this_timer; 456 457 this_timer = e->updated + delay; 458 if (time_before_eq(this_timer, jiffies)) { 459 tun_flow_delete(tun, e); 460 continue; 461 } 462 count++; 463 if (time_before(this_timer, next_timer)) 464 next_timer = this_timer; 465 } 466 } 467 468 if (count) 469 mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer)); 470 spin_unlock(&tun->lock); 471 } 472 473 static void tun_flow_update(struct tun_struct *tun, u32 rxhash, 474 struct tun_file *tfile) 475 { 476 struct hlist_head *head; 477 struct tun_flow_entry *e; 478 unsigned long delay = tun->ageing_time; 479 u16 queue_index = tfile->queue_index; 480 481 head = &tun->flows[tun_hashfn(rxhash)]; 482 483 rcu_read_lock(); 484 485 e = tun_flow_find(head, rxhash); 486 if (likely(e)) { 487 /* TODO: keep queueing to old queue until it's empty? */ 488 if (READ_ONCE(e->queue_index) != queue_index) 489 WRITE_ONCE(e->queue_index, queue_index); 490 if (e->updated != jiffies) 491 e->updated = jiffies; 492 sock_rps_record_flow_hash(e->rps_rxhash); 493 } else { 494 spin_lock_bh(&tun->lock); 495 if (!tun_flow_find(head, rxhash) && 496 tun->flow_count < MAX_TAP_FLOWS) 497 tun_flow_create(tun, head, rxhash, queue_index); 498 499 if (!timer_pending(&tun->flow_gc_timer)) 500 mod_timer(&tun->flow_gc_timer, 501 round_jiffies_up(jiffies + delay)); 502 spin_unlock_bh(&tun->lock); 503 } 504 505 rcu_read_unlock(); 506 } 507 508 /* Save the hash received in the stack receive path and update the 509 * flow_hash table accordingly. 510 */ 511 static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash) 512 { 513 if (unlikely(e->rps_rxhash != hash)) 514 e->rps_rxhash = hash; 515 } 516 517 /* We try to identify a flow through its rxhash. The reason that 518 * we do not check rxq no. is because some cards(e.g 82599), chooses 519 * the rxq based on the txq where the last packet of the flow comes. As 520 * the userspace application move between processors, we may get a 521 * different rxq no. here. 522 */ 523 static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb) 524 { 525 struct tun_flow_entry *e; 526 u32 txq = 0; 527 u32 numqueues = 0; 528 529 numqueues = READ_ONCE(tun->numqueues); 530 531 txq = __skb_get_hash_symmetric(skb); 532 e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq); 533 if (e) { 534 tun_flow_save_rps_rxhash(e, txq); 535 txq = e->queue_index; 536 } else { 537 /* use multiply and shift instead of expensive divide */ 538 txq = ((u64)txq * numqueues) >> 32; 539 } 540 541 return txq; 542 } 543 544 static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb) 545 { 546 struct tun_prog *prog; 547 u32 numqueues; 548 u16 ret = 0; 549 550 numqueues = READ_ONCE(tun->numqueues); 551 if (!numqueues) 552 return 0; 553 554 prog = rcu_dereference(tun->steering_prog); 555 if (prog) 556 ret = bpf_prog_run_clear_cb(prog->prog, skb); 557 558 return ret % numqueues; 559 } 560 561 static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb, 562 struct net_device *sb_dev) 563 { 564 struct tun_struct *tun = netdev_priv(dev); 565 u16 ret; 566 567 rcu_read_lock(); 568 if (rcu_dereference(tun->steering_prog)) 569 ret = tun_ebpf_select_queue(tun, skb); 570 else 571 ret = tun_automq_select_queue(tun, skb); 572 rcu_read_unlock(); 573 574 return ret; 575 } 576 577 static inline bool tun_not_capable(struct tun_struct *tun) 578 { 579 const struct cred *cred = current_cred(); 580 struct net *net = dev_net(tun->dev); 581 582 return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) || 583 (gid_valid(tun->group) && !in_egroup_p(tun->group))) && 584 !ns_capable(net->user_ns, CAP_NET_ADMIN); 585 } 586 587 static void tun_set_real_num_queues(struct tun_struct *tun) 588 { 589 netif_set_real_num_tx_queues(tun->dev, tun->numqueues); 590 netif_set_real_num_rx_queues(tun->dev, tun->numqueues); 591 } 592 593 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile) 594 { 595 tfile->detached = tun; 596 list_add_tail(&tfile->next, &tun->disabled); 597 ++tun->numdisabled; 598 } 599 600 static struct tun_struct *tun_enable_queue(struct tun_file *tfile) 601 { 602 struct tun_struct *tun = tfile->detached; 603 604 tfile->detached = NULL; 605 list_del_init(&tfile->next); 606 --tun->numdisabled; 607 return tun; 608 } 609 610 void tun_ptr_free(void *ptr) 611 { 612 if (!ptr) 613 return; 614 if (tun_is_xdp_frame(ptr)) { 615 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 616 617 xdp_return_frame(xdpf); 618 } else { 619 __skb_array_destroy_skb(ptr); 620 } 621 } 622 EXPORT_SYMBOL_GPL(tun_ptr_free); 623 624 static void tun_queue_purge(struct tun_file *tfile) 625 { 626 void *ptr; 627 628 while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL) 629 tun_ptr_free(ptr); 630 631 skb_queue_purge(&tfile->sk.sk_write_queue); 632 skb_queue_purge(&tfile->sk.sk_error_queue); 633 } 634 635 static void __tun_detach(struct tun_file *tfile, bool clean) 636 { 637 struct tun_file *ntfile; 638 struct tun_struct *tun; 639 640 tun = rtnl_dereference(tfile->tun); 641 642 if (tun && clean) { 643 if (!tfile->detached) 644 tun_napi_disable(tfile); 645 tun_napi_del(tfile); 646 } 647 648 if (tun && !tfile->detached) { 649 u16 index = tfile->queue_index; 650 BUG_ON(index >= tun->numqueues); 651 652 rcu_assign_pointer(tun->tfiles[index], 653 tun->tfiles[tun->numqueues - 1]); 654 ntfile = rtnl_dereference(tun->tfiles[index]); 655 ntfile->queue_index = index; 656 rcu_assign_pointer(tun->tfiles[tun->numqueues - 1], 657 NULL); 658 659 --tun->numqueues; 660 if (clean) { 661 RCU_INIT_POINTER(tfile->tun, NULL); 662 sock_put(&tfile->sk); 663 } else { 664 tun_disable_queue(tun, tfile); 665 tun_napi_disable(tfile); 666 } 667 668 synchronize_net(); 669 tun_flow_delete_by_queue(tun, tun->numqueues + 1); 670 /* Drop read queue */ 671 tun_queue_purge(tfile); 672 tun_set_real_num_queues(tun); 673 } else if (tfile->detached && clean) { 674 tun = tun_enable_queue(tfile); 675 sock_put(&tfile->sk); 676 } 677 678 if (clean) { 679 if (tun && tun->numqueues == 0 && tun->numdisabled == 0) { 680 netif_carrier_off(tun->dev); 681 682 if (!(tun->flags & IFF_PERSIST) && 683 tun->dev->reg_state == NETREG_REGISTERED) 684 unregister_netdevice(tun->dev); 685 } 686 if (tun) 687 xdp_rxq_info_unreg(&tfile->xdp_rxq); 688 ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free); 689 } 690 } 691 692 static void tun_detach(struct tun_file *tfile, bool clean) 693 { 694 struct tun_struct *tun; 695 struct net_device *dev; 696 697 rtnl_lock(); 698 tun = rtnl_dereference(tfile->tun); 699 dev = tun ? tun->dev : NULL; 700 __tun_detach(tfile, clean); 701 if (dev) 702 netdev_state_change(dev); 703 rtnl_unlock(); 704 705 if (clean) 706 sock_put(&tfile->sk); 707 } 708 709 static void tun_detach_all(struct net_device *dev) 710 { 711 struct tun_struct *tun = netdev_priv(dev); 712 struct tun_file *tfile, *tmp; 713 int i, n = tun->numqueues; 714 715 for (i = 0; i < n; i++) { 716 tfile = rtnl_dereference(tun->tfiles[i]); 717 BUG_ON(!tfile); 718 tun_napi_disable(tfile); 719 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; 720 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 721 RCU_INIT_POINTER(tfile->tun, NULL); 722 --tun->numqueues; 723 } 724 list_for_each_entry(tfile, &tun->disabled, next) { 725 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; 726 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 727 RCU_INIT_POINTER(tfile->tun, NULL); 728 } 729 BUG_ON(tun->numqueues != 0); 730 731 synchronize_net(); 732 for (i = 0; i < n; i++) { 733 tfile = rtnl_dereference(tun->tfiles[i]); 734 tun_napi_del(tfile); 735 /* Drop read queue */ 736 tun_queue_purge(tfile); 737 xdp_rxq_info_unreg(&tfile->xdp_rxq); 738 sock_put(&tfile->sk); 739 } 740 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) { 741 tun_napi_del(tfile); 742 tun_enable_queue(tfile); 743 tun_queue_purge(tfile); 744 xdp_rxq_info_unreg(&tfile->xdp_rxq); 745 sock_put(&tfile->sk); 746 } 747 BUG_ON(tun->numdisabled != 0); 748 749 if (tun->flags & IFF_PERSIST) 750 module_put(THIS_MODULE); 751 } 752 753 static int tun_attach(struct tun_struct *tun, struct file *file, 754 bool skip_filter, bool napi, bool napi_frags, 755 bool publish_tun) 756 { 757 struct tun_file *tfile = file->private_data; 758 struct net_device *dev = tun->dev; 759 int err; 760 761 err = security_tun_dev_attach(tfile->socket.sk, tun->security); 762 if (err < 0) 763 goto out; 764 765 err = -EINVAL; 766 if (rtnl_dereference(tfile->tun) && !tfile->detached) 767 goto out; 768 769 err = -EBUSY; 770 if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1) 771 goto out; 772 773 err = -E2BIG; 774 if (!tfile->detached && 775 tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES) 776 goto out; 777 778 err = 0; 779 780 /* Re-attach the filter to persist device */ 781 if (!skip_filter && (tun->filter_attached == true)) { 782 lock_sock(tfile->socket.sk); 783 err = sk_attach_filter(&tun->fprog, tfile->socket.sk); 784 release_sock(tfile->socket.sk); 785 if (!err) 786 goto out; 787 } 788 789 if (!tfile->detached && 790 ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len, 791 GFP_KERNEL, tun_ptr_free)) { 792 err = -ENOMEM; 793 goto out; 794 } 795 796 tfile->queue_index = tun->numqueues; 797 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN; 798 799 if (tfile->detached) { 800 /* Re-attach detached tfile, updating XDP queue_index */ 801 WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq)); 802 803 if (tfile->xdp_rxq.queue_index != tfile->queue_index) 804 tfile->xdp_rxq.queue_index = tfile->queue_index; 805 } else { 806 /* Setup XDP RX-queue info, for new tfile getting attached */ 807 err = xdp_rxq_info_reg(&tfile->xdp_rxq, 808 tun->dev, tfile->queue_index, 0); 809 if (err < 0) 810 goto out; 811 err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq, 812 MEM_TYPE_PAGE_SHARED, NULL); 813 if (err < 0) { 814 xdp_rxq_info_unreg(&tfile->xdp_rxq); 815 goto out; 816 } 817 err = 0; 818 } 819 820 if (tfile->detached) { 821 tun_enable_queue(tfile); 822 tun_napi_enable(tfile); 823 } else { 824 sock_hold(&tfile->sk); 825 tun_napi_init(tun, tfile, napi, napi_frags); 826 } 827 828 if (rtnl_dereference(tun->xdp_prog)) 829 sock_set_flag(&tfile->sk, SOCK_XDP); 830 831 /* device is allowed to go away first, so no need to hold extra 832 * refcnt. 833 */ 834 835 /* Publish tfile->tun and tun->tfiles only after we've fully 836 * initialized tfile; otherwise we risk using half-initialized 837 * object. 838 */ 839 if (publish_tun) 840 rcu_assign_pointer(tfile->tun, tun); 841 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile); 842 tun->numqueues++; 843 tun_set_real_num_queues(tun); 844 out: 845 return err; 846 } 847 848 static struct tun_struct *tun_get(struct tun_file *tfile) 849 { 850 struct tun_struct *tun; 851 852 rcu_read_lock(); 853 tun = rcu_dereference(tfile->tun); 854 if (tun) 855 dev_hold(tun->dev); 856 rcu_read_unlock(); 857 858 return tun; 859 } 860 861 static void tun_put(struct tun_struct *tun) 862 { 863 dev_put(tun->dev); 864 } 865 866 /* TAP filtering */ 867 static void addr_hash_set(u32 *mask, const u8 *addr) 868 { 869 int n = ether_crc(ETH_ALEN, addr) >> 26; 870 mask[n >> 5] |= (1 << (n & 31)); 871 } 872 873 static unsigned int addr_hash_test(const u32 *mask, const u8 *addr) 874 { 875 int n = ether_crc(ETH_ALEN, addr) >> 26; 876 return mask[n >> 5] & (1 << (n & 31)); 877 } 878 879 static int update_filter(struct tap_filter *filter, void __user *arg) 880 { 881 struct { u8 u[ETH_ALEN]; } *addr; 882 struct tun_filter uf; 883 int err, alen, n, nexact; 884 885 if (copy_from_user(&uf, arg, sizeof(uf))) 886 return -EFAULT; 887 888 if (!uf.count) { 889 /* Disabled */ 890 filter->count = 0; 891 return 0; 892 } 893 894 alen = ETH_ALEN * uf.count; 895 addr = memdup_user(arg + sizeof(uf), alen); 896 if (IS_ERR(addr)) 897 return PTR_ERR(addr); 898 899 /* The filter is updated without holding any locks. Which is 900 * perfectly safe. We disable it first and in the worst 901 * case we'll accept a few undesired packets. */ 902 filter->count = 0; 903 wmb(); 904 905 /* Use first set of addresses as an exact filter */ 906 for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++) 907 memcpy(filter->addr[n], addr[n].u, ETH_ALEN); 908 909 nexact = n; 910 911 /* Remaining multicast addresses are hashed, 912 * unicast will leave the filter disabled. */ 913 memset(filter->mask, 0, sizeof(filter->mask)); 914 for (; n < uf.count; n++) { 915 if (!is_multicast_ether_addr(addr[n].u)) { 916 err = 0; /* no filter */ 917 goto free_addr; 918 } 919 addr_hash_set(filter->mask, addr[n].u); 920 } 921 922 /* For ALLMULTI just set the mask to all ones. 923 * This overrides the mask populated above. */ 924 if ((uf.flags & TUN_FLT_ALLMULTI)) 925 memset(filter->mask, ~0, sizeof(filter->mask)); 926 927 /* Now enable the filter */ 928 wmb(); 929 filter->count = nexact; 930 931 /* Return the number of exact filters */ 932 err = nexact; 933 free_addr: 934 kfree(addr); 935 return err; 936 } 937 938 /* Returns: 0 - drop, !=0 - accept */ 939 static int run_filter(struct tap_filter *filter, const struct sk_buff *skb) 940 { 941 /* Cannot use eth_hdr(skb) here because skb_mac_hdr() is incorrect 942 * at this point. */ 943 struct ethhdr *eh = (struct ethhdr *) skb->data; 944 int i; 945 946 /* Exact match */ 947 for (i = 0; i < filter->count; i++) 948 if (ether_addr_equal(eh->h_dest, filter->addr[i])) 949 return 1; 950 951 /* Inexact match (multicast only) */ 952 if (is_multicast_ether_addr(eh->h_dest)) 953 return addr_hash_test(filter->mask, eh->h_dest); 954 955 return 0; 956 } 957 958 /* 959 * Checks whether the packet is accepted or not. 960 * Returns: 0 - drop, !=0 - accept 961 */ 962 static int check_filter(struct tap_filter *filter, const struct sk_buff *skb) 963 { 964 if (!filter->count) 965 return 1; 966 967 return run_filter(filter, skb); 968 } 969 970 /* Network device part of the driver */ 971 972 static const struct ethtool_ops tun_ethtool_ops; 973 974 static int tun_net_init(struct net_device *dev) 975 { 976 struct tun_struct *tun = netdev_priv(dev); 977 struct ifreq *ifr = tun->ifr; 978 int err; 979 980 dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); 981 if (!dev->tstats) 982 return -ENOMEM; 983 984 spin_lock_init(&tun->lock); 985 986 err = security_tun_dev_alloc_security(&tun->security); 987 if (err < 0) { 988 free_percpu(dev->tstats); 989 return err; 990 } 991 992 tun_flow_init(tun); 993 994 dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST | 995 TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX | 996 NETIF_F_HW_VLAN_STAG_TX; 997 dev->features = dev->hw_features | NETIF_F_LLTX; 998 dev->vlan_features = dev->features & 999 ~(NETIF_F_HW_VLAN_CTAG_TX | 1000 NETIF_F_HW_VLAN_STAG_TX); 1001 1002 tun->flags = (tun->flags & ~TUN_FEATURES) | 1003 (ifr->ifr_flags & TUN_FEATURES); 1004 1005 INIT_LIST_HEAD(&tun->disabled); 1006 err = tun_attach(tun, tun->file, false, ifr->ifr_flags & IFF_NAPI, 1007 ifr->ifr_flags & IFF_NAPI_FRAGS, false); 1008 if (err < 0) { 1009 tun_flow_uninit(tun); 1010 security_tun_dev_free_security(tun->security); 1011 free_percpu(dev->tstats); 1012 return err; 1013 } 1014 return 0; 1015 } 1016 1017 /* Net device detach from fd. */ 1018 static void tun_net_uninit(struct net_device *dev) 1019 { 1020 tun_detach_all(dev); 1021 } 1022 1023 /* Net device open. */ 1024 static int tun_net_open(struct net_device *dev) 1025 { 1026 netif_tx_start_all_queues(dev); 1027 1028 return 0; 1029 } 1030 1031 /* Net device close. */ 1032 static int tun_net_close(struct net_device *dev) 1033 { 1034 netif_tx_stop_all_queues(dev); 1035 return 0; 1036 } 1037 1038 /* Net device start xmit */ 1039 static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb) 1040 { 1041 #ifdef CONFIG_RPS 1042 if (tun->numqueues == 1 && static_branch_unlikely(&rps_needed)) { 1043 /* Select queue was not called for the skbuff, so we extract the 1044 * RPS hash and save it into the flow_table here. 1045 */ 1046 struct tun_flow_entry *e; 1047 __u32 rxhash; 1048 1049 rxhash = __skb_get_hash_symmetric(skb); 1050 e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash); 1051 if (e) 1052 tun_flow_save_rps_rxhash(e, rxhash); 1053 } 1054 #endif 1055 } 1056 1057 static unsigned int run_ebpf_filter(struct tun_struct *tun, 1058 struct sk_buff *skb, 1059 int len) 1060 { 1061 struct tun_prog *prog = rcu_dereference(tun->filter_prog); 1062 1063 if (prog) 1064 len = bpf_prog_run_clear_cb(prog->prog, skb); 1065 1066 return len; 1067 } 1068 1069 /* Net device start xmit */ 1070 static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) 1071 { 1072 struct tun_struct *tun = netdev_priv(dev); 1073 enum skb_drop_reason drop_reason; 1074 int txq = skb->queue_mapping; 1075 struct netdev_queue *queue; 1076 struct tun_file *tfile; 1077 int len = skb->len; 1078 1079 rcu_read_lock(); 1080 tfile = rcu_dereference(tun->tfiles[txq]); 1081 1082 /* Drop packet if interface is not attached */ 1083 if (!tfile) { 1084 drop_reason = SKB_DROP_REASON_DEV_READY; 1085 goto drop; 1086 } 1087 1088 if (!rcu_dereference(tun->steering_prog)) 1089 tun_automq_xmit(tun, skb); 1090 1091 netif_info(tun, tx_queued, tun->dev, "%s %d\n", __func__, skb->len); 1092 1093 /* Drop if the filter does not like it. 1094 * This is a noop if the filter is disabled. 1095 * Filter can be enabled only for the TAP devices. */ 1096 if (!check_filter(&tun->txflt, skb)) { 1097 drop_reason = SKB_DROP_REASON_TAP_TXFILTER; 1098 goto drop; 1099 } 1100 1101 if (tfile->socket.sk->sk_filter && 1102 sk_filter(tfile->socket.sk, skb)) { 1103 drop_reason = SKB_DROP_REASON_SOCKET_FILTER; 1104 goto drop; 1105 } 1106 1107 len = run_ebpf_filter(tun, skb, len); 1108 if (len == 0) { 1109 drop_reason = SKB_DROP_REASON_TAP_FILTER; 1110 goto drop; 1111 } 1112 1113 if (pskb_trim(skb, len)) { 1114 drop_reason = SKB_DROP_REASON_NOMEM; 1115 goto drop; 1116 } 1117 1118 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) { 1119 drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT; 1120 goto drop; 1121 } 1122 1123 skb_tx_timestamp(skb); 1124 1125 /* Orphan the skb - required as we might hang on to it 1126 * for indefinite time. 1127 */ 1128 skb_orphan(skb); 1129 1130 nf_reset_ct(skb); 1131 1132 if (ptr_ring_produce(&tfile->tx_ring, skb)) { 1133 drop_reason = SKB_DROP_REASON_FULL_RING; 1134 goto drop; 1135 } 1136 1137 /* NETIF_F_LLTX requires to do our own update of trans_start */ 1138 queue = netdev_get_tx_queue(dev, txq); 1139 txq_trans_cond_update(queue); 1140 1141 /* Notify and wake up reader process */ 1142 if (tfile->flags & TUN_FASYNC) 1143 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); 1144 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 1145 1146 rcu_read_unlock(); 1147 return NETDEV_TX_OK; 1148 1149 drop: 1150 dev_core_stats_tx_dropped_inc(dev); 1151 skb_tx_error(skb); 1152 kfree_skb_reason(skb, drop_reason); 1153 rcu_read_unlock(); 1154 return NET_XMIT_DROP; 1155 } 1156 1157 static void tun_net_mclist(struct net_device *dev) 1158 { 1159 /* 1160 * This callback is supposed to deal with mc filter in 1161 * _rx_ path and has nothing to do with the _tx_ path. 1162 * In rx path we always accept everything userspace gives us. 1163 */ 1164 } 1165 1166 static netdev_features_t tun_net_fix_features(struct net_device *dev, 1167 netdev_features_t features) 1168 { 1169 struct tun_struct *tun = netdev_priv(dev); 1170 1171 return (features & tun->set_features) | (features & ~TUN_USER_FEATURES); 1172 } 1173 1174 static void tun_set_headroom(struct net_device *dev, int new_hr) 1175 { 1176 struct tun_struct *tun = netdev_priv(dev); 1177 1178 if (new_hr < NET_SKB_PAD) 1179 new_hr = NET_SKB_PAD; 1180 1181 tun->align = new_hr; 1182 } 1183 1184 static void 1185 tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 1186 { 1187 struct tun_struct *tun = netdev_priv(dev); 1188 1189 dev_get_tstats64(dev, stats); 1190 1191 stats->rx_frame_errors += 1192 (unsigned long)atomic_long_read(&tun->rx_frame_errors); 1193 } 1194 1195 static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1196 struct netlink_ext_ack *extack) 1197 { 1198 struct tun_struct *tun = netdev_priv(dev); 1199 struct tun_file *tfile; 1200 struct bpf_prog *old_prog; 1201 int i; 1202 1203 old_prog = rtnl_dereference(tun->xdp_prog); 1204 rcu_assign_pointer(tun->xdp_prog, prog); 1205 if (old_prog) 1206 bpf_prog_put(old_prog); 1207 1208 for (i = 0; i < tun->numqueues; i++) { 1209 tfile = rtnl_dereference(tun->tfiles[i]); 1210 if (prog) 1211 sock_set_flag(&tfile->sk, SOCK_XDP); 1212 else 1213 sock_reset_flag(&tfile->sk, SOCK_XDP); 1214 } 1215 list_for_each_entry(tfile, &tun->disabled, next) { 1216 if (prog) 1217 sock_set_flag(&tfile->sk, SOCK_XDP); 1218 else 1219 sock_reset_flag(&tfile->sk, SOCK_XDP); 1220 } 1221 1222 return 0; 1223 } 1224 1225 static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1226 { 1227 switch (xdp->command) { 1228 case XDP_SETUP_PROG: 1229 return tun_xdp_set(dev, xdp->prog, xdp->extack); 1230 default: 1231 return -EINVAL; 1232 } 1233 } 1234 1235 static int tun_net_change_carrier(struct net_device *dev, bool new_carrier) 1236 { 1237 if (new_carrier) { 1238 struct tun_struct *tun = netdev_priv(dev); 1239 1240 if (!tun->numqueues) 1241 return -EPERM; 1242 1243 netif_carrier_on(dev); 1244 } else { 1245 netif_carrier_off(dev); 1246 } 1247 return 0; 1248 } 1249 1250 static const struct net_device_ops tun_netdev_ops = { 1251 .ndo_init = tun_net_init, 1252 .ndo_uninit = tun_net_uninit, 1253 .ndo_open = tun_net_open, 1254 .ndo_stop = tun_net_close, 1255 .ndo_start_xmit = tun_net_xmit, 1256 .ndo_fix_features = tun_net_fix_features, 1257 .ndo_select_queue = tun_select_queue, 1258 .ndo_set_rx_headroom = tun_set_headroom, 1259 .ndo_get_stats64 = tun_net_get_stats64, 1260 .ndo_change_carrier = tun_net_change_carrier, 1261 }; 1262 1263 static void __tun_xdp_flush_tfile(struct tun_file *tfile) 1264 { 1265 /* Notify and wake up reader process */ 1266 if (tfile->flags & TUN_FASYNC) 1267 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); 1268 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 1269 } 1270 1271 static int tun_xdp_xmit(struct net_device *dev, int n, 1272 struct xdp_frame **frames, u32 flags) 1273 { 1274 struct tun_struct *tun = netdev_priv(dev); 1275 struct tun_file *tfile; 1276 u32 numqueues; 1277 int nxmit = 0; 1278 int i; 1279 1280 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1281 return -EINVAL; 1282 1283 rcu_read_lock(); 1284 1285 resample: 1286 numqueues = READ_ONCE(tun->numqueues); 1287 if (!numqueues) { 1288 rcu_read_unlock(); 1289 return -ENXIO; /* Caller will free/return all frames */ 1290 } 1291 1292 tfile = rcu_dereference(tun->tfiles[smp_processor_id() % 1293 numqueues]); 1294 if (unlikely(!tfile)) 1295 goto resample; 1296 1297 spin_lock(&tfile->tx_ring.producer_lock); 1298 for (i = 0; i < n; i++) { 1299 struct xdp_frame *xdp = frames[i]; 1300 /* Encode the XDP flag into lowest bit for consumer to differ 1301 * XDP buffer from sk_buff. 1302 */ 1303 void *frame = tun_xdp_to_ptr(xdp); 1304 1305 if (__ptr_ring_produce(&tfile->tx_ring, frame)) { 1306 dev_core_stats_tx_dropped_inc(dev); 1307 break; 1308 } 1309 nxmit++; 1310 } 1311 spin_unlock(&tfile->tx_ring.producer_lock); 1312 1313 if (flags & XDP_XMIT_FLUSH) 1314 __tun_xdp_flush_tfile(tfile); 1315 1316 rcu_read_unlock(); 1317 return nxmit; 1318 } 1319 1320 static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp) 1321 { 1322 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); 1323 int nxmit; 1324 1325 if (unlikely(!frame)) 1326 return -EOVERFLOW; 1327 1328 nxmit = tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH); 1329 if (!nxmit) 1330 xdp_return_frame_rx_napi(frame); 1331 return nxmit; 1332 } 1333 1334 static const struct net_device_ops tap_netdev_ops = { 1335 .ndo_init = tun_net_init, 1336 .ndo_uninit = tun_net_uninit, 1337 .ndo_open = tun_net_open, 1338 .ndo_stop = tun_net_close, 1339 .ndo_start_xmit = tun_net_xmit, 1340 .ndo_fix_features = tun_net_fix_features, 1341 .ndo_set_rx_mode = tun_net_mclist, 1342 .ndo_set_mac_address = eth_mac_addr, 1343 .ndo_validate_addr = eth_validate_addr, 1344 .ndo_select_queue = tun_select_queue, 1345 .ndo_features_check = passthru_features_check, 1346 .ndo_set_rx_headroom = tun_set_headroom, 1347 .ndo_get_stats64 = dev_get_tstats64, 1348 .ndo_bpf = tun_xdp, 1349 .ndo_xdp_xmit = tun_xdp_xmit, 1350 .ndo_change_carrier = tun_net_change_carrier, 1351 }; 1352 1353 static void tun_flow_init(struct tun_struct *tun) 1354 { 1355 int i; 1356 1357 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) 1358 INIT_HLIST_HEAD(&tun->flows[i]); 1359 1360 tun->ageing_time = TUN_FLOW_EXPIRE; 1361 timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0); 1362 mod_timer(&tun->flow_gc_timer, 1363 round_jiffies_up(jiffies + tun->ageing_time)); 1364 } 1365 1366 static void tun_flow_uninit(struct tun_struct *tun) 1367 { 1368 del_timer_sync(&tun->flow_gc_timer); 1369 tun_flow_flush(tun); 1370 } 1371 1372 #define MIN_MTU 68 1373 #define MAX_MTU 65535 1374 1375 /* Initialize net device. */ 1376 static void tun_net_initialize(struct net_device *dev) 1377 { 1378 struct tun_struct *tun = netdev_priv(dev); 1379 1380 switch (tun->flags & TUN_TYPE_MASK) { 1381 case IFF_TUN: 1382 dev->netdev_ops = &tun_netdev_ops; 1383 dev->header_ops = &ip_tunnel_header_ops; 1384 1385 /* Point-to-Point TUN Device */ 1386 dev->hard_header_len = 0; 1387 dev->addr_len = 0; 1388 dev->mtu = 1500; 1389 1390 /* Zero header length */ 1391 dev->type = ARPHRD_NONE; 1392 dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; 1393 break; 1394 1395 case IFF_TAP: 1396 dev->netdev_ops = &tap_netdev_ops; 1397 /* Ethernet TAP Device */ 1398 ether_setup(dev); 1399 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 1400 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 1401 1402 eth_hw_addr_random(dev); 1403 1404 break; 1405 } 1406 1407 dev->min_mtu = MIN_MTU; 1408 dev->max_mtu = MAX_MTU - dev->hard_header_len; 1409 } 1410 1411 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile) 1412 { 1413 struct sock *sk = tfile->socket.sk; 1414 1415 return (tun->dev->flags & IFF_UP) && sock_writeable(sk); 1416 } 1417 1418 /* Character device part */ 1419 1420 /* Poll */ 1421 static __poll_t tun_chr_poll(struct file *file, poll_table *wait) 1422 { 1423 struct tun_file *tfile = file->private_data; 1424 struct tun_struct *tun = tun_get(tfile); 1425 struct sock *sk; 1426 __poll_t mask = 0; 1427 1428 if (!tun) 1429 return EPOLLERR; 1430 1431 sk = tfile->socket.sk; 1432 1433 poll_wait(file, sk_sleep(sk), wait); 1434 1435 if (!ptr_ring_empty(&tfile->tx_ring)) 1436 mask |= EPOLLIN | EPOLLRDNORM; 1437 1438 /* Make sure SOCKWQ_ASYNC_NOSPACE is set if not writable to 1439 * guarantee EPOLLOUT to be raised by either here or 1440 * tun_sock_write_space(). Then process could get notification 1441 * after it writes to a down device and meets -EIO. 1442 */ 1443 if (tun_sock_writeable(tun, tfile) || 1444 (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) && 1445 tun_sock_writeable(tun, tfile))) 1446 mask |= EPOLLOUT | EPOLLWRNORM; 1447 1448 if (tun->dev->reg_state != NETREG_REGISTERED) 1449 mask = EPOLLERR; 1450 1451 tun_put(tun); 1452 return mask; 1453 } 1454 1455 static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile, 1456 size_t len, 1457 const struct iov_iter *it) 1458 { 1459 struct sk_buff *skb; 1460 size_t linear; 1461 int err; 1462 int i; 1463 1464 if (it->nr_segs > MAX_SKB_FRAGS + 1 || 1465 len > (ETH_MAX_MTU - NET_SKB_PAD - NET_IP_ALIGN)) 1466 return ERR_PTR(-EMSGSIZE); 1467 1468 local_bh_disable(); 1469 skb = napi_get_frags(&tfile->napi); 1470 local_bh_enable(); 1471 if (!skb) 1472 return ERR_PTR(-ENOMEM); 1473 1474 linear = iov_iter_single_seg_count(it); 1475 err = __skb_grow(skb, linear); 1476 if (err) 1477 goto free; 1478 1479 skb->len = len; 1480 skb->data_len = len - linear; 1481 skb->truesize += skb->data_len; 1482 1483 for (i = 1; i < it->nr_segs; i++) { 1484 size_t fragsz = it->iov[i].iov_len; 1485 struct page *page; 1486 void *frag; 1487 1488 if (fragsz == 0 || fragsz > PAGE_SIZE) { 1489 err = -EINVAL; 1490 goto free; 1491 } 1492 frag = netdev_alloc_frag(fragsz); 1493 if (!frag) { 1494 err = -ENOMEM; 1495 goto free; 1496 } 1497 page = virt_to_head_page(frag); 1498 skb_fill_page_desc(skb, i - 1, page, 1499 frag - page_address(page), fragsz); 1500 } 1501 1502 return skb; 1503 free: 1504 /* frees skb and all frags allocated with napi_alloc_frag() */ 1505 napi_free_frags(&tfile->napi); 1506 return ERR_PTR(err); 1507 } 1508 1509 /* prepad is the amount to reserve at front. len is length after that. 1510 * linear is a hint as to how much to copy (usually headers). */ 1511 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile, 1512 size_t prepad, size_t len, 1513 size_t linear, int noblock) 1514 { 1515 struct sock *sk = tfile->socket.sk; 1516 struct sk_buff *skb; 1517 int err; 1518 1519 /* Under a page? Don't bother with paged skb. */ 1520 if (prepad + len < PAGE_SIZE || !linear) 1521 linear = len; 1522 1523 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, 1524 &err, 0); 1525 if (!skb) 1526 return ERR_PTR(err); 1527 1528 skb_reserve(skb, prepad); 1529 skb_put(skb, linear); 1530 skb->data_len = len - linear; 1531 skb->len += len - linear; 1532 1533 return skb; 1534 } 1535 1536 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile, 1537 struct sk_buff *skb, int more) 1538 { 1539 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 1540 struct sk_buff_head process_queue; 1541 u32 rx_batched = tun->rx_batched; 1542 bool rcv = false; 1543 1544 if (!rx_batched || (!more && skb_queue_empty(queue))) { 1545 local_bh_disable(); 1546 skb_record_rx_queue(skb, tfile->queue_index); 1547 netif_receive_skb(skb); 1548 local_bh_enable(); 1549 return; 1550 } 1551 1552 spin_lock(&queue->lock); 1553 if (!more || skb_queue_len(queue) == rx_batched) { 1554 __skb_queue_head_init(&process_queue); 1555 skb_queue_splice_tail_init(queue, &process_queue); 1556 rcv = true; 1557 } else { 1558 __skb_queue_tail(queue, skb); 1559 } 1560 spin_unlock(&queue->lock); 1561 1562 if (rcv) { 1563 struct sk_buff *nskb; 1564 1565 local_bh_disable(); 1566 while ((nskb = __skb_dequeue(&process_queue))) { 1567 skb_record_rx_queue(nskb, tfile->queue_index); 1568 netif_receive_skb(nskb); 1569 } 1570 skb_record_rx_queue(skb, tfile->queue_index); 1571 netif_receive_skb(skb); 1572 local_bh_enable(); 1573 } 1574 } 1575 1576 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile, 1577 int len, int noblock, bool zerocopy) 1578 { 1579 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 1580 return false; 1581 1582 if (tfile->socket.sk->sk_sndbuf != INT_MAX) 1583 return false; 1584 1585 if (!noblock) 1586 return false; 1587 1588 if (zerocopy) 1589 return false; 1590 1591 if (SKB_DATA_ALIGN(len + TUN_RX_PAD) + 1592 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE) 1593 return false; 1594 1595 return true; 1596 } 1597 1598 static struct sk_buff *__tun_build_skb(struct tun_file *tfile, 1599 struct page_frag *alloc_frag, char *buf, 1600 int buflen, int len, int pad) 1601 { 1602 struct sk_buff *skb = build_skb(buf, buflen); 1603 1604 if (!skb) 1605 return ERR_PTR(-ENOMEM); 1606 1607 skb_reserve(skb, pad); 1608 skb_put(skb, len); 1609 skb_set_owner_w(skb, tfile->socket.sk); 1610 1611 get_page(alloc_frag->page); 1612 alloc_frag->offset += buflen; 1613 1614 return skb; 1615 } 1616 1617 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog, 1618 struct xdp_buff *xdp, u32 act) 1619 { 1620 int err; 1621 1622 switch (act) { 1623 case XDP_REDIRECT: 1624 err = xdp_do_redirect(tun->dev, xdp, xdp_prog); 1625 if (err) 1626 return err; 1627 break; 1628 case XDP_TX: 1629 err = tun_xdp_tx(tun->dev, xdp); 1630 if (err < 0) 1631 return err; 1632 break; 1633 case XDP_PASS: 1634 break; 1635 default: 1636 bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act); 1637 fallthrough; 1638 case XDP_ABORTED: 1639 trace_xdp_exception(tun->dev, xdp_prog, act); 1640 fallthrough; 1641 case XDP_DROP: 1642 dev_core_stats_rx_dropped_inc(tun->dev); 1643 break; 1644 } 1645 1646 return act; 1647 } 1648 1649 static struct sk_buff *tun_build_skb(struct tun_struct *tun, 1650 struct tun_file *tfile, 1651 struct iov_iter *from, 1652 struct virtio_net_hdr *hdr, 1653 int len, int *skb_xdp) 1654 { 1655 struct page_frag *alloc_frag = ¤t->task_frag; 1656 struct bpf_prog *xdp_prog; 1657 int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1658 char *buf; 1659 size_t copied; 1660 int pad = TUN_RX_PAD; 1661 int err = 0; 1662 1663 rcu_read_lock(); 1664 xdp_prog = rcu_dereference(tun->xdp_prog); 1665 if (xdp_prog) 1666 pad += XDP_PACKET_HEADROOM; 1667 buflen += SKB_DATA_ALIGN(len + pad); 1668 rcu_read_unlock(); 1669 1670 alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES); 1671 if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL))) 1672 return ERR_PTR(-ENOMEM); 1673 1674 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 1675 copied = copy_page_from_iter(alloc_frag->page, 1676 alloc_frag->offset + pad, 1677 len, from); 1678 if (copied != len) 1679 return ERR_PTR(-EFAULT); 1680 1681 /* There's a small window that XDP may be set after the check 1682 * of xdp_prog above, this should be rare and for simplicity 1683 * we do XDP on skb in case the headroom is not enough. 1684 */ 1685 if (hdr->gso_type || !xdp_prog) { 1686 *skb_xdp = 1; 1687 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, 1688 pad); 1689 } 1690 1691 *skb_xdp = 0; 1692 1693 local_bh_disable(); 1694 rcu_read_lock(); 1695 xdp_prog = rcu_dereference(tun->xdp_prog); 1696 if (xdp_prog) { 1697 struct xdp_buff xdp; 1698 u32 act; 1699 1700 xdp_init_buff(&xdp, buflen, &tfile->xdp_rxq); 1701 xdp_prepare_buff(&xdp, buf, pad, len, false); 1702 1703 act = bpf_prog_run_xdp(xdp_prog, &xdp); 1704 if (act == XDP_REDIRECT || act == XDP_TX) { 1705 get_page(alloc_frag->page); 1706 alloc_frag->offset += buflen; 1707 } 1708 err = tun_xdp_act(tun, xdp_prog, &xdp, act); 1709 if (err < 0) { 1710 if (act == XDP_REDIRECT || act == XDP_TX) 1711 put_page(alloc_frag->page); 1712 goto out; 1713 } 1714 1715 if (err == XDP_REDIRECT) 1716 xdp_do_flush(); 1717 if (err != XDP_PASS) 1718 goto out; 1719 1720 pad = xdp.data - xdp.data_hard_start; 1721 len = xdp.data_end - xdp.data; 1722 } 1723 rcu_read_unlock(); 1724 local_bh_enable(); 1725 1726 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, pad); 1727 1728 out: 1729 rcu_read_unlock(); 1730 local_bh_enable(); 1731 return NULL; 1732 } 1733 1734 /* Get packet from user space buffer */ 1735 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 1736 void *msg_control, struct iov_iter *from, 1737 int noblock, bool more) 1738 { 1739 struct tun_pi pi = { 0, cpu_to_be16(ETH_P_IP) }; 1740 struct sk_buff *skb; 1741 size_t total_len = iov_iter_count(from); 1742 size_t len = total_len, align = tun->align, linear; 1743 struct virtio_net_hdr gso = { 0 }; 1744 int good_linear; 1745 int copylen; 1746 bool zerocopy = false; 1747 int err; 1748 u32 rxhash = 0; 1749 int skb_xdp = 1; 1750 bool frags = tun_napi_frags_enabled(tfile); 1751 enum skb_drop_reason drop_reason; 1752 1753 if (!(tun->flags & IFF_NO_PI)) { 1754 if (len < sizeof(pi)) 1755 return -EINVAL; 1756 len -= sizeof(pi); 1757 1758 if (!copy_from_iter_full(&pi, sizeof(pi), from)) 1759 return -EFAULT; 1760 } 1761 1762 if (tun->flags & IFF_VNET_HDR) { 1763 int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 1764 1765 if (len < vnet_hdr_sz) 1766 return -EINVAL; 1767 len -= vnet_hdr_sz; 1768 1769 if (!copy_from_iter_full(&gso, sizeof(gso), from)) 1770 return -EFAULT; 1771 1772 if ((gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 1773 tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2 > tun16_to_cpu(tun, gso.hdr_len)) 1774 gso.hdr_len = cpu_to_tun16(tun, tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2); 1775 1776 if (tun16_to_cpu(tun, gso.hdr_len) > len) 1777 return -EINVAL; 1778 iov_iter_advance(from, vnet_hdr_sz - sizeof(gso)); 1779 } 1780 1781 if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) { 1782 align += NET_IP_ALIGN; 1783 if (unlikely(len < ETH_HLEN || 1784 (gso.hdr_len && tun16_to_cpu(tun, gso.hdr_len) < ETH_HLEN))) 1785 return -EINVAL; 1786 } 1787 1788 good_linear = SKB_MAX_HEAD(align); 1789 1790 if (msg_control) { 1791 struct iov_iter i = *from; 1792 1793 /* There are 256 bytes to be copied in skb, so there is 1794 * enough room for skb expand head in case it is used. 1795 * The rest of the buffer is mapped from userspace. 1796 */ 1797 copylen = gso.hdr_len ? tun16_to_cpu(tun, gso.hdr_len) : GOODCOPY_LEN; 1798 if (copylen > good_linear) 1799 copylen = good_linear; 1800 linear = copylen; 1801 iov_iter_advance(&i, copylen); 1802 if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS) 1803 zerocopy = true; 1804 } 1805 1806 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) { 1807 /* For the packet that is not easy to be processed 1808 * (e.g gso or jumbo packet), we will do it at after 1809 * skb was created with generic XDP routine. 1810 */ 1811 skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp); 1812 if (IS_ERR(skb)) { 1813 dev_core_stats_rx_dropped_inc(tun->dev); 1814 return PTR_ERR(skb); 1815 } 1816 if (!skb) 1817 return total_len; 1818 } else { 1819 if (!zerocopy) { 1820 copylen = len; 1821 if (tun16_to_cpu(tun, gso.hdr_len) > good_linear) 1822 linear = good_linear; 1823 else 1824 linear = tun16_to_cpu(tun, gso.hdr_len); 1825 } 1826 1827 if (frags) { 1828 mutex_lock(&tfile->napi_mutex); 1829 skb = tun_napi_alloc_frags(tfile, copylen, from); 1830 /* tun_napi_alloc_frags() enforces a layout for the skb. 1831 * If zerocopy is enabled, then this layout will be 1832 * overwritten by zerocopy_sg_from_iter(). 1833 */ 1834 zerocopy = false; 1835 } else { 1836 skb = tun_alloc_skb(tfile, align, copylen, linear, 1837 noblock); 1838 } 1839 1840 if (IS_ERR(skb)) { 1841 if (PTR_ERR(skb) != -EAGAIN) 1842 dev_core_stats_rx_dropped_inc(tun->dev); 1843 if (frags) 1844 mutex_unlock(&tfile->napi_mutex); 1845 return PTR_ERR(skb); 1846 } 1847 1848 if (zerocopy) 1849 err = zerocopy_sg_from_iter(skb, from); 1850 else 1851 err = skb_copy_datagram_from_iter(skb, 0, from, len); 1852 1853 if (err) { 1854 err = -EFAULT; 1855 drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT; 1856 drop: 1857 dev_core_stats_rx_dropped_inc(tun->dev); 1858 kfree_skb_reason(skb, drop_reason); 1859 if (frags) { 1860 tfile->napi.skb = NULL; 1861 mutex_unlock(&tfile->napi_mutex); 1862 } 1863 1864 return err; 1865 } 1866 } 1867 1868 if (virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun))) { 1869 atomic_long_inc(&tun->rx_frame_errors); 1870 kfree_skb(skb); 1871 if (frags) { 1872 tfile->napi.skb = NULL; 1873 mutex_unlock(&tfile->napi_mutex); 1874 } 1875 1876 return -EINVAL; 1877 } 1878 1879 switch (tun->flags & TUN_TYPE_MASK) { 1880 case IFF_TUN: 1881 if (tun->flags & IFF_NO_PI) { 1882 u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0; 1883 1884 switch (ip_version) { 1885 case 4: 1886 pi.proto = htons(ETH_P_IP); 1887 break; 1888 case 6: 1889 pi.proto = htons(ETH_P_IPV6); 1890 break; 1891 default: 1892 dev_core_stats_rx_dropped_inc(tun->dev); 1893 kfree_skb(skb); 1894 return -EINVAL; 1895 } 1896 } 1897 1898 skb_reset_mac_header(skb); 1899 skb->protocol = pi.proto; 1900 skb->dev = tun->dev; 1901 break; 1902 case IFF_TAP: 1903 if (frags && !pskb_may_pull(skb, ETH_HLEN)) { 1904 err = -ENOMEM; 1905 drop_reason = SKB_DROP_REASON_HDR_TRUNC; 1906 goto drop; 1907 } 1908 skb->protocol = eth_type_trans(skb, tun->dev); 1909 break; 1910 } 1911 1912 /* copy skb_ubuf_info for callback when skb has no error */ 1913 if (zerocopy) { 1914 skb_zcopy_init(skb, msg_control); 1915 } else if (msg_control) { 1916 struct ubuf_info *uarg = msg_control; 1917 uarg->callback(NULL, uarg, false); 1918 } 1919 1920 skb_reset_network_header(skb); 1921 skb_probe_transport_header(skb); 1922 skb_record_rx_queue(skb, tfile->queue_index); 1923 1924 if (skb_xdp) { 1925 struct bpf_prog *xdp_prog; 1926 int ret; 1927 1928 local_bh_disable(); 1929 rcu_read_lock(); 1930 xdp_prog = rcu_dereference(tun->xdp_prog); 1931 if (xdp_prog) { 1932 ret = do_xdp_generic(xdp_prog, skb); 1933 if (ret != XDP_PASS) { 1934 rcu_read_unlock(); 1935 local_bh_enable(); 1936 if (frags) { 1937 tfile->napi.skb = NULL; 1938 mutex_unlock(&tfile->napi_mutex); 1939 } 1940 return total_len; 1941 } 1942 } 1943 rcu_read_unlock(); 1944 local_bh_enable(); 1945 } 1946 1947 /* Compute the costly rx hash only if needed for flow updates. 1948 * We may get a very small possibility of OOO during switching, not 1949 * worth to optimize. 1950 */ 1951 if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 && 1952 !tfile->detached) 1953 rxhash = __skb_get_hash_symmetric(skb); 1954 1955 rcu_read_lock(); 1956 if (unlikely(!(tun->dev->flags & IFF_UP))) { 1957 err = -EIO; 1958 rcu_read_unlock(); 1959 drop_reason = SKB_DROP_REASON_DEV_READY; 1960 goto drop; 1961 } 1962 1963 if (frags) { 1964 u32 headlen; 1965 1966 /* Exercise flow dissector code path. */ 1967 skb_push(skb, ETH_HLEN); 1968 headlen = eth_get_headlen(tun->dev, skb->data, 1969 skb_headlen(skb)); 1970 1971 if (unlikely(headlen > skb_headlen(skb))) { 1972 WARN_ON_ONCE(1); 1973 err = -ENOMEM; 1974 dev_core_stats_rx_dropped_inc(tun->dev); 1975 napi_busy: 1976 napi_free_frags(&tfile->napi); 1977 rcu_read_unlock(); 1978 mutex_unlock(&tfile->napi_mutex); 1979 return err; 1980 } 1981 1982 if (likely(napi_schedule_prep(&tfile->napi))) { 1983 local_bh_disable(); 1984 napi_gro_frags(&tfile->napi); 1985 napi_complete(&tfile->napi); 1986 local_bh_enable(); 1987 } else { 1988 err = -EBUSY; 1989 goto napi_busy; 1990 } 1991 mutex_unlock(&tfile->napi_mutex); 1992 } else if (tfile->napi_enabled) { 1993 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 1994 int queue_len; 1995 1996 spin_lock_bh(&queue->lock); 1997 __skb_queue_tail(queue, skb); 1998 queue_len = skb_queue_len(queue); 1999 spin_unlock(&queue->lock); 2000 2001 if (!more || queue_len > NAPI_POLL_WEIGHT) 2002 napi_schedule(&tfile->napi); 2003 2004 local_bh_enable(); 2005 } else if (!IS_ENABLED(CONFIG_4KSTACKS)) { 2006 tun_rx_batched(tun, tfile, skb, more); 2007 } else { 2008 netif_rx(skb); 2009 } 2010 rcu_read_unlock(); 2011 2012 preempt_disable(); 2013 dev_sw_netstats_rx_add(tun->dev, len); 2014 preempt_enable(); 2015 2016 if (rxhash) 2017 tun_flow_update(tun, rxhash, tfile); 2018 2019 return total_len; 2020 } 2021 2022 static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from) 2023 { 2024 struct file *file = iocb->ki_filp; 2025 struct tun_file *tfile = file->private_data; 2026 struct tun_struct *tun = tun_get(tfile); 2027 ssize_t result; 2028 int noblock = 0; 2029 2030 if (!tun) 2031 return -EBADFD; 2032 2033 if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT)) 2034 noblock = 1; 2035 2036 result = tun_get_user(tun, tfile, NULL, from, noblock, false); 2037 2038 tun_put(tun); 2039 return result; 2040 } 2041 2042 static ssize_t tun_put_user_xdp(struct tun_struct *tun, 2043 struct tun_file *tfile, 2044 struct xdp_frame *xdp_frame, 2045 struct iov_iter *iter) 2046 { 2047 int vnet_hdr_sz = 0; 2048 size_t size = xdp_frame->len; 2049 size_t ret; 2050 2051 if (tun->flags & IFF_VNET_HDR) { 2052 struct virtio_net_hdr gso = { 0 }; 2053 2054 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 2055 if (unlikely(iov_iter_count(iter) < vnet_hdr_sz)) 2056 return -EINVAL; 2057 if (unlikely(copy_to_iter(&gso, sizeof(gso), iter) != 2058 sizeof(gso))) 2059 return -EFAULT; 2060 iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso)); 2061 } 2062 2063 ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz; 2064 2065 preempt_disable(); 2066 dev_sw_netstats_tx_add(tun->dev, 1, ret); 2067 preempt_enable(); 2068 2069 return ret; 2070 } 2071 2072 /* Put packet to the user space buffer */ 2073 static ssize_t tun_put_user(struct tun_struct *tun, 2074 struct tun_file *tfile, 2075 struct sk_buff *skb, 2076 struct iov_iter *iter) 2077 { 2078 struct tun_pi pi = { 0, skb->protocol }; 2079 ssize_t total; 2080 int vlan_offset = 0; 2081 int vlan_hlen = 0; 2082 int vnet_hdr_sz = 0; 2083 2084 if (skb_vlan_tag_present(skb)) 2085 vlan_hlen = VLAN_HLEN; 2086 2087 if (tun->flags & IFF_VNET_HDR) 2088 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 2089 2090 total = skb->len + vlan_hlen + vnet_hdr_sz; 2091 2092 if (!(tun->flags & IFF_NO_PI)) { 2093 if (iov_iter_count(iter) < sizeof(pi)) 2094 return -EINVAL; 2095 2096 total += sizeof(pi); 2097 if (iov_iter_count(iter) < total) { 2098 /* Packet will be striped */ 2099 pi.flags |= TUN_PKT_STRIP; 2100 } 2101 2102 if (copy_to_iter(&pi, sizeof(pi), iter) != sizeof(pi)) 2103 return -EFAULT; 2104 } 2105 2106 if (vnet_hdr_sz) { 2107 struct virtio_net_hdr gso; 2108 2109 if (iov_iter_count(iter) < vnet_hdr_sz) 2110 return -EINVAL; 2111 2112 if (virtio_net_hdr_from_skb(skb, &gso, 2113 tun_is_little_endian(tun), true, 2114 vlan_hlen)) { 2115 struct skb_shared_info *sinfo = skb_shinfo(skb); 2116 pr_err("unexpected GSO type: " 2117 "0x%x, gso_size %d, hdr_len %d\n", 2118 sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size), 2119 tun16_to_cpu(tun, gso.hdr_len)); 2120 print_hex_dump(KERN_ERR, "tun: ", 2121 DUMP_PREFIX_NONE, 2122 16, 1, skb->head, 2123 min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true); 2124 WARN_ON_ONCE(1); 2125 return -EINVAL; 2126 } 2127 2128 if (copy_to_iter(&gso, sizeof(gso), iter) != sizeof(gso)) 2129 return -EFAULT; 2130 2131 iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso)); 2132 } 2133 2134 if (vlan_hlen) { 2135 int ret; 2136 struct veth veth; 2137 2138 veth.h_vlan_proto = skb->vlan_proto; 2139 veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb)); 2140 2141 vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto); 2142 2143 ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset); 2144 if (ret || !iov_iter_count(iter)) 2145 goto done; 2146 2147 ret = copy_to_iter(&veth, sizeof(veth), iter); 2148 if (ret != sizeof(veth) || !iov_iter_count(iter)) 2149 goto done; 2150 } 2151 2152 skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset); 2153 2154 done: 2155 /* caller is in process context, */ 2156 preempt_disable(); 2157 dev_sw_netstats_tx_add(tun->dev, 1, skb->len + vlan_hlen); 2158 preempt_enable(); 2159 2160 return total; 2161 } 2162 2163 static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) 2164 { 2165 DECLARE_WAITQUEUE(wait, current); 2166 void *ptr = NULL; 2167 int error = 0; 2168 2169 ptr = ptr_ring_consume(&tfile->tx_ring); 2170 if (ptr) 2171 goto out; 2172 if (noblock) { 2173 error = -EAGAIN; 2174 goto out; 2175 } 2176 2177 add_wait_queue(&tfile->socket.wq.wait, &wait); 2178 2179 while (1) { 2180 set_current_state(TASK_INTERRUPTIBLE); 2181 ptr = ptr_ring_consume(&tfile->tx_ring); 2182 if (ptr) 2183 break; 2184 if (signal_pending(current)) { 2185 error = -ERESTARTSYS; 2186 break; 2187 } 2188 if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) { 2189 error = -EFAULT; 2190 break; 2191 } 2192 2193 schedule(); 2194 } 2195 2196 __set_current_state(TASK_RUNNING); 2197 remove_wait_queue(&tfile->socket.wq.wait, &wait); 2198 2199 out: 2200 *err = error; 2201 return ptr; 2202 } 2203 2204 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, 2205 struct iov_iter *to, 2206 int noblock, void *ptr) 2207 { 2208 ssize_t ret; 2209 int err; 2210 2211 if (!iov_iter_count(to)) { 2212 tun_ptr_free(ptr); 2213 return 0; 2214 } 2215 2216 if (!ptr) { 2217 /* Read frames from ring */ 2218 ptr = tun_ring_recv(tfile, noblock, &err); 2219 if (!ptr) 2220 return err; 2221 } 2222 2223 if (tun_is_xdp_frame(ptr)) { 2224 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 2225 2226 ret = tun_put_user_xdp(tun, tfile, xdpf, to); 2227 xdp_return_frame(xdpf); 2228 } else { 2229 struct sk_buff *skb = ptr; 2230 2231 ret = tun_put_user(tun, tfile, skb, to); 2232 if (unlikely(ret < 0)) 2233 kfree_skb(skb); 2234 else 2235 consume_skb(skb); 2236 } 2237 2238 return ret; 2239 } 2240 2241 static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to) 2242 { 2243 struct file *file = iocb->ki_filp; 2244 struct tun_file *tfile = file->private_data; 2245 struct tun_struct *tun = tun_get(tfile); 2246 ssize_t len = iov_iter_count(to), ret; 2247 int noblock = 0; 2248 2249 if (!tun) 2250 return -EBADFD; 2251 2252 if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT)) 2253 noblock = 1; 2254 2255 ret = tun_do_read(tun, tfile, to, noblock, NULL); 2256 ret = min_t(ssize_t, ret, len); 2257 if (ret > 0) 2258 iocb->ki_pos = ret; 2259 tun_put(tun); 2260 return ret; 2261 } 2262 2263 static void tun_prog_free(struct rcu_head *rcu) 2264 { 2265 struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu); 2266 2267 bpf_prog_destroy(prog->prog); 2268 kfree(prog); 2269 } 2270 2271 static int __tun_set_ebpf(struct tun_struct *tun, 2272 struct tun_prog __rcu **prog_p, 2273 struct bpf_prog *prog) 2274 { 2275 struct tun_prog *old, *new = NULL; 2276 2277 if (prog) { 2278 new = kmalloc(sizeof(*new), GFP_KERNEL); 2279 if (!new) 2280 return -ENOMEM; 2281 new->prog = prog; 2282 } 2283 2284 spin_lock_bh(&tun->lock); 2285 old = rcu_dereference_protected(*prog_p, 2286 lockdep_is_held(&tun->lock)); 2287 rcu_assign_pointer(*prog_p, new); 2288 spin_unlock_bh(&tun->lock); 2289 2290 if (old) 2291 call_rcu(&old->rcu, tun_prog_free); 2292 2293 return 0; 2294 } 2295 2296 static void tun_free_netdev(struct net_device *dev) 2297 { 2298 struct tun_struct *tun = netdev_priv(dev); 2299 2300 BUG_ON(!(list_empty(&tun->disabled))); 2301 2302 free_percpu(dev->tstats); 2303 tun_flow_uninit(tun); 2304 security_tun_dev_free_security(tun->security); 2305 __tun_set_ebpf(tun, &tun->steering_prog, NULL); 2306 __tun_set_ebpf(tun, &tun->filter_prog, NULL); 2307 } 2308 2309 static void tun_setup(struct net_device *dev) 2310 { 2311 struct tun_struct *tun = netdev_priv(dev); 2312 2313 tun->owner = INVALID_UID; 2314 tun->group = INVALID_GID; 2315 tun_default_link_ksettings(dev, &tun->link_ksettings); 2316 2317 dev->ethtool_ops = &tun_ethtool_ops; 2318 dev->needs_free_netdev = true; 2319 dev->priv_destructor = tun_free_netdev; 2320 /* We prefer our own queue length */ 2321 dev->tx_queue_len = TUN_READQ_SIZE; 2322 } 2323 2324 /* Trivial set of netlink ops to allow deleting tun or tap 2325 * device with netlink. 2326 */ 2327 static int tun_validate(struct nlattr *tb[], struct nlattr *data[], 2328 struct netlink_ext_ack *extack) 2329 { 2330 NL_SET_ERR_MSG(extack, 2331 "tun/tap creation via rtnetlink is not supported."); 2332 return -EOPNOTSUPP; 2333 } 2334 2335 static size_t tun_get_size(const struct net_device *dev) 2336 { 2337 BUILD_BUG_ON(sizeof(u32) != sizeof(uid_t)); 2338 BUILD_BUG_ON(sizeof(u32) != sizeof(gid_t)); 2339 2340 return nla_total_size(sizeof(uid_t)) + /* OWNER */ 2341 nla_total_size(sizeof(gid_t)) + /* GROUP */ 2342 nla_total_size(sizeof(u8)) + /* TYPE */ 2343 nla_total_size(sizeof(u8)) + /* PI */ 2344 nla_total_size(sizeof(u8)) + /* VNET_HDR */ 2345 nla_total_size(sizeof(u8)) + /* PERSIST */ 2346 nla_total_size(sizeof(u8)) + /* MULTI_QUEUE */ 2347 nla_total_size(sizeof(u32)) + /* NUM_QUEUES */ 2348 nla_total_size(sizeof(u32)) + /* NUM_DISABLED_QUEUES */ 2349 0; 2350 } 2351 2352 static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev) 2353 { 2354 struct tun_struct *tun = netdev_priv(dev); 2355 2356 if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK)) 2357 goto nla_put_failure; 2358 if (uid_valid(tun->owner) && 2359 nla_put_u32(skb, IFLA_TUN_OWNER, 2360 from_kuid_munged(current_user_ns(), tun->owner))) 2361 goto nla_put_failure; 2362 if (gid_valid(tun->group) && 2363 nla_put_u32(skb, IFLA_TUN_GROUP, 2364 from_kgid_munged(current_user_ns(), tun->group))) 2365 goto nla_put_failure; 2366 if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI))) 2367 goto nla_put_failure; 2368 if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR))) 2369 goto nla_put_failure; 2370 if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST))) 2371 goto nla_put_failure; 2372 if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE, 2373 !!(tun->flags & IFF_MULTI_QUEUE))) 2374 goto nla_put_failure; 2375 if (tun->flags & IFF_MULTI_QUEUE) { 2376 if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues)) 2377 goto nla_put_failure; 2378 if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES, 2379 tun->numdisabled)) 2380 goto nla_put_failure; 2381 } 2382 2383 return 0; 2384 2385 nla_put_failure: 2386 return -EMSGSIZE; 2387 } 2388 2389 static struct rtnl_link_ops tun_link_ops __read_mostly = { 2390 .kind = DRV_NAME, 2391 .priv_size = sizeof(struct tun_struct), 2392 .setup = tun_setup, 2393 .validate = tun_validate, 2394 .get_size = tun_get_size, 2395 .fill_info = tun_fill_info, 2396 }; 2397 2398 static void tun_sock_write_space(struct sock *sk) 2399 { 2400 struct tun_file *tfile; 2401 wait_queue_head_t *wqueue; 2402 2403 if (!sock_writeable(sk)) 2404 return; 2405 2406 if (!test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags)) 2407 return; 2408 2409 wqueue = sk_sleep(sk); 2410 if (wqueue && waitqueue_active(wqueue)) 2411 wake_up_interruptible_sync_poll(wqueue, EPOLLOUT | 2412 EPOLLWRNORM | EPOLLWRBAND); 2413 2414 tfile = container_of(sk, struct tun_file, sk); 2415 kill_fasync(&tfile->fasync, SIGIO, POLL_OUT); 2416 } 2417 2418 static void tun_put_page(struct tun_page *tpage) 2419 { 2420 if (tpage->page) 2421 __page_frag_cache_drain(tpage->page, tpage->count); 2422 } 2423 2424 static int tun_xdp_one(struct tun_struct *tun, 2425 struct tun_file *tfile, 2426 struct xdp_buff *xdp, int *flush, 2427 struct tun_page *tpage) 2428 { 2429 unsigned int datasize = xdp->data_end - xdp->data; 2430 struct tun_xdp_hdr *hdr = xdp->data_hard_start; 2431 struct virtio_net_hdr *gso = &hdr->gso; 2432 struct bpf_prog *xdp_prog; 2433 struct sk_buff *skb = NULL; 2434 struct sk_buff_head *queue; 2435 u32 rxhash = 0, act; 2436 int buflen = hdr->buflen; 2437 int ret = 0; 2438 bool skb_xdp = false; 2439 struct page *page; 2440 2441 xdp_prog = rcu_dereference(tun->xdp_prog); 2442 if (xdp_prog) { 2443 if (gso->gso_type) { 2444 skb_xdp = true; 2445 goto build; 2446 } 2447 2448 xdp_init_buff(xdp, buflen, &tfile->xdp_rxq); 2449 xdp_set_data_meta_invalid(xdp); 2450 2451 act = bpf_prog_run_xdp(xdp_prog, xdp); 2452 ret = tun_xdp_act(tun, xdp_prog, xdp, act); 2453 if (ret < 0) { 2454 put_page(virt_to_head_page(xdp->data)); 2455 return ret; 2456 } 2457 2458 switch (ret) { 2459 case XDP_REDIRECT: 2460 *flush = true; 2461 fallthrough; 2462 case XDP_TX: 2463 return 0; 2464 case XDP_PASS: 2465 break; 2466 default: 2467 page = virt_to_head_page(xdp->data); 2468 if (tpage->page == page) { 2469 ++tpage->count; 2470 } else { 2471 tun_put_page(tpage); 2472 tpage->page = page; 2473 tpage->count = 1; 2474 } 2475 return 0; 2476 } 2477 } 2478 2479 build: 2480 skb = build_skb(xdp->data_hard_start, buflen); 2481 if (!skb) { 2482 ret = -ENOMEM; 2483 goto out; 2484 } 2485 2486 skb_reserve(skb, xdp->data - xdp->data_hard_start); 2487 skb_put(skb, xdp->data_end - xdp->data); 2488 2489 if (virtio_net_hdr_to_skb(skb, gso, tun_is_little_endian(tun))) { 2490 atomic_long_inc(&tun->rx_frame_errors); 2491 kfree_skb(skb); 2492 ret = -EINVAL; 2493 goto out; 2494 } 2495 2496 skb->protocol = eth_type_trans(skb, tun->dev); 2497 skb_reset_network_header(skb); 2498 skb_probe_transport_header(skb); 2499 skb_record_rx_queue(skb, tfile->queue_index); 2500 2501 if (skb_xdp) { 2502 ret = do_xdp_generic(xdp_prog, skb); 2503 if (ret != XDP_PASS) { 2504 ret = 0; 2505 goto out; 2506 } 2507 } 2508 2509 if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 && 2510 !tfile->detached) 2511 rxhash = __skb_get_hash_symmetric(skb); 2512 2513 if (tfile->napi_enabled) { 2514 queue = &tfile->sk.sk_write_queue; 2515 spin_lock(&queue->lock); 2516 __skb_queue_tail(queue, skb); 2517 spin_unlock(&queue->lock); 2518 ret = 1; 2519 } else { 2520 netif_receive_skb(skb); 2521 ret = 0; 2522 } 2523 2524 /* No need to disable preemption here since this function is 2525 * always called with bh disabled 2526 */ 2527 dev_sw_netstats_rx_add(tun->dev, datasize); 2528 2529 if (rxhash) 2530 tun_flow_update(tun, rxhash, tfile); 2531 2532 out: 2533 return ret; 2534 } 2535 2536 static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len) 2537 { 2538 int ret, i; 2539 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2540 struct tun_struct *tun = tun_get(tfile); 2541 struct tun_msg_ctl *ctl = m->msg_control; 2542 struct xdp_buff *xdp; 2543 2544 if (!tun) 2545 return -EBADFD; 2546 2547 if (m->msg_controllen == sizeof(struct tun_msg_ctl) && 2548 ctl && ctl->type == TUN_MSG_PTR) { 2549 struct tun_page tpage; 2550 int n = ctl->num; 2551 int flush = 0, queued = 0; 2552 2553 memset(&tpage, 0, sizeof(tpage)); 2554 2555 local_bh_disable(); 2556 rcu_read_lock(); 2557 2558 for (i = 0; i < n; i++) { 2559 xdp = &((struct xdp_buff *)ctl->ptr)[i]; 2560 ret = tun_xdp_one(tun, tfile, xdp, &flush, &tpage); 2561 if (ret > 0) 2562 queued += ret; 2563 } 2564 2565 if (flush) 2566 xdp_do_flush(); 2567 2568 if (tfile->napi_enabled && queued > 0) 2569 napi_schedule(&tfile->napi); 2570 2571 rcu_read_unlock(); 2572 local_bh_enable(); 2573 2574 tun_put_page(&tpage); 2575 2576 ret = total_len; 2577 goto out; 2578 } 2579 2580 ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter, 2581 m->msg_flags & MSG_DONTWAIT, 2582 m->msg_flags & MSG_MORE); 2583 out: 2584 tun_put(tun); 2585 return ret; 2586 } 2587 2588 static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len, 2589 int flags) 2590 { 2591 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2592 struct tun_struct *tun = tun_get(tfile); 2593 void *ptr = m->msg_control; 2594 int ret; 2595 2596 if (!tun) { 2597 ret = -EBADFD; 2598 goto out_free; 2599 } 2600 2601 if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) { 2602 ret = -EINVAL; 2603 goto out_put_tun; 2604 } 2605 if (flags & MSG_ERRQUEUE) { 2606 ret = sock_recv_errqueue(sock->sk, m, total_len, 2607 SOL_PACKET, TUN_TX_TIMESTAMP); 2608 goto out; 2609 } 2610 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr); 2611 if (ret > (ssize_t)total_len) { 2612 m->msg_flags |= MSG_TRUNC; 2613 ret = flags & MSG_TRUNC ? ret : total_len; 2614 } 2615 out: 2616 tun_put(tun); 2617 return ret; 2618 2619 out_put_tun: 2620 tun_put(tun); 2621 out_free: 2622 tun_ptr_free(ptr); 2623 return ret; 2624 } 2625 2626 static int tun_ptr_peek_len(void *ptr) 2627 { 2628 if (likely(ptr)) { 2629 if (tun_is_xdp_frame(ptr)) { 2630 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 2631 2632 return xdpf->len; 2633 } 2634 return __skb_array_len_with_tag(ptr); 2635 } else { 2636 return 0; 2637 } 2638 } 2639 2640 static int tun_peek_len(struct socket *sock) 2641 { 2642 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2643 struct tun_struct *tun; 2644 int ret = 0; 2645 2646 tun = tun_get(tfile); 2647 if (!tun) 2648 return 0; 2649 2650 ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len); 2651 tun_put(tun); 2652 2653 return ret; 2654 } 2655 2656 /* Ops structure to mimic raw sockets with tun */ 2657 static const struct proto_ops tun_socket_ops = { 2658 .peek_len = tun_peek_len, 2659 .sendmsg = tun_sendmsg, 2660 .recvmsg = tun_recvmsg, 2661 }; 2662 2663 static struct proto tun_proto = { 2664 .name = "tun", 2665 .owner = THIS_MODULE, 2666 .obj_size = sizeof(struct tun_file), 2667 }; 2668 2669 static int tun_flags(struct tun_struct *tun) 2670 { 2671 return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP); 2672 } 2673 2674 static ssize_t tun_flags_show(struct device *dev, struct device_attribute *attr, 2675 char *buf) 2676 { 2677 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2678 return sysfs_emit(buf, "0x%x\n", tun_flags(tun)); 2679 } 2680 2681 static ssize_t owner_show(struct device *dev, struct device_attribute *attr, 2682 char *buf) 2683 { 2684 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2685 return uid_valid(tun->owner)? 2686 sysfs_emit(buf, "%u\n", 2687 from_kuid_munged(current_user_ns(), tun->owner)) : 2688 sysfs_emit(buf, "-1\n"); 2689 } 2690 2691 static ssize_t group_show(struct device *dev, struct device_attribute *attr, 2692 char *buf) 2693 { 2694 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2695 return gid_valid(tun->group) ? 2696 sysfs_emit(buf, "%u\n", 2697 from_kgid_munged(current_user_ns(), tun->group)) : 2698 sysfs_emit(buf, "-1\n"); 2699 } 2700 2701 static DEVICE_ATTR_RO(tun_flags); 2702 static DEVICE_ATTR_RO(owner); 2703 static DEVICE_ATTR_RO(group); 2704 2705 static struct attribute *tun_dev_attrs[] = { 2706 &dev_attr_tun_flags.attr, 2707 &dev_attr_owner.attr, 2708 &dev_attr_group.attr, 2709 NULL 2710 }; 2711 2712 static const struct attribute_group tun_attr_group = { 2713 .attrs = tun_dev_attrs 2714 }; 2715 2716 static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr) 2717 { 2718 struct tun_struct *tun; 2719 struct tun_file *tfile = file->private_data; 2720 struct net_device *dev; 2721 int err; 2722 2723 if (tfile->detached) 2724 return -EINVAL; 2725 2726 if ((ifr->ifr_flags & IFF_NAPI_FRAGS)) { 2727 if (!capable(CAP_NET_ADMIN)) 2728 return -EPERM; 2729 2730 if (!(ifr->ifr_flags & IFF_NAPI) || 2731 (ifr->ifr_flags & TUN_TYPE_MASK) != IFF_TAP) 2732 return -EINVAL; 2733 } 2734 2735 dev = __dev_get_by_name(net, ifr->ifr_name); 2736 if (dev) { 2737 if (ifr->ifr_flags & IFF_TUN_EXCL) 2738 return -EBUSY; 2739 if ((ifr->ifr_flags & IFF_TUN) && dev->netdev_ops == &tun_netdev_ops) 2740 tun = netdev_priv(dev); 2741 else if ((ifr->ifr_flags & IFF_TAP) && dev->netdev_ops == &tap_netdev_ops) 2742 tun = netdev_priv(dev); 2743 else 2744 return -EINVAL; 2745 2746 if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) != 2747 !!(tun->flags & IFF_MULTI_QUEUE)) 2748 return -EINVAL; 2749 2750 if (tun_not_capable(tun)) 2751 return -EPERM; 2752 err = security_tun_dev_open(tun->security); 2753 if (err < 0) 2754 return err; 2755 2756 err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER, 2757 ifr->ifr_flags & IFF_NAPI, 2758 ifr->ifr_flags & IFF_NAPI_FRAGS, true); 2759 if (err < 0) 2760 return err; 2761 2762 if (tun->flags & IFF_MULTI_QUEUE && 2763 (tun->numqueues + tun->numdisabled > 1)) { 2764 /* One or more queue has already been attached, no need 2765 * to initialize the device again. 2766 */ 2767 netdev_state_change(dev); 2768 return 0; 2769 } 2770 2771 tun->flags = (tun->flags & ~TUN_FEATURES) | 2772 (ifr->ifr_flags & TUN_FEATURES); 2773 2774 netdev_state_change(dev); 2775 } else { 2776 char *name; 2777 unsigned long flags = 0; 2778 int queues = ifr->ifr_flags & IFF_MULTI_QUEUE ? 2779 MAX_TAP_QUEUES : 1; 2780 2781 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 2782 return -EPERM; 2783 err = security_tun_dev_create(); 2784 if (err < 0) 2785 return err; 2786 2787 /* Set dev type */ 2788 if (ifr->ifr_flags & IFF_TUN) { 2789 /* TUN device */ 2790 flags |= IFF_TUN; 2791 name = "tun%d"; 2792 } else if (ifr->ifr_flags & IFF_TAP) { 2793 /* TAP device */ 2794 flags |= IFF_TAP; 2795 name = "tap%d"; 2796 } else 2797 return -EINVAL; 2798 2799 if (*ifr->ifr_name) 2800 name = ifr->ifr_name; 2801 2802 dev = alloc_netdev_mqs(sizeof(struct tun_struct), name, 2803 NET_NAME_UNKNOWN, tun_setup, queues, 2804 queues); 2805 2806 if (!dev) 2807 return -ENOMEM; 2808 2809 dev_net_set(dev, net); 2810 dev->rtnl_link_ops = &tun_link_ops; 2811 dev->ifindex = tfile->ifindex; 2812 dev->sysfs_groups[0] = &tun_attr_group; 2813 2814 tun = netdev_priv(dev); 2815 tun->dev = dev; 2816 tun->flags = flags; 2817 tun->txflt.count = 0; 2818 tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr); 2819 2820 tun->align = NET_SKB_PAD; 2821 tun->filter_attached = false; 2822 tun->sndbuf = tfile->socket.sk->sk_sndbuf; 2823 tun->rx_batched = 0; 2824 RCU_INIT_POINTER(tun->steering_prog, NULL); 2825 2826 tun->ifr = ifr; 2827 tun->file = file; 2828 2829 tun_net_initialize(dev); 2830 2831 err = register_netdevice(tun->dev); 2832 if (err < 0) { 2833 free_netdev(dev); 2834 return err; 2835 } 2836 /* free_netdev() won't check refcnt, to avoid race 2837 * with dev_put() we need publish tun after registration. 2838 */ 2839 rcu_assign_pointer(tfile->tun, tun); 2840 } 2841 2842 if (ifr->ifr_flags & IFF_NO_CARRIER) 2843 netif_carrier_off(tun->dev); 2844 else 2845 netif_carrier_on(tun->dev); 2846 2847 /* Make sure persistent devices do not get stuck in 2848 * xoff state. 2849 */ 2850 if (netif_running(tun->dev)) 2851 netif_tx_wake_all_queues(tun->dev); 2852 2853 strcpy(ifr->ifr_name, tun->dev->name); 2854 return 0; 2855 } 2856 2857 static void tun_get_iff(struct tun_struct *tun, struct ifreq *ifr) 2858 { 2859 strcpy(ifr->ifr_name, tun->dev->name); 2860 2861 ifr->ifr_flags = tun_flags(tun); 2862 2863 } 2864 2865 /* This is like a cut-down ethtool ops, except done via tun fd so no 2866 * privs required. */ 2867 static int set_offload(struct tun_struct *tun, unsigned long arg) 2868 { 2869 netdev_features_t features = 0; 2870 2871 if (arg & TUN_F_CSUM) { 2872 features |= NETIF_F_HW_CSUM; 2873 arg &= ~TUN_F_CSUM; 2874 2875 if (arg & (TUN_F_TSO4|TUN_F_TSO6)) { 2876 if (arg & TUN_F_TSO_ECN) { 2877 features |= NETIF_F_TSO_ECN; 2878 arg &= ~TUN_F_TSO_ECN; 2879 } 2880 if (arg & TUN_F_TSO4) 2881 features |= NETIF_F_TSO; 2882 if (arg & TUN_F_TSO6) 2883 features |= NETIF_F_TSO6; 2884 arg &= ~(TUN_F_TSO4|TUN_F_TSO6); 2885 } 2886 2887 arg &= ~TUN_F_UFO; 2888 } 2889 2890 /* This gives the user a way to test for new features in future by 2891 * trying to set them. */ 2892 if (arg) 2893 return -EINVAL; 2894 2895 tun->set_features = features; 2896 tun->dev->wanted_features &= ~TUN_USER_FEATURES; 2897 tun->dev->wanted_features |= features; 2898 netdev_update_features(tun->dev); 2899 2900 return 0; 2901 } 2902 2903 static void tun_detach_filter(struct tun_struct *tun, int n) 2904 { 2905 int i; 2906 struct tun_file *tfile; 2907 2908 for (i = 0; i < n; i++) { 2909 tfile = rtnl_dereference(tun->tfiles[i]); 2910 lock_sock(tfile->socket.sk); 2911 sk_detach_filter(tfile->socket.sk); 2912 release_sock(tfile->socket.sk); 2913 } 2914 2915 tun->filter_attached = false; 2916 } 2917 2918 static int tun_attach_filter(struct tun_struct *tun) 2919 { 2920 int i, ret = 0; 2921 struct tun_file *tfile; 2922 2923 for (i = 0; i < tun->numqueues; i++) { 2924 tfile = rtnl_dereference(tun->tfiles[i]); 2925 lock_sock(tfile->socket.sk); 2926 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk); 2927 release_sock(tfile->socket.sk); 2928 if (ret) { 2929 tun_detach_filter(tun, i); 2930 return ret; 2931 } 2932 } 2933 2934 tun->filter_attached = true; 2935 return ret; 2936 } 2937 2938 static void tun_set_sndbuf(struct tun_struct *tun) 2939 { 2940 struct tun_file *tfile; 2941 int i; 2942 2943 for (i = 0; i < tun->numqueues; i++) { 2944 tfile = rtnl_dereference(tun->tfiles[i]); 2945 tfile->socket.sk->sk_sndbuf = tun->sndbuf; 2946 } 2947 } 2948 2949 static int tun_set_queue(struct file *file, struct ifreq *ifr) 2950 { 2951 struct tun_file *tfile = file->private_data; 2952 struct tun_struct *tun; 2953 int ret = 0; 2954 2955 rtnl_lock(); 2956 2957 if (ifr->ifr_flags & IFF_ATTACH_QUEUE) { 2958 tun = tfile->detached; 2959 if (!tun) { 2960 ret = -EINVAL; 2961 goto unlock; 2962 } 2963 ret = security_tun_dev_attach_queue(tun->security); 2964 if (ret < 0) 2965 goto unlock; 2966 ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI, 2967 tun->flags & IFF_NAPI_FRAGS, true); 2968 } else if (ifr->ifr_flags & IFF_DETACH_QUEUE) { 2969 tun = rtnl_dereference(tfile->tun); 2970 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached) 2971 ret = -EINVAL; 2972 else 2973 __tun_detach(tfile, false); 2974 } else 2975 ret = -EINVAL; 2976 2977 if (ret >= 0) 2978 netdev_state_change(tun->dev); 2979 2980 unlock: 2981 rtnl_unlock(); 2982 return ret; 2983 } 2984 2985 static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog __rcu **prog_p, 2986 void __user *data) 2987 { 2988 struct bpf_prog *prog; 2989 int fd; 2990 2991 if (copy_from_user(&fd, data, sizeof(fd))) 2992 return -EFAULT; 2993 2994 if (fd == -1) { 2995 prog = NULL; 2996 } else { 2997 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER); 2998 if (IS_ERR(prog)) 2999 return PTR_ERR(prog); 3000 } 3001 3002 return __tun_set_ebpf(tun, prog_p, prog); 3003 } 3004 3005 /* Return correct value for tun->dev->addr_len based on tun->dev->type. */ 3006 static unsigned char tun_get_addr_len(unsigned short type) 3007 { 3008 switch (type) { 3009 case ARPHRD_IP6GRE: 3010 case ARPHRD_TUNNEL6: 3011 return sizeof(struct in6_addr); 3012 case ARPHRD_IPGRE: 3013 case ARPHRD_TUNNEL: 3014 case ARPHRD_SIT: 3015 return 4; 3016 case ARPHRD_ETHER: 3017 return ETH_ALEN; 3018 case ARPHRD_IEEE802154: 3019 case ARPHRD_IEEE802154_MONITOR: 3020 return IEEE802154_EXTENDED_ADDR_LEN; 3021 case ARPHRD_PHONET_PIPE: 3022 case ARPHRD_PPP: 3023 case ARPHRD_NONE: 3024 return 0; 3025 case ARPHRD_6LOWPAN: 3026 return EUI64_ADDR_LEN; 3027 case ARPHRD_FDDI: 3028 return FDDI_K_ALEN; 3029 case ARPHRD_HIPPI: 3030 return HIPPI_ALEN; 3031 case ARPHRD_IEEE802: 3032 return FC_ALEN; 3033 case ARPHRD_ROSE: 3034 return ROSE_ADDR_LEN; 3035 case ARPHRD_NETROM: 3036 return AX25_ADDR_LEN; 3037 case ARPHRD_LOCALTLK: 3038 return LTALK_ALEN; 3039 default: 3040 return 0; 3041 } 3042 } 3043 3044 static long __tun_chr_ioctl(struct file *file, unsigned int cmd, 3045 unsigned long arg, int ifreq_len) 3046 { 3047 struct tun_file *tfile = file->private_data; 3048 struct net *net = sock_net(&tfile->sk); 3049 struct tun_struct *tun; 3050 void __user* argp = (void __user*)arg; 3051 unsigned int ifindex, carrier; 3052 struct ifreq ifr; 3053 kuid_t owner; 3054 kgid_t group; 3055 int sndbuf; 3056 int vnet_hdr_sz; 3057 int le; 3058 int ret; 3059 bool do_notify = false; 3060 3061 if (cmd == TUNSETIFF || cmd == TUNSETQUEUE || 3062 (_IOC_TYPE(cmd) == SOCK_IOC_TYPE && cmd != SIOCGSKNS)) { 3063 if (copy_from_user(&ifr, argp, ifreq_len)) 3064 return -EFAULT; 3065 } else { 3066 memset(&ifr, 0, sizeof(ifr)); 3067 } 3068 if (cmd == TUNGETFEATURES) { 3069 /* Currently this just means: "what IFF flags are valid?". 3070 * This is needed because we never checked for invalid flags on 3071 * TUNSETIFF. 3072 */ 3073 return put_user(IFF_TUN | IFF_TAP | IFF_NO_CARRIER | 3074 TUN_FEATURES, (unsigned int __user*)argp); 3075 } else if (cmd == TUNSETQUEUE) { 3076 return tun_set_queue(file, &ifr); 3077 } else if (cmd == SIOCGSKNS) { 3078 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 3079 return -EPERM; 3080 return open_related_ns(&net->ns, get_net_ns); 3081 } 3082 3083 rtnl_lock(); 3084 3085 tun = tun_get(tfile); 3086 if (cmd == TUNSETIFF) { 3087 ret = -EEXIST; 3088 if (tun) 3089 goto unlock; 3090 3091 ifr.ifr_name[IFNAMSIZ-1] = '\0'; 3092 3093 ret = tun_set_iff(net, file, &ifr); 3094 3095 if (ret) 3096 goto unlock; 3097 3098 if (copy_to_user(argp, &ifr, ifreq_len)) 3099 ret = -EFAULT; 3100 goto unlock; 3101 } 3102 if (cmd == TUNSETIFINDEX) { 3103 ret = -EPERM; 3104 if (tun) 3105 goto unlock; 3106 3107 ret = -EFAULT; 3108 if (copy_from_user(&ifindex, argp, sizeof(ifindex))) 3109 goto unlock; 3110 3111 ret = 0; 3112 tfile->ifindex = ifindex; 3113 goto unlock; 3114 } 3115 3116 ret = -EBADFD; 3117 if (!tun) 3118 goto unlock; 3119 3120 netif_info(tun, drv, tun->dev, "tun_chr_ioctl cmd %u\n", cmd); 3121 3122 net = dev_net(tun->dev); 3123 ret = 0; 3124 switch (cmd) { 3125 case TUNGETIFF: 3126 tun_get_iff(tun, &ifr); 3127 3128 if (tfile->detached) 3129 ifr.ifr_flags |= IFF_DETACH_QUEUE; 3130 if (!tfile->socket.sk->sk_filter) 3131 ifr.ifr_flags |= IFF_NOFILTER; 3132 3133 if (copy_to_user(argp, &ifr, ifreq_len)) 3134 ret = -EFAULT; 3135 break; 3136 3137 case TUNSETNOCSUM: 3138 /* Disable/Enable checksum */ 3139 3140 /* [unimplemented] */ 3141 netif_info(tun, drv, tun->dev, "ignored: set checksum %s\n", 3142 arg ? "disabled" : "enabled"); 3143 break; 3144 3145 case TUNSETPERSIST: 3146 /* Disable/Enable persist mode. Keep an extra reference to the 3147 * module to prevent the module being unprobed. 3148 */ 3149 if (arg && !(tun->flags & IFF_PERSIST)) { 3150 tun->flags |= IFF_PERSIST; 3151 __module_get(THIS_MODULE); 3152 do_notify = true; 3153 } 3154 if (!arg && (tun->flags & IFF_PERSIST)) { 3155 tun->flags &= ~IFF_PERSIST; 3156 module_put(THIS_MODULE); 3157 do_notify = true; 3158 } 3159 3160 netif_info(tun, drv, tun->dev, "persist %s\n", 3161 arg ? "enabled" : "disabled"); 3162 break; 3163 3164 case TUNSETOWNER: 3165 /* Set owner of the device */ 3166 owner = make_kuid(current_user_ns(), arg); 3167 if (!uid_valid(owner)) { 3168 ret = -EINVAL; 3169 break; 3170 } 3171 tun->owner = owner; 3172 do_notify = true; 3173 netif_info(tun, drv, tun->dev, "owner set to %u\n", 3174 from_kuid(&init_user_ns, tun->owner)); 3175 break; 3176 3177 case TUNSETGROUP: 3178 /* Set group of the device */ 3179 group = make_kgid(current_user_ns(), arg); 3180 if (!gid_valid(group)) { 3181 ret = -EINVAL; 3182 break; 3183 } 3184 tun->group = group; 3185 do_notify = true; 3186 netif_info(tun, drv, tun->dev, "group set to %u\n", 3187 from_kgid(&init_user_ns, tun->group)); 3188 break; 3189 3190 case TUNSETLINK: 3191 /* Only allow setting the type when the interface is down */ 3192 if (tun->dev->flags & IFF_UP) { 3193 netif_info(tun, drv, tun->dev, 3194 "Linktype set failed because interface is up\n"); 3195 ret = -EBUSY; 3196 } else { 3197 ret = call_netdevice_notifiers(NETDEV_PRE_TYPE_CHANGE, 3198 tun->dev); 3199 ret = notifier_to_errno(ret); 3200 if (ret) { 3201 netif_info(tun, drv, tun->dev, 3202 "Refused to change device type\n"); 3203 break; 3204 } 3205 tun->dev->type = (int) arg; 3206 tun->dev->addr_len = tun_get_addr_len(tun->dev->type); 3207 netif_info(tun, drv, tun->dev, "linktype set to %d\n", 3208 tun->dev->type); 3209 call_netdevice_notifiers(NETDEV_POST_TYPE_CHANGE, 3210 tun->dev); 3211 } 3212 break; 3213 3214 case TUNSETDEBUG: 3215 tun->msg_enable = (u32)arg; 3216 break; 3217 3218 case TUNSETOFFLOAD: 3219 ret = set_offload(tun, arg); 3220 break; 3221 3222 case TUNSETTXFILTER: 3223 /* Can be set only for TAPs */ 3224 ret = -EINVAL; 3225 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3226 break; 3227 ret = update_filter(&tun->txflt, (void __user *)arg); 3228 break; 3229 3230 case SIOCGIFHWADDR: 3231 /* Get hw address */ 3232 dev_get_mac_address(&ifr.ifr_hwaddr, net, tun->dev->name); 3233 if (copy_to_user(argp, &ifr, ifreq_len)) 3234 ret = -EFAULT; 3235 break; 3236 3237 case SIOCSIFHWADDR: 3238 /* Set hw address */ 3239 ret = dev_set_mac_address_user(tun->dev, &ifr.ifr_hwaddr, NULL); 3240 break; 3241 3242 case TUNGETSNDBUF: 3243 sndbuf = tfile->socket.sk->sk_sndbuf; 3244 if (copy_to_user(argp, &sndbuf, sizeof(sndbuf))) 3245 ret = -EFAULT; 3246 break; 3247 3248 case TUNSETSNDBUF: 3249 if (copy_from_user(&sndbuf, argp, sizeof(sndbuf))) { 3250 ret = -EFAULT; 3251 break; 3252 } 3253 if (sndbuf <= 0) { 3254 ret = -EINVAL; 3255 break; 3256 } 3257 3258 tun->sndbuf = sndbuf; 3259 tun_set_sndbuf(tun); 3260 break; 3261 3262 case TUNGETVNETHDRSZ: 3263 vnet_hdr_sz = tun->vnet_hdr_sz; 3264 if (copy_to_user(argp, &vnet_hdr_sz, sizeof(vnet_hdr_sz))) 3265 ret = -EFAULT; 3266 break; 3267 3268 case TUNSETVNETHDRSZ: 3269 if (copy_from_user(&vnet_hdr_sz, argp, sizeof(vnet_hdr_sz))) { 3270 ret = -EFAULT; 3271 break; 3272 } 3273 if (vnet_hdr_sz < (int)sizeof(struct virtio_net_hdr)) { 3274 ret = -EINVAL; 3275 break; 3276 } 3277 3278 tun->vnet_hdr_sz = vnet_hdr_sz; 3279 break; 3280 3281 case TUNGETVNETLE: 3282 le = !!(tun->flags & TUN_VNET_LE); 3283 if (put_user(le, (int __user *)argp)) 3284 ret = -EFAULT; 3285 break; 3286 3287 case TUNSETVNETLE: 3288 if (get_user(le, (int __user *)argp)) { 3289 ret = -EFAULT; 3290 break; 3291 } 3292 if (le) 3293 tun->flags |= TUN_VNET_LE; 3294 else 3295 tun->flags &= ~TUN_VNET_LE; 3296 break; 3297 3298 case TUNGETVNETBE: 3299 ret = tun_get_vnet_be(tun, argp); 3300 break; 3301 3302 case TUNSETVNETBE: 3303 ret = tun_set_vnet_be(tun, argp); 3304 break; 3305 3306 case TUNATTACHFILTER: 3307 /* Can be set only for TAPs */ 3308 ret = -EINVAL; 3309 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3310 break; 3311 ret = -EFAULT; 3312 if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog))) 3313 break; 3314 3315 ret = tun_attach_filter(tun); 3316 break; 3317 3318 case TUNDETACHFILTER: 3319 /* Can be set only for TAPs */ 3320 ret = -EINVAL; 3321 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3322 break; 3323 ret = 0; 3324 tun_detach_filter(tun, tun->numqueues); 3325 break; 3326 3327 case TUNGETFILTER: 3328 ret = -EINVAL; 3329 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3330 break; 3331 ret = -EFAULT; 3332 if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog))) 3333 break; 3334 ret = 0; 3335 break; 3336 3337 case TUNSETSTEERINGEBPF: 3338 ret = tun_set_ebpf(tun, &tun->steering_prog, argp); 3339 break; 3340 3341 case TUNSETFILTEREBPF: 3342 ret = tun_set_ebpf(tun, &tun->filter_prog, argp); 3343 break; 3344 3345 case TUNSETCARRIER: 3346 ret = -EFAULT; 3347 if (copy_from_user(&carrier, argp, sizeof(carrier))) 3348 goto unlock; 3349 3350 ret = tun_net_change_carrier(tun->dev, (bool)carrier); 3351 break; 3352 3353 case TUNGETDEVNETNS: 3354 ret = -EPERM; 3355 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 3356 goto unlock; 3357 ret = open_related_ns(&net->ns, get_net_ns); 3358 break; 3359 3360 default: 3361 ret = -EINVAL; 3362 break; 3363 } 3364 3365 if (do_notify) 3366 netdev_state_change(tun->dev); 3367 3368 unlock: 3369 rtnl_unlock(); 3370 if (tun) 3371 tun_put(tun); 3372 return ret; 3373 } 3374 3375 static long tun_chr_ioctl(struct file *file, 3376 unsigned int cmd, unsigned long arg) 3377 { 3378 return __tun_chr_ioctl(file, cmd, arg, sizeof (struct ifreq)); 3379 } 3380 3381 #ifdef CONFIG_COMPAT 3382 static long tun_chr_compat_ioctl(struct file *file, 3383 unsigned int cmd, unsigned long arg) 3384 { 3385 switch (cmd) { 3386 case TUNSETIFF: 3387 case TUNGETIFF: 3388 case TUNSETTXFILTER: 3389 case TUNGETSNDBUF: 3390 case TUNSETSNDBUF: 3391 case SIOCGIFHWADDR: 3392 case SIOCSIFHWADDR: 3393 arg = (unsigned long)compat_ptr(arg); 3394 break; 3395 default: 3396 arg = (compat_ulong_t)arg; 3397 break; 3398 } 3399 3400 /* 3401 * compat_ifreq is shorter than ifreq, so we must not access beyond 3402 * the end of that structure. All fields that are used in this 3403 * driver are compatible though, we don't need to convert the 3404 * contents. 3405 */ 3406 return __tun_chr_ioctl(file, cmd, arg, sizeof(struct compat_ifreq)); 3407 } 3408 #endif /* CONFIG_COMPAT */ 3409 3410 static int tun_chr_fasync(int fd, struct file *file, int on) 3411 { 3412 struct tun_file *tfile = file->private_data; 3413 int ret; 3414 3415 if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0) 3416 goto out; 3417 3418 if (on) { 3419 __f_setown(file, task_pid(current), PIDTYPE_TGID, 0); 3420 tfile->flags |= TUN_FASYNC; 3421 } else 3422 tfile->flags &= ~TUN_FASYNC; 3423 ret = 0; 3424 out: 3425 return ret; 3426 } 3427 3428 static int tun_chr_open(struct inode *inode, struct file * file) 3429 { 3430 struct net *net = current->nsproxy->net_ns; 3431 struct tun_file *tfile; 3432 3433 tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, 3434 &tun_proto, 0); 3435 if (!tfile) 3436 return -ENOMEM; 3437 if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) { 3438 sk_free(&tfile->sk); 3439 return -ENOMEM; 3440 } 3441 3442 mutex_init(&tfile->napi_mutex); 3443 RCU_INIT_POINTER(tfile->tun, NULL); 3444 tfile->flags = 0; 3445 tfile->ifindex = 0; 3446 3447 init_waitqueue_head(&tfile->socket.wq.wait); 3448 3449 tfile->socket.file = file; 3450 tfile->socket.ops = &tun_socket_ops; 3451 3452 sock_init_data(&tfile->socket, &tfile->sk); 3453 3454 tfile->sk.sk_write_space = tun_sock_write_space; 3455 tfile->sk.sk_sndbuf = INT_MAX; 3456 3457 file->private_data = tfile; 3458 INIT_LIST_HEAD(&tfile->next); 3459 3460 sock_set_flag(&tfile->sk, SOCK_ZEROCOPY); 3461 3462 return 0; 3463 } 3464 3465 static int tun_chr_close(struct inode *inode, struct file *file) 3466 { 3467 struct tun_file *tfile = file->private_data; 3468 3469 tun_detach(tfile, true); 3470 3471 return 0; 3472 } 3473 3474 #ifdef CONFIG_PROC_FS 3475 static void tun_chr_show_fdinfo(struct seq_file *m, struct file *file) 3476 { 3477 struct tun_file *tfile = file->private_data; 3478 struct tun_struct *tun; 3479 struct ifreq ifr; 3480 3481 memset(&ifr, 0, sizeof(ifr)); 3482 3483 rtnl_lock(); 3484 tun = tun_get(tfile); 3485 if (tun) 3486 tun_get_iff(tun, &ifr); 3487 rtnl_unlock(); 3488 3489 if (tun) 3490 tun_put(tun); 3491 3492 seq_printf(m, "iff:\t%s\n", ifr.ifr_name); 3493 } 3494 #endif 3495 3496 static const struct file_operations tun_fops = { 3497 .owner = THIS_MODULE, 3498 .llseek = no_llseek, 3499 .read_iter = tun_chr_read_iter, 3500 .write_iter = tun_chr_write_iter, 3501 .poll = tun_chr_poll, 3502 .unlocked_ioctl = tun_chr_ioctl, 3503 #ifdef CONFIG_COMPAT 3504 .compat_ioctl = tun_chr_compat_ioctl, 3505 #endif 3506 .open = tun_chr_open, 3507 .release = tun_chr_close, 3508 .fasync = tun_chr_fasync, 3509 #ifdef CONFIG_PROC_FS 3510 .show_fdinfo = tun_chr_show_fdinfo, 3511 #endif 3512 }; 3513 3514 static struct miscdevice tun_miscdev = { 3515 .minor = TUN_MINOR, 3516 .name = "tun", 3517 .nodename = "net/tun", 3518 .fops = &tun_fops, 3519 }; 3520 3521 /* ethtool interface */ 3522 3523 static void tun_default_link_ksettings(struct net_device *dev, 3524 struct ethtool_link_ksettings *cmd) 3525 { 3526 ethtool_link_ksettings_zero_link_mode(cmd, supported); 3527 ethtool_link_ksettings_zero_link_mode(cmd, advertising); 3528 cmd->base.speed = SPEED_10; 3529 cmd->base.duplex = DUPLEX_FULL; 3530 cmd->base.port = PORT_TP; 3531 cmd->base.phy_address = 0; 3532 cmd->base.autoneg = AUTONEG_DISABLE; 3533 } 3534 3535 static int tun_get_link_ksettings(struct net_device *dev, 3536 struct ethtool_link_ksettings *cmd) 3537 { 3538 struct tun_struct *tun = netdev_priv(dev); 3539 3540 memcpy(cmd, &tun->link_ksettings, sizeof(*cmd)); 3541 return 0; 3542 } 3543 3544 static int tun_set_link_ksettings(struct net_device *dev, 3545 const struct ethtool_link_ksettings *cmd) 3546 { 3547 struct tun_struct *tun = netdev_priv(dev); 3548 3549 memcpy(&tun->link_ksettings, cmd, sizeof(*cmd)); 3550 return 0; 3551 } 3552 3553 static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 3554 { 3555 struct tun_struct *tun = netdev_priv(dev); 3556 3557 strscpy(info->driver, DRV_NAME, sizeof(info->driver)); 3558 strscpy(info->version, DRV_VERSION, sizeof(info->version)); 3559 3560 switch (tun->flags & TUN_TYPE_MASK) { 3561 case IFF_TUN: 3562 strscpy(info->bus_info, "tun", sizeof(info->bus_info)); 3563 break; 3564 case IFF_TAP: 3565 strscpy(info->bus_info, "tap", sizeof(info->bus_info)); 3566 break; 3567 } 3568 } 3569 3570 static u32 tun_get_msglevel(struct net_device *dev) 3571 { 3572 struct tun_struct *tun = netdev_priv(dev); 3573 3574 return tun->msg_enable; 3575 } 3576 3577 static void tun_set_msglevel(struct net_device *dev, u32 value) 3578 { 3579 struct tun_struct *tun = netdev_priv(dev); 3580 3581 tun->msg_enable = value; 3582 } 3583 3584 static int tun_get_coalesce(struct net_device *dev, 3585 struct ethtool_coalesce *ec, 3586 struct kernel_ethtool_coalesce *kernel_coal, 3587 struct netlink_ext_ack *extack) 3588 { 3589 struct tun_struct *tun = netdev_priv(dev); 3590 3591 ec->rx_max_coalesced_frames = tun->rx_batched; 3592 3593 return 0; 3594 } 3595 3596 static int tun_set_coalesce(struct net_device *dev, 3597 struct ethtool_coalesce *ec, 3598 struct kernel_ethtool_coalesce *kernel_coal, 3599 struct netlink_ext_ack *extack) 3600 { 3601 struct tun_struct *tun = netdev_priv(dev); 3602 3603 if (ec->rx_max_coalesced_frames > NAPI_POLL_WEIGHT) 3604 tun->rx_batched = NAPI_POLL_WEIGHT; 3605 else 3606 tun->rx_batched = ec->rx_max_coalesced_frames; 3607 3608 return 0; 3609 } 3610 3611 static const struct ethtool_ops tun_ethtool_ops = { 3612 .supported_coalesce_params = ETHTOOL_COALESCE_RX_MAX_FRAMES, 3613 .get_drvinfo = tun_get_drvinfo, 3614 .get_msglevel = tun_get_msglevel, 3615 .set_msglevel = tun_set_msglevel, 3616 .get_link = ethtool_op_get_link, 3617 .get_ts_info = ethtool_op_get_ts_info, 3618 .get_coalesce = tun_get_coalesce, 3619 .set_coalesce = tun_set_coalesce, 3620 .get_link_ksettings = tun_get_link_ksettings, 3621 .set_link_ksettings = tun_set_link_ksettings, 3622 }; 3623 3624 static int tun_queue_resize(struct tun_struct *tun) 3625 { 3626 struct net_device *dev = tun->dev; 3627 struct tun_file *tfile; 3628 struct ptr_ring **rings; 3629 int n = tun->numqueues + tun->numdisabled; 3630 int ret, i; 3631 3632 rings = kmalloc_array(n, sizeof(*rings), GFP_KERNEL); 3633 if (!rings) 3634 return -ENOMEM; 3635 3636 for (i = 0; i < tun->numqueues; i++) { 3637 tfile = rtnl_dereference(tun->tfiles[i]); 3638 rings[i] = &tfile->tx_ring; 3639 } 3640 list_for_each_entry(tfile, &tun->disabled, next) 3641 rings[i++] = &tfile->tx_ring; 3642 3643 ret = ptr_ring_resize_multiple(rings, n, 3644 dev->tx_queue_len, GFP_KERNEL, 3645 tun_ptr_free); 3646 3647 kfree(rings); 3648 return ret; 3649 } 3650 3651 static int tun_device_event(struct notifier_block *unused, 3652 unsigned long event, void *ptr) 3653 { 3654 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 3655 struct tun_struct *tun = netdev_priv(dev); 3656 int i; 3657 3658 if (dev->rtnl_link_ops != &tun_link_ops) 3659 return NOTIFY_DONE; 3660 3661 switch (event) { 3662 case NETDEV_CHANGE_TX_QUEUE_LEN: 3663 if (tun_queue_resize(tun)) 3664 return NOTIFY_BAD; 3665 break; 3666 case NETDEV_UP: 3667 for (i = 0; i < tun->numqueues; i++) { 3668 struct tun_file *tfile; 3669 3670 tfile = rtnl_dereference(tun->tfiles[i]); 3671 tfile->socket.sk->sk_write_space(tfile->socket.sk); 3672 } 3673 break; 3674 default: 3675 break; 3676 } 3677 3678 return NOTIFY_DONE; 3679 } 3680 3681 static struct notifier_block tun_notifier_block __read_mostly = { 3682 .notifier_call = tun_device_event, 3683 }; 3684 3685 static int __init tun_init(void) 3686 { 3687 int ret = 0; 3688 3689 pr_info("%s, %s\n", DRV_DESCRIPTION, DRV_VERSION); 3690 3691 ret = rtnl_link_register(&tun_link_ops); 3692 if (ret) { 3693 pr_err("Can't register link_ops\n"); 3694 goto err_linkops; 3695 } 3696 3697 ret = misc_register(&tun_miscdev); 3698 if (ret) { 3699 pr_err("Can't register misc device %d\n", TUN_MINOR); 3700 goto err_misc; 3701 } 3702 3703 ret = register_netdevice_notifier(&tun_notifier_block); 3704 if (ret) { 3705 pr_err("Can't register netdevice notifier\n"); 3706 goto err_notifier; 3707 } 3708 3709 return 0; 3710 3711 err_notifier: 3712 misc_deregister(&tun_miscdev); 3713 err_misc: 3714 rtnl_link_unregister(&tun_link_ops); 3715 err_linkops: 3716 return ret; 3717 } 3718 3719 static void tun_cleanup(void) 3720 { 3721 misc_deregister(&tun_miscdev); 3722 rtnl_link_unregister(&tun_link_ops); 3723 unregister_netdevice_notifier(&tun_notifier_block); 3724 } 3725 3726 /* Get an underlying socket object from tun file. Returns error unless file is 3727 * attached to a device. The returned object works like a packet socket, it 3728 * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for 3729 * holding a reference to the file for as long as the socket is in use. */ 3730 struct socket *tun_get_socket(struct file *file) 3731 { 3732 struct tun_file *tfile; 3733 if (file->f_op != &tun_fops) 3734 return ERR_PTR(-EINVAL); 3735 tfile = file->private_data; 3736 if (!tfile) 3737 return ERR_PTR(-EBADFD); 3738 return &tfile->socket; 3739 } 3740 EXPORT_SYMBOL_GPL(tun_get_socket); 3741 3742 struct ptr_ring *tun_get_tx_ring(struct file *file) 3743 { 3744 struct tun_file *tfile; 3745 3746 if (file->f_op != &tun_fops) 3747 return ERR_PTR(-EINVAL); 3748 tfile = file->private_data; 3749 if (!tfile) 3750 return ERR_PTR(-EBADFD); 3751 return &tfile->tx_ring; 3752 } 3753 EXPORT_SYMBOL_GPL(tun_get_tx_ring); 3754 3755 module_init(tun_init); 3756 module_exit(tun_cleanup); 3757 MODULE_DESCRIPTION(DRV_DESCRIPTION); 3758 MODULE_AUTHOR(DRV_COPYRIGHT); 3759 MODULE_LICENSE("GPL"); 3760 MODULE_ALIAS_MISCDEV(TUN_MINOR); 3761 MODULE_ALIAS("devname:net/tun"); 3762