1 /* 2 * TUN - Universal TUN/TAP device driver. 3 * Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $ 16 */ 17 18 /* 19 * Changes: 20 * 21 * Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14 22 * Add TUNSETLINK ioctl to set the link encapsulation 23 * 24 * Mark Smith <markzzzsmith@yahoo.com.au> 25 * Use eth_random_addr() for tap MAC address. 26 * 27 * Harald Roelle <harald.roelle@ifi.lmu.de> 2004/04/20 28 * Fixes in packet dropping, queue length setting and queue wakeup. 29 * Increased default tx queue length. 30 * Added ethtool API. 31 * Minor cleanups 32 * 33 * Daniel Podlejski <underley@underley.eu.org> 34 * Modifications for 2.3.99-pre5 kernel. 35 */ 36 37 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 38 39 #define DRV_NAME "tun" 40 #define DRV_VERSION "1.6" 41 #define DRV_DESCRIPTION "Universal TUN/TAP device driver" 42 #define DRV_COPYRIGHT "(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>" 43 44 #include <linux/module.h> 45 #include <linux/errno.h> 46 #include <linux/kernel.h> 47 #include <linux/sched/signal.h> 48 #include <linux/major.h> 49 #include <linux/slab.h> 50 #include <linux/poll.h> 51 #include <linux/fcntl.h> 52 #include <linux/init.h> 53 #include <linux/skbuff.h> 54 #include <linux/netdevice.h> 55 #include <linux/etherdevice.h> 56 #include <linux/miscdevice.h> 57 #include <linux/ethtool.h> 58 #include <linux/rtnetlink.h> 59 #include <linux/compat.h> 60 #include <linux/if.h> 61 #include <linux/if_arp.h> 62 #include <linux/if_ether.h> 63 #include <linux/if_tun.h> 64 #include <linux/if_vlan.h> 65 #include <linux/crc32.h> 66 #include <linux/nsproxy.h> 67 #include <linux/virtio_net.h> 68 #include <linux/rcupdate.h> 69 #include <net/net_namespace.h> 70 #include <net/netns/generic.h> 71 #include <net/rtnetlink.h> 72 #include <net/sock.h> 73 #include <net/xdp.h> 74 #include <linux/seq_file.h> 75 #include <linux/uio.h> 76 #include <linux/skb_array.h> 77 #include <linux/bpf.h> 78 #include <linux/bpf_trace.h> 79 #include <linux/mutex.h> 80 81 #include <linux/uaccess.h> 82 #include <linux/proc_fs.h> 83 84 static void tun_default_link_ksettings(struct net_device *dev, 85 struct ethtool_link_ksettings *cmd); 86 87 /* Uncomment to enable debugging */ 88 /* #define TUN_DEBUG 1 */ 89 90 #ifdef TUN_DEBUG 91 static int debug; 92 93 #define tun_debug(level, tun, fmt, args...) \ 94 do { \ 95 if (tun->debug) \ 96 netdev_printk(level, tun->dev, fmt, ##args); \ 97 } while (0) 98 #define DBG1(level, fmt, args...) \ 99 do { \ 100 if (debug == 2) \ 101 printk(level fmt, ##args); \ 102 } while (0) 103 #else 104 #define tun_debug(level, tun, fmt, args...) \ 105 do { \ 106 if (0) \ 107 netdev_printk(level, tun->dev, fmt, ##args); \ 108 } while (0) 109 #define DBG1(level, fmt, args...) \ 110 do { \ 111 if (0) \ 112 printk(level fmt, ##args); \ 113 } while (0) 114 #endif 115 116 #define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 117 118 /* TUN device flags */ 119 120 /* IFF_ATTACH_QUEUE is never stored in device flags, 121 * overload it to mean fasync when stored there. 122 */ 123 #define TUN_FASYNC IFF_ATTACH_QUEUE 124 /* High bits in flags field are unused. */ 125 #define TUN_VNET_LE 0x80000000 126 #define TUN_VNET_BE 0x40000000 127 128 #define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \ 129 IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS) 130 131 #define GOODCOPY_LEN 128 132 133 #define FLT_EXACT_COUNT 8 134 struct tap_filter { 135 unsigned int count; /* Number of addrs. Zero means disabled */ 136 u32 mask[2]; /* Mask of the hashed addrs */ 137 unsigned char addr[FLT_EXACT_COUNT][ETH_ALEN]; 138 }; 139 140 /* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal 141 * to max number of VCPUs in guest. */ 142 #define MAX_TAP_QUEUES 256 143 #define MAX_TAP_FLOWS 4096 144 145 #define TUN_FLOW_EXPIRE (3 * HZ) 146 147 struct tun_pcpu_stats { 148 u64 rx_packets; 149 u64 rx_bytes; 150 u64 tx_packets; 151 u64 tx_bytes; 152 struct u64_stats_sync syncp; 153 u32 rx_dropped; 154 u32 tx_dropped; 155 u32 rx_frame_errors; 156 }; 157 158 /* A tun_file connects an open character device to a tuntap netdevice. It 159 * also contains all socket related structures (except sock_fprog and tap_filter) 160 * to serve as one transmit queue for tuntap device. The sock_fprog and 161 * tap_filter were kept in tun_struct since they were used for filtering for the 162 * netdevice not for a specific queue (at least I didn't see the requirement for 163 * this). 164 * 165 * RCU usage: 166 * The tun_file and tun_struct are loosely coupled, the pointer from one to the 167 * other can only be read while rcu_read_lock or rtnl_lock is held. 168 */ 169 struct tun_file { 170 struct sock sk; 171 struct socket socket; 172 struct socket_wq wq; 173 struct tun_struct __rcu *tun; 174 struct fasync_struct *fasync; 175 /* only used for fasnyc */ 176 unsigned int flags; 177 union { 178 u16 queue_index; 179 unsigned int ifindex; 180 }; 181 struct napi_struct napi; 182 bool napi_enabled; 183 bool napi_frags_enabled; 184 struct mutex napi_mutex; /* Protects access to the above napi */ 185 struct list_head next; 186 struct tun_struct *detached; 187 struct ptr_ring tx_ring; 188 struct xdp_rxq_info xdp_rxq; 189 }; 190 191 struct tun_flow_entry { 192 struct hlist_node hash_link; 193 struct rcu_head rcu; 194 struct tun_struct *tun; 195 196 u32 rxhash; 197 u32 rps_rxhash; 198 int queue_index; 199 unsigned long updated; 200 }; 201 202 #define TUN_NUM_FLOW_ENTRIES 1024 203 #define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1) 204 205 struct tun_prog { 206 struct rcu_head rcu; 207 struct bpf_prog *prog; 208 }; 209 210 /* Since the socket were moved to tun_file, to preserve the behavior of persist 211 * device, socket filter, sndbuf and vnet header size were restore when the 212 * file were attached to a persist device. 213 */ 214 struct tun_struct { 215 struct tun_file __rcu *tfiles[MAX_TAP_QUEUES]; 216 unsigned int numqueues; 217 unsigned int flags; 218 kuid_t owner; 219 kgid_t group; 220 221 struct net_device *dev; 222 netdev_features_t set_features; 223 #define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \ 224 NETIF_F_TSO6) 225 226 int align; 227 int vnet_hdr_sz; 228 int sndbuf; 229 struct tap_filter txflt; 230 struct sock_fprog fprog; 231 /* protected by rtnl lock */ 232 bool filter_attached; 233 #ifdef TUN_DEBUG 234 int debug; 235 #endif 236 spinlock_t lock; 237 struct hlist_head flows[TUN_NUM_FLOW_ENTRIES]; 238 struct timer_list flow_gc_timer; 239 unsigned long ageing_time; 240 unsigned int numdisabled; 241 struct list_head disabled; 242 void *security; 243 u32 flow_count; 244 u32 rx_batched; 245 struct tun_pcpu_stats __percpu *pcpu_stats; 246 struct bpf_prog __rcu *xdp_prog; 247 struct tun_prog __rcu *steering_prog; 248 struct tun_prog __rcu *filter_prog; 249 struct ethtool_link_ksettings link_ksettings; 250 }; 251 252 struct veth { 253 __be16 h_vlan_proto; 254 __be16 h_vlan_TCI; 255 }; 256 257 bool tun_is_xdp_frame(void *ptr) 258 { 259 return (unsigned long)ptr & TUN_XDP_FLAG; 260 } 261 EXPORT_SYMBOL(tun_is_xdp_frame); 262 263 void *tun_xdp_to_ptr(void *ptr) 264 { 265 return (void *)((unsigned long)ptr | TUN_XDP_FLAG); 266 } 267 EXPORT_SYMBOL(tun_xdp_to_ptr); 268 269 void *tun_ptr_to_xdp(void *ptr) 270 { 271 return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG); 272 } 273 EXPORT_SYMBOL(tun_ptr_to_xdp); 274 275 static int tun_napi_receive(struct napi_struct *napi, int budget) 276 { 277 struct tun_file *tfile = container_of(napi, struct tun_file, napi); 278 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 279 struct sk_buff_head process_queue; 280 struct sk_buff *skb; 281 int received = 0; 282 283 __skb_queue_head_init(&process_queue); 284 285 spin_lock(&queue->lock); 286 skb_queue_splice_tail_init(queue, &process_queue); 287 spin_unlock(&queue->lock); 288 289 while (received < budget && (skb = __skb_dequeue(&process_queue))) { 290 napi_gro_receive(napi, skb); 291 ++received; 292 } 293 294 if (!skb_queue_empty(&process_queue)) { 295 spin_lock(&queue->lock); 296 skb_queue_splice(&process_queue, queue); 297 spin_unlock(&queue->lock); 298 } 299 300 return received; 301 } 302 303 static int tun_napi_poll(struct napi_struct *napi, int budget) 304 { 305 unsigned int received; 306 307 received = tun_napi_receive(napi, budget); 308 309 if (received < budget) 310 napi_complete_done(napi, received); 311 312 return received; 313 } 314 315 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile, 316 bool napi_en, bool napi_frags) 317 { 318 tfile->napi_enabled = napi_en; 319 tfile->napi_frags_enabled = napi_en && napi_frags; 320 if (napi_en) { 321 netif_napi_add(tun->dev, &tfile->napi, tun_napi_poll, 322 NAPI_POLL_WEIGHT); 323 napi_enable(&tfile->napi); 324 } 325 } 326 327 static void tun_napi_disable(struct tun_file *tfile) 328 { 329 if (tfile->napi_enabled) 330 napi_disable(&tfile->napi); 331 } 332 333 static void tun_napi_del(struct tun_file *tfile) 334 { 335 if (tfile->napi_enabled) 336 netif_napi_del(&tfile->napi); 337 } 338 339 static bool tun_napi_frags_enabled(const struct tun_file *tfile) 340 { 341 return tfile->napi_frags_enabled; 342 } 343 344 #ifdef CONFIG_TUN_VNET_CROSS_LE 345 static inline bool tun_legacy_is_little_endian(struct tun_struct *tun) 346 { 347 return tun->flags & TUN_VNET_BE ? false : 348 virtio_legacy_is_little_endian(); 349 } 350 351 static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp) 352 { 353 int be = !!(tun->flags & TUN_VNET_BE); 354 355 if (put_user(be, argp)) 356 return -EFAULT; 357 358 return 0; 359 } 360 361 static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp) 362 { 363 int be; 364 365 if (get_user(be, argp)) 366 return -EFAULT; 367 368 if (be) 369 tun->flags |= TUN_VNET_BE; 370 else 371 tun->flags &= ~TUN_VNET_BE; 372 373 return 0; 374 } 375 #else 376 static inline bool tun_legacy_is_little_endian(struct tun_struct *tun) 377 { 378 return virtio_legacy_is_little_endian(); 379 } 380 381 static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp) 382 { 383 return -EINVAL; 384 } 385 386 static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp) 387 { 388 return -EINVAL; 389 } 390 #endif /* CONFIG_TUN_VNET_CROSS_LE */ 391 392 static inline bool tun_is_little_endian(struct tun_struct *tun) 393 { 394 return tun->flags & TUN_VNET_LE || 395 tun_legacy_is_little_endian(tun); 396 } 397 398 static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val) 399 { 400 return __virtio16_to_cpu(tun_is_little_endian(tun), val); 401 } 402 403 static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val) 404 { 405 return __cpu_to_virtio16(tun_is_little_endian(tun), val); 406 } 407 408 static inline u32 tun_hashfn(u32 rxhash) 409 { 410 return rxhash & TUN_MASK_FLOW_ENTRIES; 411 } 412 413 static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash) 414 { 415 struct tun_flow_entry *e; 416 417 hlist_for_each_entry_rcu(e, head, hash_link) { 418 if (e->rxhash == rxhash) 419 return e; 420 } 421 return NULL; 422 } 423 424 static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun, 425 struct hlist_head *head, 426 u32 rxhash, u16 queue_index) 427 { 428 struct tun_flow_entry *e = kmalloc(sizeof(*e), GFP_ATOMIC); 429 430 if (e) { 431 tun_debug(KERN_INFO, tun, "create flow: hash %u index %u\n", 432 rxhash, queue_index); 433 e->updated = jiffies; 434 e->rxhash = rxhash; 435 e->rps_rxhash = 0; 436 e->queue_index = queue_index; 437 e->tun = tun; 438 hlist_add_head_rcu(&e->hash_link, head); 439 ++tun->flow_count; 440 } 441 return e; 442 } 443 444 static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e) 445 { 446 tun_debug(KERN_INFO, tun, "delete flow: hash %u index %u\n", 447 e->rxhash, e->queue_index); 448 hlist_del_rcu(&e->hash_link); 449 kfree_rcu(e, rcu); 450 --tun->flow_count; 451 } 452 453 static void tun_flow_flush(struct tun_struct *tun) 454 { 455 int i; 456 457 spin_lock_bh(&tun->lock); 458 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 459 struct tun_flow_entry *e; 460 struct hlist_node *n; 461 462 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) 463 tun_flow_delete(tun, e); 464 } 465 spin_unlock_bh(&tun->lock); 466 } 467 468 static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index) 469 { 470 int i; 471 472 spin_lock_bh(&tun->lock); 473 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 474 struct tun_flow_entry *e; 475 struct hlist_node *n; 476 477 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) { 478 if (e->queue_index == queue_index) 479 tun_flow_delete(tun, e); 480 } 481 } 482 spin_unlock_bh(&tun->lock); 483 } 484 485 static void tun_flow_cleanup(struct timer_list *t) 486 { 487 struct tun_struct *tun = from_timer(tun, t, flow_gc_timer); 488 unsigned long delay = tun->ageing_time; 489 unsigned long next_timer = jiffies + delay; 490 unsigned long count = 0; 491 int i; 492 493 tun_debug(KERN_INFO, tun, "tun_flow_cleanup\n"); 494 495 spin_lock(&tun->lock); 496 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 497 struct tun_flow_entry *e; 498 struct hlist_node *n; 499 500 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) { 501 unsigned long this_timer; 502 503 this_timer = e->updated + delay; 504 if (time_before_eq(this_timer, jiffies)) { 505 tun_flow_delete(tun, e); 506 continue; 507 } 508 count++; 509 if (time_before(this_timer, next_timer)) 510 next_timer = this_timer; 511 } 512 } 513 514 if (count) 515 mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer)); 516 spin_unlock(&tun->lock); 517 } 518 519 static void tun_flow_update(struct tun_struct *tun, u32 rxhash, 520 struct tun_file *tfile) 521 { 522 struct hlist_head *head; 523 struct tun_flow_entry *e; 524 unsigned long delay = tun->ageing_time; 525 u16 queue_index = tfile->queue_index; 526 527 if (!rxhash) 528 return; 529 else 530 head = &tun->flows[tun_hashfn(rxhash)]; 531 532 rcu_read_lock(); 533 534 e = tun_flow_find(head, rxhash); 535 if (likely(e)) { 536 /* TODO: keep queueing to old queue until it's empty? */ 537 e->queue_index = queue_index; 538 e->updated = jiffies; 539 sock_rps_record_flow_hash(e->rps_rxhash); 540 } else { 541 spin_lock_bh(&tun->lock); 542 if (!tun_flow_find(head, rxhash) && 543 tun->flow_count < MAX_TAP_FLOWS) 544 tun_flow_create(tun, head, rxhash, queue_index); 545 546 if (!timer_pending(&tun->flow_gc_timer)) 547 mod_timer(&tun->flow_gc_timer, 548 round_jiffies_up(jiffies + delay)); 549 spin_unlock_bh(&tun->lock); 550 } 551 552 rcu_read_unlock(); 553 } 554 555 /** 556 * Save the hash received in the stack receive path and update the 557 * flow_hash table accordingly. 558 */ 559 static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash) 560 { 561 if (unlikely(e->rps_rxhash != hash)) 562 e->rps_rxhash = hash; 563 } 564 565 /* We try to identify a flow through its rxhash first. The reason that 566 * we do not check rxq no. is because some cards(e.g 82599), chooses 567 * the rxq based on the txq where the last packet of the flow comes. As 568 * the userspace application move between processors, we may get a 569 * different rxq no. here. If we could not get rxhash, then we would 570 * hope the rxq no. may help here. 571 */ 572 static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb) 573 { 574 struct tun_flow_entry *e; 575 u32 txq = 0; 576 u32 numqueues = 0; 577 578 numqueues = READ_ONCE(tun->numqueues); 579 580 txq = __skb_get_hash_symmetric(skb); 581 if (txq) { 582 e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq); 583 if (e) { 584 tun_flow_save_rps_rxhash(e, txq); 585 txq = e->queue_index; 586 } else 587 /* use multiply and shift instead of expensive divide */ 588 txq = ((u64)txq * numqueues) >> 32; 589 } else if (likely(skb_rx_queue_recorded(skb))) { 590 txq = skb_get_rx_queue(skb); 591 while (unlikely(txq >= numqueues)) 592 txq -= numqueues; 593 } 594 595 return txq; 596 } 597 598 static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb) 599 { 600 struct tun_prog *prog; 601 u16 ret = 0; 602 603 prog = rcu_dereference(tun->steering_prog); 604 if (prog) 605 ret = bpf_prog_run_clear_cb(prog->prog, skb); 606 607 return ret % tun->numqueues; 608 } 609 610 static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb, 611 struct net_device *sb_dev, 612 select_queue_fallback_t fallback) 613 { 614 struct tun_struct *tun = netdev_priv(dev); 615 u16 ret; 616 617 rcu_read_lock(); 618 if (rcu_dereference(tun->steering_prog)) 619 ret = tun_ebpf_select_queue(tun, skb); 620 else 621 ret = tun_automq_select_queue(tun, skb); 622 rcu_read_unlock(); 623 624 return ret; 625 } 626 627 static inline bool tun_not_capable(struct tun_struct *tun) 628 { 629 const struct cred *cred = current_cred(); 630 struct net *net = dev_net(tun->dev); 631 632 return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) || 633 (gid_valid(tun->group) && !in_egroup_p(tun->group))) && 634 !ns_capable(net->user_ns, CAP_NET_ADMIN); 635 } 636 637 static void tun_set_real_num_queues(struct tun_struct *tun) 638 { 639 netif_set_real_num_tx_queues(tun->dev, tun->numqueues); 640 netif_set_real_num_rx_queues(tun->dev, tun->numqueues); 641 } 642 643 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile) 644 { 645 tfile->detached = tun; 646 list_add_tail(&tfile->next, &tun->disabled); 647 ++tun->numdisabled; 648 } 649 650 static struct tun_struct *tun_enable_queue(struct tun_file *tfile) 651 { 652 struct tun_struct *tun = tfile->detached; 653 654 tfile->detached = NULL; 655 list_del_init(&tfile->next); 656 --tun->numdisabled; 657 return tun; 658 } 659 660 void tun_ptr_free(void *ptr) 661 { 662 if (!ptr) 663 return; 664 if (tun_is_xdp_frame(ptr)) { 665 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 666 667 xdp_return_frame(xdpf); 668 } else { 669 __skb_array_destroy_skb(ptr); 670 } 671 } 672 EXPORT_SYMBOL_GPL(tun_ptr_free); 673 674 static void tun_queue_purge(struct tun_file *tfile) 675 { 676 void *ptr; 677 678 while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL) 679 tun_ptr_free(ptr); 680 681 skb_queue_purge(&tfile->sk.sk_write_queue); 682 skb_queue_purge(&tfile->sk.sk_error_queue); 683 } 684 685 static void __tun_detach(struct tun_file *tfile, bool clean) 686 { 687 struct tun_file *ntfile; 688 struct tun_struct *tun; 689 690 tun = rtnl_dereference(tfile->tun); 691 692 if (tun && clean) { 693 tun_napi_disable(tfile); 694 tun_napi_del(tfile); 695 } 696 697 if (tun && !tfile->detached) { 698 u16 index = tfile->queue_index; 699 BUG_ON(index >= tun->numqueues); 700 701 rcu_assign_pointer(tun->tfiles[index], 702 tun->tfiles[tun->numqueues - 1]); 703 ntfile = rtnl_dereference(tun->tfiles[index]); 704 ntfile->queue_index = index; 705 706 --tun->numqueues; 707 if (clean) { 708 RCU_INIT_POINTER(tfile->tun, NULL); 709 sock_put(&tfile->sk); 710 } else 711 tun_disable_queue(tun, tfile); 712 713 synchronize_net(); 714 tun_flow_delete_by_queue(tun, tun->numqueues + 1); 715 /* Drop read queue */ 716 tun_queue_purge(tfile); 717 tun_set_real_num_queues(tun); 718 } else if (tfile->detached && clean) { 719 tun = tun_enable_queue(tfile); 720 sock_put(&tfile->sk); 721 } 722 723 if (clean) { 724 if (tun && tun->numqueues == 0 && tun->numdisabled == 0) { 725 netif_carrier_off(tun->dev); 726 727 if (!(tun->flags & IFF_PERSIST) && 728 tun->dev->reg_state == NETREG_REGISTERED) 729 unregister_netdevice(tun->dev); 730 } 731 if (tun) 732 xdp_rxq_info_unreg(&tfile->xdp_rxq); 733 ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free); 734 sock_put(&tfile->sk); 735 } 736 } 737 738 static void tun_detach(struct tun_file *tfile, bool clean) 739 { 740 struct tun_struct *tun; 741 struct net_device *dev; 742 743 rtnl_lock(); 744 tun = rtnl_dereference(tfile->tun); 745 dev = tun ? tun->dev : NULL; 746 __tun_detach(tfile, clean); 747 if (dev) 748 netdev_state_change(dev); 749 rtnl_unlock(); 750 } 751 752 static void tun_detach_all(struct net_device *dev) 753 { 754 struct tun_struct *tun = netdev_priv(dev); 755 struct tun_file *tfile, *tmp; 756 int i, n = tun->numqueues; 757 758 for (i = 0; i < n; i++) { 759 tfile = rtnl_dereference(tun->tfiles[i]); 760 BUG_ON(!tfile); 761 tun_napi_disable(tfile); 762 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; 763 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 764 RCU_INIT_POINTER(tfile->tun, NULL); 765 --tun->numqueues; 766 } 767 list_for_each_entry(tfile, &tun->disabled, next) { 768 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; 769 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 770 RCU_INIT_POINTER(tfile->tun, NULL); 771 } 772 BUG_ON(tun->numqueues != 0); 773 774 synchronize_net(); 775 for (i = 0; i < n; i++) { 776 tfile = rtnl_dereference(tun->tfiles[i]); 777 tun_napi_del(tfile); 778 /* Drop read queue */ 779 tun_queue_purge(tfile); 780 xdp_rxq_info_unreg(&tfile->xdp_rxq); 781 sock_put(&tfile->sk); 782 } 783 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) { 784 tun_enable_queue(tfile); 785 tun_queue_purge(tfile); 786 xdp_rxq_info_unreg(&tfile->xdp_rxq); 787 sock_put(&tfile->sk); 788 } 789 BUG_ON(tun->numdisabled != 0); 790 791 if (tun->flags & IFF_PERSIST) 792 module_put(THIS_MODULE); 793 } 794 795 static int tun_attach(struct tun_struct *tun, struct file *file, 796 bool skip_filter, bool napi, bool napi_frags) 797 { 798 struct tun_file *tfile = file->private_data; 799 struct net_device *dev = tun->dev; 800 int err; 801 802 err = security_tun_dev_attach(tfile->socket.sk, tun->security); 803 if (err < 0) 804 goto out; 805 806 err = -EINVAL; 807 if (rtnl_dereference(tfile->tun) && !tfile->detached) 808 goto out; 809 810 err = -EBUSY; 811 if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1) 812 goto out; 813 814 err = -E2BIG; 815 if (!tfile->detached && 816 tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES) 817 goto out; 818 819 err = 0; 820 821 /* Re-attach the filter to persist device */ 822 if (!skip_filter && (tun->filter_attached == true)) { 823 lock_sock(tfile->socket.sk); 824 err = sk_attach_filter(&tun->fprog, tfile->socket.sk); 825 release_sock(tfile->socket.sk); 826 if (!err) 827 goto out; 828 } 829 830 if (!tfile->detached && 831 ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len, 832 GFP_KERNEL, tun_ptr_free)) { 833 err = -ENOMEM; 834 goto out; 835 } 836 837 tfile->queue_index = tun->numqueues; 838 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN; 839 840 if (tfile->detached) { 841 /* Re-attach detached tfile, updating XDP queue_index */ 842 WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq)); 843 844 if (tfile->xdp_rxq.queue_index != tfile->queue_index) 845 tfile->xdp_rxq.queue_index = tfile->queue_index; 846 } else { 847 /* Setup XDP RX-queue info, for new tfile getting attached */ 848 err = xdp_rxq_info_reg(&tfile->xdp_rxq, 849 tun->dev, tfile->queue_index); 850 if (err < 0) 851 goto out; 852 err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq, 853 MEM_TYPE_PAGE_SHARED, NULL); 854 if (err < 0) { 855 xdp_rxq_info_unreg(&tfile->xdp_rxq); 856 goto out; 857 } 858 err = 0; 859 } 860 861 rcu_assign_pointer(tfile->tun, tun); 862 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile); 863 tun->numqueues++; 864 865 if (tfile->detached) { 866 tun_enable_queue(tfile); 867 } else { 868 sock_hold(&tfile->sk); 869 tun_napi_init(tun, tfile, napi, napi_frags); 870 } 871 872 if (rtnl_dereference(tun->xdp_prog)) 873 sock_set_flag(&tfile->sk, SOCK_XDP); 874 875 tun_set_real_num_queues(tun); 876 877 /* device is allowed to go away first, so no need to hold extra 878 * refcnt. 879 */ 880 881 out: 882 return err; 883 } 884 885 static struct tun_struct *tun_get(struct tun_file *tfile) 886 { 887 struct tun_struct *tun; 888 889 rcu_read_lock(); 890 tun = rcu_dereference(tfile->tun); 891 if (tun) 892 dev_hold(tun->dev); 893 rcu_read_unlock(); 894 895 return tun; 896 } 897 898 static void tun_put(struct tun_struct *tun) 899 { 900 dev_put(tun->dev); 901 } 902 903 /* TAP filtering */ 904 static void addr_hash_set(u32 *mask, const u8 *addr) 905 { 906 int n = ether_crc(ETH_ALEN, addr) >> 26; 907 mask[n >> 5] |= (1 << (n & 31)); 908 } 909 910 static unsigned int addr_hash_test(const u32 *mask, const u8 *addr) 911 { 912 int n = ether_crc(ETH_ALEN, addr) >> 26; 913 return mask[n >> 5] & (1 << (n & 31)); 914 } 915 916 static int update_filter(struct tap_filter *filter, void __user *arg) 917 { 918 struct { u8 u[ETH_ALEN]; } *addr; 919 struct tun_filter uf; 920 int err, alen, n, nexact; 921 922 if (copy_from_user(&uf, arg, sizeof(uf))) 923 return -EFAULT; 924 925 if (!uf.count) { 926 /* Disabled */ 927 filter->count = 0; 928 return 0; 929 } 930 931 alen = ETH_ALEN * uf.count; 932 addr = memdup_user(arg + sizeof(uf), alen); 933 if (IS_ERR(addr)) 934 return PTR_ERR(addr); 935 936 /* The filter is updated without holding any locks. Which is 937 * perfectly safe. We disable it first and in the worst 938 * case we'll accept a few undesired packets. */ 939 filter->count = 0; 940 wmb(); 941 942 /* Use first set of addresses as an exact filter */ 943 for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++) 944 memcpy(filter->addr[n], addr[n].u, ETH_ALEN); 945 946 nexact = n; 947 948 /* Remaining multicast addresses are hashed, 949 * unicast will leave the filter disabled. */ 950 memset(filter->mask, 0, sizeof(filter->mask)); 951 for (; n < uf.count; n++) { 952 if (!is_multicast_ether_addr(addr[n].u)) { 953 err = 0; /* no filter */ 954 goto free_addr; 955 } 956 addr_hash_set(filter->mask, addr[n].u); 957 } 958 959 /* For ALLMULTI just set the mask to all ones. 960 * This overrides the mask populated above. */ 961 if ((uf.flags & TUN_FLT_ALLMULTI)) 962 memset(filter->mask, ~0, sizeof(filter->mask)); 963 964 /* Now enable the filter */ 965 wmb(); 966 filter->count = nexact; 967 968 /* Return the number of exact filters */ 969 err = nexact; 970 free_addr: 971 kfree(addr); 972 return err; 973 } 974 975 /* Returns: 0 - drop, !=0 - accept */ 976 static int run_filter(struct tap_filter *filter, const struct sk_buff *skb) 977 { 978 /* Cannot use eth_hdr(skb) here because skb_mac_hdr() is incorrect 979 * at this point. */ 980 struct ethhdr *eh = (struct ethhdr *) skb->data; 981 int i; 982 983 /* Exact match */ 984 for (i = 0; i < filter->count; i++) 985 if (ether_addr_equal(eh->h_dest, filter->addr[i])) 986 return 1; 987 988 /* Inexact match (multicast only) */ 989 if (is_multicast_ether_addr(eh->h_dest)) 990 return addr_hash_test(filter->mask, eh->h_dest); 991 992 return 0; 993 } 994 995 /* 996 * Checks whether the packet is accepted or not. 997 * Returns: 0 - drop, !=0 - accept 998 */ 999 static int check_filter(struct tap_filter *filter, const struct sk_buff *skb) 1000 { 1001 if (!filter->count) 1002 return 1; 1003 1004 return run_filter(filter, skb); 1005 } 1006 1007 /* Network device part of the driver */ 1008 1009 static const struct ethtool_ops tun_ethtool_ops; 1010 1011 /* Net device detach from fd. */ 1012 static void tun_net_uninit(struct net_device *dev) 1013 { 1014 tun_detach_all(dev); 1015 } 1016 1017 /* Net device open. */ 1018 static int tun_net_open(struct net_device *dev) 1019 { 1020 struct tun_struct *tun = netdev_priv(dev); 1021 int i; 1022 1023 netif_tx_start_all_queues(dev); 1024 1025 for (i = 0; i < tun->numqueues; i++) { 1026 struct tun_file *tfile; 1027 1028 tfile = rtnl_dereference(tun->tfiles[i]); 1029 tfile->socket.sk->sk_write_space(tfile->socket.sk); 1030 } 1031 1032 return 0; 1033 } 1034 1035 /* Net device close. */ 1036 static int tun_net_close(struct net_device *dev) 1037 { 1038 netif_tx_stop_all_queues(dev); 1039 return 0; 1040 } 1041 1042 /* Net device start xmit */ 1043 static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb) 1044 { 1045 #ifdef CONFIG_RPS 1046 if (tun->numqueues == 1 && static_key_false(&rps_needed)) { 1047 /* Select queue was not called for the skbuff, so we extract the 1048 * RPS hash and save it into the flow_table here. 1049 */ 1050 __u32 rxhash; 1051 1052 rxhash = __skb_get_hash_symmetric(skb); 1053 if (rxhash) { 1054 struct tun_flow_entry *e; 1055 e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], 1056 rxhash); 1057 if (e) 1058 tun_flow_save_rps_rxhash(e, rxhash); 1059 } 1060 } 1061 #endif 1062 } 1063 1064 static unsigned int run_ebpf_filter(struct tun_struct *tun, 1065 struct sk_buff *skb, 1066 int len) 1067 { 1068 struct tun_prog *prog = rcu_dereference(tun->filter_prog); 1069 1070 if (prog) 1071 len = bpf_prog_run_clear_cb(prog->prog, skb); 1072 1073 return len; 1074 } 1075 1076 /* Net device start xmit */ 1077 static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) 1078 { 1079 struct tun_struct *tun = netdev_priv(dev); 1080 int txq = skb->queue_mapping; 1081 struct tun_file *tfile; 1082 int len = skb->len; 1083 1084 rcu_read_lock(); 1085 tfile = rcu_dereference(tun->tfiles[txq]); 1086 1087 /* Drop packet if interface is not attached */ 1088 if (txq >= tun->numqueues) 1089 goto drop; 1090 1091 if (!rcu_dereference(tun->steering_prog)) 1092 tun_automq_xmit(tun, skb); 1093 1094 tun_debug(KERN_INFO, tun, "tun_net_xmit %d\n", skb->len); 1095 1096 BUG_ON(!tfile); 1097 1098 /* Drop if the filter does not like it. 1099 * This is a noop if the filter is disabled. 1100 * Filter can be enabled only for the TAP devices. */ 1101 if (!check_filter(&tun->txflt, skb)) 1102 goto drop; 1103 1104 if (tfile->socket.sk->sk_filter && 1105 sk_filter(tfile->socket.sk, skb)) 1106 goto drop; 1107 1108 len = run_ebpf_filter(tun, skb, len); 1109 if (len == 0 || pskb_trim(skb, len)) 1110 goto drop; 1111 1112 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) 1113 goto drop; 1114 1115 skb_tx_timestamp(skb); 1116 1117 /* Orphan the skb - required as we might hang on to it 1118 * for indefinite time. 1119 */ 1120 skb_orphan(skb); 1121 1122 nf_reset(skb); 1123 1124 if (ptr_ring_produce(&tfile->tx_ring, skb)) 1125 goto drop; 1126 1127 /* Notify and wake up reader process */ 1128 if (tfile->flags & TUN_FASYNC) 1129 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); 1130 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 1131 1132 rcu_read_unlock(); 1133 return NETDEV_TX_OK; 1134 1135 drop: 1136 this_cpu_inc(tun->pcpu_stats->tx_dropped); 1137 skb_tx_error(skb); 1138 kfree_skb(skb); 1139 rcu_read_unlock(); 1140 return NET_XMIT_DROP; 1141 } 1142 1143 static void tun_net_mclist(struct net_device *dev) 1144 { 1145 /* 1146 * This callback is supposed to deal with mc filter in 1147 * _rx_ path and has nothing to do with the _tx_ path. 1148 * In rx path we always accept everything userspace gives us. 1149 */ 1150 } 1151 1152 static netdev_features_t tun_net_fix_features(struct net_device *dev, 1153 netdev_features_t features) 1154 { 1155 struct tun_struct *tun = netdev_priv(dev); 1156 1157 return (features & tun->set_features) | (features & ~TUN_USER_FEATURES); 1158 } 1159 1160 static void tun_set_headroom(struct net_device *dev, int new_hr) 1161 { 1162 struct tun_struct *tun = netdev_priv(dev); 1163 1164 if (new_hr < NET_SKB_PAD) 1165 new_hr = NET_SKB_PAD; 1166 1167 tun->align = new_hr; 1168 } 1169 1170 static void 1171 tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 1172 { 1173 u32 rx_dropped = 0, tx_dropped = 0, rx_frame_errors = 0; 1174 struct tun_struct *tun = netdev_priv(dev); 1175 struct tun_pcpu_stats *p; 1176 int i; 1177 1178 for_each_possible_cpu(i) { 1179 u64 rxpackets, rxbytes, txpackets, txbytes; 1180 unsigned int start; 1181 1182 p = per_cpu_ptr(tun->pcpu_stats, i); 1183 do { 1184 start = u64_stats_fetch_begin(&p->syncp); 1185 rxpackets = p->rx_packets; 1186 rxbytes = p->rx_bytes; 1187 txpackets = p->tx_packets; 1188 txbytes = p->tx_bytes; 1189 } while (u64_stats_fetch_retry(&p->syncp, start)); 1190 1191 stats->rx_packets += rxpackets; 1192 stats->rx_bytes += rxbytes; 1193 stats->tx_packets += txpackets; 1194 stats->tx_bytes += txbytes; 1195 1196 /* u32 counters */ 1197 rx_dropped += p->rx_dropped; 1198 rx_frame_errors += p->rx_frame_errors; 1199 tx_dropped += p->tx_dropped; 1200 } 1201 stats->rx_dropped = rx_dropped; 1202 stats->rx_frame_errors = rx_frame_errors; 1203 stats->tx_dropped = tx_dropped; 1204 } 1205 1206 static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1207 struct netlink_ext_ack *extack) 1208 { 1209 struct tun_struct *tun = netdev_priv(dev); 1210 struct tun_file *tfile; 1211 struct bpf_prog *old_prog; 1212 int i; 1213 1214 old_prog = rtnl_dereference(tun->xdp_prog); 1215 rcu_assign_pointer(tun->xdp_prog, prog); 1216 if (old_prog) 1217 bpf_prog_put(old_prog); 1218 1219 for (i = 0; i < tun->numqueues; i++) { 1220 tfile = rtnl_dereference(tun->tfiles[i]); 1221 if (prog) 1222 sock_set_flag(&tfile->sk, SOCK_XDP); 1223 else 1224 sock_reset_flag(&tfile->sk, SOCK_XDP); 1225 } 1226 list_for_each_entry(tfile, &tun->disabled, next) { 1227 if (prog) 1228 sock_set_flag(&tfile->sk, SOCK_XDP); 1229 else 1230 sock_reset_flag(&tfile->sk, SOCK_XDP); 1231 } 1232 1233 return 0; 1234 } 1235 1236 static u32 tun_xdp_query(struct net_device *dev) 1237 { 1238 struct tun_struct *tun = netdev_priv(dev); 1239 const struct bpf_prog *xdp_prog; 1240 1241 xdp_prog = rtnl_dereference(tun->xdp_prog); 1242 if (xdp_prog) 1243 return xdp_prog->aux->id; 1244 1245 return 0; 1246 } 1247 1248 static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1249 { 1250 switch (xdp->command) { 1251 case XDP_SETUP_PROG: 1252 return tun_xdp_set(dev, xdp->prog, xdp->extack); 1253 case XDP_QUERY_PROG: 1254 xdp->prog_id = tun_xdp_query(dev); 1255 return 0; 1256 default: 1257 return -EINVAL; 1258 } 1259 } 1260 1261 static const struct net_device_ops tun_netdev_ops = { 1262 .ndo_uninit = tun_net_uninit, 1263 .ndo_open = tun_net_open, 1264 .ndo_stop = tun_net_close, 1265 .ndo_start_xmit = tun_net_xmit, 1266 .ndo_fix_features = tun_net_fix_features, 1267 .ndo_select_queue = tun_select_queue, 1268 .ndo_set_rx_headroom = tun_set_headroom, 1269 .ndo_get_stats64 = tun_net_get_stats64, 1270 }; 1271 1272 static void __tun_xdp_flush_tfile(struct tun_file *tfile) 1273 { 1274 /* Notify and wake up reader process */ 1275 if (tfile->flags & TUN_FASYNC) 1276 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); 1277 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 1278 } 1279 1280 static int tun_xdp_xmit(struct net_device *dev, int n, 1281 struct xdp_frame **frames, u32 flags) 1282 { 1283 struct tun_struct *tun = netdev_priv(dev); 1284 struct tun_file *tfile; 1285 u32 numqueues; 1286 int drops = 0; 1287 int cnt = n; 1288 int i; 1289 1290 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1291 return -EINVAL; 1292 1293 rcu_read_lock(); 1294 1295 numqueues = READ_ONCE(tun->numqueues); 1296 if (!numqueues) { 1297 rcu_read_unlock(); 1298 return -ENXIO; /* Caller will free/return all frames */ 1299 } 1300 1301 tfile = rcu_dereference(tun->tfiles[smp_processor_id() % 1302 numqueues]); 1303 1304 spin_lock(&tfile->tx_ring.producer_lock); 1305 for (i = 0; i < n; i++) { 1306 struct xdp_frame *xdp = frames[i]; 1307 /* Encode the XDP flag into lowest bit for consumer to differ 1308 * XDP buffer from sk_buff. 1309 */ 1310 void *frame = tun_xdp_to_ptr(xdp); 1311 1312 if (__ptr_ring_produce(&tfile->tx_ring, frame)) { 1313 this_cpu_inc(tun->pcpu_stats->tx_dropped); 1314 xdp_return_frame_rx_napi(xdp); 1315 drops++; 1316 } 1317 } 1318 spin_unlock(&tfile->tx_ring.producer_lock); 1319 1320 if (flags & XDP_XMIT_FLUSH) 1321 __tun_xdp_flush_tfile(tfile); 1322 1323 rcu_read_unlock(); 1324 return cnt - drops; 1325 } 1326 1327 static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp) 1328 { 1329 struct xdp_frame *frame = convert_to_xdp_frame(xdp); 1330 1331 if (unlikely(!frame)) 1332 return -EOVERFLOW; 1333 1334 return tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH); 1335 } 1336 1337 static const struct net_device_ops tap_netdev_ops = { 1338 .ndo_uninit = tun_net_uninit, 1339 .ndo_open = tun_net_open, 1340 .ndo_stop = tun_net_close, 1341 .ndo_start_xmit = tun_net_xmit, 1342 .ndo_fix_features = tun_net_fix_features, 1343 .ndo_set_rx_mode = tun_net_mclist, 1344 .ndo_set_mac_address = eth_mac_addr, 1345 .ndo_validate_addr = eth_validate_addr, 1346 .ndo_select_queue = tun_select_queue, 1347 .ndo_features_check = passthru_features_check, 1348 .ndo_set_rx_headroom = tun_set_headroom, 1349 .ndo_get_stats64 = tun_net_get_stats64, 1350 .ndo_bpf = tun_xdp, 1351 .ndo_xdp_xmit = tun_xdp_xmit, 1352 }; 1353 1354 static void tun_flow_init(struct tun_struct *tun) 1355 { 1356 int i; 1357 1358 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) 1359 INIT_HLIST_HEAD(&tun->flows[i]); 1360 1361 tun->ageing_time = TUN_FLOW_EXPIRE; 1362 timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0); 1363 mod_timer(&tun->flow_gc_timer, 1364 round_jiffies_up(jiffies + tun->ageing_time)); 1365 } 1366 1367 static void tun_flow_uninit(struct tun_struct *tun) 1368 { 1369 del_timer_sync(&tun->flow_gc_timer); 1370 tun_flow_flush(tun); 1371 } 1372 1373 #define MIN_MTU 68 1374 #define MAX_MTU 65535 1375 1376 /* Initialize net device. */ 1377 static void tun_net_init(struct net_device *dev) 1378 { 1379 struct tun_struct *tun = netdev_priv(dev); 1380 1381 switch (tun->flags & TUN_TYPE_MASK) { 1382 case IFF_TUN: 1383 dev->netdev_ops = &tun_netdev_ops; 1384 1385 /* Point-to-Point TUN Device */ 1386 dev->hard_header_len = 0; 1387 dev->addr_len = 0; 1388 dev->mtu = 1500; 1389 1390 /* Zero header length */ 1391 dev->type = ARPHRD_NONE; 1392 dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; 1393 break; 1394 1395 case IFF_TAP: 1396 dev->netdev_ops = &tap_netdev_ops; 1397 /* Ethernet TAP Device */ 1398 ether_setup(dev); 1399 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 1400 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 1401 1402 eth_hw_addr_random(dev); 1403 1404 break; 1405 } 1406 1407 dev->min_mtu = MIN_MTU; 1408 dev->max_mtu = MAX_MTU - dev->hard_header_len; 1409 } 1410 1411 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile) 1412 { 1413 struct sock *sk = tfile->socket.sk; 1414 1415 return (tun->dev->flags & IFF_UP) && sock_writeable(sk); 1416 } 1417 1418 /* Character device part */ 1419 1420 /* Poll */ 1421 static __poll_t tun_chr_poll(struct file *file, poll_table *wait) 1422 { 1423 struct tun_file *tfile = file->private_data; 1424 struct tun_struct *tun = tun_get(tfile); 1425 struct sock *sk; 1426 __poll_t mask = 0; 1427 1428 if (!tun) 1429 return EPOLLERR; 1430 1431 sk = tfile->socket.sk; 1432 1433 tun_debug(KERN_INFO, tun, "tun_chr_poll\n"); 1434 1435 poll_wait(file, sk_sleep(sk), wait); 1436 1437 if (!ptr_ring_empty(&tfile->tx_ring)) 1438 mask |= EPOLLIN | EPOLLRDNORM; 1439 1440 /* Make sure SOCKWQ_ASYNC_NOSPACE is set if not writable to 1441 * guarantee EPOLLOUT to be raised by either here or 1442 * tun_sock_write_space(). Then process could get notification 1443 * after it writes to a down device and meets -EIO. 1444 */ 1445 if (tun_sock_writeable(tun, tfile) || 1446 (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) && 1447 tun_sock_writeable(tun, tfile))) 1448 mask |= EPOLLOUT | EPOLLWRNORM; 1449 1450 if (tun->dev->reg_state != NETREG_REGISTERED) 1451 mask = EPOLLERR; 1452 1453 tun_put(tun); 1454 return mask; 1455 } 1456 1457 static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile, 1458 size_t len, 1459 const struct iov_iter *it) 1460 { 1461 struct sk_buff *skb; 1462 size_t linear; 1463 int err; 1464 int i; 1465 1466 if (it->nr_segs > MAX_SKB_FRAGS + 1) 1467 return ERR_PTR(-ENOMEM); 1468 1469 local_bh_disable(); 1470 skb = napi_get_frags(&tfile->napi); 1471 local_bh_enable(); 1472 if (!skb) 1473 return ERR_PTR(-ENOMEM); 1474 1475 linear = iov_iter_single_seg_count(it); 1476 err = __skb_grow(skb, linear); 1477 if (err) 1478 goto free; 1479 1480 skb->len = len; 1481 skb->data_len = len - linear; 1482 skb->truesize += skb->data_len; 1483 1484 for (i = 1; i < it->nr_segs; i++) { 1485 struct page_frag *pfrag = ¤t->task_frag; 1486 size_t fragsz = it->iov[i].iov_len; 1487 1488 if (fragsz == 0 || fragsz > PAGE_SIZE) { 1489 err = -EINVAL; 1490 goto free; 1491 } 1492 1493 if (!skb_page_frag_refill(fragsz, pfrag, GFP_KERNEL)) { 1494 err = -ENOMEM; 1495 goto free; 1496 } 1497 1498 skb_fill_page_desc(skb, i - 1, pfrag->page, 1499 pfrag->offset, fragsz); 1500 page_ref_inc(pfrag->page); 1501 pfrag->offset += fragsz; 1502 } 1503 1504 return skb; 1505 free: 1506 /* frees skb and all frags allocated with napi_alloc_frag() */ 1507 napi_free_frags(&tfile->napi); 1508 return ERR_PTR(err); 1509 } 1510 1511 /* prepad is the amount to reserve at front. len is length after that. 1512 * linear is a hint as to how much to copy (usually headers). */ 1513 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile, 1514 size_t prepad, size_t len, 1515 size_t linear, int noblock) 1516 { 1517 struct sock *sk = tfile->socket.sk; 1518 struct sk_buff *skb; 1519 int err; 1520 1521 /* Under a page? Don't bother with paged skb. */ 1522 if (prepad + len < PAGE_SIZE || !linear) 1523 linear = len; 1524 1525 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, 1526 &err, 0); 1527 if (!skb) 1528 return ERR_PTR(err); 1529 1530 skb_reserve(skb, prepad); 1531 skb_put(skb, linear); 1532 skb->data_len = len - linear; 1533 skb->len += len - linear; 1534 1535 return skb; 1536 } 1537 1538 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile, 1539 struct sk_buff *skb, int more) 1540 { 1541 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 1542 struct sk_buff_head process_queue; 1543 u32 rx_batched = tun->rx_batched; 1544 bool rcv = false; 1545 1546 if (!rx_batched || (!more && skb_queue_empty(queue))) { 1547 local_bh_disable(); 1548 netif_receive_skb(skb); 1549 local_bh_enable(); 1550 return; 1551 } 1552 1553 spin_lock(&queue->lock); 1554 if (!more || skb_queue_len(queue) == rx_batched) { 1555 __skb_queue_head_init(&process_queue); 1556 skb_queue_splice_tail_init(queue, &process_queue); 1557 rcv = true; 1558 } else { 1559 __skb_queue_tail(queue, skb); 1560 } 1561 spin_unlock(&queue->lock); 1562 1563 if (rcv) { 1564 struct sk_buff *nskb; 1565 1566 local_bh_disable(); 1567 while ((nskb = __skb_dequeue(&process_queue))) 1568 netif_receive_skb(nskb); 1569 netif_receive_skb(skb); 1570 local_bh_enable(); 1571 } 1572 } 1573 1574 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile, 1575 int len, int noblock, bool zerocopy) 1576 { 1577 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 1578 return false; 1579 1580 if (tfile->socket.sk->sk_sndbuf != INT_MAX) 1581 return false; 1582 1583 if (!noblock) 1584 return false; 1585 1586 if (zerocopy) 1587 return false; 1588 1589 if (SKB_DATA_ALIGN(len + TUN_RX_PAD) + 1590 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE) 1591 return false; 1592 1593 return true; 1594 } 1595 1596 static struct sk_buff *__tun_build_skb(struct page_frag *alloc_frag, char *buf, 1597 int buflen, int len, int pad) 1598 { 1599 struct sk_buff *skb = build_skb(buf, buflen); 1600 1601 if (!skb) 1602 return ERR_PTR(-ENOMEM); 1603 1604 skb_reserve(skb, pad); 1605 skb_put(skb, len); 1606 1607 get_page(alloc_frag->page); 1608 alloc_frag->offset += buflen; 1609 1610 return skb; 1611 } 1612 1613 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog, 1614 struct xdp_buff *xdp, u32 act) 1615 { 1616 int err; 1617 1618 switch (act) { 1619 case XDP_REDIRECT: 1620 err = xdp_do_redirect(tun->dev, xdp, xdp_prog); 1621 if (err) 1622 return err; 1623 break; 1624 case XDP_TX: 1625 err = tun_xdp_tx(tun->dev, xdp); 1626 if (err < 0) 1627 return err; 1628 break; 1629 case XDP_PASS: 1630 break; 1631 default: 1632 bpf_warn_invalid_xdp_action(act); 1633 /* fall through */ 1634 case XDP_ABORTED: 1635 trace_xdp_exception(tun->dev, xdp_prog, act); 1636 /* fall through */ 1637 case XDP_DROP: 1638 this_cpu_inc(tun->pcpu_stats->rx_dropped); 1639 break; 1640 } 1641 1642 return act; 1643 } 1644 1645 static struct sk_buff *tun_build_skb(struct tun_struct *tun, 1646 struct tun_file *tfile, 1647 struct iov_iter *from, 1648 struct virtio_net_hdr *hdr, 1649 int len, int *skb_xdp) 1650 { 1651 struct page_frag *alloc_frag = ¤t->task_frag; 1652 struct bpf_prog *xdp_prog; 1653 int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1654 char *buf; 1655 size_t copied; 1656 int pad = TUN_RX_PAD; 1657 int err = 0; 1658 1659 rcu_read_lock(); 1660 xdp_prog = rcu_dereference(tun->xdp_prog); 1661 if (xdp_prog) 1662 pad += XDP_PACKET_HEADROOM; 1663 buflen += SKB_DATA_ALIGN(len + pad); 1664 rcu_read_unlock(); 1665 1666 alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES); 1667 if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL))) 1668 return ERR_PTR(-ENOMEM); 1669 1670 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 1671 copied = copy_page_from_iter(alloc_frag->page, 1672 alloc_frag->offset + pad, 1673 len, from); 1674 if (copied != len) 1675 return ERR_PTR(-EFAULT); 1676 1677 /* There's a small window that XDP may be set after the check 1678 * of xdp_prog above, this should be rare and for simplicity 1679 * we do XDP on skb in case the headroom is not enough. 1680 */ 1681 if (hdr->gso_type || !xdp_prog) { 1682 *skb_xdp = 1; 1683 return __tun_build_skb(alloc_frag, buf, buflen, len, pad); 1684 } 1685 1686 *skb_xdp = 0; 1687 1688 local_bh_disable(); 1689 rcu_read_lock(); 1690 xdp_prog = rcu_dereference(tun->xdp_prog); 1691 if (xdp_prog) { 1692 struct xdp_buff xdp; 1693 u32 act; 1694 1695 xdp.data_hard_start = buf; 1696 xdp.data = buf + pad; 1697 xdp_set_data_meta_invalid(&xdp); 1698 xdp.data_end = xdp.data + len; 1699 xdp.rxq = &tfile->xdp_rxq; 1700 1701 act = bpf_prog_run_xdp(xdp_prog, &xdp); 1702 if (act == XDP_REDIRECT || act == XDP_TX) { 1703 get_page(alloc_frag->page); 1704 alloc_frag->offset += buflen; 1705 } 1706 err = tun_xdp_act(tun, xdp_prog, &xdp, act); 1707 if (err < 0) 1708 goto err_xdp; 1709 if (err == XDP_REDIRECT) 1710 xdp_do_flush_map(); 1711 if (err != XDP_PASS) 1712 goto out; 1713 1714 pad = xdp.data - xdp.data_hard_start; 1715 len = xdp.data_end - xdp.data; 1716 } 1717 rcu_read_unlock(); 1718 local_bh_enable(); 1719 1720 return __tun_build_skb(alloc_frag, buf, buflen, len, pad); 1721 1722 err_xdp: 1723 put_page(alloc_frag->page); 1724 out: 1725 rcu_read_unlock(); 1726 local_bh_enable(); 1727 return NULL; 1728 } 1729 1730 /* Get packet from user space buffer */ 1731 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 1732 void *msg_control, struct iov_iter *from, 1733 int noblock, bool more) 1734 { 1735 struct tun_pi pi = { 0, cpu_to_be16(ETH_P_IP) }; 1736 struct sk_buff *skb; 1737 size_t total_len = iov_iter_count(from); 1738 size_t len = total_len, align = tun->align, linear; 1739 struct virtio_net_hdr gso = { 0 }; 1740 struct tun_pcpu_stats *stats; 1741 int good_linear; 1742 int copylen; 1743 bool zerocopy = false; 1744 int err; 1745 u32 rxhash = 0; 1746 int skb_xdp = 1; 1747 bool frags = tun_napi_frags_enabled(tfile); 1748 1749 if (!(tun->dev->flags & IFF_UP)) 1750 return -EIO; 1751 1752 if (!(tun->flags & IFF_NO_PI)) { 1753 if (len < sizeof(pi)) 1754 return -EINVAL; 1755 len -= sizeof(pi); 1756 1757 if (!copy_from_iter_full(&pi, sizeof(pi), from)) 1758 return -EFAULT; 1759 } 1760 1761 if (tun->flags & IFF_VNET_HDR) { 1762 int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 1763 1764 if (len < vnet_hdr_sz) 1765 return -EINVAL; 1766 len -= vnet_hdr_sz; 1767 1768 if (!copy_from_iter_full(&gso, sizeof(gso), from)) 1769 return -EFAULT; 1770 1771 if ((gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 1772 tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2 > tun16_to_cpu(tun, gso.hdr_len)) 1773 gso.hdr_len = cpu_to_tun16(tun, tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2); 1774 1775 if (tun16_to_cpu(tun, gso.hdr_len) > len) 1776 return -EINVAL; 1777 iov_iter_advance(from, vnet_hdr_sz - sizeof(gso)); 1778 } 1779 1780 if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) { 1781 align += NET_IP_ALIGN; 1782 if (unlikely(len < ETH_HLEN || 1783 (gso.hdr_len && tun16_to_cpu(tun, gso.hdr_len) < ETH_HLEN))) 1784 return -EINVAL; 1785 } 1786 1787 good_linear = SKB_MAX_HEAD(align); 1788 1789 if (msg_control) { 1790 struct iov_iter i = *from; 1791 1792 /* There are 256 bytes to be copied in skb, so there is 1793 * enough room for skb expand head in case it is used. 1794 * The rest of the buffer is mapped from userspace. 1795 */ 1796 copylen = gso.hdr_len ? tun16_to_cpu(tun, gso.hdr_len) : GOODCOPY_LEN; 1797 if (copylen > good_linear) 1798 copylen = good_linear; 1799 linear = copylen; 1800 iov_iter_advance(&i, copylen); 1801 if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS) 1802 zerocopy = true; 1803 } 1804 1805 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) { 1806 /* For the packet that is not easy to be processed 1807 * (e.g gso or jumbo packet), we will do it at after 1808 * skb was created with generic XDP routine. 1809 */ 1810 skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp); 1811 if (IS_ERR(skb)) { 1812 this_cpu_inc(tun->pcpu_stats->rx_dropped); 1813 return PTR_ERR(skb); 1814 } 1815 if (!skb) 1816 return total_len; 1817 } else { 1818 if (!zerocopy) { 1819 copylen = len; 1820 if (tun16_to_cpu(tun, gso.hdr_len) > good_linear) 1821 linear = good_linear; 1822 else 1823 linear = tun16_to_cpu(tun, gso.hdr_len); 1824 } 1825 1826 if (frags) { 1827 mutex_lock(&tfile->napi_mutex); 1828 skb = tun_napi_alloc_frags(tfile, copylen, from); 1829 /* tun_napi_alloc_frags() enforces a layout for the skb. 1830 * If zerocopy is enabled, then this layout will be 1831 * overwritten by zerocopy_sg_from_iter(). 1832 */ 1833 zerocopy = false; 1834 } else { 1835 skb = tun_alloc_skb(tfile, align, copylen, linear, 1836 noblock); 1837 } 1838 1839 if (IS_ERR(skb)) { 1840 if (PTR_ERR(skb) != -EAGAIN) 1841 this_cpu_inc(tun->pcpu_stats->rx_dropped); 1842 if (frags) 1843 mutex_unlock(&tfile->napi_mutex); 1844 return PTR_ERR(skb); 1845 } 1846 1847 if (zerocopy) 1848 err = zerocopy_sg_from_iter(skb, from); 1849 else 1850 err = skb_copy_datagram_from_iter(skb, 0, from, len); 1851 1852 if (err) { 1853 this_cpu_inc(tun->pcpu_stats->rx_dropped); 1854 kfree_skb(skb); 1855 if (frags) { 1856 tfile->napi.skb = NULL; 1857 mutex_unlock(&tfile->napi_mutex); 1858 } 1859 1860 return -EFAULT; 1861 } 1862 } 1863 1864 if (virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun))) { 1865 this_cpu_inc(tun->pcpu_stats->rx_frame_errors); 1866 kfree_skb(skb); 1867 if (frags) { 1868 tfile->napi.skb = NULL; 1869 mutex_unlock(&tfile->napi_mutex); 1870 } 1871 1872 return -EINVAL; 1873 } 1874 1875 switch (tun->flags & TUN_TYPE_MASK) { 1876 case IFF_TUN: 1877 if (tun->flags & IFF_NO_PI) { 1878 u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0; 1879 1880 switch (ip_version) { 1881 case 4: 1882 pi.proto = htons(ETH_P_IP); 1883 break; 1884 case 6: 1885 pi.proto = htons(ETH_P_IPV6); 1886 break; 1887 default: 1888 this_cpu_inc(tun->pcpu_stats->rx_dropped); 1889 kfree_skb(skb); 1890 return -EINVAL; 1891 } 1892 } 1893 1894 skb_reset_mac_header(skb); 1895 skb->protocol = pi.proto; 1896 skb->dev = tun->dev; 1897 break; 1898 case IFF_TAP: 1899 if (!frags) 1900 skb->protocol = eth_type_trans(skb, tun->dev); 1901 break; 1902 } 1903 1904 /* copy skb_ubuf_info for callback when skb has no error */ 1905 if (zerocopy) { 1906 skb_shinfo(skb)->destructor_arg = msg_control; 1907 skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; 1908 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; 1909 } else if (msg_control) { 1910 struct ubuf_info *uarg = msg_control; 1911 uarg->callback(uarg, false); 1912 } 1913 1914 skb_reset_network_header(skb); 1915 skb_probe_transport_header(skb, 0); 1916 1917 if (skb_xdp) { 1918 struct bpf_prog *xdp_prog; 1919 int ret; 1920 1921 local_bh_disable(); 1922 rcu_read_lock(); 1923 xdp_prog = rcu_dereference(tun->xdp_prog); 1924 if (xdp_prog) { 1925 ret = do_xdp_generic(xdp_prog, skb); 1926 if (ret != XDP_PASS) { 1927 rcu_read_unlock(); 1928 local_bh_enable(); 1929 return total_len; 1930 } 1931 } 1932 rcu_read_unlock(); 1933 local_bh_enable(); 1934 } 1935 1936 /* Compute the costly rx hash only if needed for flow updates. 1937 * We may get a very small possibility of OOO during switching, not 1938 * worth to optimize. 1939 */ 1940 if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 && 1941 !tfile->detached) 1942 rxhash = __skb_get_hash_symmetric(skb); 1943 1944 if (frags) { 1945 /* Exercise flow dissector code path. */ 1946 u32 headlen = eth_get_headlen(skb->data, skb_headlen(skb)); 1947 1948 if (unlikely(headlen > skb_headlen(skb))) { 1949 this_cpu_inc(tun->pcpu_stats->rx_dropped); 1950 napi_free_frags(&tfile->napi); 1951 mutex_unlock(&tfile->napi_mutex); 1952 WARN_ON(1); 1953 return -ENOMEM; 1954 } 1955 1956 local_bh_disable(); 1957 napi_gro_frags(&tfile->napi); 1958 local_bh_enable(); 1959 mutex_unlock(&tfile->napi_mutex); 1960 } else if (tfile->napi_enabled) { 1961 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 1962 int queue_len; 1963 1964 spin_lock_bh(&queue->lock); 1965 __skb_queue_tail(queue, skb); 1966 queue_len = skb_queue_len(queue); 1967 spin_unlock(&queue->lock); 1968 1969 if (!more || queue_len > NAPI_POLL_WEIGHT) 1970 napi_schedule(&tfile->napi); 1971 1972 local_bh_enable(); 1973 } else if (!IS_ENABLED(CONFIG_4KSTACKS)) { 1974 tun_rx_batched(tun, tfile, skb, more); 1975 } else { 1976 netif_rx_ni(skb); 1977 } 1978 1979 stats = get_cpu_ptr(tun->pcpu_stats); 1980 u64_stats_update_begin(&stats->syncp); 1981 stats->rx_packets++; 1982 stats->rx_bytes += len; 1983 u64_stats_update_end(&stats->syncp); 1984 put_cpu_ptr(stats); 1985 1986 if (rxhash) 1987 tun_flow_update(tun, rxhash, tfile); 1988 1989 return total_len; 1990 } 1991 1992 static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from) 1993 { 1994 struct file *file = iocb->ki_filp; 1995 struct tun_file *tfile = file->private_data; 1996 struct tun_struct *tun = tun_get(tfile); 1997 ssize_t result; 1998 1999 if (!tun) 2000 return -EBADFD; 2001 2002 result = tun_get_user(tun, tfile, NULL, from, 2003 file->f_flags & O_NONBLOCK, false); 2004 2005 tun_put(tun); 2006 return result; 2007 } 2008 2009 static ssize_t tun_put_user_xdp(struct tun_struct *tun, 2010 struct tun_file *tfile, 2011 struct xdp_frame *xdp_frame, 2012 struct iov_iter *iter) 2013 { 2014 int vnet_hdr_sz = 0; 2015 size_t size = xdp_frame->len; 2016 struct tun_pcpu_stats *stats; 2017 size_t ret; 2018 2019 if (tun->flags & IFF_VNET_HDR) { 2020 struct virtio_net_hdr gso = { 0 }; 2021 2022 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 2023 if (unlikely(iov_iter_count(iter) < vnet_hdr_sz)) 2024 return -EINVAL; 2025 if (unlikely(copy_to_iter(&gso, sizeof(gso), iter) != 2026 sizeof(gso))) 2027 return -EFAULT; 2028 iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso)); 2029 } 2030 2031 ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz; 2032 2033 stats = get_cpu_ptr(tun->pcpu_stats); 2034 u64_stats_update_begin(&stats->syncp); 2035 stats->tx_packets++; 2036 stats->tx_bytes += ret; 2037 u64_stats_update_end(&stats->syncp); 2038 put_cpu_ptr(tun->pcpu_stats); 2039 2040 return ret; 2041 } 2042 2043 /* Put packet to the user space buffer */ 2044 static ssize_t tun_put_user(struct tun_struct *tun, 2045 struct tun_file *tfile, 2046 struct sk_buff *skb, 2047 struct iov_iter *iter) 2048 { 2049 struct tun_pi pi = { 0, skb->protocol }; 2050 struct tun_pcpu_stats *stats; 2051 ssize_t total; 2052 int vlan_offset = 0; 2053 int vlan_hlen = 0; 2054 int vnet_hdr_sz = 0; 2055 2056 if (skb_vlan_tag_present(skb)) 2057 vlan_hlen = VLAN_HLEN; 2058 2059 if (tun->flags & IFF_VNET_HDR) 2060 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 2061 2062 total = skb->len + vlan_hlen + vnet_hdr_sz; 2063 2064 if (!(tun->flags & IFF_NO_PI)) { 2065 if (iov_iter_count(iter) < sizeof(pi)) 2066 return -EINVAL; 2067 2068 total += sizeof(pi); 2069 if (iov_iter_count(iter) < total) { 2070 /* Packet will be striped */ 2071 pi.flags |= TUN_PKT_STRIP; 2072 } 2073 2074 if (copy_to_iter(&pi, sizeof(pi), iter) != sizeof(pi)) 2075 return -EFAULT; 2076 } 2077 2078 if (vnet_hdr_sz) { 2079 struct virtio_net_hdr gso; 2080 2081 if (iov_iter_count(iter) < vnet_hdr_sz) 2082 return -EINVAL; 2083 2084 if (virtio_net_hdr_from_skb(skb, &gso, 2085 tun_is_little_endian(tun), true, 2086 vlan_hlen)) { 2087 struct skb_shared_info *sinfo = skb_shinfo(skb); 2088 pr_err("unexpected GSO type: " 2089 "0x%x, gso_size %d, hdr_len %d\n", 2090 sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size), 2091 tun16_to_cpu(tun, gso.hdr_len)); 2092 print_hex_dump(KERN_ERR, "tun: ", 2093 DUMP_PREFIX_NONE, 2094 16, 1, skb->head, 2095 min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true); 2096 WARN_ON_ONCE(1); 2097 return -EINVAL; 2098 } 2099 2100 if (copy_to_iter(&gso, sizeof(gso), iter) != sizeof(gso)) 2101 return -EFAULT; 2102 2103 iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso)); 2104 } 2105 2106 if (vlan_hlen) { 2107 int ret; 2108 struct veth veth; 2109 2110 veth.h_vlan_proto = skb->vlan_proto; 2111 veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb)); 2112 2113 vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto); 2114 2115 ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset); 2116 if (ret || !iov_iter_count(iter)) 2117 goto done; 2118 2119 ret = copy_to_iter(&veth, sizeof(veth), iter); 2120 if (ret != sizeof(veth) || !iov_iter_count(iter)) 2121 goto done; 2122 } 2123 2124 skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset); 2125 2126 done: 2127 /* caller is in process context, */ 2128 stats = get_cpu_ptr(tun->pcpu_stats); 2129 u64_stats_update_begin(&stats->syncp); 2130 stats->tx_packets++; 2131 stats->tx_bytes += skb->len + vlan_hlen; 2132 u64_stats_update_end(&stats->syncp); 2133 put_cpu_ptr(tun->pcpu_stats); 2134 2135 return total; 2136 } 2137 2138 static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) 2139 { 2140 DECLARE_WAITQUEUE(wait, current); 2141 void *ptr = NULL; 2142 int error = 0; 2143 2144 ptr = ptr_ring_consume(&tfile->tx_ring); 2145 if (ptr) 2146 goto out; 2147 if (noblock) { 2148 error = -EAGAIN; 2149 goto out; 2150 } 2151 2152 add_wait_queue(&tfile->wq.wait, &wait); 2153 current->state = TASK_INTERRUPTIBLE; 2154 2155 while (1) { 2156 ptr = ptr_ring_consume(&tfile->tx_ring); 2157 if (ptr) 2158 break; 2159 if (signal_pending(current)) { 2160 error = -ERESTARTSYS; 2161 break; 2162 } 2163 if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) { 2164 error = -EFAULT; 2165 break; 2166 } 2167 2168 schedule(); 2169 } 2170 2171 current->state = TASK_RUNNING; 2172 remove_wait_queue(&tfile->wq.wait, &wait); 2173 2174 out: 2175 *err = error; 2176 return ptr; 2177 } 2178 2179 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, 2180 struct iov_iter *to, 2181 int noblock, void *ptr) 2182 { 2183 ssize_t ret; 2184 int err; 2185 2186 tun_debug(KERN_INFO, tun, "tun_do_read\n"); 2187 2188 if (!iov_iter_count(to)) { 2189 tun_ptr_free(ptr); 2190 return 0; 2191 } 2192 2193 if (!ptr) { 2194 /* Read frames from ring */ 2195 ptr = tun_ring_recv(tfile, noblock, &err); 2196 if (!ptr) 2197 return err; 2198 } 2199 2200 if (tun_is_xdp_frame(ptr)) { 2201 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 2202 2203 ret = tun_put_user_xdp(tun, tfile, xdpf, to); 2204 xdp_return_frame(xdpf); 2205 } else { 2206 struct sk_buff *skb = ptr; 2207 2208 ret = tun_put_user(tun, tfile, skb, to); 2209 if (unlikely(ret < 0)) 2210 kfree_skb(skb); 2211 else 2212 consume_skb(skb); 2213 } 2214 2215 return ret; 2216 } 2217 2218 static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to) 2219 { 2220 struct file *file = iocb->ki_filp; 2221 struct tun_file *tfile = file->private_data; 2222 struct tun_struct *tun = tun_get(tfile); 2223 ssize_t len = iov_iter_count(to), ret; 2224 2225 if (!tun) 2226 return -EBADFD; 2227 ret = tun_do_read(tun, tfile, to, file->f_flags & O_NONBLOCK, NULL); 2228 ret = min_t(ssize_t, ret, len); 2229 if (ret > 0) 2230 iocb->ki_pos = ret; 2231 tun_put(tun); 2232 return ret; 2233 } 2234 2235 static void tun_prog_free(struct rcu_head *rcu) 2236 { 2237 struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu); 2238 2239 bpf_prog_destroy(prog->prog); 2240 kfree(prog); 2241 } 2242 2243 static int __tun_set_ebpf(struct tun_struct *tun, 2244 struct tun_prog __rcu **prog_p, 2245 struct bpf_prog *prog) 2246 { 2247 struct tun_prog *old, *new = NULL; 2248 2249 if (prog) { 2250 new = kmalloc(sizeof(*new), GFP_KERNEL); 2251 if (!new) 2252 return -ENOMEM; 2253 new->prog = prog; 2254 } 2255 2256 spin_lock_bh(&tun->lock); 2257 old = rcu_dereference_protected(*prog_p, 2258 lockdep_is_held(&tun->lock)); 2259 rcu_assign_pointer(*prog_p, new); 2260 spin_unlock_bh(&tun->lock); 2261 2262 if (old) 2263 call_rcu(&old->rcu, tun_prog_free); 2264 2265 return 0; 2266 } 2267 2268 static void tun_free_netdev(struct net_device *dev) 2269 { 2270 struct tun_struct *tun = netdev_priv(dev); 2271 2272 BUG_ON(!(list_empty(&tun->disabled))); 2273 free_percpu(tun->pcpu_stats); 2274 tun_flow_uninit(tun); 2275 security_tun_dev_free_security(tun->security); 2276 __tun_set_ebpf(tun, &tun->steering_prog, NULL); 2277 __tun_set_ebpf(tun, &tun->filter_prog, NULL); 2278 } 2279 2280 static void tun_setup(struct net_device *dev) 2281 { 2282 struct tun_struct *tun = netdev_priv(dev); 2283 2284 tun->owner = INVALID_UID; 2285 tun->group = INVALID_GID; 2286 tun_default_link_ksettings(dev, &tun->link_ksettings); 2287 2288 dev->ethtool_ops = &tun_ethtool_ops; 2289 dev->needs_free_netdev = true; 2290 dev->priv_destructor = tun_free_netdev; 2291 /* We prefer our own queue length */ 2292 dev->tx_queue_len = TUN_READQ_SIZE; 2293 } 2294 2295 /* Trivial set of netlink ops to allow deleting tun or tap 2296 * device with netlink. 2297 */ 2298 static int tun_validate(struct nlattr *tb[], struct nlattr *data[], 2299 struct netlink_ext_ack *extack) 2300 { 2301 return -EINVAL; 2302 } 2303 2304 static size_t tun_get_size(const struct net_device *dev) 2305 { 2306 BUILD_BUG_ON(sizeof(u32) != sizeof(uid_t)); 2307 BUILD_BUG_ON(sizeof(u32) != sizeof(gid_t)); 2308 2309 return nla_total_size(sizeof(uid_t)) + /* OWNER */ 2310 nla_total_size(sizeof(gid_t)) + /* GROUP */ 2311 nla_total_size(sizeof(u8)) + /* TYPE */ 2312 nla_total_size(sizeof(u8)) + /* PI */ 2313 nla_total_size(sizeof(u8)) + /* VNET_HDR */ 2314 nla_total_size(sizeof(u8)) + /* PERSIST */ 2315 nla_total_size(sizeof(u8)) + /* MULTI_QUEUE */ 2316 nla_total_size(sizeof(u32)) + /* NUM_QUEUES */ 2317 nla_total_size(sizeof(u32)) + /* NUM_DISABLED_QUEUES */ 2318 0; 2319 } 2320 2321 static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev) 2322 { 2323 struct tun_struct *tun = netdev_priv(dev); 2324 2325 if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK)) 2326 goto nla_put_failure; 2327 if (uid_valid(tun->owner) && 2328 nla_put_u32(skb, IFLA_TUN_OWNER, 2329 from_kuid_munged(current_user_ns(), tun->owner))) 2330 goto nla_put_failure; 2331 if (gid_valid(tun->group) && 2332 nla_put_u32(skb, IFLA_TUN_GROUP, 2333 from_kgid_munged(current_user_ns(), tun->group))) 2334 goto nla_put_failure; 2335 if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI))) 2336 goto nla_put_failure; 2337 if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR))) 2338 goto nla_put_failure; 2339 if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST))) 2340 goto nla_put_failure; 2341 if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE, 2342 !!(tun->flags & IFF_MULTI_QUEUE))) 2343 goto nla_put_failure; 2344 if (tun->flags & IFF_MULTI_QUEUE) { 2345 if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues)) 2346 goto nla_put_failure; 2347 if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES, 2348 tun->numdisabled)) 2349 goto nla_put_failure; 2350 } 2351 2352 return 0; 2353 2354 nla_put_failure: 2355 return -EMSGSIZE; 2356 } 2357 2358 static struct rtnl_link_ops tun_link_ops __read_mostly = { 2359 .kind = DRV_NAME, 2360 .priv_size = sizeof(struct tun_struct), 2361 .setup = tun_setup, 2362 .validate = tun_validate, 2363 .get_size = tun_get_size, 2364 .fill_info = tun_fill_info, 2365 }; 2366 2367 static void tun_sock_write_space(struct sock *sk) 2368 { 2369 struct tun_file *tfile; 2370 wait_queue_head_t *wqueue; 2371 2372 if (!sock_writeable(sk)) 2373 return; 2374 2375 if (!test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags)) 2376 return; 2377 2378 wqueue = sk_sleep(sk); 2379 if (wqueue && waitqueue_active(wqueue)) 2380 wake_up_interruptible_sync_poll(wqueue, EPOLLOUT | 2381 EPOLLWRNORM | EPOLLWRBAND); 2382 2383 tfile = container_of(sk, struct tun_file, sk); 2384 kill_fasync(&tfile->fasync, SIGIO, POLL_OUT); 2385 } 2386 2387 static int tun_xdp_one(struct tun_struct *tun, 2388 struct tun_file *tfile, 2389 struct xdp_buff *xdp, int *flush) 2390 { 2391 struct tun_xdp_hdr *hdr = xdp->data_hard_start; 2392 struct virtio_net_hdr *gso = &hdr->gso; 2393 struct tun_pcpu_stats *stats; 2394 struct bpf_prog *xdp_prog; 2395 struct sk_buff *skb = NULL; 2396 u32 rxhash = 0, act; 2397 int buflen = hdr->buflen; 2398 int err = 0; 2399 bool skb_xdp = false; 2400 2401 xdp_prog = rcu_dereference(tun->xdp_prog); 2402 if (xdp_prog) { 2403 if (gso->gso_type) { 2404 skb_xdp = true; 2405 goto build; 2406 } 2407 xdp_set_data_meta_invalid(xdp); 2408 xdp->rxq = &tfile->xdp_rxq; 2409 2410 act = bpf_prog_run_xdp(xdp_prog, xdp); 2411 err = tun_xdp_act(tun, xdp_prog, xdp, act); 2412 if (err < 0) { 2413 put_page(virt_to_head_page(xdp->data)); 2414 return err; 2415 } 2416 2417 switch (err) { 2418 case XDP_REDIRECT: 2419 *flush = true; 2420 /* fall through */ 2421 case XDP_TX: 2422 return 0; 2423 case XDP_PASS: 2424 break; 2425 default: 2426 put_page(virt_to_head_page(xdp->data)); 2427 return 0; 2428 } 2429 } 2430 2431 build: 2432 skb = build_skb(xdp->data_hard_start, buflen); 2433 if (!skb) { 2434 err = -ENOMEM; 2435 goto out; 2436 } 2437 2438 skb_reserve(skb, xdp->data - xdp->data_hard_start); 2439 skb_put(skb, xdp->data_end - xdp->data); 2440 2441 if (virtio_net_hdr_to_skb(skb, gso, tun_is_little_endian(tun))) { 2442 this_cpu_inc(tun->pcpu_stats->rx_frame_errors); 2443 kfree_skb(skb); 2444 err = -EINVAL; 2445 goto out; 2446 } 2447 2448 skb->protocol = eth_type_trans(skb, tun->dev); 2449 skb_reset_network_header(skb); 2450 skb_probe_transport_header(skb, 0); 2451 2452 if (skb_xdp) { 2453 err = do_xdp_generic(xdp_prog, skb); 2454 if (err != XDP_PASS) 2455 goto out; 2456 } 2457 2458 if (!rcu_dereference(tun->steering_prog)) 2459 rxhash = __skb_get_hash_symmetric(skb); 2460 2461 netif_receive_skb(skb); 2462 2463 stats = get_cpu_ptr(tun->pcpu_stats); 2464 u64_stats_update_begin(&stats->syncp); 2465 stats->rx_packets++; 2466 stats->rx_bytes += skb->len; 2467 u64_stats_update_end(&stats->syncp); 2468 put_cpu_ptr(stats); 2469 2470 if (rxhash) 2471 tun_flow_update(tun, rxhash, tfile); 2472 2473 out: 2474 return err; 2475 } 2476 2477 static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len) 2478 { 2479 int ret, i; 2480 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2481 struct tun_struct *tun = tun_get(tfile); 2482 struct tun_msg_ctl *ctl = m->msg_control; 2483 struct xdp_buff *xdp; 2484 2485 if (!tun) 2486 return -EBADFD; 2487 2488 if (ctl && (ctl->type == TUN_MSG_PTR)) { 2489 int n = ctl->num; 2490 int flush = 0; 2491 2492 local_bh_disable(); 2493 rcu_read_lock(); 2494 2495 for (i = 0; i < n; i++) { 2496 xdp = &((struct xdp_buff *)ctl->ptr)[i]; 2497 tun_xdp_one(tun, tfile, xdp, &flush); 2498 } 2499 2500 if (flush) 2501 xdp_do_flush_map(); 2502 2503 rcu_read_unlock(); 2504 local_bh_enable(); 2505 2506 ret = total_len; 2507 goto out; 2508 } 2509 2510 ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter, 2511 m->msg_flags & MSG_DONTWAIT, 2512 m->msg_flags & MSG_MORE); 2513 out: 2514 tun_put(tun); 2515 return ret; 2516 } 2517 2518 static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len, 2519 int flags) 2520 { 2521 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2522 struct tun_struct *tun = tun_get(tfile); 2523 void *ptr = m->msg_control; 2524 int ret; 2525 2526 if (!tun) { 2527 ret = -EBADFD; 2528 goto out_free; 2529 } 2530 2531 if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) { 2532 ret = -EINVAL; 2533 goto out_put_tun; 2534 } 2535 if (flags & MSG_ERRQUEUE) { 2536 ret = sock_recv_errqueue(sock->sk, m, total_len, 2537 SOL_PACKET, TUN_TX_TIMESTAMP); 2538 goto out; 2539 } 2540 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr); 2541 if (ret > (ssize_t)total_len) { 2542 m->msg_flags |= MSG_TRUNC; 2543 ret = flags & MSG_TRUNC ? ret : total_len; 2544 } 2545 out: 2546 tun_put(tun); 2547 return ret; 2548 2549 out_put_tun: 2550 tun_put(tun); 2551 out_free: 2552 tun_ptr_free(ptr); 2553 return ret; 2554 } 2555 2556 static int tun_ptr_peek_len(void *ptr) 2557 { 2558 if (likely(ptr)) { 2559 if (tun_is_xdp_frame(ptr)) { 2560 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 2561 2562 return xdpf->len; 2563 } 2564 return __skb_array_len_with_tag(ptr); 2565 } else { 2566 return 0; 2567 } 2568 } 2569 2570 static int tun_peek_len(struct socket *sock) 2571 { 2572 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2573 struct tun_struct *tun; 2574 int ret = 0; 2575 2576 tun = tun_get(tfile); 2577 if (!tun) 2578 return 0; 2579 2580 ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len); 2581 tun_put(tun); 2582 2583 return ret; 2584 } 2585 2586 /* Ops structure to mimic raw sockets with tun */ 2587 static const struct proto_ops tun_socket_ops = { 2588 .peek_len = tun_peek_len, 2589 .sendmsg = tun_sendmsg, 2590 .recvmsg = tun_recvmsg, 2591 }; 2592 2593 static struct proto tun_proto = { 2594 .name = "tun", 2595 .owner = THIS_MODULE, 2596 .obj_size = sizeof(struct tun_file), 2597 }; 2598 2599 static int tun_flags(struct tun_struct *tun) 2600 { 2601 return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP); 2602 } 2603 2604 static ssize_t tun_show_flags(struct device *dev, struct device_attribute *attr, 2605 char *buf) 2606 { 2607 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2608 return sprintf(buf, "0x%x\n", tun_flags(tun)); 2609 } 2610 2611 static ssize_t tun_show_owner(struct device *dev, struct device_attribute *attr, 2612 char *buf) 2613 { 2614 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2615 return uid_valid(tun->owner)? 2616 sprintf(buf, "%u\n", 2617 from_kuid_munged(current_user_ns(), tun->owner)): 2618 sprintf(buf, "-1\n"); 2619 } 2620 2621 static ssize_t tun_show_group(struct device *dev, struct device_attribute *attr, 2622 char *buf) 2623 { 2624 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2625 return gid_valid(tun->group) ? 2626 sprintf(buf, "%u\n", 2627 from_kgid_munged(current_user_ns(), tun->group)): 2628 sprintf(buf, "-1\n"); 2629 } 2630 2631 static DEVICE_ATTR(tun_flags, 0444, tun_show_flags, NULL); 2632 static DEVICE_ATTR(owner, 0444, tun_show_owner, NULL); 2633 static DEVICE_ATTR(group, 0444, tun_show_group, NULL); 2634 2635 static struct attribute *tun_dev_attrs[] = { 2636 &dev_attr_tun_flags.attr, 2637 &dev_attr_owner.attr, 2638 &dev_attr_group.attr, 2639 NULL 2640 }; 2641 2642 static const struct attribute_group tun_attr_group = { 2643 .attrs = tun_dev_attrs 2644 }; 2645 2646 static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr) 2647 { 2648 struct tun_struct *tun; 2649 struct tun_file *tfile = file->private_data; 2650 struct net_device *dev; 2651 int err; 2652 2653 if (tfile->detached) 2654 return -EINVAL; 2655 2656 if ((ifr->ifr_flags & IFF_NAPI_FRAGS)) { 2657 if (!capable(CAP_NET_ADMIN)) 2658 return -EPERM; 2659 2660 if (!(ifr->ifr_flags & IFF_NAPI) || 2661 (ifr->ifr_flags & TUN_TYPE_MASK) != IFF_TAP) 2662 return -EINVAL; 2663 } 2664 2665 dev = __dev_get_by_name(net, ifr->ifr_name); 2666 if (dev) { 2667 if (ifr->ifr_flags & IFF_TUN_EXCL) 2668 return -EBUSY; 2669 if ((ifr->ifr_flags & IFF_TUN) && dev->netdev_ops == &tun_netdev_ops) 2670 tun = netdev_priv(dev); 2671 else if ((ifr->ifr_flags & IFF_TAP) && dev->netdev_ops == &tap_netdev_ops) 2672 tun = netdev_priv(dev); 2673 else 2674 return -EINVAL; 2675 2676 if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) != 2677 !!(tun->flags & IFF_MULTI_QUEUE)) 2678 return -EINVAL; 2679 2680 if (tun_not_capable(tun)) 2681 return -EPERM; 2682 err = security_tun_dev_open(tun->security); 2683 if (err < 0) 2684 return err; 2685 2686 err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER, 2687 ifr->ifr_flags & IFF_NAPI, 2688 ifr->ifr_flags & IFF_NAPI_FRAGS); 2689 if (err < 0) 2690 return err; 2691 2692 if (tun->flags & IFF_MULTI_QUEUE && 2693 (tun->numqueues + tun->numdisabled > 1)) { 2694 /* One or more queue has already been attached, no need 2695 * to initialize the device again. 2696 */ 2697 netdev_state_change(dev); 2698 return 0; 2699 } 2700 2701 tun->flags = (tun->flags & ~TUN_FEATURES) | 2702 (ifr->ifr_flags & TUN_FEATURES); 2703 2704 netdev_state_change(dev); 2705 } else { 2706 char *name; 2707 unsigned long flags = 0; 2708 int queues = ifr->ifr_flags & IFF_MULTI_QUEUE ? 2709 MAX_TAP_QUEUES : 1; 2710 2711 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 2712 return -EPERM; 2713 err = security_tun_dev_create(); 2714 if (err < 0) 2715 return err; 2716 2717 /* Set dev type */ 2718 if (ifr->ifr_flags & IFF_TUN) { 2719 /* TUN device */ 2720 flags |= IFF_TUN; 2721 name = "tun%d"; 2722 } else if (ifr->ifr_flags & IFF_TAP) { 2723 /* TAP device */ 2724 flags |= IFF_TAP; 2725 name = "tap%d"; 2726 } else 2727 return -EINVAL; 2728 2729 if (*ifr->ifr_name) 2730 name = ifr->ifr_name; 2731 2732 dev = alloc_netdev_mqs(sizeof(struct tun_struct), name, 2733 NET_NAME_UNKNOWN, tun_setup, queues, 2734 queues); 2735 2736 if (!dev) 2737 return -ENOMEM; 2738 err = dev_get_valid_name(net, dev, name); 2739 if (err < 0) 2740 goto err_free_dev; 2741 2742 dev_net_set(dev, net); 2743 dev->rtnl_link_ops = &tun_link_ops; 2744 dev->ifindex = tfile->ifindex; 2745 dev->sysfs_groups[0] = &tun_attr_group; 2746 2747 tun = netdev_priv(dev); 2748 tun->dev = dev; 2749 tun->flags = flags; 2750 tun->txflt.count = 0; 2751 tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr); 2752 2753 tun->align = NET_SKB_PAD; 2754 tun->filter_attached = false; 2755 tun->sndbuf = tfile->socket.sk->sk_sndbuf; 2756 tun->rx_batched = 0; 2757 RCU_INIT_POINTER(tun->steering_prog, NULL); 2758 2759 tun->pcpu_stats = netdev_alloc_pcpu_stats(struct tun_pcpu_stats); 2760 if (!tun->pcpu_stats) { 2761 err = -ENOMEM; 2762 goto err_free_dev; 2763 } 2764 2765 spin_lock_init(&tun->lock); 2766 2767 err = security_tun_dev_alloc_security(&tun->security); 2768 if (err < 0) 2769 goto err_free_stat; 2770 2771 tun_net_init(dev); 2772 tun_flow_init(tun); 2773 2774 dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST | 2775 TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX | 2776 NETIF_F_HW_VLAN_STAG_TX; 2777 dev->features = dev->hw_features | NETIF_F_LLTX; 2778 dev->vlan_features = dev->features & 2779 ~(NETIF_F_HW_VLAN_CTAG_TX | 2780 NETIF_F_HW_VLAN_STAG_TX); 2781 2782 tun->flags = (tun->flags & ~TUN_FEATURES) | 2783 (ifr->ifr_flags & TUN_FEATURES); 2784 2785 INIT_LIST_HEAD(&tun->disabled); 2786 err = tun_attach(tun, file, false, ifr->ifr_flags & IFF_NAPI, 2787 ifr->ifr_flags & IFF_NAPI_FRAGS); 2788 if (err < 0) 2789 goto err_free_flow; 2790 2791 err = register_netdevice(tun->dev); 2792 if (err < 0) 2793 goto err_detach; 2794 } 2795 2796 netif_carrier_on(tun->dev); 2797 2798 tun_debug(KERN_INFO, tun, "tun_set_iff\n"); 2799 2800 /* Make sure persistent devices do not get stuck in 2801 * xoff state. 2802 */ 2803 if (netif_running(tun->dev)) 2804 netif_tx_wake_all_queues(tun->dev); 2805 2806 strcpy(ifr->ifr_name, tun->dev->name); 2807 return 0; 2808 2809 err_detach: 2810 tun_detach_all(dev); 2811 /* register_netdevice() already called tun_free_netdev() */ 2812 goto err_free_dev; 2813 2814 err_free_flow: 2815 tun_flow_uninit(tun); 2816 security_tun_dev_free_security(tun->security); 2817 err_free_stat: 2818 free_percpu(tun->pcpu_stats); 2819 err_free_dev: 2820 free_netdev(dev); 2821 return err; 2822 } 2823 2824 static void tun_get_iff(struct net *net, struct tun_struct *tun, 2825 struct ifreq *ifr) 2826 { 2827 tun_debug(KERN_INFO, tun, "tun_get_iff\n"); 2828 2829 strcpy(ifr->ifr_name, tun->dev->name); 2830 2831 ifr->ifr_flags = tun_flags(tun); 2832 2833 } 2834 2835 /* This is like a cut-down ethtool ops, except done via tun fd so no 2836 * privs required. */ 2837 static int set_offload(struct tun_struct *tun, unsigned long arg) 2838 { 2839 netdev_features_t features = 0; 2840 2841 if (arg & TUN_F_CSUM) { 2842 features |= NETIF_F_HW_CSUM; 2843 arg &= ~TUN_F_CSUM; 2844 2845 if (arg & (TUN_F_TSO4|TUN_F_TSO6)) { 2846 if (arg & TUN_F_TSO_ECN) { 2847 features |= NETIF_F_TSO_ECN; 2848 arg &= ~TUN_F_TSO_ECN; 2849 } 2850 if (arg & TUN_F_TSO4) 2851 features |= NETIF_F_TSO; 2852 if (arg & TUN_F_TSO6) 2853 features |= NETIF_F_TSO6; 2854 arg &= ~(TUN_F_TSO4|TUN_F_TSO6); 2855 } 2856 2857 arg &= ~TUN_F_UFO; 2858 } 2859 2860 /* This gives the user a way to test for new features in future by 2861 * trying to set them. */ 2862 if (arg) 2863 return -EINVAL; 2864 2865 tun->set_features = features; 2866 tun->dev->wanted_features &= ~TUN_USER_FEATURES; 2867 tun->dev->wanted_features |= features; 2868 netdev_update_features(tun->dev); 2869 2870 return 0; 2871 } 2872 2873 static void tun_detach_filter(struct tun_struct *tun, int n) 2874 { 2875 int i; 2876 struct tun_file *tfile; 2877 2878 for (i = 0; i < n; i++) { 2879 tfile = rtnl_dereference(tun->tfiles[i]); 2880 lock_sock(tfile->socket.sk); 2881 sk_detach_filter(tfile->socket.sk); 2882 release_sock(tfile->socket.sk); 2883 } 2884 2885 tun->filter_attached = false; 2886 } 2887 2888 static int tun_attach_filter(struct tun_struct *tun) 2889 { 2890 int i, ret = 0; 2891 struct tun_file *tfile; 2892 2893 for (i = 0; i < tun->numqueues; i++) { 2894 tfile = rtnl_dereference(tun->tfiles[i]); 2895 lock_sock(tfile->socket.sk); 2896 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk); 2897 release_sock(tfile->socket.sk); 2898 if (ret) { 2899 tun_detach_filter(tun, i); 2900 return ret; 2901 } 2902 } 2903 2904 tun->filter_attached = true; 2905 return ret; 2906 } 2907 2908 static void tun_set_sndbuf(struct tun_struct *tun) 2909 { 2910 struct tun_file *tfile; 2911 int i; 2912 2913 for (i = 0; i < tun->numqueues; i++) { 2914 tfile = rtnl_dereference(tun->tfiles[i]); 2915 tfile->socket.sk->sk_sndbuf = tun->sndbuf; 2916 } 2917 } 2918 2919 static int tun_set_queue(struct file *file, struct ifreq *ifr) 2920 { 2921 struct tun_file *tfile = file->private_data; 2922 struct tun_struct *tun; 2923 int ret = 0; 2924 2925 rtnl_lock(); 2926 2927 if (ifr->ifr_flags & IFF_ATTACH_QUEUE) { 2928 tun = tfile->detached; 2929 if (!tun) { 2930 ret = -EINVAL; 2931 goto unlock; 2932 } 2933 ret = security_tun_dev_attach_queue(tun->security); 2934 if (ret < 0) 2935 goto unlock; 2936 ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI, 2937 tun->flags & IFF_NAPI_FRAGS); 2938 } else if (ifr->ifr_flags & IFF_DETACH_QUEUE) { 2939 tun = rtnl_dereference(tfile->tun); 2940 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached) 2941 ret = -EINVAL; 2942 else 2943 __tun_detach(tfile, false); 2944 } else 2945 ret = -EINVAL; 2946 2947 if (ret >= 0) 2948 netdev_state_change(tun->dev); 2949 2950 unlock: 2951 rtnl_unlock(); 2952 return ret; 2953 } 2954 2955 static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog **prog_p, 2956 void __user *data) 2957 { 2958 struct bpf_prog *prog; 2959 int fd; 2960 2961 if (copy_from_user(&fd, data, sizeof(fd))) 2962 return -EFAULT; 2963 2964 if (fd == -1) { 2965 prog = NULL; 2966 } else { 2967 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER); 2968 if (IS_ERR(prog)) 2969 return PTR_ERR(prog); 2970 } 2971 2972 return __tun_set_ebpf(tun, prog_p, prog); 2973 } 2974 2975 static long __tun_chr_ioctl(struct file *file, unsigned int cmd, 2976 unsigned long arg, int ifreq_len) 2977 { 2978 struct tun_file *tfile = file->private_data; 2979 struct net *net = sock_net(&tfile->sk); 2980 struct tun_struct *tun; 2981 void __user* argp = (void __user*)arg; 2982 struct ifreq ifr; 2983 kuid_t owner; 2984 kgid_t group; 2985 int sndbuf; 2986 int vnet_hdr_sz; 2987 unsigned int ifindex; 2988 int le; 2989 int ret; 2990 bool do_notify = false; 2991 2992 if (cmd == TUNSETIFF || cmd == TUNSETQUEUE || 2993 (_IOC_TYPE(cmd) == SOCK_IOC_TYPE && cmd != SIOCGSKNS)) { 2994 if (copy_from_user(&ifr, argp, ifreq_len)) 2995 return -EFAULT; 2996 } else { 2997 memset(&ifr, 0, sizeof(ifr)); 2998 } 2999 if (cmd == TUNGETFEATURES) { 3000 /* Currently this just means: "what IFF flags are valid?". 3001 * This is needed because we never checked for invalid flags on 3002 * TUNSETIFF. 3003 */ 3004 return put_user(IFF_TUN | IFF_TAP | TUN_FEATURES, 3005 (unsigned int __user*)argp); 3006 } else if (cmd == TUNSETQUEUE) { 3007 return tun_set_queue(file, &ifr); 3008 } else if (cmd == SIOCGSKNS) { 3009 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 3010 return -EPERM; 3011 return open_related_ns(&net->ns, get_net_ns); 3012 } 3013 3014 ret = 0; 3015 rtnl_lock(); 3016 3017 tun = tun_get(tfile); 3018 if (cmd == TUNSETIFF) { 3019 ret = -EEXIST; 3020 if (tun) 3021 goto unlock; 3022 3023 ifr.ifr_name[IFNAMSIZ-1] = '\0'; 3024 3025 ret = tun_set_iff(net, file, &ifr); 3026 3027 if (ret) 3028 goto unlock; 3029 3030 if (copy_to_user(argp, &ifr, ifreq_len)) 3031 ret = -EFAULT; 3032 goto unlock; 3033 } 3034 if (cmd == TUNSETIFINDEX) { 3035 ret = -EPERM; 3036 if (tun) 3037 goto unlock; 3038 3039 ret = -EFAULT; 3040 if (copy_from_user(&ifindex, argp, sizeof(ifindex))) 3041 goto unlock; 3042 3043 ret = 0; 3044 tfile->ifindex = ifindex; 3045 goto unlock; 3046 } 3047 3048 ret = -EBADFD; 3049 if (!tun) 3050 goto unlock; 3051 3052 tun_debug(KERN_INFO, tun, "tun_chr_ioctl cmd %u\n", cmd); 3053 3054 ret = 0; 3055 switch (cmd) { 3056 case TUNGETIFF: 3057 tun_get_iff(current->nsproxy->net_ns, tun, &ifr); 3058 3059 if (tfile->detached) 3060 ifr.ifr_flags |= IFF_DETACH_QUEUE; 3061 if (!tfile->socket.sk->sk_filter) 3062 ifr.ifr_flags |= IFF_NOFILTER; 3063 3064 if (copy_to_user(argp, &ifr, ifreq_len)) 3065 ret = -EFAULT; 3066 break; 3067 3068 case TUNSETNOCSUM: 3069 /* Disable/Enable checksum */ 3070 3071 /* [unimplemented] */ 3072 tun_debug(KERN_INFO, tun, "ignored: set checksum %s\n", 3073 arg ? "disabled" : "enabled"); 3074 break; 3075 3076 case TUNSETPERSIST: 3077 /* Disable/Enable persist mode. Keep an extra reference to the 3078 * module to prevent the module being unprobed. 3079 */ 3080 if (arg && !(tun->flags & IFF_PERSIST)) { 3081 tun->flags |= IFF_PERSIST; 3082 __module_get(THIS_MODULE); 3083 do_notify = true; 3084 } 3085 if (!arg && (tun->flags & IFF_PERSIST)) { 3086 tun->flags &= ~IFF_PERSIST; 3087 module_put(THIS_MODULE); 3088 do_notify = true; 3089 } 3090 3091 tun_debug(KERN_INFO, tun, "persist %s\n", 3092 arg ? "enabled" : "disabled"); 3093 break; 3094 3095 case TUNSETOWNER: 3096 /* Set owner of the device */ 3097 owner = make_kuid(current_user_ns(), arg); 3098 if (!uid_valid(owner)) { 3099 ret = -EINVAL; 3100 break; 3101 } 3102 tun->owner = owner; 3103 do_notify = true; 3104 tun_debug(KERN_INFO, tun, "owner set to %u\n", 3105 from_kuid(&init_user_ns, tun->owner)); 3106 break; 3107 3108 case TUNSETGROUP: 3109 /* Set group of the device */ 3110 group = make_kgid(current_user_ns(), arg); 3111 if (!gid_valid(group)) { 3112 ret = -EINVAL; 3113 break; 3114 } 3115 tun->group = group; 3116 do_notify = true; 3117 tun_debug(KERN_INFO, tun, "group set to %u\n", 3118 from_kgid(&init_user_ns, tun->group)); 3119 break; 3120 3121 case TUNSETLINK: 3122 /* Only allow setting the type when the interface is down */ 3123 if (tun->dev->flags & IFF_UP) { 3124 tun_debug(KERN_INFO, tun, 3125 "Linktype set failed because interface is up\n"); 3126 ret = -EBUSY; 3127 } else { 3128 tun->dev->type = (int) arg; 3129 tun_debug(KERN_INFO, tun, "linktype set to %d\n", 3130 tun->dev->type); 3131 ret = 0; 3132 } 3133 break; 3134 3135 #ifdef TUN_DEBUG 3136 case TUNSETDEBUG: 3137 tun->debug = arg; 3138 break; 3139 #endif 3140 case TUNSETOFFLOAD: 3141 ret = set_offload(tun, arg); 3142 break; 3143 3144 case TUNSETTXFILTER: 3145 /* Can be set only for TAPs */ 3146 ret = -EINVAL; 3147 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3148 break; 3149 ret = update_filter(&tun->txflt, (void __user *)arg); 3150 break; 3151 3152 case SIOCGIFHWADDR: 3153 /* Get hw address */ 3154 memcpy(ifr.ifr_hwaddr.sa_data, tun->dev->dev_addr, ETH_ALEN); 3155 ifr.ifr_hwaddr.sa_family = tun->dev->type; 3156 if (copy_to_user(argp, &ifr, ifreq_len)) 3157 ret = -EFAULT; 3158 break; 3159 3160 case SIOCSIFHWADDR: 3161 /* Set hw address */ 3162 tun_debug(KERN_DEBUG, tun, "set hw address: %pM\n", 3163 ifr.ifr_hwaddr.sa_data); 3164 3165 ret = dev_set_mac_address(tun->dev, &ifr.ifr_hwaddr); 3166 break; 3167 3168 case TUNGETSNDBUF: 3169 sndbuf = tfile->socket.sk->sk_sndbuf; 3170 if (copy_to_user(argp, &sndbuf, sizeof(sndbuf))) 3171 ret = -EFAULT; 3172 break; 3173 3174 case TUNSETSNDBUF: 3175 if (copy_from_user(&sndbuf, argp, sizeof(sndbuf))) { 3176 ret = -EFAULT; 3177 break; 3178 } 3179 if (sndbuf <= 0) { 3180 ret = -EINVAL; 3181 break; 3182 } 3183 3184 tun->sndbuf = sndbuf; 3185 tun_set_sndbuf(tun); 3186 break; 3187 3188 case TUNGETVNETHDRSZ: 3189 vnet_hdr_sz = tun->vnet_hdr_sz; 3190 if (copy_to_user(argp, &vnet_hdr_sz, sizeof(vnet_hdr_sz))) 3191 ret = -EFAULT; 3192 break; 3193 3194 case TUNSETVNETHDRSZ: 3195 if (copy_from_user(&vnet_hdr_sz, argp, sizeof(vnet_hdr_sz))) { 3196 ret = -EFAULT; 3197 break; 3198 } 3199 if (vnet_hdr_sz < (int)sizeof(struct virtio_net_hdr)) { 3200 ret = -EINVAL; 3201 break; 3202 } 3203 3204 tun->vnet_hdr_sz = vnet_hdr_sz; 3205 break; 3206 3207 case TUNGETVNETLE: 3208 le = !!(tun->flags & TUN_VNET_LE); 3209 if (put_user(le, (int __user *)argp)) 3210 ret = -EFAULT; 3211 break; 3212 3213 case TUNSETVNETLE: 3214 if (get_user(le, (int __user *)argp)) { 3215 ret = -EFAULT; 3216 break; 3217 } 3218 if (le) 3219 tun->flags |= TUN_VNET_LE; 3220 else 3221 tun->flags &= ~TUN_VNET_LE; 3222 break; 3223 3224 case TUNGETVNETBE: 3225 ret = tun_get_vnet_be(tun, argp); 3226 break; 3227 3228 case TUNSETVNETBE: 3229 ret = tun_set_vnet_be(tun, argp); 3230 break; 3231 3232 case TUNATTACHFILTER: 3233 /* Can be set only for TAPs */ 3234 ret = -EINVAL; 3235 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3236 break; 3237 ret = -EFAULT; 3238 if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog))) 3239 break; 3240 3241 ret = tun_attach_filter(tun); 3242 break; 3243 3244 case TUNDETACHFILTER: 3245 /* Can be set only for TAPs */ 3246 ret = -EINVAL; 3247 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3248 break; 3249 ret = 0; 3250 tun_detach_filter(tun, tun->numqueues); 3251 break; 3252 3253 case TUNGETFILTER: 3254 ret = -EINVAL; 3255 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3256 break; 3257 ret = -EFAULT; 3258 if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog))) 3259 break; 3260 ret = 0; 3261 break; 3262 3263 case TUNSETSTEERINGEBPF: 3264 ret = tun_set_ebpf(tun, &tun->steering_prog, argp); 3265 break; 3266 3267 case TUNSETFILTEREBPF: 3268 ret = tun_set_ebpf(tun, &tun->filter_prog, argp); 3269 break; 3270 3271 default: 3272 ret = -EINVAL; 3273 break; 3274 } 3275 3276 if (do_notify) 3277 netdev_state_change(tun->dev); 3278 3279 unlock: 3280 rtnl_unlock(); 3281 if (tun) 3282 tun_put(tun); 3283 return ret; 3284 } 3285 3286 static long tun_chr_ioctl(struct file *file, 3287 unsigned int cmd, unsigned long arg) 3288 { 3289 return __tun_chr_ioctl(file, cmd, arg, sizeof (struct ifreq)); 3290 } 3291 3292 #ifdef CONFIG_COMPAT 3293 static long tun_chr_compat_ioctl(struct file *file, 3294 unsigned int cmd, unsigned long arg) 3295 { 3296 switch (cmd) { 3297 case TUNSETIFF: 3298 case TUNGETIFF: 3299 case TUNSETTXFILTER: 3300 case TUNGETSNDBUF: 3301 case TUNSETSNDBUF: 3302 case SIOCGIFHWADDR: 3303 case SIOCSIFHWADDR: 3304 arg = (unsigned long)compat_ptr(arg); 3305 break; 3306 default: 3307 arg = (compat_ulong_t)arg; 3308 break; 3309 } 3310 3311 /* 3312 * compat_ifreq is shorter than ifreq, so we must not access beyond 3313 * the end of that structure. All fields that are used in this 3314 * driver are compatible though, we don't need to convert the 3315 * contents. 3316 */ 3317 return __tun_chr_ioctl(file, cmd, arg, sizeof(struct compat_ifreq)); 3318 } 3319 #endif /* CONFIG_COMPAT */ 3320 3321 static int tun_chr_fasync(int fd, struct file *file, int on) 3322 { 3323 struct tun_file *tfile = file->private_data; 3324 int ret; 3325 3326 if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0) 3327 goto out; 3328 3329 if (on) { 3330 __f_setown(file, task_pid(current), PIDTYPE_TGID, 0); 3331 tfile->flags |= TUN_FASYNC; 3332 } else 3333 tfile->flags &= ~TUN_FASYNC; 3334 ret = 0; 3335 out: 3336 return ret; 3337 } 3338 3339 static int tun_chr_open(struct inode *inode, struct file * file) 3340 { 3341 struct net *net = current->nsproxy->net_ns; 3342 struct tun_file *tfile; 3343 3344 DBG1(KERN_INFO, "tunX: tun_chr_open\n"); 3345 3346 tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, 3347 &tun_proto, 0); 3348 if (!tfile) 3349 return -ENOMEM; 3350 if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) { 3351 sk_free(&tfile->sk); 3352 return -ENOMEM; 3353 } 3354 3355 mutex_init(&tfile->napi_mutex); 3356 RCU_INIT_POINTER(tfile->tun, NULL); 3357 tfile->flags = 0; 3358 tfile->ifindex = 0; 3359 3360 init_waitqueue_head(&tfile->wq.wait); 3361 RCU_INIT_POINTER(tfile->socket.wq, &tfile->wq); 3362 3363 tfile->socket.file = file; 3364 tfile->socket.ops = &tun_socket_ops; 3365 3366 sock_init_data(&tfile->socket, &tfile->sk); 3367 3368 tfile->sk.sk_write_space = tun_sock_write_space; 3369 tfile->sk.sk_sndbuf = INT_MAX; 3370 3371 file->private_data = tfile; 3372 INIT_LIST_HEAD(&tfile->next); 3373 3374 sock_set_flag(&tfile->sk, SOCK_ZEROCOPY); 3375 3376 return 0; 3377 } 3378 3379 static int tun_chr_close(struct inode *inode, struct file *file) 3380 { 3381 struct tun_file *tfile = file->private_data; 3382 3383 tun_detach(tfile, true); 3384 3385 return 0; 3386 } 3387 3388 #ifdef CONFIG_PROC_FS 3389 static void tun_chr_show_fdinfo(struct seq_file *m, struct file *file) 3390 { 3391 struct tun_file *tfile = file->private_data; 3392 struct tun_struct *tun; 3393 struct ifreq ifr; 3394 3395 memset(&ifr, 0, sizeof(ifr)); 3396 3397 rtnl_lock(); 3398 tun = tun_get(tfile); 3399 if (tun) 3400 tun_get_iff(current->nsproxy->net_ns, tun, &ifr); 3401 rtnl_unlock(); 3402 3403 if (tun) 3404 tun_put(tun); 3405 3406 seq_printf(m, "iff:\t%s\n", ifr.ifr_name); 3407 } 3408 #endif 3409 3410 static const struct file_operations tun_fops = { 3411 .owner = THIS_MODULE, 3412 .llseek = no_llseek, 3413 .read_iter = tun_chr_read_iter, 3414 .write_iter = tun_chr_write_iter, 3415 .poll = tun_chr_poll, 3416 .unlocked_ioctl = tun_chr_ioctl, 3417 #ifdef CONFIG_COMPAT 3418 .compat_ioctl = tun_chr_compat_ioctl, 3419 #endif 3420 .open = tun_chr_open, 3421 .release = tun_chr_close, 3422 .fasync = tun_chr_fasync, 3423 #ifdef CONFIG_PROC_FS 3424 .show_fdinfo = tun_chr_show_fdinfo, 3425 #endif 3426 }; 3427 3428 static struct miscdevice tun_miscdev = { 3429 .minor = TUN_MINOR, 3430 .name = "tun", 3431 .nodename = "net/tun", 3432 .fops = &tun_fops, 3433 }; 3434 3435 /* ethtool interface */ 3436 3437 static void tun_default_link_ksettings(struct net_device *dev, 3438 struct ethtool_link_ksettings *cmd) 3439 { 3440 ethtool_link_ksettings_zero_link_mode(cmd, supported); 3441 ethtool_link_ksettings_zero_link_mode(cmd, advertising); 3442 cmd->base.speed = SPEED_10; 3443 cmd->base.duplex = DUPLEX_FULL; 3444 cmd->base.port = PORT_TP; 3445 cmd->base.phy_address = 0; 3446 cmd->base.autoneg = AUTONEG_DISABLE; 3447 } 3448 3449 static int tun_get_link_ksettings(struct net_device *dev, 3450 struct ethtool_link_ksettings *cmd) 3451 { 3452 struct tun_struct *tun = netdev_priv(dev); 3453 3454 memcpy(cmd, &tun->link_ksettings, sizeof(*cmd)); 3455 return 0; 3456 } 3457 3458 static int tun_set_link_ksettings(struct net_device *dev, 3459 const struct ethtool_link_ksettings *cmd) 3460 { 3461 struct tun_struct *tun = netdev_priv(dev); 3462 3463 memcpy(&tun->link_ksettings, cmd, sizeof(*cmd)); 3464 return 0; 3465 } 3466 3467 static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 3468 { 3469 struct tun_struct *tun = netdev_priv(dev); 3470 3471 strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); 3472 strlcpy(info->version, DRV_VERSION, sizeof(info->version)); 3473 3474 switch (tun->flags & TUN_TYPE_MASK) { 3475 case IFF_TUN: 3476 strlcpy(info->bus_info, "tun", sizeof(info->bus_info)); 3477 break; 3478 case IFF_TAP: 3479 strlcpy(info->bus_info, "tap", sizeof(info->bus_info)); 3480 break; 3481 } 3482 } 3483 3484 static u32 tun_get_msglevel(struct net_device *dev) 3485 { 3486 #ifdef TUN_DEBUG 3487 struct tun_struct *tun = netdev_priv(dev); 3488 return tun->debug; 3489 #else 3490 return -EOPNOTSUPP; 3491 #endif 3492 } 3493 3494 static void tun_set_msglevel(struct net_device *dev, u32 value) 3495 { 3496 #ifdef TUN_DEBUG 3497 struct tun_struct *tun = netdev_priv(dev); 3498 tun->debug = value; 3499 #endif 3500 } 3501 3502 static int tun_get_coalesce(struct net_device *dev, 3503 struct ethtool_coalesce *ec) 3504 { 3505 struct tun_struct *tun = netdev_priv(dev); 3506 3507 ec->rx_max_coalesced_frames = tun->rx_batched; 3508 3509 return 0; 3510 } 3511 3512 static int tun_set_coalesce(struct net_device *dev, 3513 struct ethtool_coalesce *ec) 3514 { 3515 struct tun_struct *tun = netdev_priv(dev); 3516 3517 if (ec->rx_max_coalesced_frames > NAPI_POLL_WEIGHT) 3518 tun->rx_batched = NAPI_POLL_WEIGHT; 3519 else 3520 tun->rx_batched = ec->rx_max_coalesced_frames; 3521 3522 return 0; 3523 } 3524 3525 static const struct ethtool_ops tun_ethtool_ops = { 3526 .get_drvinfo = tun_get_drvinfo, 3527 .get_msglevel = tun_get_msglevel, 3528 .set_msglevel = tun_set_msglevel, 3529 .get_link = ethtool_op_get_link, 3530 .get_ts_info = ethtool_op_get_ts_info, 3531 .get_coalesce = tun_get_coalesce, 3532 .set_coalesce = tun_set_coalesce, 3533 .get_link_ksettings = tun_get_link_ksettings, 3534 .set_link_ksettings = tun_set_link_ksettings, 3535 }; 3536 3537 static int tun_queue_resize(struct tun_struct *tun) 3538 { 3539 struct net_device *dev = tun->dev; 3540 struct tun_file *tfile; 3541 struct ptr_ring **rings; 3542 int n = tun->numqueues + tun->numdisabled; 3543 int ret, i; 3544 3545 rings = kmalloc_array(n, sizeof(*rings), GFP_KERNEL); 3546 if (!rings) 3547 return -ENOMEM; 3548 3549 for (i = 0; i < tun->numqueues; i++) { 3550 tfile = rtnl_dereference(tun->tfiles[i]); 3551 rings[i] = &tfile->tx_ring; 3552 } 3553 list_for_each_entry(tfile, &tun->disabled, next) 3554 rings[i++] = &tfile->tx_ring; 3555 3556 ret = ptr_ring_resize_multiple(rings, n, 3557 dev->tx_queue_len, GFP_KERNEL, 3558 tun_ptr_free); 3559 3560 kfree(rings); 3561 return ret; 3562 } 3563 3564 static int tun_device_event(struct notifier_block *unused, 3565 unsigned long event, void *ptr) 3566 { 3567 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 3568 struct tun_struct *tun = netdev_priv(dev); 3569 3570 if (dev->rtnl_link_ops != &tun_link_ops) 3571 return NOTIFY_DONE; 3572 3573 switch (event) { 3574 case NETDEV_CHANGE_TX_QUEUE_LEN: 3575 if (tun_queue_resize(tun)) 3576 return NOTIFY_BAD; 3577 break; 3578 default: 3579 break; 3580 } 3581 3582 return NOTIFY_DONE; 3583 } 3584 3585 static struct notifier_block tun_notifier_block __read_mostly = { 3586 .notifier_call = tun_device_event, 3587 }; 3588 3589 static int __init tun_init(void) 3590 { 3591 int ret = 0; 3592 3593 pr_info("%s, %s\n", DRV_DESCRIPTION, DRV_VERSION); 3594 3595 ret = rtnl_link_register(&tun_link_ops); 3596 if (ret) { 3597 pr_err("Can't register link_ops\n"); 3598 goto err_linkops; 3599 } 3600 3601 ret = misc_register(&tun_miscdev); 3602 if (ret) { 3603 pr_err("Can't register misc device %d\n", TUN_MINOR); 3604 goto err_misc; 3605 } 3606 3607 ret = register_netdevice_notifier(&tun_notifier_block); 3608 if (ret) { 3609 pr_err("Can't register netdevice notifier\n"); 3610 goto err_notifier; 3611 } 3612 3613 return 0; 3614 3615 err_notifier: 3616 misc_deregister(&tun_miscdev); 3617 err_misc: 3618 rtnl_link_unregister(&tun_link_ops); 3619 err_linkops: 3620 return ret; 3621 } 3622 3623 static void tun_cleanup(void) 3624 { 3625 misc_deregister(&tun_miscdev); 3626 rtnl_link_unregister(&tun_link_ops); 3627 unregister_netdevice_notifier(&tun_notifier_block); 3628 } 3629 3630 /* Get an underlying socket object from tun file. Returns error unless file is 3631 * attached to a device. The returned object works like a packet socket, it 3632 * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for 3633 * holding a reference to the file for as long as the socket is in use. */ 3634 struct socket *tun_get_socket(struct file *file) 3635 { 3636 struct tun_file *tfile; 3637 if (file->f_op != &tun_fops) 3638 return ERR_PTR(-EINVAL); 3639 tfile = file->private_data; 3640 if (!tfile) 3641 return ERR_PTR(-EBADFD); 3642 return &tfile->socket; 3643 } 3644 EXPORT_SYMBOL_GPL(tun_get_socket); 3645 3646 struct ptr_ring *tun_get_tx_ring(struct file *file) 3647 { 3648 struct tun_file *tfile; 3649 3650 if (file->f_op != &tun_fops) 3651 return ERR_PTR(-EINVAL); 3652 tfile = file->private_data; 3653 if (!tfile) 3654 return ERR_PTR(-EBADFD); 3655 return &tfile->tx_ring; 3656 } 3657 EXPORT_SYMBOL_GPL(tun_get_tx_ring); 3658 3659 module_init(tun_init); 3660 module_exit(tun_cleanup); 3661 MODULE_DESCRIPTION(DRV_DESCRIPTION); 3662 MODULE_AUTHOR(DRV_COPYRIGHT); 3663 MODULE_LICENSE("GPL"); 3664 MODULE_ALIAS_MISCDEV(TUN_MINOR); 3665 MODULE_ALIAS("devname:net/tun"); 3666