1 #include <linux/etherdevice.h> 2 #include <linux/if_tap.h> 3 #include <linux/if_vlan.h> 4 #include <linux/interrupt.h> 5 #include <linux/nsproxy.h> 6 #include <linux/compat.h> 7 #include <linux/if_tun.h> 8 #include <linux/module.h> 9 #include <linux/skbuff.h> 10 #include <linux/cache.h> 11 #include <linux/sched.h> 12 #include <linux/types.h> 13 #include <linux/slab.h> 14 #include <linux/wait.h> 15 #include <linux/cdev.h> 16 #include <linux/idr.h> 17 #include <linux/fs.h> 18 #include <linux/uio.h> 19 20 #include <net/net_namespace.h> 21 #include <net/rtnetlink.h> 22 #include <net/sock.h> 23 #include <linux/virtio_net.h> 24 #include <linux/skb_array.h> 25 26 #define TAP_IFFEATURES (IFF_VNET_HDR | IFF_MULTI_QUEUE) 27 28 #define TAP_VNET_LE 0x80000000 29 #define TAP_VNET_BE 0x40000000 30 31 #ifdef CONFIG_TUN_VNET_CROSS_LE 32 static inline bool tap_legacy_is_little_endian(struct tap_queue *q) 33 { 34 return q->flags & TAP_VNET_BE ? false : 35 virtio_legacy_is_little_endian(); 36 } 37 38 static long tap_get_vnet_be(struct tap_queue *q, int __user *sp) 39 { 40 int s = !!(q->flags & TAP_VNET_BE); 41 42 if (put_user(s, sp)) 43 return -EFAULT; 44 45 return 0; 46 } 47 48 static long tap_set_vnet_be(struct tap_queue *q, int __user *sp) 49 { 50 int s; 51 52 if (get_user(s, sp)) 53 return -EFAULT; 54 55 if (s) 56 q->flags |= TAP_VNET_BE; 57 else 58 q->flags &= ~TAP_VNET_BE; 59 60 return 0; 61 } 62 #else 63 static inline bool tap_legacy_is_little_endian(struct tap_queue *q) 64 { 65 return virtio_legacy_is_little_endian(); 66 } 67 68 static long tap_get_vnet_be(struct tap_queue *q, int __user *argp) 69 { 70 return -EINVAL; 71 } 72 73 static long tap_set_vnet_be(struct tap_queue *q, int __user *argp) 74 { 75 return -EINVAL; 76 } 77 #endif /* CONFIG_TUN_VNET_CROSS_LE */ 78 79 static inline bool tap_is_little_endian(struct tap_queue *q) 80 { 81 return q->flags & TAP_VNET_LE || 82 tap_legacy_is_little_endian(q); 83 } 84 85 static inline u16 tap16_to_cpu(struct tap_queue *q, __virtio16 val) 86 { 87 return __virtio16_to_cpu(tap_is_little_endian(q), val); 88 } 89 90 static inline __virtio16 cpu_to_tap16(struct tap_queue *q, u16 val) 91 { 92 return __cpu_to_virtio16(tap_is_little_endian(q), val); 93 } 94 95 static struct proto tap_proto = { 96 .name = "tap", 97 .owner = THIS_MODULE, 98 .obj_size = sizeof(struct tap_queue), 99 }; 100 101 #define TAP_NUM_DEVS (1U << MINORBITS) 102 struct major_info { 103 dev_t major; 104 struct idr minor_idr; 105 struct mutex minor_lock; 106 const char *device_name; 107 } macvtap_major; 108 109 #define GOODCOPY_LEN 128 110 111 static const struct proto_ops tap_socket_ops; 112 113 #define RX_OFFLOADS (NETIF_F_GRO | NETIF_F_LRO) 114 #define TAP_FEATURES (NETIF_F_GSO | NETIF_F_SG | NETIF_F_FRAGLIST) 115 116 static struct tap_dev *tap_dev_get_rcu(const struct net_device *dev) 117 { 118 return rcu_dereference(dev->rx_handler_data); 119 } 120 121 /* 122 * RCU usage: 123 * The tap_queue and the macvlan_dev are loosely coupled, the 124 * pointers from one to the other can only be read while rcu_read_lock 125 * or rtnl is held. 126 * 127 * Both the file and the macvlan_dev hold a reference on the tap_queue 128 * through sock_hold(&q->sk). When the macvlan_dev goes away first, 129 * q->vlan becomes inaccessible. When the files gets closed, 130 * tap_get_queue() fails. 131 * 132 * There may still be references to the struct sock inside of the 133 * queue from outbound SKBs, but these never reference back to the 134 * file or the dev. The data structure is freed through __sk_free 135 * when both our references and any pending SKBs are gone. 136 */ 137 138 static int tap_enable_queue(struct tap_dev *tap, struct file *file, 139 struct tap_queue *q) 140 { 141 int err = -EINVAL; 142 143 ASSERT_RTNL(); 144 145 if (q->enabled) 146 goto out; 147 148 err = 0; 149 rcu_assign_pointer(tap->taps[tap->numvtaps], q); 150 q->queue_index = tap->numvtaps; 151 q->enabled = true; 152 153 tap->numvtaps++; 154 out: 155 return err; 156 } 157 158 /* Requires RTNL */ 159 static int tap_set_queue(struct tap_dev *tap, struct file *file, 160 struct tap_queue *q) 161 { 162 if (tap->numqueues == MAX_TAP_QUEUES) 163 return -EBUSY; 164 165 rcu_assign_pointer(q->tap, tap); 166 rcu_assign_pointer(tap->taps[tap->numvtaps], q); 167 sock_hold(&q->sk); 168 169 q->file = file; 170 q->queue_index = tap->numvtaps; 171 q->enabled = true; 172 file->private_data = q; 173 list_add_tail(&q->next, &tap->queue_list); 174 175 tap->numvtaps++; 176 tap->numqueues++; 177 178 return 0; 179 } 180 181 static int tap_disable_queue(struct tap_queue *q) 182 { 183 struct tap_dev *tap; 184 struct tap_queue *nq; 185 186 ASSERT_RTNL(); 187 if (!q->enabled) 188 return -EINVAL; 189 190 tap = rtnl_dereference(q->tap); 191 192 if (tap) { 193 int index = q->queue_index; 194 BUG_ON(index >= tap->numvtaps); 195 nq = rtnl_dereference(tap->taps[tap->numvtaps - 1]); 196 nq->queue_index = index; 197 198 rcu_assign_pointer(tap->taps[index], nq); 199 RCU_INIT_POINTER(tap->taps[tap->numvtaps - 1], NULL); 200 q->enabled = false; 201 202 tap->numvtaps--; 203 } 204 205 return 0; 206 } 207 208 /* 209 * The file owning the queue got closed, give up both 210 * the reference that the files holds as well as the 211 * one from the macvlan_dev if that still exists. 212 * 213 * Using the spinlock makes sure that we don't get 214 * to the queue again after destroying it. 215 */ 216 static void tap_put_queue(struct tap_queue *q) 217 { 218 struct tap_dev *tap; 219 220 rtnl_lock(); 221 tap = rtnl_dereference(q->tap); 222 223 if (tap) { 224 if (q->enabled) 225 BUG_ON(tap_disable_queue(q)); 226 227 tap->numqueues--; 228 RCU_INIT_POINTER(q->tap, NULL); 229 sock_put(&q->sk); 230 list_del_init(&q->next); 231 } 232 233 rtnl_unlock(); 234 235 synchronize_rcu(); 236 sock_put(&q->sk); 237 } 238 239 /* 240 * Select a queue based on the rxq of the device on which this packet 241 * arrived. If the incoming device is not mq, calculate a flow hash 242 * to select a queue. If all fails, find the first available queue. 243 * Cache vlan->numvtaps since it can become zero during the execution 244 * of this function. 245 */ 246 static struct tap_queue *tap_get_queue(struct tap_dev *tap, 247 struct sk_buff *skb) 248 { 249 struct tap_queue *queue = NULL; 250 /* Access to taps array is protected by rcu, but access to numvtaps 251 * isn't. Below we use it to lookup a queue, but treat it as a hint 252 * and validate that the result isn't NULL - in case we are 253 * racing against queue removal. 254 */ 255 int numvtaps = ACCESS_ONCE(tap->numvtaps); 256 __u32 rxq; 257 258 if (!numvtaps) 259 goto out; 260 261 if (numvtaps == 1) 262 goto single; 263 264 /* Check if we can use flow to select a queue */ 265 rxq = skb_get_hash(skb); 266 if (rxq) { 267 queue = rcu_dereference(tap->taps[rxq % numvtaps]); 268 goto out; 269 } 270 271 if (likely(skb_rx_queue_recorded(skb))) { 272 rxq = skb_get_rx_queue(skb); 273 274 while (unlikely(rxq >= numvtaps)) 275 rxq -= numvtaps; 276 277 queue = rcu_dereference(tap->taps[rxq]); 278 goto out; 279 } 280 281 single: 282 queue = rcu_dereference(tap->taps[0]); 283 out: 284 return queue; 285 } 286 287 /* 288 * The net_device is going away, give up the reference 289 * that it holds on all queues and safely set the pointer 290 * from the queues to NULL. 291 */ 292 void tap_del_queues(struct tap_dev *tap) 293 { 294 struct tap_queue *q, *tmp; 295 296 ASSERT_RTNL(); 297 list_for_each_entry_safe(q, tmp, &tap->queue_list, next) { 298 list_del_init(&q->next); 299 RCU_INIT_POINTER(q->tap, NULL); 300 if (q->enabled) 301 tap->numvtaps--; 302 tap->numqueues--; 303 sock_put(&q->sk); 304 } 305 BUG_ON(tap->numvtaps); 306 BUG_ON(tap->numqueues); 307 /* guarantee that any future tap_set_queue will fail */ 308 tap->numvtaps = MAX_TAP_QUEUES; 309 } 310 311 rx_handler_result_t tap_handle_frame(struct sk_buff **pskb) 312 { 313 struct sk_buff *skb = *pskb; 314 struct net_device *dev = skb->dev; 315 struct tap_dev *tap; 316 struct tap_queue *q; 317 netdev_features_t features = TAP_FEATURES; 318 319 tap = tap_dev_get_rcu(dev); 320 if (!tap) 321 return RX_HANDLER_PASS; 322 323 q = tap_get_queue(tap, skb); 324 if (!q) 325 return RX_HANDLER_PASS; 326 327 if (__skb_array_full(&q->skb_array)) 328 goto drop; 329 330 skb_push(skb, ETH_HLEN); 331 332 /* Apply the forward feature mask so that we perform segmentation 333 * according to users wishes. This only works if VNET_HDR is 334 * enabled. 335 */ 336 if (q->flags & IFF_VNET_HDR) 337 features |= tap->tap_features; 338 if (netif_needs_gso(skb, features)) { 339 struct sk_buff *segs = __skb_gso_segment(skb, features, false); 340 341 if (IS_ERR(segs)) 342 goto drop; 343 344 if (!segs) { 345 if (skb_array_produce(&q->skb_array, skb)) 346 goto drop; 347 goto wake_up; 348 } 349 350 consume_skb(skb); 351 while (segs) { 352 struct sk_buff *nskb = segs->next; 353 354 segs->next = NULL; 355 if (skb_array_produce(&q->skb_array, segs)) { 356 kfree_skb(segs); 357 kfree_skb_list(nskb); 358 break; 359 } 360 segs = nskb; 361 } 362 } else { 363 /* If we receive a partial checksum and the tap side 364 * doesn't support checksum offload, compute the checksum. 365 * Note: it doesn't matter which checksum feature to 366 * check, we either support them all or none. 367 */ 368 if (skb->ip_summed == CHECKSUM_PARTIAL && 369 !(features & NETIF_F_CSUM_MASK) && 370 skb_checksum_help(skb)) 371 goto drop; 372 if (skb_array_produce(&q->skb_array, skb)) 373 goto drop; 374 } 375 376 wake_up: 377 wake_up_interruptible_poll(sk_sleep(&q->sk), POLLIN | POLLRDNORM | POLLRDBAND); 378 return RX_HANDLER_CONSUMED; 379 380 drop: 381 /* Count errors/drops only here, thus don't care about args. */ 382 if (tap->count_rx_dropped) 383 tap->count_rx_dropped(tap); 384 kfree_skb(skb); 385 return RX_HANDLER_CONSUMED; 386 } 387 388 int tap_get_minor(struct tap_dev *tap) 389 { 390 int retval = -ENOMEM; 391 392 mutex_lock(&macvtap_major.minor_lock); 393 retval = idr_alloc(&macvtap_major.minor_idr, tap, 1, TAP_NUM_DEVS, GFP_KERNEL); 394 if (retval >= 0) { 395 tap->minor = retval; 396 } else if (retval == -ENOSPC) { 397 netdev_err(tap->dev, "Too many tap devices\n"); 398 retval = -EINVAL; 399 } 400 mutex_unlock(&macvtap_major.minor_lock); 401 return retval < 0 ? retval : 0; 402 } 403 404 void tap_free_minor(struct tap_dev *tap) 405 { 406 mutex_lock(&macvtap_major.minor_lock); 407 if (tap->minor) { 408 idr_remove(&macvtap_major.minor_idr, tap->minor); 409 tap->minor = 0; 410 } 411 mutex_unlock(&macvtap_major.minor_lock); 412 } 413 414 static struct tap_dev *dev_get_by_tap_minor(int minor) 415 { 416 struct net_device *dev = NULL; 417 struct tap_dev *tap; 418 419 mutex_lock(&macvtap_major.minor_lock); 420 tap = idr_find(&macvtap_major.minor_idr, minor); 421 if (tap) { 422 dev = tap->dev; 423 dev_hold(dev); 424 } 425 mutex_unlock(&macvtap_major.minor_lock); 426 return tap; 427 } 428 429 static void tap_sock_write_space(struct sock *sk) 430 { 431 wait_queue_head_t *wqueue; 432 433 if (!sock_writeable(sk) || 434 !test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags)) 435 return; 436 437 wqueue = sk_sleep(sk); 438 if (wqueue && waitqueue_active(wqueue)) 439 wake_up_interruptible_poll(wqueue, POLLOUT | POLLWRNORM | POLLWRBAND); 440 } 441 442 static void tap_sock_destruct(struct sock *sk) 443 { 444 struct tap_queue *q = container_of(sk, struct tap_queue, sk); 445 446 skb_array_cleanup(&q->skb_array); 447 } 448 449 static int tap_open(struct inode *inode, struct file *file) 450 { 451 struct net *net = current->nsproxy->net_ns; 452 struct tap_dev *tap; 453 struct tap_queue *q; 454 int err = -ENODEV; 455 456 rtnl_lock(); 457 tap = dev_get_by_tap_minor(iminor(inode)); 458 if (!tap) 459 goto err; 460 461 err = -ENOMEM; 462 q = (struct tap_queue *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, 463 &tap_proto, 0); 464 if (!q) 465 goto err; 466 467 RCU_INIT_POINTER(q->sock.wq, &q->wq); 468 init_waitqueue_head(&q->wq.wait); 469 q->sock.type = SOCK_RAW; 470 q->sock.state = SS_CONNECTED; 471 q->sock.file = file; 472 q->sock.ops = &tap_socket_ops; 473 sock_init_data(&q->sock, &q->sk); 474 q->sk.sk_write_space = tap_sock_write_space; 475 q->sk.sk_destruct = tap_sock_destruct; 476 q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP; 477 q->vnet_hdr_sz = sizeof(struct virtio_net_hdr); 478 479 /* 480 * so far only KVM virtio_net uses tap, enable zero copy between 481 * guest kernel and host kernel when lower device supports zerocopy 482 * 483 * The macvlan supports zerocopy iff the lower device supports zero 484 * copy so we don't have to look at the lower device directly. 485 */ 486 if ((tap->dev->features & NETIF_F_HIGHDMA) && (tap->dev->features & NETIF_F_SG)) 487 sock_set_flag(&q->sk, SOCK_ZEROCOPY); 488 489 err = -ENOMEM; 490 if (skb_array_init(&q->skb_array, tap->dev->tx_queue_len, GFP_KERNEL)) 491 goto err_array; 492 493 err = tap_set_queue(tap, file, q); 494 if (err) 495 goto err_queue; 496 497 dev_put(tap->dev); 498 499 rtnl_unlock(); 500 return err; 501 502 err_queue: 503 skb_array_cleanup(&q->skb_array); 504 err_array: 505 sock_put(&q->sk); 506 err: 507 if (tap) 508 dev_put(tap->dev); 509 510 rtnl_unlock(); 511 return err; 512 } 513 514 static int tap_release(struct inode *inode, struct file *file) 515 { 516 struct tap_queue *q = file->private_data; 517 tap_put_queue(q); 518 return 0; 519 } 520 521 static unsigned int tap_poll(struct file *file, poll_table *wait) 522 { 523 struct tap_queue *q = file->private_data; 524 unsigned int mask = POLLERR; 525 526 if (!q) 527 goto out; 528 529 mask = 0; 530 poll_wait(file, &q->wq.wait, wait); 531 532 if (!skb_array_empty(&q->skb_array)) 533 mask |= POLLIN | POLLRDNORM; 534 535 if (sock_writeable(&q->sk) || 536 (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &q->sock.flags) && 537 sock_writeable(&q->sk))) 538 mask |= POLLOUT | POLLWRNORM; 539 540 out: 541 return mask; 542 } 543 544 static inline struct sk_buff *tap_alloc_skb(struct sock *sk, size_t prepad, 545 size_t len, size_t linear, 546 int noblock, int *err) 547 { 548 struct sk_buff *skb; 549 550 /* Under a page? Don't bother with paged skb. */ 551 if (prepad + len < PAGE_SIZE || !linear) 552 linear = len; 553 554 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, 555 err, 0); 556 if (!skb) 557 return NULL; 558 559 skb_reserve(skb, prepad); 560 skb_put(skb, linear); 561 skb->data_len = len - linear; 562 skb->len += len - linear; 563 564 return skb; 565 } 566 567 /* Neighbour code has some assumptions on HH_DATA_MOD alignment */ 568 #define TAP_RESERVE HH_DATA_OFF(ETH_HLEN) 569 570 /* Get packet from user space buffer */ 571 static ssize_t tap_get_user(struct tap_queue *q, struct msghdr *m, 572 struct iov_iter *from, int noblock) 573 { 574 int good_linear = SKB_MAX_HEAD(TAP_RESERVE); 575 struct sk_buff *skb; 576 struct tap_dev *tap; 577 unsigned long total_len = iov_iter_count(from); 578 unsigned long len = total_len; 579 int err; 580 struct virtio_net_hdr vnet_hdr = { 0 }; 581 int vnet_hdr_len = 0; 582 int copylen = 0; 583 int depth; 584 bool zerocopy = false; 585 size_t linear; 586 587 if (q->flags & IFF_VNET_HDR) { 588 vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz); 589 590 err = -EINVAL; 591 if (len < vnet_hdr_len) 592 goto err; 593 len -= vnet_hdr_len; 594 595 err = -EFAULT; 596 if (!copy_from_iter_full(&vnet_hdr, sizeof(vnet_hdr), from)) 597 goto err; 598 iov_iter_advance(from, vnet_hdr_len - sizeof(vnet_hdr)); 599 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 600 tap16_to_cpu(q, vnet_hdr.csum_start) + 601 tap16_to_cpu(q, vnet_hdr.csum_offset) + 2 > 602 tap16_to_cpu(q, vnet_hdr.hdr_len)) 603 vnet_hdr.hdr_len = cpu_to_tap16(q, 604 tap16_to_cpu(q, vnet_hdr.csum_start) + 605 tap16_to_cpu(q, vnet_hdr.csum_offset) + 2); 606 err = -EINVAL; 607 if (tap16_to_cpu(q, vnet_hdr.hdr_len) > len) 608 goto err; 609 } 610 611 err = -EINVAL; 612 if (unlikely(len < ETH_HLEN)) 613 goto err; 614 615 if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) { 616 struct iov_iter i; 617 618 copylen = vnet_hdr.hdr_len ? 619 tap16_to_cpu(q, vnet_hdr.hdr_len) : GOODCOPY_LEN; 620 if (copylen > good_linear) 621 copylen = good_linear; 622 else if (copylen < ETH_HLEN) 623 copylen = ETH_HLEN; 624 linear = copylen; 625 i = *from; 626 iov_iter_advance(&i, copylen); 627 if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS) 628 zerocopy = true; 629 } 630 631 if (!zerocopy) { 632 copylen = len; 633 linear = tap16_to_cpu(q, vnet_hdr.hdr_len); 634 if (linear > good_linear) 635 linear = good_linear; 636 else if (linear < ETH_HLEN) 637 linear = ETH_HLEN; 638 } 639 640 skb = tap_alloc_skb(&q->sk, TAP_RESERVE, copylen, 641 linear, noblock, &err); 642 if (!skb) 643 goto err; 644 645 if (zerocopy) 646 err = zerocopy_sg_from_iter(skb, from); 647 else 648 err = skb_copy_datagram_from_iter(skb, 0, from, len); 649 650 if (err) 651 goto err_kfree; 652 653 skb_set_network_header(skb, ETH_HLEN); 654 skb_reset_mac_header(skb); 655 skb->protocol = eth_hdr(skb)->h_proto; 656 657 if (vnet_hdr_len) { 658 err = virtio_net_hdr_to_skb(skb, &vnet_hdr, 659 tap_is_little_endian(q)); 660 if (err) 661 goto err_kfree; 662 } 663 664 skb_probe_transport_header(skb, ETH_HLEN); 665 666 /* Move network header to the right position for VLAN tagged packets */ 667 if ((skb->protocol == htons(ETH_P_8021Q) || 668 skb->protocol == htons(ETH_P_8021AD)) && 669 __vlan_get_protocol(skb, skb->protocol, &depth) != 0) 670 skb_set_network_header(skb, depth); 671 672 rcu_read_lock(); 673 tap = rcu_dereference(q->tap); 674 /* copy skb_ubuf_info for callback when skb has no error */ 675 if (zerocopy) { 676 skb_shinfo(skb)->destructor_arg = m->msg_control; 677 skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; 678 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; 679 } else if (m && m->msg_control) { 680 struct ubuf_info *uarg = m->msg_control; 681 uarg->callback(uarg, false); 682 } 683 684 if (tap) { 685 skb->dev = tap->dev; 686 dev_queue_xmit(skb); 687 } else { 688 kfree_skb(skb); 689 } 690 rcu_read_unlock(); 691 692 return total_len; 693 694 err_kfree: 695 kfree_skb(skb); 696 697 err: 698 rcu_read_lock(); 699 tap = rcu_dereference(q->tap); 700 if (tap && tap->count_tx_dropped) 701 tap->count_tx_dropped(tap); 702 rcu_read_unlock(); 703 704 return err; 705 } 706 707 static ssize_t tap_write_iter(struct kiocb *iocb, struct iov_iter *from) 708 { 709 struct file *file = iocb->ki_filp; 710 struct tap_queue *q = file->private_data; 711 712 return tap_get_user(q, NULL, from, file->f_flags & O_NONBLOCK); 713 } 714 715 /* Put packet to the user space buffer */ 716 static ssize_t tap_put_user(struct tap_queue *q, 717 const struct sk_buff *skb, 718 struct iov_iter *iter) 719 { 720 int ret; 721 int vnet_hdr_len = 0; 722 int vlan_offset = 0; 723 int total; 724 725 if (q->flags & IFF_VNET_HDR) { 726 struct virtio_net_hdr vnet_hdr; 727 vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz); 728 if (iov_iter_count(iter) < vnet_hdr_len) 729 return -EINVAL; 730 731 if (virtio_net_hdr_from_skb(skb, &vnet_hdr, 732 tap_is_little_endian(q), true)) 733 BUG(); 734 735 if (copy_to_iter(&vnet_hdr, sizeof(vnet_hdr), iter) != 736 sizeof(vnet_hdr)) 737 return -EFAULT; 738 739 iov_iter_advance(iter, vnet_hdr_len - sizeof(vnet_hdr)); 740 } 741 total = vnet_hdr_len; 742 total += skb->len; 743 744 if (skb_vlan_tag_present(skb)) { 745 struct { 746 __be16 h_vlan_proto; 747 __be16 h_vlan_TCI; 748 } veth; 749 veth.h_vlan_proto = skb->vlan_proto; 750 veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb)); 751 752 vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto); 753 total += VLAN_HLEN; 754 755 ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset); 756 if (ret || !iov_iter_count(iter)) 757 goto done; 758 759 ret = copy_to_iter(&veth, sizeof(veth), iter); 760 if (ret != sizeof(veth) || !iov_iter_count(iter)) 761 goto done; 762 } 763 764 ret = skb_copy_datagram_iter(skb, vlan_offset, iter, 765 skb->len - vlan_offset); 766 767 done: 768 return ret ? ret : total; 769 } 770 771 static ssize_t tap_do_read(struct tap_queue *q, 772 struct iov_iter *to, 773 int noblock) 774 { 775 DEFINE_WAIT(wait); 776 struct sk_buff *skb; 777 ssize_t ret = 0; 778 779 if (!iov_iter_count(to)) 780 return 0; 781 782 while (1) { 783 if (!noblock) 784 prepare_to_wait(sk_sleep(&q->sk), &wait, 785 TASK_INTERRUPTIBLE); 786 787 /* Read frames from the queue */ 788 skb = skb_array_consume(&q->skb_array); 789 if (skb) 790 break; 791 if (noblock) { 792 ret = -EAGAIN; 793 break; 794 } 795 if (signal_pending(current)) { 796 ret = -ERESTARTSYS; 797 break; 798 } 799 /* Nothing to read, let's sleep */ 800 schedule(); 801 } 802 if (!noblock) 803 finish_wait(sk_sleep(&q->sk), &wait); 804 805 if (skb) { 806 ret = tap_put_user(q, skb, to); 807 if (unlikely(ret < 0)) 808 kfree_skb(skb); 809 else 810 consume_skb(skb); 811 } 812 return ret; 813 } 814 815 static ssize_t tap_read_iter(struct kiocb *iocb, struct iov_iter *to) 816 { 817 struct file *file = iocb->ki_filp; 818 struct tap_queue *q = file->private_data; 819 ssize_t len = iov_iter_count(to), ret; 820 821 ret = tap_do_read(q, to, file->f_flags & O_NONBLOCK); 822 ret = min_t(ssize_t, ret, len); 823 if (ret > 0) 824 iocb->ki_pos = ret; 825 return ret; 826 } 827 828 static struct tap_dev *tap_get_tap_dev(struct tap_queue *q) 829 { 830 struct tap_dev *tap; 831 832 ASSERT_RTNL(); 833 tap = rtnl_dereference(q->tap); 834 if (tap) 835 dev_hold(tap->dev); 836 837 return tap; 838 } 839 840 static void tap_put_tap_dev(struct tap_dev *tap) 841 { 842 dev_put(tap->dev); 843 } 844 845 static int tap_ioctl_set_queue(struct file *file, unsigned int flags) 846 { 847 struct tap_queue *q = file->private_data; 848 struct tap_dev *tap; 849 int ret; 850 851 tap = tap_get_tap_dev(q); 852 if (!tap) 853 return -EINVAL; 854 855 if (flags & IFF_ATTACH_QUEUE) 856 ret = tap_enable_queue(tap, file, q); 857 else if (flags & IFF_DETACH_QUEUE) 858 ret = tap_disable_queue(q); 859 else 860 ret = -EINVAL; 861 862 tap_put_tap_dev(tap); 863 return ret; 864 } 865 866 static int set_offload(struct tap_queue *q, unsigned long arg) 867 { 868 struct tap_dev *tap; 869 netdev_features_t features; 870 netdev_features_t feature_mask = 0; 871 872 tap = rtnl_dereference(q->tap); 873 if (!tap) 874 return -ENOLINK; 875 876 features = tap->dev->features; 877 878 if (arg & TUN_F_CSUM) { 879 feature_mask = NETIF_F_HW_CSUM; 880 881 if (arg & (TUN_F_TSO4 | TUN_F_TSO6)) { 882 if (arg & TUN_F_TSO_ECN) 883 feature_mask |= NETIF_F_TSO_ECN; 884 if (arg & TUN_F_TSO4) 885 feature_mask |= NETIF_F_TSO; 886 if (arg & TUN_F_TSO6) 887 feature_mask |= NETIF_F_TSO6; 888 } 889 890 if (arg & TUN_F_UFO) 891 feature_mask |= NETIF_F_UFO; 892 } 893 894 /* tun/tap driver inverts the usage for TSO offloads, where 895 * setting the TSO bit means that the userspace wants to 896 * accept TSO frames and turning it off means that user space 897 * does not support TSO. 898 * For tap, we have to invert it to mean the same thing. 899 * When user space turns off TSO, we turn off GSO/LRO so that 900 * user-space will not receive TSO frames. 901 */ 902 if (feature_mask & (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_UFO)) 903 features |= RX_OFFLOADS; 904 else 905 features &= ~RX_OFFLOADS; 906 907 /* tap_features are the same as features on tun/tap and 908 * reflect user expectations. 909 */ 910 tap->tap_features = feature_mask; 911 if (tap->update_features) 912 tap->update_features(tap, features); 913 914 return 0; 915 } 916 917 /* 918 * provide compatibility with generic tun/tap interface 919 */ 920 static long tap_ioctl(struct file *file, unsigned int cmd, 921 unsigned long arg) 922 { 923 struct tap_queue *q = file->private_data; 924 struct tap_dev *tap; 925 void __user *argp = (void __user *)arg; 926 struct ifreq __user *ifr = argp; 927 unsigned int __user *up = argp; 928 unsigned short u; 929 int __user *sp = argp; 930 struct sockaddr sa; 931 int s; 932 int ret; 933 934 switch (cmd) { 935 case TUNSETIFF: 936 /* ignore the name, just look at flags */ 937 if (get_user(u, &ifr->ifr_flags)) 938 return -EFAULT; 939 940 ret = 0; 941 if ((u & ~TAP_IFFEATURES) != (IFF_NO_PI | IFF_TAP)) 942 ret = -EINVAL; 943 else 944 q->flags = (q->flags & ~TAP_IFFEATURES) | u; 945 946 return ret; 947 948 case TUNGETIFF: 949 rtnl_lock(); 950 tap = tap_get_tap_dev(q); 951 if (!tap) { 952 rtnl_unlock(); 953 return -ENOLINK; 954 } 955 956 ret = 0; 957 u = q->flags; 958 if (copy_to_user(&ifr->ifr_name, tap->dev->name, IFNAMSIZ) || 959 put_user(u, &ifr->ifr_flags)) 960 ret = -EFAULT; 961 tap_put_tap_dev(tap); 962 rtnl_unlock(); 963 return ret; 964 965 case TUNSETQUEUE: 966 if (get_user(u, &ifr->ifr_flags)) 967 return -EFAULT; 968 rtnl_lock(); 969 ret = tap_ioctl_set_queue(file, u); 970 rtnl_unlock(); 971 return ret; 972 973 case TUNGETFEATURES: 974 if (put_user(IFF_TAP | IFF_NO_PI | TAP_IFFEATURES, up)) 975 return -EFAULT; 976 return 0; 977 978 case TUNSETSNDBUF: 979 if (get_user(s, sp)) 980 return -EFAULT; 981 982 q->sk.sk_sndbuf = s; 983 return 0; 984 985 case TUNGETVNETHDRSZ: 986 s = q->vnet_hdr_sz; 987 if (put_user(s, sp)) 988 return -EFAULT; 989 return 0; 990 991 case TUNSETVNETHDRSZ: 992 if (get_user(s, sp)) 993 return -EFAULT; 994 if (s < (int)sizeof(struct virtio_net_hdr)) 995 return -EINVAL; 996 997 q->vnet_hdr_sz = s; 998 return 0; 999 1000 case TUNGETVNETLE: 1001 s = !!(q->flags & TAP_VNET_LE); 1002 if (put_user(s, sp)) 1003 return -EFAULT; 1004 return 0; 1005 1006 case TUNSETVNETLE: 1007 if (get_user(s, sp)) 1008 return -EFAULT; 1009 if (s) 1010 q->flags |= TAP_VNET_LE; 1011 else 1012 q->flags &= ~TAP_VNET_LE; 1013 return 0; 1014 1015 case TUNGETVNETBE: 1016 return tap_get_vnet_be(q, sp); 1017 1018 case TUNSETVNETBE: 1019 return tap_set_vnet_be(q, sp); 1020 1021 case TUNSETOFFLOAD: 1022 /* let the user check for future flags */ 1023 if (arg & ~(TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | 1024 TUN_F_TSO_ECN | TUN_F_UFO)) 1025 return -EINVAL; 1026 1027 rtnl_lock(); 1028 ret = set_offload(q, arg); 1029 rtnl_unlock(); 1030 return ret; 1031 1032 case SIOCGIFHWADDR: 1033 rtnl_lock(); 1034 tap = tap_get_tap_dev(q); 1035 if (!tap) { 1036 rtnl_unlock(); 1037 return -ENOLINK; 1038 } 1039 ret = 0; 1040 u = tap->dev->type; 1041 if (copy_to_user(&ifr->ifr_name, tap->dev->name, IFNAMSIZ) || 1042 copy_to_user(&ifr->ifr_hwaddr.sa_data, tap->dev->dev_addr, ETH_ALEN) || 1043 put_user(u, &ifr->ifr_hwaddr.sa_family)) 1044 ret = -EFAULT; 1045 tap_put_tap_dev(tap); 1046 rtnl_unlock(); 1047 return ret; 1048 1049 case SIOCSIFHWADDR: 1050 if (copy_from_user(&sa, &ifr->ifr_hwaddr, sizeof(sa))) 1051 return -EFAULT; 1052 rtnl_lock(); 1053 tap = tap_get_tap_dev(q); 1054 if (!tap) { 1055 rtnl_unlock(); 1056 return -ENOLINK; 1057 } 1058 ret = dev_set_mac_address(tap->dev, &sa); 1059 tap_put_tap_dev(tap); 1060 rtnl_unlock(); 1061 return ret; 1062 1063 default: 1064 return -EINVAL; 1065 } 1066 } 1067 1068 #ifdef CONFIG_COMPAT 1069 static long tap_compat_ioctl(struct file *file, unsigned int cmd, 1070 unsigned long arg) 1071 { 1072 return tap_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); 1073 } 1074 #endif 1075 1076 const struct file_operations tap_fops = { 1077 .owner = THIS_MODULE, 1078 .open = tap_open, 1079 .release = tap_release, 1080 .read_iter = tap_read_iter, 1081 .write_iter = tap_write_iter, 1082 .poll = tap_poll, 1083 .llseek = no_llseek, 1084 .unlocked_ioctl = tap_ioctl, 1085 #ifdef CONFIG_COMPAT 1086 .compat_ioctl = tap_compat_ioctl, 1087 #endif 1088 }; 1089 1090 static int tap_sendmsg(struct socket *sock, struct msghdr *m, 1091 size_t total_len) 1092 { 1093 struct tap_queue *q = container_of(sock, struct tap_queue, sock); 1094 return tap_get_user(q, m, &m->msg_iter, m->msg_flags & MSG_DONTWAIT); 1095 } 1096 1097 static int tap_recvmsg(struct socket *sock, struct msghdr *m, 1098 size_t total_len, int flags) 1099 { 1100 struct tap_queue *q = container_of(sock, struct tap_queue, sock); 1101 int ret; 1102 if (flags & ~(MSG_DONTWAIT|MSG_TRUNC)) 1103 return -EINVAL; 1104 ret = tap_do_read(q, &m->msg_iter, flags & MSG_DONTWAIT); 1105 if (ret > total_len) { 1106 m->msg_flags |= MSG_TRUNC; 1107 ret = flags & MSG_TRUNC ? ret : total_len; 1108 } 1109 return ret; 1110 } 1111 1112 static int tap_peek_len(struct socket *sock) 1113 { 1114 struct tap_queue *q = container_of(sock, struct tap_queue, 1115 sock); 1116 return skb_array_peek_len(&q->skb_array); 1117 } 1118 1119 /* Ops structure to mimic raw sockets with tun */ 1120 static const struct proto_ops tap_socket_ops = { 1121 .sendmsg = tap_sendmsg, 1122 .recvmsg = tap_recvmsg, 1123 .peek_len = tap_peek_len, 1124 }; 1125 1126 /* Get an underlying socket object from tun file. Returns error unless file is 1127 * attached to a device. The returned object works like a packet socket, it 1128 * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for 1129 * holding a reference to the file for as long as the socket is in use. */ 1130 struct socket *tap_get_socket(struct file *file) 1131 { 1132 struct tap_queue *q; 1133 if (file->f_op != &tap_fops) 1134 return ERR_PTR(-EINVAL); 1135 q = file->private_data; 1136 if (!q) 1137 return ERR_PTR(-EBADFD); 1138 return &q->sock; 1139 } 1140 EXPORT_SYMBOL_GPL(tap_get_socket); 1141 1142 int tap_queue_resize(struct tap_dev *tap) 1143 { 1144 struct net_device *dev = tap->dev; 1145 struct tap_queue *q; 1146 struct skb_array **arrays; 1147 int n = tap->numqueues; 1148 int ret, i = 0; 1149 1150 arrays = kmalloc(sizeof *arrays * n, GFP_KERNEL); 1151 if (!arrays) 1152 return -ENOMEM; 1153 1154 list_for_each_entry(q, &tap->queue_list, next) 1155 arrays[i++] = &q->skb_array; 1156 1157 ret = skb_array_resize_multiple(arrays, n, 1158 dev->tx_queue_len, GFP_KERNEL); 1159 1160 kfree(arrays); 1161 return ret; 1162 } 1163 1164 int tap_create_cdev(struct cdev *tap_cdev, 1165 dev_t *tap_major, const char *device_name) 1166 { 1167 int err; 1168 1169 err = alloc_chrdev_region(tap_major, 0, TAP_NUM_DEVS, device_name); 1170 if (err) 1171 goto out1; 1172 1173 cdev_init(tap_cdev, &tap_fops); 1174 err = cdev_add(tap_cdev, *tap_major, TAP_NUM_DEVS); 1175 if (err) 1176 goto out2; 1177 1178 macvtap_major.major = MAJOR(*tap_major); 1179 1180 idr_init(&macvtap_major.minor_idr); 1181 mutex_init(&macvtap_major.minor_lock); 1182 1183 macvtap_major.device_name = device_name; 1184 1185 return 0; 1186 1187 out2: 1188 unregister_chrdev_region(*tap_major, TAP_NUM_DEVS); 1189 out1: 1190 return err; 1191 } 1192 1193 void tap_destroy_cdev(dev_t major, struct cdev *tap_cdev) 1194 { 1195 cdev_del(tap_cdev); 1196 unregister_chrdev_region(major, TAP_NUM_DEVS); 1197 idr_destroy(&macvtap_major.minor_idr); 1198 } 1199