xref: /openbmc/linux/drivers/net/macvtap.c (revision b34e08d5)
1 #include <linux/etherdevice.h>
2 #include <linux/if_macvlan.h>
3 #include <linux/if_vlan.h>
4 #include <linux/interrupt.h>
5 #include <linux/nsproxy.h>
6 #include <linux/compat.h>
7 #include <linux/if_tun.h>
8 #include <linux/module.h>
9 #include <linux/skbuff.h>
10 #include <linux/cache.h>
11 #include <linux/sched.h>
12 #include <linux/types.h>
13 #include <linux/slab.h>
14 #include <linux/wait.h>
15 #include <linux/cdev.h>
16 #include <linux/idr.h>
17 #include <linux/fs.h>
18 
19 #include <net/net_namespace.h>
20 #include <net/rtnetlink.h>
21 #include <net/sock.h>
22 #include <linux/virtio_net.h>
23 
24 /*
25  * A macvtap queue is the central object of this driver, it connects
26  * an open character device to a macvlan interface. There can be
27  * multiple queues on one interface, which map back to queues
28  * implemented in hardware on the underlying device.
29  *
30  * macvtap_proto is used to allocate queues through the sock allocation
31  * mechanism.
32  *
33  */
34 struct macvtap_queue {
35 	struct sock sk;
36 	struct socket sock;
37 	struct socket_wq wq;
38 	int vnet_hdr_sz;
39 	struct macvlan_dev __rcu *vlan;
40 	struct file *file;
41 	unsigned int flags;
42 	u16 queue_index;
43 	bool enabled;
44 	struct list_head next;
45 };
46 
47 static struct proto macvtap_proto = {
48 	.name = "macvtap",
49 	.owner = THIS_MODULE,
50 	.obj_size = sizeof (struct macvtap_queue),
51 };
52 
53 /*
54  * Variables for dealing with macvtaps device numbers.
55  */
56 static dev_t macvtap_major;
57 #define MACVTAP_NUM_DEVS (1U << MINORBITS)
58 static DEFINE_MUTEX(minor_lock);
59 static DEFINE_IDR(minor_idr);
60 
61 #define GOODCOPY_LEN 128
62 static struct class *macvtap_class;
63 static struct cdev macvtap_cdev;
64 
65 static const struct proto_ops macvtap_socket_ops;
66 
67 #define TUN_OFFLOADS (NETIF_F_HW_CSUM | NETIF_F_TSO_ECN | NETIF_F_TSO | \
68 		      NETIF_F_TSO6 | NETIF_F_UFO)
69 #define RX_OFFLOADS (NETIF_F_GRO | NETIF_F_LRO)
70 #define TAP_FEATURES (NETIF_F_GSO | NETIF_F_SG)
71 
72 static struct macvlan_dev *macvtap_get_vlan_rcu(const struct net_device *dev)
73 {
74 	return rcu_dereference(dev->rx_handler_data);
75 }
76 
77 /*
78  * RCU usage:
79  * The macvtap_queue and the macvlan_dev are loosely coupled, the
80  * pointers from one to the other can only be read while rcu_read_lock
81  * or rtnl is held.
82  *
83  * Both the file and the macvlan_dev hold a reference on the macvtap_queue
84  * through sock_hold(&q->sk). When the macvlan_dev goes away first,
85  * q->vlan becomes inaccessible. When the files gets closed,
86  * macvtap_get_queue() fails.
87  *
88  * There may still be references to the struct sock inside of the
89  * queue from outbound SKBs, but these never reference back to the
90  * file or the dev. The data structure is freed through __sk_free
91  * when both our references and any pending SKBs are gone.
92  */
93 
94 static int macvtap_enable_queue(struct net_device *dev, struct file *file,
95 				struct macvtap_queue *q)
96 {
97 	struct macvlan_dev *vlan = netdev_priv(dev);
98 	int err = -EINVAL;
99 
100 	ASSERT_RTNL();
101 
102 	if (q->enabled)
103 		goto out;
104 
105 	err = 0;
106 	rcu_assign_pointer(vlan->taps[vlan->numvtaps], q);
107 	q->queue_index = vlan->numvtaps;
108 	q->enabled = true;
109 
110 	vlan->numvtaps++;
111 out:
112 	return err;
113 }
114 
115 static int macvtap_set_queue(struct net_device *dev, struct file *file,
116 			     struct macvtap_queue *q)
117 {
118 	struct macvlan_dev *vlan = netdev_priv(dev);
119 	int err = -EBUSY;
120 
121 	rtnl_lock();
122 	if (vlan->numqueues == MAX_MACVTAP_QUEUES)
123 		goto out;
124 
125 	err = 0;
126 	rcu_assign_pointer(q->vlan, vlan);
127 	rcu_assign_pointer(vlan->taps[vlan->numvtaps], q);
128 	sock_hold(&q->sk);
129 
130 	q->file = file;
131 	q->queue_index = vlan->numvtaps;
132 	q->enabled = true;
133 	file->private_data = q;
134 	list_add_tail(&q->next, &vlan->queue_list);
135 
136 	vlan->numvtaps++;
137 	vlan->numqueues++;
138 
139 out:
140 	rtnl_unlock();
141 	return err;
142 }
143 
144 static int macvtap_disable_queue(struct macvtap_queue *q)
145 {
146 	struct macvlan_dev *vlan;
147 	struct macvtap_queue *nq;
148 
149 	ASSERT_RTNL();
150 	if (!q->enabled)
151 		return -EINVAL;
152 
153 	vlan = rtnl_dereference(q->vlan);
154 
155 	if (vlan) {
156 		int index = q->queue_index;
157 		BUG_ON(index >= vlan->numvtaps);
158 		nq = rtnl_dereference(vlan->taps[vlan->numvtaps - 1]);
159 		nq->queue_index = index;
160 
161 		rcu_assign_pointer(vlan->taps[index], nq);
162 		RCU_INIT_POINTER(vlan->taps[vlan->numvtaps - 1], NULL);
163 		q->enabled = false;
164 
165 		vlan->numvtaps--;
166 	}
167 
168 	return 0;
169 }
170 
171 /*
172  * The file owning the queue got closed, give up both
173  * the reference that the files holds as well as the
174  * one from the macvlan_dev if that still exists.
175  *
176  * Using the spinlock makes sure that we don't get
177  * to the queue again after destroying it.
178  */
179 static void macvtap_put_queue(struct macvtap_queue *q)
180 {
181 	struct macvlan_dev *vlan;
182 
183 	rtnl_lock();
184 	vlan = rtnl_dereference(q->vlan);
185 
186 	if (vlan) {
187 		if (q->enabled)
188 			BUG_ON(macvtap_disable_queue(q));
189 
190 		vlan->numqueues--;
191 		RCU_INIT_POINTER(q->vlan, NULL);
192 		sock_put(&q->sk);
193 		list_del_init(&q->next);
194 	}
195 
196 	rtnl_unlock();
197 
198 	synchronize_rcu();
199 	sock_put(&q->sk);
200 }
201 
202 /*
203  * Select a queue based on the rxq of the device on which this packet
204  * arrived. If the incoming device is not mq, calculate a flow hash
205  * to select a queue. If all fails, find the first available queue.
206  * Cache vlan->numvtaps since it can become zero during the execution
207  * of this function.
208  */
209 static struct macvtap_queue *macvtap_get_queue(struct net_device *dev,
210 					       struct sk_buff *skb)
211 {
212 	struct macvlan_dev *vlan = netdev_priv(dev);
213 	struct macvtap_queue *tap = NULL;
214 	/* Access to taps array is protected by rcu, but access to numvtaps
215 	 * isn't. Below we use it to lookup a queue, but treat it as a hint
216 	 * and validate that the result isn't NULL - in case we are
217 	 * racing against queue removal.
218 	 */
219 	int numvtaps = ACCESS_ONCE(vlan->numvtaps);
220 	__u32 rxq;
221 
222 	if (!numvtaps)
223 		goto out;
224 
225 	/* Check if we can use flow to select a queue */
226 	rxq = skb_get_hash(skb);
227 	if (rxq) {
228 		tap = rcu_dereference(vlan->taps[rxq % numvtaps]);
229 		goto out;
230 	}
231 
232 	if (likely(skb_rx_queue_recorded(skb))) {
233 		rxq = skb_get_rx_queue(skb);
234 
235 		while (unlikely(rxq >= numvtaps))
236 			rxq -= numvtaps;
237 
238 		tap = rcu_dereference(vlan->taps[rxq]);
239 		goto out;
240 	}
241 
242 	tap = rcu_dereference(vlan->taps[0]);
243 out:
244 	return tap;
245 }
246 
247 /*
248  * The net_device is going away, give up the reference
249  * that it holds on all queues and safely set the pointer
250  * from the queues to NULL.
251  */
252 static void macvtap_del_queues(struct net_device *dev)
253 {
254 	struct macvlan_dev *vlan = netdev_priv(dev);
255 	struct macvtap_queue *q, *tmp, *qlist[MAX_MACVTAP_QUEUES];
256 	int i, j = 0;
257 
258 	ASSERT_RTNL();
259 	list_for_each_entry_safe(q, tmp, &vlan->queue_list, next) {
260 		list_del_init(&q->next);
261 		qlist[j++] = q;
262 		RCU_INIT_POINTER(q->vlan, NULL);
263 		if (q->enabled)
264 			vlan->numvtaps--;
265 		vlan->numqueues--;
266 	}
267 	for (i = 0; i < vlan->numvtaps; i++)
268 		RCU_INIT_POINTER(vlan->taps[i], NULL);
269 	BUG_ON(vlan->numvtaps);
270 	BUG_ON(vlan->numqueues);
271 	/* guarantee that any future macvtap_set_queue will fail */
272 	vlan->numvtaps = MAX_MACVTAP_QUEUES;
273 
274 	for (--j; j >= 0; j--)
275 		sock_put(&qlist[j]->sk);
276 }
277 
278 static rx_handler_result_t macvtap_handle_frame(struct sk_buff **pskb)
279 {
280 	struct sk_buff *skb = *pskb;
281 	struct net_device *dev = skb->dev;
282 	struct macvlan_dev *vlan;
283 	struct macvtap_queue *q;
284 	netdev_features_t features = TAP_FEATURES;
285 
286 	vlan = macvtap_get_vlan_rcu(dev);
287 	if (!vlan)
288 		return RX_HANDLER_PASS;
289 
290 	q = macvtap_get_queue(dev, skb);
291 	if (!q)
292 		return RX_HANDLER_PASS;
293 
294 	if (skb_queue_len(&q->sk.sk_receive_queue) >= dev->tx_queue_len)
295 		goto drop;
296 
297 	skb_push(skb, ETH_HLEN);
298 
299 	/* Apply the forward feature mask so that we perform segmentation
300 	 * according to users wishes.  This only works if VNET_HDR is
301 	 * enabled.
302 	 */
303 	if (q->flags & IFF_VNET_HDR)
304 		features |= vlan->tap_features;
305 	if (netif_needs_gso(skb, features)) {
306 		struct sk_buff *segs = __skb_gso_segment(skb, features, false);
307 
308 		if (IS_ERR(segs))
309 			goto drop;
310 
311 		if (!segs) {
312 			skb_queue_tail(&q->sk.sk_receive_queue, skb);
313 			goto wake_up;
314 		}
315 
316 		kfree_skb(skb);
317 		while (segs) {
318 			struct sk_buff *nskb = segs->next;
319 
320 			segs->next = NULL;
321 			skb_queue_tail(&q->sk.sk_receive_queue, segs);
322 			segs = nskb;
323 		}
324 	} else {
325 		skb_queue_tail(&q->sk.sk_receive_queue, skb);
326 	}
327 
328 wake_up:
329 	wake_up_interruptible_poll(sk_sleep(&q->sk), POLLIN | POLLRDNORM | POLLRDBAND);
330 	return RX_HANDLER_CONSUMED;
331 
332 drop:
333 	/* Count errors/drops only here, thus don't care about args. */
334 	macvlan_count_rx(vlan, 0, 0, 0);
335 	kfree_skb(skb);
336 	return RX_HANDLER_CONSUMED;
337 }
338 
339 static int macvtap_get_minor(struct macvlan_dev *vlan)
340 {
341 	int retval = -ENOMEM;
342 
343 	mutex_lock(&minor_lock);
344 	retval = idr_alloc(&minor_idr, vlan, 1, MACVTAP_NUM_DEVS, GFP_KERNEL);
345 	if (retval >= 0) {
346 		vlan->minor = retval;
347 	} else if (retval == -ENOSPC) {
348 		printk(KERN_ERR "too many macvtap devices\n");
349 		retval = -EINVAL;
350 	}
351 	mutex_unlock(&minor_lock);
352 	return retval < 0 ? retval : 0;
353 }
354 
355 static void macvtap_free_minor(struct macvlan_dev *vlan)
356 {
357 	mutex_lock(&minor_lock);
358 	if (vlan->minor) {
359 		idr_remove(&minor_idr, vlan->minor);
360 		vlan->minor = 0;
361 	}
362 	mutex_unlock(&minor_lock);
363 }
364 
365 static struct net_device *dev_get_by_macvtap_minor(int minor)
366 {
367 	struct net_device *dev = NULL;
368 	struct macvlan_dev *vlan;
369 
370 	mutex_lock(&minor_lock);
371 	vlan = idr_find(&minor_idr, minor);
372 	if (vlan) {
373 		dev = vlan->dev;
374 		dev_hold(dev);
375 	}
376 	mutex_unlock(&minor_lock);
377 	return dev;
378 }
379 
380 static int macvtap_newlink(struct net *src_net,
381 			   struct net_device *dev,
382 			   struct nlattr *tb[],
383 			   struct nlattr *data[])
384 {
385 	struct macvlan_dev *vlan = netdev_priv(dev);
386 	int err;
387 
388 	INIT_LIST_HEAD(&vlan->queue_list);
389 
390 	/* Since macvlan supports all offloads by default, make
391 	 * tap support all offloads also.
392 	 */
393 	vlan->tap_features = TUN_OFFLOADS;
394 
395 	err = netdev_rx_handler_register(dev, macvtap_handle_frame, vlan);
396 	if (err)
397 		return err;
398 
399 	/* Don't put anything that may fail after macvlan_common_newlink
400 	 * because we can't undo what it does.
401 	 */
402 	return macvlan_common_newlink(src_net, dev, tb, data);
403 }
404 
405 static void macvtap_dellink(struct net_device *dev,
406 			    struct list_head *head)
407 {
408 	netdev_rx_handler_unregister(dev);
409 	macvtap_del_queues(dev);
410 	macvlan_dellink(dev, head);
411 }
412 
413 static void macvtap_setup(struct net_device *dev)
414 {
415 	macvlan_common_setup(dev);
416 	dev->tx_queue_len = TUN_READQ_SIZE;
417 }
418 
419 static struct rtnl_link_ops macvtap_link_ops __read_mostly = {
420 	.kind		= "macvtap",
421 	.setup		= macvtap_setup,
422 	.newlink	= macvtap_newlink,
423 	.dellink	= macvtap_dellink,
424 };
425 
426 
427 static void macvtap_sock_write_space(struct sock *sk)
428 {
429 	wait_queue_head_t *wqueue;
430 
431 	if (!sock_writeable(sk) ||
432 	    !test_and_clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags))
433 		return;
434 
435 	wqueue = sk_sleep(sk);
436 	if (wqueue && waitqueue_active(wqueue))
437 		wake_up_interruptible_poll(wqueue, POLLOUT | POLLWRNORM | POLLWRBAND);
438 }
439 
440 static void macvtap_sock_destruct(struct sock *sk)
441 {
442 	skb_queue_purge(&sk->sk_receive_queue);
443 }
444 
445 static int macvtap_open(struct inode *inode, struct file *file)
446 {
447 	struct net *net = current->nsproxy->net_ns;
448 	struct net_device *dev = dev_get_by_macvtap_minor(iminor(inode));
449 	struct macvtap_queue *q;
450 	int err;
451 
452 	err = -ENODEV;
453 	if (!dev)
454 		goto out;
455 
456 	err = -ENOMEM;
457 	q = (struct macvtap_queue *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
458 					     &macvtap_proto);
459 	if (!q)
460 		goto out;
461 
462 	RCU_INIT_POINTER(q->sock.wq, &q->wq);
463 	init_waitqueue_head(&q->wq.wait);
464 	q->sock.type = SOCK_RAW;
465 	q->sock.state = SS_CONNECTED;
466 	q->sock.file = file;
467 	q->sock.ops = &macvtap_socket_ops;
468 	sock_init_data(&q->sock, &q->sk);
469 	q->sk.sk_write_space = macvtap_sock_write_space;
470 	q->sk.sk_destruct = macvtap_sock_destruct;
471 	q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP;
472 	q->vnet_hdr_sz = sizeof(struct virtio_net_hdr);
473 
474 	/*
475 	 * so far only KVM virtio_net uses macvtap, enable zero copy between
476 	 * guest kernel and host kernel when lower device supports zerocopy
477 	 *
478 	 * The macvlan supports zerocopy iff the lower device supports zero
479 	 * copy so we don't have to look at the lower device directly.
480 	 */
481 	if ((dev->features & NETIF_F_HIGHDMA) && (dev->features & NETIF_F_SG))
482 		sock_set_flag(&q->sk, SOCK_ZEROCOPY);
483 
484 	err = macvtap_set_queue(dev, file, q);
485 	if (err)
486 		sock_put(&q->sk);
487 
488 out:
489 	if (dev)
490 		dev_put(dev);
491 
492 	return err;
493 }
494 
495 static int macvtap_release(struct inode *inode, struct file *file)
496 {
497 	struct macvtap_queue *q = file->private_data;
498 	macvtap_put_queue(q);
499 	return 0;
500 }
501 
502 static unsigned int macvtap_poll(struct file *file, poll_table * wait)
503 {
504 	struct macvtap_queue *q = file->private_data;
505 	unsigned int mask = POLLERR;
506 
507 	if (!q)
508 		goto out;
509 
510 	mask = 0;
511 	poll_wait(file, &q->wq.wait, wait);
512 
513 	if (!skb_queue_empty(&q->sk.sk_receive_queue))
514 		mask |= POLLIN | POLLRDNORM;
515 
516 	if (sock_writeable(&q->sk) ||
517 	    (!test_and_set_bit(SOCK_ASYNC_NOSPACE, &q->sock.flags) &&
518 	     sock_writeable(&q->sk)))
519 		mask |= POLLOUT | POLLWRNORM;
520 
521 out:
522 	return mask;
523 }
524 
525 static inline struct sk_buff *macvtap_alloc_skb(struct sock *sk, size_t prepad,
526 						size_t len, size_t linear,
527 						int noblock, int *err)
528 {
529 	struct sk_buff *skb;
530 
531 	/* Under a page?  Don't bother with paged skb. */
532 	if (prepad + len < PAGE_SIZE || !linear)
533 		linear = len;
534 
535 	skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
536 				   err, 0);
537 	if (!skb)
538 		return NULL;
539 
540 	skb_reserve(skb, prepad);
541 	skb_put(skb, linear);
542 	skb->data_len = len - linear;
543 	skb->len += len - linear;
544 
545 	return skb;
546 }
547 
548 /*
549  * macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should
550  * be shared with the tun/tap driver.
551  */
552 static int macvtap_skb_from_vnet_hdr(struct sk_buff *skb,
553 				     struct virtio_net_hdr *vnet_hdr)
554 {
555 	unsigned short gso_type = 0;
556 	if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
557 		switch (vnet_hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
558 		case VIRTIO_NET_HDR_GSO_TCPV4:
559 			gso_type = SKB_GSO_TCPV4;
560 			break;
561 		case VIRTIO_NET_HDR_GSO_TCPV6:
562 			gso_type = SKB_GSO_TCPV6;
563 			break;
564 		case VIRTIO_NET_HDR_GSO_UDP:
565 			gso_type = SKB_GSO_UDP;
566 			break;
567 		default:
568 			return -EINVAL;
569 		}
570 
571 		if (vnet_hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN)
572 			gso_type |= SKB_GSO_TCP_ECN;
573 
574 		if (vnet_hdr->gso_size == 0)
575 			return -EINVAL;
576 	}
577 
578 	if (vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
579 		if (!skb_partial_csum_set(skb, vnet_hdr->csum_start,
580 					  vnet_hdr->csum_offset))
581 			return -EINVAL;
582 	}
583 
584 	if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
585 		skb_shinfo(skb)->gso_size = vnet_hdr->gso_size;
586 		skb_shinfo(skb)->gso_type = gso_type;
587 
588 		/* Header must be checked, and gso_segs computed. */
589 		skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
590 		skb_shinfo(skb)->gso_segs = 0;
591 	}
592 	return 0;
593 }
594 
595 static void macvtap_skb_to_vnet_hdr(const struct sk_buff *skb,
596 				   struct virtio_net_hdr *vnet_hdr)
597 {
598 	memset(vnet_hdr, 0, sizeof(*vnet_hdr));
599 
600 	if (skb_is_gso(skb)) {
601 		struct skb_shared_info *sinfo = skb_shinfo(skb);
602 
603 		/* This is a hint as to how much should be linear. */
604 		vnet_hdr->hdr_len = skb_headlen(skb);
605 		vnet_hdr->gso_size = sinfo->gso_size;
606 		if (sinfo->gso_type & SKB_GSO_TCPV4)
607 			vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
608 		else if (sinfo->gso_type & SKB_GSO_TCPV6)
609 			vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
610 		else if (sinfo->gso_type & SKB_GSO_UDP)
611 			vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
612 		else
613 			BUG();
614 		if (sinfo->gso_type & SKB_GSO_TCP_ECN)
615 			vnet_hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
616 	} else
617 		vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE;
618 
619 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
620 		vnet_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
621 		vnet_hdr->csum_start = skb_checksum_start_offset(skb);
622 		vnet_hdr->csum_offset = skb->csum_offset;
623 	} else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
624 		vnet_hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID;
625 	} /* else everything is zero */
626 }
627 
628 /* Get packet from user space buffer */
629 static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
630 				const struct iovec *iv, unsigned long total_len,
631 				size_t count, int noblock)
632 {
633 	int good_linear = SKB_MAX_HEAD(NET_IP_ALIGN);
634 	struct sk_buff *skb;
635 	struct macvlan_dev *vlan;
636 	unsigned long len = total_len;
637 	int err;
638 	struct virtio_net_hdr vnet_hdr = { 0 };
639 	int vnet_hdr_len = 0;
640 	int copylen = 0;
641 	bool zerocopy = false;
642 	size_t linear;
643 
644 	if (q->flags & IFF_VNET_HDR) {
645 		vnet_hdr_len = q->vnet_hdr_sz;
646 
647 		err = -EINVAL;
648 		if (len < vnet_hdr_len)
649 			goto err;
650 		len -= vnet_hdr_len;
651 
652 		err = memcpy_fromiovecend((void *)&vnet_hdr, iv, 0,
653 					   sizeof(vnet_hdr));
654 		if (err < 0)
655 			goto err;
656 		if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
657 		     vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
658 							vnet_hdr.hdr_len)
659 			vnet_hdr.hdr_len = vnet_hdr.csum_start +
660 						vnet_hdr.csum_offset + 2;
661 		err = -EINVAL;
662 		if (vnet_hdr.hdr_len > len)
663 			goto err;
664 	}
665 
666 	err = -EINVAL;
667 	if (unlikely(len < ETH_HLEN))
668 		goto err;
669 
670 	err = -EMSGSIZE;
671 	if (unlikely(count > UIO_MAXIOV))
672 		goto err;
673 
674 	if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) {
675 		copylen = vnet_hdr.hdr_len ? vnet_hdr.hdr_len : GOODCOPY_LEN;
676 		if (copylen > good_linear)
677 			copylen = good_linear;
678 		linear = copylen;
679 		if (iov_pages(iv, vnet_hdr_len + copylen, count)
680 		    <= MAX_SKB_FRAGS)
681 			zerocopy = true;
682 	}
683 
684 	if (!zerocopy) {
685 		copylen = len;
686 		if (vnet_hdr.hdr_len > good_linear)
687 			linear = good_linear;
688 		else
689 			linear = vnet_hdr.hdr_len;
690 	}
691 
692 	skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, copylen,
693 				linear, noblock, &err);
694 	if (!skb)
695 		goto err;
696 
697 	if (zerocopy)
698 		err = zerocopy_sg_from_iovec(skb, iv, vnet_hdr_len, count);
699 	else {
700 		err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len,
701 						   len);
702 		if (!err && m && m->msg_control) {
703 			struct ubuf_info *uarg = m->msg_control;
704 			uarg->callback(uarg, false);
705 		}
706 	}
707 
708 	if (err)
709 		goto err_kfree;
710 
711 	skb_set_network_header(skb, ETH_HLEN);
712 	skb_reset_mac_header(skb);
713 	skb->protocol = eth_hdr(skb)->h_proto;
714 
715 	if (vnet_hdr_len) {
716 		err = macvtap_skb_from_vnet_hdr(skb, &vnet_hdr);
717 		if (err)
718 			goto err_kfree;
719 	}
720 
721 	skb_probe_transport_header(skb, ETH_HLEN);
722 
723 	rcu_read_lock();
724 	vlan = rcu_dereference(q->vlan);
725 	/* copy skb_ubuf_info for callback when skb has no error */
726 	if (zerocopy) {
727 		skb_shinfo(skb)->destructor_arg = m->msg_control;
728 		skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
729 		skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
730 	}
731 	if (vlan) {
732 		skb->dev = vlan->dev;
733 		dev_queue_xmit(skb);
734 	} else {
735 		kfree_skb(skb);
736 	}
737 	rcu_read_unlock();
738 
739 	return total_len;
740 
741 err_kfree:
742 	kfree_skb(skb);
743 
744 err:
745 	rcu_read_lock();
746 	vlan = rcu_dereference(q->vlan);
747 	if (vlan)
748 		this_cpu_inc(vlan->pcpu_stats->tx_dropped);
749 	rcu_read_unlock();
750 
751 	return err;
752 }
753 
754 static ssize_t macvtap_aio_write(struct kiocb *iocb, const struct iovec *iv,
755 				 unsigned long count, loff_t pos)
756 {
757 	struct file *file = iocb->ki_filp;
758 	ssize_t result = -ENOLINK;
759 	struct macvtap_queue *q = file->private_data;
760 
761 	result = macvtap_get_user(q, NULL, iv, iov_length(iv, count), count,
762 				  file->f_flags & O_NONBLOCK);
763 	return result;
764 }
765 
766 /* Put packet to the user space buffer */
767 static ssize_t macvtap_put_user(struct macvtap_queue *q,
768 				const struct sk_buff *skb,
769 				const struct iovec *iv, int len)
770 {
771 	int ret;
772 	int vnet_hdr_len = 0;
773 	int vlan_offset = 0;
774 	int copied, total;
775 
776 	if (q->flags & IFF_VNET_HDR) {
777 		struct virtio_net_hdr vnet_hdr;
778 		vnet_hdr_len = q->vnet_hdr_sz;
779 		if ((len -= vnet_hdr_len) < 0)
780 			return -EINVAL;
781 
782 		macvtap_skb_to_vnet_hdr(skb, &vnet_hdr);
783 
784 		if (memcpy_toiovecend(iv, (void *)&vnet_hdr, 0, sizeof(vnet_hdr)))
785 			return -EFAULT;
786 	}
787 	total = copied = vnet_hdr_len;
788 	total += skb->len;
789 
790 	if (!vlan_tx_tag_present(skb))
791 		len = min_t(int, skb->len, len);
792 	else {
793 		int copy;
794 		struct {
795 			__be16 h_vlan_proto;
796 			__be16 h_vlan_TCI;
797 		} veth;
798 		veth.h_vlan_proto = skb->vlan_proto;
799 		veth.h_vlan_TCI = htons(vlan_tx_tag_get(skb));
800 
801 		vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto);
802 		len = min_t(int, skb->len + VLAN_HLEN, len);
803 		total += VLAN_HLEN;
804 
805 		copy = min_t(int, vlan_offset, len);
806 		ret = skb_copy_datagram_const_iovec(skb, 0, iv, copied, copy);
807 		len -= copy;
808 		copied += copy;
809 		if (ret || !len)
810 			goto done;
811 
812 		copy = min_t(int, sizeof(veth), len);
813 		ret = memcpy_toiovecend(iv, (void *)&veth, copied, copy);
814 		len -= copy;
815 		copied += copy;
816 		if (ret || !len)
817 			goto done;
818 	}
819 
820 	ret = skb_copy_datagram_const_iovec(skb, vlan_offset, iv, copied, len);
821 
822 done:
823 	return ret ? ret : total;
824 }
825 
826 static ssize_t macvtap_do_read(struct macvtap_queue *q,
827 			       const struct iovec *iv, unsigned long len,
828 			       int noblock)
829 {
830 	DEFINE_WAIT(wait);
831 	struct sk_buff *skb;
832 	ssize_t ret = 0;
833 
834 	while (len) {
835 		if (!noblock)
836 			prepare_to_wait(sk_sleep(&q->sk), &wait,
837 					TASK_INTERRUPTIBLE);
838 
839 		/* Read frames from the queue */
840 		skb = skb_dequeue(&q->sk.sk_receive_queue);
841 		if (!skb) {
842 			if (noblock) {
843 				ret = -EAGAIN;
844 				break;
845 			}
846 			if (signal_pending(current)) {
847 				ret = -ERESTARTSYS;
848 				break;
849 			}
850 			/* Nothing to read, let's sleep */
851 			schedule();
852 			continue;
853 		}
854 		ret = macvtap_put_user(q, skb, iv, len);
855 		kfree_skb(skb);
856 		break;
857 	}
858 
859 	if (!noblock)
860 		finish_wait(sk_sleep(&q->sk), &wait);
861 	return ret;
862 }
863 
864 static ssize_t macvtap_aio_read(struct kiocb *iocb, const struct iovec *iv,
865 				unsigned long count, loff_t pos)
866 {
867 	struct file *file = iocb->ki_filp;
868 	struct macvtap_queue *q = file->private_data;
869 	ssize_t len, ret = 0;
870 
871 	len = iov_length(iv, count);
872 	if (len < 0) {
873 		ret = -EINVAL;
874 		goto out;
875 	}
876 
877 	ret = macvtap_do_read(q, iv, len, file->f_flags & O_NONBLOCK);
878 	ret = min_t(ssize_t, ret, len);
879 	if (ret > 0)
880 		iocb->ki_pos = ret;
881 out:
882 	return ret;
883 }
884 
885 static struct macvlan_dev *macvtap_get_vlan(struct macvtap_queue *q)
886 {
887 	struct macvlan_dev *vlan;
888 
889 	ASSERT_RTNL();
890 	vlan = rtnl_dereference(q->vlan);
891 	if (vlan)
892 		dev_hold(vlan->dev);
893 
894 	return vlan;
895 }
896 
897 static void macvtap_put_vlan(struct macvlan_dev *vlan)
898 {
899 	dev_put(vlan->dev);
900 }
901 
902 static int macvtap_ioctl_set_queue(struct file *file, unsigned int flags)
903 {
904 	struct macvtap_queue *q = file->private_data;
905 	struct macvlan_dev *vlan;
906 	int ret;
907 
908 	vlan = macvtap_get_vlan(q);
909 	if (!vlan)
910 		return -EINVAL;
911 
912 	if (flags & IFF_ATTACH_QUEUE)
913 		ret = macvtap_enable_queue(vlan->dev, file, q);
914 	else if (flags & IFF_DETACH_QUEUE)
915 		ret = macvtap_disable_queue(q);
916 	else
917 		ret = -EINVAL;
918 
919 	macvtap_put_vlan(vlan);
920 	return ret;
921 }
922 
923 static int set_offload(struct macvtap_queue *q, unsigned long arg)
924 {
925 	struct macvlan_dev *vlan;
926 	netdev_features_t features;
927 	netdev_features_t feature_mask = 0;
928 
929 	vlan = rtnl_dereference(q->vlan);
930 	if (!vlan)
931 		return -ENOLINK;
932 
933 	features = vlan->dev->features;
934 
935 	if (arg & TUN_F_CSUM) {
936 		feature_mask = NETIF_F_HW_CSUM;
937 
938 		if (arg & (TUN_F_TSO4 | TUN_F_TSO6)) {
939 			if (arg & TUN_F_TSO_ECN)
940 				feature_mask |= NETIF_F_TSO_ECN;
941 			if (arg & TUN_F_TSO4)
942 				feature_mask |= NETIF_F_TSO;
943 			if (arg & TUN_F_TSO6)
944 				feature_mask |= NETIF_F_TSO6;
945 		}
946 
947 		if (arg & TUN_F_UFO)
948 			feature_mask |= NETIF_F_UFO;
949 	}
950 
951 	/* tun/tap driver inverts the usage for TSO offloads, where
952 	 * setting the TSO bit means that the userspace wants to
953 	 * accept TSO frames and turning it off means that user space
954 	 * does not support TSO.
955 	 * For macvtap, we have to invert it to mean the same thing.
956 	 * When user space turns off TSO, we turn off GSO/LRO so that
957 	 * user-space will not receive TSO frames.
958 	 */
959 	if (feature_mask & (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_UFO))
960 		features |= RX_OFFLOADS;
961 	else
962 		features &= ~RX_OFFLOADS;
963 
964 	/* tap_features are the same as features on tun/tap and
965 	 * reflect user expectations.
966 	 */
967 	vlan->tap_features = feature_mask;
968 	vlan->set_features = features;
969 	netdev_update_features(vlan->dev);
970 
971 	return 0;
972 }
973 
974 /*
975  * provide compatibility with generic tun/tap interface
976  */
977 static long macvtap_ioctl(struct file *file, unsigned int cmd,
978 			  unsigned long arg)
979 {
980 	struct macvtap_queue *q = file->private_data;
981 	struct macvlan_dev *vlan;
982 	void __user *argp = (void __user *)arg;
983 	struct ifreq __user *ifr = argp;
984 	unsigned int __user *up = argp;
985 	unsigned int u;
986 	int __user *sp = argp;
987 	int s;
988 	int ret;
989 
990 	switch (cmd) {
991 	case TUNSETIFF:
992 		/* ignore the name, just look at flags */
993 		if (get_user(u, &ifr->ifr_flags))
994 			return -EFAULT;
995 
996 		ret = 0;
997 		if ((u & ~(IFF_VNET_HDR | IFF_MULTI_QUEUE)) !=
998 		    (IFF_NO_PI | IFF_TAP))
999 			ret = -EINVAL;
1000 		else
1001 			q->flags = u;
1002 
1003 		return ret;
1004 
1005 	case TUNGETIFF:
1006 		rtnl_lock();
1007 		vlan = macvtap_get_vlan(q);
1008 		if (!vlan) {
1009 			rtnl_unlock();
1010 			return -ENOLINK;
1011 		}
1012 
1013 		ret = 0;
1014 		if (copy_to_user(&ifr->ifr_name, vlan->dev->name, IFNAMSIZ) ||
1015 		    put_user(q->flags, &ifr->ifr_flags))
1016 			ret = -EFAULT;
1017 		macvtap_put_vlan(vlan);
1018 		rtnl_unlock();
1019 		return ret;
1020 
1021 	case TUNSETQUEUE:
1022 		if (get_user(u, &ifr->ifr_flags))
1023 			return -EFAULT;
1024 		rtnl_lock();
1025 		ret = macvtap_ioctl_set_queue(file, u);
1026 		rtnl_unlock();
1027 		return ret;
1028 
1029 	case TUNGETFEATURES:
1030 		if (put_user(IFF_TAP | IFF_NO_PI | IFF_VNET_HDR |
1031 			     IFF_MULTI_QUEUE, up))
1032 			return -EFAULT;
1033 		return 0;
1034 
1035 	case TUNSETSNDBUF:
1036 		if (get_user(u, up))
1037 			return -EFAULT;
1038 
1039 		q->sk.sk_sndbuf = u;
1040 		return 0;
1041 
1042 	case TUNGETVNETHDRSZ:
1043 		s = q->vnet_hdr_sz;
1044 		if (put_user(s, sp))
1045 			return -EFAULT;
1046 		return 0;
1047 
1048 	case TUNSETVNETHDRSZ:
1049 		if (get_user(s, sp))
1050 			return -EFAULT;
1051 		if (s < (int)sizeof(struct virtio_net_hdr))
1052 			return -EINVAL;
1053 
1054 		q->vnet_hdr_sz = s;
1055 		return 0;
1056 
1057 	case TUNSETOFFLOAD:
1058 		/* let the user check for future flags */
1059 		if (arg & ~(TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 |
1060 			    TUN_F_TSO_ECN | TUN_F_UFO))
1061 			return -EINVAL;
1062 
1063 		rtnl_lock();
1064 		ret = set_offload(q, arg);
1065 		rtnl_unlock();
1066 		return ret;
1067 
1068 	default:
1069 		return -EINVAL;
1070 	}
1071 }
1072 
1073 #ifdef CONFIG_COMPAT
1074 static long macvtap_compat_ioctl(struct file *file, unsigned int cmd,
1075 				 unsigned long arg)
1076 {
1077 	return macvtap_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
1078 }
1079 #endif
1080 
1081 static const struct file_operations macvtap_fops = {
1082 	.owner		= THIS_MODULE,
1083 	.open		= macvtap_open,
1084 	.release	= macvtap_release,
1085 	.aio_read	= macvtap_aio_read,
1086 	.aio_write	= macvtap_aio_write,
1087 	.poll		= macvtap_poll,
1088 	.llseek		= no_llseek,
1089 	.unlocked_ioctl	= macvtap_ioctl,
1090 #ifdef CONFIG_COMPAT
1091 	.compat_ioctl	= macvtap_compat_ioctl,
1092 #endif
1093 };
1094 
1095 static int macvtap_sendmsg(struct kiocb *iocb, struct socket *sock,
1096 			   struct msghdr *m, size_t total_len)
1097 {
1098 	struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock);
1099 	return macvtap_get_user(q, m, m->msg_iov, total_len, m->msg_iovlen,
1100 			    m->msg_flags & MSG_DONTWAIT);
1101 }
1102 
1103 static int macvtap_recvmsg(struct kiocb *iocb, struct socket *sock,
1104 			   struct msghdr *m, size_t total_len,
1105 			   int flags)
1106 {
1107 	struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock);
1108 	int ret;
1109 	if (flags & ~(MSG_DONTWAIT|MSG_TRUNC))
1110 		return -EINVAL;
1111 	ret = macvtap_do_read(q, m->msg_iov, total_len,
1112 			  flags & MSG_DONTWAIT);
1113 	if (ret > total_len) {
1114 		m->msg_flags |= MSG_TRUNC;
1115 		ret = flags & MSG_TRUNC ? ret : total_len;
1116 	}
1117 	return ret;
1118 }
1119 
1120 /* Ops structure to mimic raw sockets with tun */
1121 static const struct proto_ops macvtap_socket_ops = {
1122 	.sendmsg = macvtap_sendmsg,
1123 	.recvmsg = macvtap_recvmsg,
1124 };
1125 
1126 /* Get an underlying socket object from tun file.  Returns error unless file is
1127  * attached to a device.  The returned object works like a packet socket, it
1128  * can be used for sock_sendmsg/sock_recvmsg.  The caller is responsible for
1129  * holding a reference to the file for as long as the socket is in use. */
1130 struct socket *macvtap_get_socket(struct file *file)
1131 {
1132 	struct macvtap_queue *q;
1133 	if (file->f_op != &macvtap_fops)
1134 		return ERR_PTR(-EINVAL);
1135 	q = file->private_data;
1136 	if (!q)
1137 		return ERR_PTR(-EBADFD);
1138 	return &q->sock;
1139 }
1140 EXPORT_SYMBOL_GPL(macvtap_get_socket);
1141 
1142 static int macvtap_device_event(struct notifier_block *unused,
1143 				unsigned long event, void *ptr)
1144 {
1145 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1146 	struct macvlan_dev *vlan;
1147 	struct device *classdev;
1148 	dev_t devt;
1149 	int err;
1150 
1151 	if (dev->rtnl_link_ops != &macvtap_link_ops)
1152 		return NOTIFY_DONE;
1153 
1154 	vlan = netdev_priv(dev);
1155 
1156 	switch (event) {
1157 	case NETDEV_REGISTER:
1158 		/* Create the device node here after the network device has
1159 		 * been registered but before register_netdevice has
1160 		 * finished running.
1161 		 */
1162 		err = macvtap_get_minor(vlan);
1163 		if (err)
1164 			return notifier_from_errno(err);
1165 
1166 		devt = MKDEV(MAJOR(macvtap_major), vlan->minor);
1167 		classdev = device_create(macvtap_class, &dev->dev, devt,
1168 					 dev, "tap%d", dev->ifindex);
1169 		if (IS_ERR(classdev)) {
1170 			macvtap_free_minor(vlan);
1171 			return notifier_from_errno(PTR_ERR(classdev));
1172 		}
1173 		break;
1174 	case NETDEV_UNREGISTER:
1175 		devt = MKDEV(MAJOR(macvtap_major), vlan->minor);
1176 		device_destroy(macvtap_class, devt);
1177 		macvtap_free_minor(vlan);
1178 		break;
1179 	}
1180 
1181 	return NOTIFY_DONE;
1182 }
1183 
1184 static struct notifier_block macvtap_notifier_block __read_mostly = {
1185 	.notifier_call	= macvtap_device_event,
1186 };
1187 
1188 static int macvtap_init(void)
1189 {
1190 	int err;
1191 
1192 	err = alloc_chrdev_region(&macvtap_major, 0,
1193 				MACVTAP_NUM_DEVS, "macvtap");
1194 	if (err)
1195 		goto out1;
1196 
1197 	cdev_init(&macvtap_cdev, &macvtap_fops);
1198 	err = cdev_add(&macvtap_cdev, macvtap_major, MACVTAP_NUM_DEVS);
1199 	if (err)
1200 		goto out2;
1201 
1202 	macvtap_class = class_create(THIS_MODULE, "macvtap");
1203 	if (IS_ERR(macvtap_class)) {
1204 		err = PTR_ERR(macvtap_class);
1205 		goto out3;
1206 	}
1207 
1208 	err = register_netdevice_notifier(&macvtap_notifier_block);
1209 	if (err)
1210 		goto out4;
1211 
1212 	err = macvlan_link_register(&macvtap_link_ops);
1213 	if (err)
1214 		goto out5;
1215 
1216 	return 0;
1217 
1218 out5:
1219 	unregister_netdevice_notifier(&macvtap_notifier_block);
1220 out4:
1221 	class_unregister(macvtap_class);
1222 out3:
1223 	cdev_del(&macvtap_cdev);
1224 out2:
1225 	unregister_chrdev_region(macvtap_major, MACVTAP_NUM_DEVS);
1226 out1:
1227 	return err;
1228 }
1229 module_init(macvtap_init);
1230 
1231 static void macvtap_exit(void)
1232 {
1233 	rtnl_link_unregister(&macvtap_link_ops);
1234 	unregister_netdevice_notifier(&macvtap_notifier_block);
1235 	class_unregister(macvtap_class);
1236 	cdev_del(&macvtap_cdev);
1237 	unregister_chrdev_region(macvtap_major, MACVTAP_NUM_DEVS);
1238 }
1239 module_exit(macvtap_exit);
1240 
1241 MODULE_ALIAS_RTNL_LINK("macvtap");
1242 MODULE_AUTHOR("Arnd Bergmann <arnd@arndb.de>");
1243 MODULE_LICENSE("GPL");
1244