virtio_net.c (f1d4884d6871ded0592604b0e72e4a7bd292eab9) virtio_net.c (503d539a6e417b018616bf3060e0b5814fafce47)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* A network driver using virtio.
3 *
4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
5 */
6//#define DEBUG
7#include <linux/netdevice.h>
8#include <linux/etherdevice.h>

--- 357 unchanged lines hidden (view full) ---

366 return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
367}
368
369/* Called from bottom half context */
370static struct sk_buff *page_to_skb(struct virtnet_info *vi,
371 struct receive_queue *rq,
372 struct page *page, unsigned int offset,
373 unsigned int len, unsigned int truesize,
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* A network driver using virtio.
3 *
4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
5 */
6//#define DEBUG
7#include <linux/netdevice.h>
8#include <linux/etherdevice.h>

--- 357 unchanged lines hidden (view full) ---

366 return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
367}
368
369/* Called from bottom half context */
370static struct sk_buff *page_to_skb(struct virtnet_info *vi,
371 struct receive_queue *rq,
372 struct page *page, unsigned int offset,
373 unsigned int len, unsigned int truesize,
374 bool hdr_valid)
374 bool hdr_valid, unsigned int metasize)
375{
376 struct sk_buff *skb;
377 struct virtio_net_hdr_mrg_rxbuf *hdr;
378 unsigned int copy, hdr_len, hdr_padded_len;
379 char *p;
380
381 p = page_address(page) + offset;
382

--- 5 unchanged lines hidden (view full) ---

388 hdr = skb_vnet_hdr(skb);
389
390 hdr_len = vi->hdr_len;
391 if (vi->mergeable_rx_bufs)
392 hdr_padded_len = sizeof(*hdr);
393 else
394 hdr_padded_len = sizeof(struct padded_vnet_hdr);
395
375{
376 struct sk_buff *skb;
377 struct virtio_net_hdr_mrg_rxbuf *hdr;
378 unsigned int copy, hdr_len, hdr_padded_len;
379 char *p;
380
381 p = page_address(page) + offset;
382

--- 5 unchanged lines hidden (view full) ---

388 hdr = skb_vnet_hdr(skb);
389
390 hdr_len = vi->hdr_len;
391 if (vi->mergeable_rx_bufs)
392 hdr_padded_len = sizeof(*hdr);
393 else
394 hdr_padded_len = sizeof(struct padded_vnet_hdr);
395
396 /* hdr_valid means no XDP, so we can copy the vnet header */
396 if (hdr_valid)
397 memcpy(hdr, p, hdr_len);
398
399 len -= hdr_len;
400 offset += hdr_padded_len;
401 p += hdr_padded_len;
402
403 copy = len;
404 if (copy > skb_tailroom(skb))
405 copy = skb_tailroom(skb);
406 skb_put_data(skb, p, copy);
407
397 if (hdr_valid)
398 memcpy(hdr, p, hdr_len);
399
400 len -= hdr_len;
401 offset += hdr_padded_len;
402 p += hdr_padded_len;
403
404 copy = len;
405 if (copy > skb_tailroom(skb))
406 copy = skb_tailroom(skb);
407 skb_put_data(skb, p, copy);
408
409 if (metasize) {
410 __skb_pull(skb, metasize);
411 skb_metadata_set(skb, metasize);
412 }
413
408 len -= copy;
409 offset += copy;
410
411 if (vi->mergeable_rx_bufs) {
412 if (len)
413 skb_add_rx_frag(skb, 0, page, offset, len, truesize);
414 else
415 put_page(page);

--- 29 unchanged lines hidden (view full) ---

445
446static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
447 struct send_queue *sq,
448 struct xdp_frame *xdpf)
449{
450 struct virtio_net_hdr_mrg_rxbuf *hdr;
451 int err;
452
414 len -= copy;
415 offset += copy;
416
417 if (vi->mergeable_rx_bufs) {
418 if (len)
419 skb_add_rx_frag(skb, 0, page, offset, len, truesize);
420 else
421 put_page(page);

--- 29 unchanged lines hidden (view full) ---

451
452static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
453 struct send_queue *sq,
454 struct xdp_frame *xdpf)
455{
456 struct virtio_net_hdr_mrg_rxbuf *hdr;
457 int err;
458
453 /* virtqueue want to use data area in-front of packet */
454 if (unlikely(xdpf->metasize > 0))
455 return -EOPNOTSUPP;
456
457 if (unlikely(xdpf->headroom < vi->hdr_len))
458 return -EOVERFLOW;
459
460 /* Make room for virtqueue hdr (also change xdpf->headroom?) */
461 xdpf->data -= vi->hdr_len;
462 /* Zero header and leave csum up to XDP layers */
463 hdr = xdpf->data;
464 memset(hdr, 0, vi->hdr_len);

--- 174 unchanged lines hidden (view full) ---

639 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
640 unsigned int headroom = vi->hdr_len + header_offset;
641 unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
642 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
643 struct page *page = virt_to_head_page(buf);
644 unsigned int delta = 0;
645 struct page *xdp_page;
646 int err;
459 if (unlikely(xdpf->headroom < vi->hdr_len))
460 return -EOVERFLOW;
461
462 /* Make room for virtqueue hdr (also change xdpf->headroom?) */
463 xdpf->data -= vi->hdr_len;
464 /* Zero header and leave csum up to XDP layers */
465 hdr = xdpf->data;
466 memset(hdr, 0, vi->hdr_len);

--- 174 unchanged lines hidden (view full) ---

641 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
642 unsigned int headroom = vi->hdr_len + header_offset;
643 unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
644 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
645 struct page *page = virt_to_head_page(buf);
646 unsigned int delta = 0;
647 struct page *xdp_page;
648 int err;
649 unsigned int metasize = 0;
647
648 len -= vi->hdr_len;
649 stats->bytes += len;
650
651 rcu_read_lock();
652 xdp_prog = rcu_dereference(rq->xdp_prog);
653 if (xdp_prog) {
654 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;

--- 23 unchanged lines hidden (view full) ---

678
679 buf = page_address(xdp_page);
680 put_page(page);
681 page = xdp_page;
682 }
683
684 xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
685 xdp.data = xdp.data_hard_start + xdp_headroom;
650
651 len -= vi->hdr_len;
652 stats->bytes += len;
653
654 rcu_read_lock();
655 xdp_prog = rcu_dereference(rq->xdp_prog);
656 if (xdp_prog) {
657 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;

--- 23 unchanged lines hidden (view full) ---

681
682 buf = page_address(xdp_page);
683 put_page(page);
684 page = xdp_page;
685 }
686
687 xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
688 xdp.data = xdp.data_hard_start + xdp_headroom;
686 xdp_set_data_meta_invalid(&xdp);
687 xdp.data_end = xdp.data + len;
689 xdp.data_end = xdp.data + len;
690 xdp.data_meta = xdp.data;
688 xdp.rxq = &rq->xdp_rxq;
689 orig_data = xdp.data;
690 act = bpf_prog_run_xdp(xdp_prog, &xdp);
691 stats->xdp_packets++;
692
693 switch (act) {
694 case XDP_PASS:
695 /* Recalculate length in case bpf program changed it */
696 delta = orig_data - xdp.data;
697 len = xdp.data_end - xdp.data;
691 xdp.rxq = &rq->xdp_rxq;
692 orig_data = xdp.data;
693 act = bpf_prog_run_xdp(xdp_prog, &xdp);
694 stats->xdp_packets++;
695
696 switch (act) {
697 case XDP_PASS:
698 /* Recalculate length in case bpf program changed it */
699 delta = orig_data - xdp.data;
700 len = xdp.data_end - xdp.data;
701 metasize = xdp.data - xdp.data_meta;
698 break;
699 case XDP_TX:
700 stats->xdp_tx++;
701 xdpf = convert_to_xdp_frame(&xdp);
702 if (unlikely(!xdpf))
703 goto err_xdp;
704 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
705 if (unlikely(err < 0)) {

--- 29 unchanged lines hidden (view full) ---

735 }
736 skb_reserve(skb, headroom - delta);
737 skb_put(skb, len);
738 if (!xdp_prog) {
739 buf += header_offset;
740 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
741 } /* keep zeroed vnet hdr since XDP is loaded */
742
702 break;
703 case XDP_TX:
704 stats->xdp_tx++;
705 xdpf = convert_to_xdp_frame(&xdp);
706 if (unlikely(!xdpf))
707 goto err_xdp;
708 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
709 if (unlikely(err < 0)) {

--- 29 unchanged lines hidden (view full) ---

739 }
740 skb_reserve(skb, headroom - delta);
741 skb_put(skb, len);
742 if (!xdp_prog) {
743 buf += header_offset;
744 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
745 } /* keep zeroed vnet hdr since XDP is loaded */
746
747 if (metasize)
748 skb_metadata_set(skb, metasize);
749
743err:
744 return skb;
745
746err_xdp:
747 rcu_read_unlock();
748 stats->xdp_drops++;
749 stats->drops++;
750 put_page(page);

--- 4 unchanged lines hidden (view full) ---

755static struct sk_buff *receive_big(struct net_device *dev,
756 struct virtnet_info *vi,
757 struct receive_queue *rq,
758 void *buf,
759 unsigned int len,
760 struct virtnet_rq_stats *stats)
761{
762 struct page *page = buf;
750err:
751 return skb;
752
753err_xdp:
754 rcu_read_unlock();
755 stats->xdp_drops++;
756 stats->drops++;
757 put_page(page);

--- 4 unchanged lines hidden (view full) ---

762static struct sk_buff *receive_big(struct net_device *dev,
763 struct virtnet_info *vi,
764 struct receive_queue *rq,
765 void *buf,
766 unsigned int len,
767 struct virtnet_rq_stats *stats)
768{
769 struct page *page = buf;
763 struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len,
764 PAGE_SIZE, true);
770 struct sk_buff *skb =
771 page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0);
765
766 stats->bytes += len - vi->hdr_len;
767 if (unlikely(!skb))
768 goto err;
769
770 return skb;
771
772err:

--- 15 unchanged lines hidden (view full) ---

788 u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
789 struct page *page = virt_to_head_page(buf);
790 int offset = buf - page_address(page);
791 struct sk_buff *head_skb, *curr_skb;
792 struct bpf_prog *xdp_prog;
793 unsigned int truesize;
794 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
795 int err;
772
773 stats->bytes += len - vi->hdr_len;
774 if (unlikely(!skb))
775 goto err;
776
777 return skb;
778
779err:

--- 15 unchanged lines hidden (view full) ---

795 u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
796 struct page *page = virt_to_head_page(buf);
797 int offset = buf - page_address(page);
798 struct sk_buff *head_skb, *curr_skb;
799 struct bpf_prog *xdp_prog;
800 unsigned int truesize;
801 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
802 int err;
803 unsigned int metasize = 0;
796
797 head_skb = NULL;
798 stats->bytes += len - vi->hdr_len;
799
800 rcu_read_lock();
801 xdp_prog = rcu_dereference(rq->xdp_prog);
802 if (xdp_prog) {
803 struct xdp_frame *xdpf;

--- 30 unchanged lines hidden (view full) ---

834 }
835
836 /* Allow consuming headroom but reserve enough space to push
837 * the descriptor on if we get an XDP_TX return code.
838 */
839 data = page_address(xdp_page) + offset;
840 xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;
841 xdp.data = data + vi->hdr_len;
804
805 head_skb = NULL;
806 stats->bytes += len - vi->hdr_len;
807
808 rcu_read_lock();
809 xdp_prog = rcu_dereference(rq->xdp_prog);
810 if (xdp_prog) {
811 struct xdp_frame *xdpf;

--- 30 unchanged lines hidden (view full) ---

842 }
843
844 /* Allow consuming headroom but reserve enough space to push
845 * the descriptor on if we get an XDP_TX return code.
846 */
847 data = page_address(xdp_page) + offset;
848 xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;
849 xdp.data = data + vi->hdr_len;
842 xdp_set_data_meta_invalid(&xdp);
843 xdp.data_end = xdp.data + (len - vi->hdr_len);
850 xdp.data_end = xdp.data + (len - vi->hdr_len);
851 xdp.data_meta = xdp.data;
844 xdp.rxq = &rq->xdp_rxq;
845
846 act = bpf_prog_run_xdp(xdp_prog, &xdp);
847 stats->xdp_packets++;
848
849 switch (act) {
850 case XDP_PASS:
852 xdp.rxq = &rq->xdp_rxq;
853
854 act = bpf_prog_run_xdp(xdp_prog, &xdp);
855 stats->xdp_packets++;
856
857 switch (act) {
858 case XDP_PASS:
859 metasize = xdp.data - xdp.data_meta;
860
851 /* recalculate offset to account for any header
861 /* recalculate offset to account for any header
852 * adjustments. Note other cases do not build an
853 * skb and avoid using offset
862 * adjustments and minus the metasize to copy the
863 * metadata in page_to_skb(). Note other cases do not
864 * build an skb and avoid using offset
854 */
865 */
855 offset = xdp.data -
856 page_address(xdp_page) - vi->hdr_len;
866 offset = xdp.data - page_address(xdp_page) -
867 vi->hdr_len - metasize;
857
868
858 /* recalculate len if xdp.data or xdp.data_end were
859 * adjusted
869 /* recalculate len if xdp.data, xdp.data_end or
870 * xdp.data_meta were adjusted
860 */
871 */
861 len = xdp.data_end - xdp.data + vi->hdr_len;
872 len = xdp.data_end - xdp.data + vi->hdr_len + metasize;
862 /* We can only create skb based on xdp_page. */
863 if (unlikely(xdp_page != page)) {
864 rcu_read_unlock();
865 put_page(page);
873 /* We can only create skb based on xdp_page. */
874 if (unlikely(xdp_page != page)) {
875 rcu_read_unlock();
876 put_page(page);
866 head_skb = page_to_skb(vi, rq, xdp_page,
867 offset, len,
868 PAGE_SIZE, false);
877 head_skb = page_to_skb(vi, rq, xdp_page, offset,
878 len, PAGE_SIZE, false,
879 metasize);
869 return head_skb;
870 }
871 break;
872 case XDP_TX:
873 stats->xdp_tx++;
874 xdpf = convert_to_xdp_frame(&xdp);
875 if (unlikely(!xdpf))
876 goto err_xdp;

--- 39 unchanged lines hidden (view full) ---

916 truesize = mergeable_ctx_to_truesize(ctx);
917 if (unlikely(len > truesize)) {
918 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
919 dev->name, len, (unsigned long)ctx);
920 dev->stats.rx_length_errors++;
921 goto err_skb;
922 }
923
880 return head_skb;
881 }
882 break;
883 case XDP_TX:
884 stats->xdp_tx++;
885 xdpf = convert_to_xdp_frame(&xdp);
886 if (unlikely(!xdpf))
887 goto err_xdp;

--- 39 unchanged lines hidden (view full) ---

927 truesize = mergeable_ctx_to_truesize(ctx);
928 if (unlikely(len > truesize)) {
929 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
930 dev->name, len, (unsigned long)ctx);
931 dev->stats.rx_length_errors++;
932 goto err_skb;
933 }
934
924 head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog);
935 head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog,
936 metasize);
925 curr_skb = head_skb;
926
927 if (unlikely(!curr_skb))
928 goto err_skb;
929 while (--num_buf) {
930 int num_skb_frags;
931
932 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);

--- 2413 unchanged lines hidden --- 		937 curr_skb = head_skb;
938
939 if (unlikely(!curr_skb))
940 goto err_skb;
941 while (--num_buf) {
942 int num_skb_frags;
943
944 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);

--- 2413 unchanged lines hidden ---