virtio_net.c (1ac731c529cd4d6adbce134754b51ff7d822b145) virtio_net.c (295525e29a5b5694a6e96864f0c1365f79639863)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* A network driver using virtio.
3 *
4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
5 */
6//#define DEBUG
7#include <linux/netdevice.h>
8#include <linux/etherdevice.h>

--- 112 unchanged lines hidden (view full) ---

121 { "xdp_redirects", VIRTNET_RQ_STAT(xdp_redirects) },
122 { "xdp_drops", VIRTNET_RQ_STAT(xdp_drops) },
123 { "kicks", VIRTNET_RQ_STAT(kicks) },
124};
125
126#define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc)
127#define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc)
128
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* A network driver using virtio.
3 *
4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
5 */
6//#define DEBUG
7#include <linux/netdevice.h>
8#include <linux/etherdevice.h>

--- 112 unchanged lines hidden (view full) ---

121 { "xdp_redirects", VIRTNET_RQ_STAT(xdp_redirects) },
122 { "xdp_drops", VIRTNET_RQ_STAT(xdp_drops) },
123 { "kicks", VIRTNET_RQ_STAT(kicks) },
124};
125
126#define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc)
127#define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc)
128
129/* The dma information of pages allocated at a time. */
130struct virtnet_rq_dma {
131 dma_addr_t addr;
132 u32 ref;
133 u16 len;
134 u16 need_sync;
135};
136
129/* Internal representation of a send virtqueue */
130struct send_queue {
131 /* Virtqueue associated with this send _queue */
132 struct virtqueue *vq;
133
134 /* TX: fragments + linear part + virtio header */
135 struct scatterlist sg[MAX_SKB_FRAGS + 2];
136

--- 33 unchanged lines hidden (view full) ---

170
171 /* Min single buffer size for mergeable buffers case. */
172 unsigned int min_buf_len;
173
174 /* Name of this receive queue: input.$index */
175 char name[16];
176
177 struct xdp_rxq_info xdp_rxq;
137/* Internal representation of a send virtqueue */
138struct send_queue {
139 /* Virtqueue associated with this send _queue */
140 struct virtqueue *vq;
141
142 /* TX: fragments + linear part + virtio header */
143 struct scatterlist sg[MAX_SKB_FRAGS + 2];
144

--- 33 unchanged lines hidden (view full) ---

178
179 /* Min single buffer size for mergeable buffers case. */
180 unsigned int min_buf_len;
181
182 /* Name of this receive queue: input.$index */
183 char name[16];
184
185 struct xdp_rxq_info xdp_rxq;
186
187 /* Record the last dma info to free after new pages is allocated. */
188 struct virtnet_rq_dma *last_dma;
189
190 /* Do dma by self */
191 bool do_dma;
178};
179
180/* This structure can contain rss message with maximum settings for indirection table and keysize
181 * Note, that default structure that describes RSS configuration virtio_net_rss_config
182 * contains same info but can't handle table values.
183 * In any case, structure would be passed to virtio hw through sg_buf split by parts
184 * because table sizes may be differ according to the device configuration.
185 */

--- 254 unchanged lines hidden (view full) ---

440 return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
441}
442
443static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
444{
445 return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
446}
447
192};
193
194/* This structure can contain rss message with maximum settings for indirection table and keysize
195 * Note, that default structure that describes RSS configuration virtio_net_rss_config
196 * contains same info but can't handle table values.
197 * In any case, structure would be passed to virtio hw through sg_buf split by parts
198 * because table sizes may be differ according to the device configuration.
199 */

--- 254 unchanged lines hidden (view full) ---

454 return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
455}
456
457static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
458{
459 return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
460}
461
462static struct sk_buff *virtnet_build_skb(void *buf, unsigned int buflen,
463 unsigned int headroom,
464 unsigned int len)
465{
466 struct sk_buff *skb;
467
468 skb = build_skb(buf, buflen);
469 if (unlikely(!skb))
470 return NULL;
471
472 skb_reserve(skb, headroom);
473 skb_put(skb, len);
474
475 return skb;
476}
477
448/* Called from bottom half context */
449static struct sk_buff *page_to_skb(struct virtnet_info *vi,
450 struct receive_queue *rq,
451 struct page *page, unsigned int offset,
452 unsigned int len, unsigned int truesize,
453 unsigned int headroom)
454{
455 struct sk_buff *skb;

--- 17 unchanged lines hidden (view full) ---

473 offset += hdr_padded_len;
474 p += hdr_padded_len;
475 tailroom = truesize - headroom - hdr_padded_len - len;
476
477 shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
478
479 /* copy small packet so we can reuse these pages */
480 if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) {
478/* Called from bottom half context */
479static struct sk_buff *page_to_skb(struct virtnet_info *vi,
480 struct receive_queue *rq,
481 struct page *page, unsigned int offset,
482 unsigned int len, unsigned int truesize,
483 unsigned int headroom)
484{
485 struct sk_buff *skb;

--- 17 unchanged lines hidden (view full) ---

503 offset += hdr_padded_len;
504 p += hdr_padded_len;
505 tailroom = truesize - headroom - hdr_padded_len - len;
506
507 shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
508
509 /* copy small packet so we can reuse these pages */
510 if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) {
481 skb = build_skb(buf, truesize);
511 skb = virtnet_build_skb(buf, truesize, p - buf, len);
482 if (unlikely(!skb))
483 return NULL;
484
512 if (unlikely(!skb))
513 return NULL;
514
485 skb_reserve(skb, p - buf);
486 skb_put(skb, len);
487
488 page = (struct page *)page->private;
489 if (page)
490 give_pages(rq, page);
491 goto ok;
492 }
493
494 /* copy small packet so we can reuse these pages for small data */
495 skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);

--- 48 unchanged lines hidden (view full) ---

544 hdr = skb_vnet_hdr(skb);
545 memcpy(hdr, hdr_p, hdr_len);
546 if (page_to_free)
547 put_page(page_to_free);
548
549 return skb;
550}
551
515 page = (struct page *)page->private;
516 if (page)
517 give_pages(rq, page);
518 goto ok;
519 }
520
521 /* copy small packet so we can reuse these pages for small data */
522 skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);

--- 48 unchanged lines hidden (view full) ---

571 hdr = skb_vnet_hdr(skb);
572 memcpy(hdr, hdr_p, hdr_len);
573 if (page_to_free)
574 put_page(page_to_free);
575
576 return skb;
577}
578
579static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len)
580{
581 struct page *page = virt_to_head_page(buf);
582 struct virtnet_rq_dma *dma;
583 void *head;
584 int offset;
585
586 head = page_address(page);
587
588 dma = head;
589
590 --dma->ref;
591
592 if (dma->ref) {
593 if (dma->need_sync && len) {
594 offset = buf - (head + sizeof(*dma));
595
596 virtqueue_dma_sync_single_range_for_cpu(rq->vq, dma->addr, offset,
597 len, DMA_FROM_DEVICE);
598 }
599
600 return;
601 }
602
603 virtqueue_dma_unmap_single_attrs(rq->vq, dma->addr, dma->len,
604 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
605 put_page(page);
606}
607
608static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx)
609{
610 void *buf;
611
612 buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
613 if (buf && rq->do_dma)
614 virtnet_rq_unmap(rq, buf, *len);
615
616 return buf;
617}
618
619static void *virtnet_rq_detach_unused_buf(struct receive_queue *rq)
620{
621 void *buf;
622
623 buf = virtqueue_detach_unused_buf(rq->vq);
624 if (buf && rq->do_dma)
625 virtnet_rq_unmap(rq, buf, 0);
626
627 return buf;
628}
629
630static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len)
631{
632 struct virtnet_rq_dma *dma;
633 dma_addr_t addr;
634 u32 offset;
635 void *head;
636
637 if (!rq->do_dma) {
638 sg_init_one(rq->sg, buf, len);
639 return;
640 }
641
642 head = page_address(rq->alloc_frag.page);
643
644 offset = buf - head;
645
646 dma = head;
647
648 addr = dma->addr - sizeof(*dma) + offset;
649
650 sg_init_table(rq->sg, 1);
651 rq->sg[0].dma_address = addr;
652 rq->sg[0].length = len;
653}
654
655static void *virtnet_rq_alloc(struct receive_queue *rq, u32 size, gfp_t gfp)
656{
657 struct page_frag *alloc_frag = &rq->alloc_frag;
658 struct virtnet_rq_dma *dma;
659 void *buf, *head;
660 dma_addr_t addr;
661
662 if (unlikely(!skb_page_frag_refill(size, alloc_frag, gfp)))
663 return NULL;
664
665 head = page_address(alloc_frag->page);
666
667 if (rq->do_dma) {
668 dma = head;
669
670 /* new pages */
671 if (!alloc_frag->offset) {
672 if (rq->last_dma) {
673 /* Now, the new page is allocated, the last dma
674 * will not be used. So the dma can be unmapped
675 * if the ref is 0.
676 */
677 virtnet_rq_unmap(rq, rq->last_dma, 0);
678 rq->last_dma = NULL;
679 }
680
681 dma->len = alloc_frag->size - sizeof(*dma);
682
683 addr = virtqueue_dma_map_single_attrs(rq->vq, dma + 1,
684 dma->len, DMA_FROM_DEVICE, 0);
685 if (virtqueue_dma_mapping_error(rq->vq, addr))
686 return NULL;
687
688 dma->addr = addr;
689 dma->need_sync = virtqueue_dma_need_sync(rq->vq, addr);
690
691 /* Add a reference to dma to prevent the entire dma from
692 * being released during error handling. This reference
693 * will be freed after the pages are no longer used.
694 */
695 get_page(alloc_frag->page);
696 dma->ref = 1;
697 alloc_frag->offset = sizeof(*dma);
698
699 rq->last_dma = dma;
700 }
701
702 ++dma->ref;
703 }
704
705 buf = head + alloc_frag->offset;
706
707 get_page(alloc_frag->page);
708 alloc_frag->offset += size;
709
710 return buf;
711}
712
713static void virtnet_rq_set_premapped(struct virtnet_info *vi)
714{
715 int i;
716
717 /* disable for big mode */
718 if (!vi->mergeable_rx_bufs && vi->big_packets)
719 return;
720
721 for (i = 0; i < vi->max_queue_pairs; i++) {
722 if (virtqueue_set_dma_premapped(vi->rq[i].vq))
723 continue;
724
725 vi->rq[i].do_dma = true;
726 }
727}
728
552static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi)
553{
554 unsigned int len;
555 unsigned int packets = 0;
556 unsigned int bytes = 0;
557 void *ptr;
558
559 while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {

--- 226 unchanged lines hidden (view full) ---

786 sq->stats.xdp_tx_drops += n - nxmit;
787 sq->stats.kicks += kicks;
788 u64_stats_update_end(&sq->stats.syncp);
789
790 virtnet_xdp_put_sq(vi, sq);
791 return ret;
792}
793
729static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi)
730{
731 unsigned int len;
732 unsigned int packets = 0;
733 unsigned int bytes = 0;
734 void *ptr;
735
736 while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {

--- 226 unchanged lines hidden (view full) ---

963 sq->stats.xdp_tx_drops += n - nxmit;
964 sq->stats.kicks += kicks;
965 u64_stats_update_end(&sq->stats.syncp);
966
967 virtnet_xdp_put_sq(vi, sq);
968 return ret;
969}
970
971static void put_xdp_frags(struct xdp_buff *xdp)
972{
973 struct skb_shared_info *shinfo;
974 struct page *xdp_page;
975 int i;
976
977 if (xdp_buff_has_frags(xdp)) {
978 shinfo = xdp_get_shared_info_from_buff(xdp);
979 for (i = 0; i < shinfo->nr_frags; i++) {
980 xdp_page = skb_frag_page(&shinfo->frags[i]);
981 put_page(xdp_page);
982 }
983 }
984}
985
986static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
987 struct net_device *dev,
988 unsigned int *xdp_xmit,
989 struct virtnet_rq_stats *stats)
990{
991 struct xdp_frame *xdpf;
992 int err;
993 u32 act;
994
995 act = bpf_prog_run_xdp(xdp_prog, xdp);
996 stats->xdp_packets++;
997
998 switch (act) {
999 case XDP_PASS:
1000 return act;
1001
1002 case XDP_TX:
1003 stats->xdp_tx++;
1004 xdpf = xdp_convert_buff_to_frame(xdp);
1005 if (unlikely(!xdpf)) {
1006 netdev_dbg(dev, "convert buff to frame failed for xdp\n");
1007 return XDP_DROP;
1008 }
1009
1010 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
1011 if (unlikely(!err)) {
1012 xdp_return_frame_rx_napi(xdpf);
1013 } else if (unlikely(err < 0)) {
1014 trace_xdp_exception(dev, xdp_prog, act);
1015 return XDP_DROP;
1016 }
1017 *xdp_xmit |= VIRTIO_XDP_TX;
1018 return act;
1019
1020 case XDP_REDIRECT:
1021 stats->xdp_redirects++;
1022 err = xdp_do_redirect(dev, xdp, xdp_prog);
1023 if (err)
1024 return XDP_DROP;
1025
1026 *xdp_xmit |= VIRTIO_XDP_REDIR;
1027 return act;
1028
1029 default:
1030 bpf_warn_invalid_xdp_action(dev, xdp_prog, act);
1031 fallthrough;
1032 case XDP_ABORTED:
1033 trace_xdp_exception(dev, xdp_prog, act);
1034 fallthrough;
1035 case XDP_DROP:
1036 return XDP_DROP;
1037 }
1038}
1039
794static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
795{
796 return vi->xdp_enabled ? VIRTIO_XDP_HEADROOM : 0;
797}
798
799/* We copy the packet for XDP in the following cases:
800 *
801 * 1) Packet is scattered across multiple rx buffers.

--- 28 unchanged lines hidden (view full) ---

830 memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
831 page_off += *len;
832
833 while (--*num_buf) {
834 unsigned int buflen;
835 void *buf;
836 int off;
837
1040static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
1041{
1042 return vi->xdp_enabled ? VIRTIO_XDP_HEADROOM : 0;
1043}
1044
1045/* We copy the packet for XDP in the following cases:
1046 *
1047 * 1) Packet is scattered across multiple rx buffers.

--- 28 unchanged lines hidden (view full) ---

1076 memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
1077 page_off += *len;
1078
1079 while (--*num_buf) {
1080 unsigned int buflen;
1081 void *buf;
1082 int off;
1083
838 buf = virtqueue_get_buf(rq->vq, &buflen);
1084 buf = virtnet_rq_get_buf(rq, &buflen, NULL);
839 if (unlikely(!buf))
840 goto err_buf;
841
842 p = virt_to_head_page(buf);
843 off = buf - page_address(p);
844
845 /* guard against a misconfigured or uncooperative backend that
846 * is sending packet larger than the MTU.

--- 12 unchanged lines hidden (view full) ---

859 /* Headroom does not contribute to packet length */
860 *len = page_off - VIRTIO_XDP_HEADROOM;
861 return page;
862err_buf:
863 __free_pages(page, 0);
864 return NULL;
865}
866
1085 if (unlikely(!buf))
1086 goto err_buf;
1087
1088 p = virt_to_head_page(buf);
1089 off = buf - page_address(p);
1090
1091 /* guard against a misconfigured or uncooperative backend that
1092 * is sending packet larger than the MTU.

--- 12 unchanged lines hidden (view full) ---

1105 /* Headroom does not contribute to packet length */
1106 *len = page_off - VIRTIO_XDP_HEADROOM;
1107 return page;
1108err_buf:
1109 __free_pages(page, 0);
1110 return NULL;
1111}
1112
867static struct sk_buff *receive_small(struct net_device *dev,
868 struct virtnet_info *vi,
869 struct receive_queue *rq,
870 void *buf, void *ctx,
871 unsigned int len,
872 unsigned int *xdp_xmit,
873 struct virtnet_rq_stats *stats)
1113static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi,
1114 unsigned int xdp_headroom,
1115 void *buf,
1116 unsigned int len)
874{
1117{
1118 unsigned int header_offset;
1119 unsigned int headroom;
1120 unsigned int buflen;
875 struct sk_buff *skb;
1121 struct sk_buff *skb;
876 struct bpf_prog *xdp_prog;
877 unsigned int xdp_headroom = (unsigned long)ctx;
1122
1123 header_offset = VIRTNET_RX_PAD + xdp_headroom;
1124 headroom = vi->hdr_len + header_offset;
1125 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
1126 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1127
1128 skb = virtnet_build_skb(buf, buflen, headroom, len);
1129 if (unlikely(!skb))
1130 return NULL;
1131
1132 buf += header_offset;
1133 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
1134
1135 return skb;
1136}
1137
1138static struct sk_buff *receive_small_xdp(struct net_device *dev,
1139 struct virtnet_info *vi,
1140 struct receive_queue *rq,
1141 struct bpf_prog *xdp_prog,
1142 void *buf,
1143 unsigned int xdp_headroom,
1144 unsigned int len,
1145 unsigned int *xdp_xmit,
1146 struct virtnet_rq_stats *stats)
1147{
878 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
879 unsigned int headroom = vi->hdr_len + header_offset;
1148 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
1149 unsigned int headroom = vi->hdr_len + header_offset;
880 unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
881 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1150 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
882 struct page *page = virt_to_head_page(buf);
1151 struct page *page = virt_to_head_page(buf);
883 unsigned int delta = 0;
884 struct page *xdp_page;
1152 struct page *xdp_page;
885 int err;
1153 unsigned int buflen;
1154 struct xdp_buff xdp;
1155 struct sk_buff *skb;
886 unsigned int metasize = 0;
1156 unsigned int metasize = 0;
1157 u32 act;
887
1158
888 len -= vi->hdr_len;
889 stats->bytes += len;
1159 if (unlikely(hdr->hdr.gso_type))
1160 goto err_xdp;
890
1161
891 if (unlikely(len > GOOD_PACKET_LEN)) {
892 pr_debug("%s: rx error: len %u exceeds max size %d\n",
893 dev->name, len, GOOD_PACKET_LEN);
894 dev->stats.rx_length_errors++;
895 goto err;
896 }
1162 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
1163 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
897
1164
898 if (likely(!vi->xdp_enabled)) {
899 xdp_prog = NULL;
900 goto skip_xdp;
901 }
1165 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
1166 int offset = buf - page_address(page) + header_offset;
1167 unsigned int tlen = len + vi->hdr_len;
1168 int num_buf = 1;
902
1169
903 rcu_read_lock();
904 xdp_prog = rcu_dereference(rq->xdp_prog);
905 if (xdp_prog) {
906 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
907 struct xdp_frame *xdpf;
908 struct xdp_buff xdp;
909 void *orig_data;
910 u32 act;
911
912 if (unlikely(hdr->hdr.gso_type))
1170 xdp_headroom = virtnet_get_headroom(vi);
1171 header_offset = VIRTNET_RX_PAD + xdp_headroom;
1172 headroom = vi->hdr_len + header_offset;
1173 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
1174 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1175 xdp_page = xdp_linearize_page(rq, &num_buf, page,
1176 offset, header_offset,
1177 &tlen);
1178 if (!xdp_page)
913 goto err_xdp;
914
1179 goto err_xdp;
1180
915 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
916 int offset = buf - page_address(page) + header_offset;
917 unsigned int tlen = len + vi->hdr_len;
918 int num_buf = 1;
1181 buf = page_address(xdp_page);
1182 put_page(page);
1183 page = xdp_page;
1184 }
919
1185
920 xdp_headroom = virtnet_get_headroom(vi);
921 header_offset = VIRTNET_RX_PAD + xdp_headroom;
922 headroom = vi->hdr_len + header_offset;
923 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
924 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
925 xdp_page = xdp_linearize_page(rq, &num_buf, page,
926 offset, header_offset,
927 &tlen);
928 if (!xdp_page)
929 goto err_xdp;
1186 xdp_init_buff(&xdp, buflen, &rq->xdp_rxq);
1187 xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len,
1188 xdp_headroom, len, true);
930
1189
931 buf = page_address(xdp_page);
932 put_page(page);
933 page = xdp_page;
934 }
1190 act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
935
1191
936 xdp_init_buff(&xdp, buflen, &rq->xdp_rxq);
937 xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len,
938 xdp_headroom, len, true);
939 orig_data = xdp.data;
940 act = bpf_prog_run_xdp(xdp_prog, &xdp);
941 stats->xdp_packets++;
1192 switch (act) {
1193 case XDP_PASS:
1194 /* Recalculate length in case bpf program changed it */
1195 len = xdp.data_end - xdp.data;
1196 metasize = xdp.data - xdp.data_meta;
1197 break;
942
1198
943 switch (act) {
944 case XDP_PASS:
945 /* Recalculate length in case bpf program changed it */
946 delta = orig_data - xdp.data;
947 len = xdp.data_end - xdp.data;
948 metasize = xdp.data - xdp.data_meta;
949 break;
950 case XDP_TX:
951 stats->xdp_tx++;
952 xdpf = xdp_convert_buff_to_frame(&xdp);
953 if (unlikely(!xdpf))
954 goto err_xdp;
955 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
956 if (unlikely(!err)) {
957 xdp_return_frame_rx_napi(xdpf);
958 } else if (unlikely(err < 0)) {
959 trace_xdp_exception(vi->dev, xdp_prog, act);
960 goto err_xdp;
961 }
962 *xdp_xmit |= VIRTIO_XDP_TX;
963 rcu_read_unlock();
964 goto xdp_xmit;
965 case XDP_REDIRECT:
966 stats->xdp_redirects++;
967 err = xdp_do_redirect(dev, &xdp, xdp_prog);
968 if (err)
969 goto err_xdp;
970 *xdp_xmit |= VIRTIO_XDP_REDIR;
971 rcu_read_unlock();
972 goto xdp_xmit;
973 default:
974 bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act);
975 fallthrough;
976 case XDP_ABORTED:
977 trace_xdp_exception(vi->dev, xdp_prog, act);
978 goto err_xdp;
979 case XDP_DROP:
980 goto err_xdp;
981 }
1199 case XDP_TX:
1200 case XDP_REDIRECT:
1201 goto xdp_xmit;
1202
1203 default:
1204 goto err_xdp;
982 }
1205 }
983 rcu_read_unlock();
984
1206
985skip_xdp:
986 skb = build_skb(buf, buflen);
987 if (!skb)
1207 skb = virtnet_build_skb(buf, buflen, xdp.data - buf, len);
1208 if (unlikely(!skb))
988 goto err;
1209 goto err;
989 skb_reserve(skb, headroom - delta);
990 skb_put(skb, len);
991 if (!xdp_prog) {
992 buf += header_offset;
993 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
994 } /* keep zeroed vnet hdr since XDP is loaded */
995
996 if (metasize)
997 skb_metadata_set(skb, metasize);
998
999 return skb;
1000
1001err_xdp:
1210
1211 if (metasize)
1212 skb_metadata_set(skb, metasize);
1213
1214 return skb;
1215
1216err_xdp:
1002 rcu_read_unlock();
1003 stats->xdp_drops++;
1004err:
1005 stats->drops++;
1006 put_page(page);
1007xdp_xmit:
1008 return NULL;
1009}
1010
1217 stats->xdp_drops++;
1218err:
1219 stats->drops++;
1220 put_page(page);
1221xdp_xmit:
1222 return NULL;
1223}
1224
1225static struct sk_buff *receive_small(struct net_device *dev,
1226 struct virtnet_info *vi,
1227 struct receive_queue *rq,
1228 void *buf, void *ctx,
1229 unsigned int len,
1230 unsigned int *xdp_xmit,
1231 struct virtnet_rq_stats *stats)
1232{
1233 unsigned int xdp_headroom = (unsigned long)ctx;
1234 struct page *page = virt_to_head_page(buf);
1235 struct sk_buff *skb;
1236
1237 len -= vi->hdr_len;
1238 stats->bytes += len;
1239
1240 if (unlikely(len > GOOD_PACKET_LEN)) {
1241 pr_debug("%s: rx error: len %u exceeds max size %d\n",
1242 dev->name, len, GOOD_PACKET_LEN);
1243 dev->stats.rx_length_errors++;
1244 goto err;
1245 }
1246
1247 if (unlikely(vi->xdp_enabled)) {
1248 struct bpf_prog *xdp_prog;
1249
1250 rcu_read_lock();
1251 xdp_prog = rcu_dereference(rq->xdp_prog);
1252 if (xdp_prog) {
1253 skb = receive_small_xdp(dev, vi, rq, xdp_prog, buf,
1254 xdp_headroom, len, xdp_xmit,
1255 stats);
1256 rcu_read_unlock();
1257 return skb;
1258 }
1259 rcu_read_unlock();
1260 }
1261
1262 skb = receive_small_build_skb(vi, xdp_headroom, buf, len);
1263 if (likely(skb))
1264 return skb;
1265
1266err:
1267 stats->drops++;
1268 put_page(page);
1269 return NULL;
1270}
1271
1011static struct sk_buff *receive_big(struct net_device *dev,
1012 struct virtnet_info *vi,
1013 struct receive_queue *rq,
1014 void *buf,
1015 unsigned int len,
1016 struct virtnet_rq_stats *stats)
1017{
1018 struct page *page = buf;

--- 7 unchanged lines hidden (view full) ---

1026 return skb;
1027
1028err:
1029 stats->drops++;
1030 give_pages(rq, page);
1031 return NULL;
1032}
1033
1272static struct sk_buff *receive_big(struct net_device *dev,
1273 struct virtnet_info *vi,
1274 struct receive_queue *rq,
1275 void *buf,
1276 unsigned int len,
1277 struct virtnet_rq_stats *stats)
1278{
1279 struct page *page = buf;

--- 7 unchanged lines hidden (view full) ---

1287 return skb;
1288
1289err:
1290 stats->drops++;
1291 give_pages(rq, page);
1292 return NULL;
1293}
1294
1295static void mergeable_buf_free(struct receive_queue *rq, int num_buf,
1296 struct net_device *dev,
1297 struct virtnet_rq_stats *stats)
1298{
1299 struct page *page;
1300 void *buf;
1301 int len;
1302
1303 while (num_buf-- > 1) {
1304 buf = virtnet_rq_get_buf(rq, &len, NULL);
1305 if (unlikely(!buf)) {
1306 pr_debug("%s: rx error: %d buffers missing\n",
1307 dev->name, num_buf);
1308 dev->stats.rx_length_errors++;
1309 break;
1310 }
1311 stats->bytes += len;
1312 page = virt_to_head_page(buf);
1313 put_page(page);
1314 }
1315}
1316
1034/* Why not use xdp_build_skb_from_frame() ?
1035 * XDP core assumes that xdp frags are PAGE_SIZE in length, while in
1036 * virtio-net there are 2 points that do not match its requirements:
1037 * 1. The size of the prefilled buffer is not fixed before xdp is set.
1038 * 2. xdp_build_skb_from_frame() does more checks that we don't need,
1039 * like eth_type_trans() (which virtio-net does in receive_buf()).
1040 */
1041static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev,

--- 79 unchanged lines hidden (view full) ---

1121 shinfo->nr_frags = 0;
1122 shinfo->xdp_frags_size = 0;
1123 }
1124
1125 if (*num_buf > MAX_SKB_FRAGS + 1)
1126 return -EINVAL;
1127
1128 while (--*num_buf > 0) {
1317/* Why not use xdp_build_skb_from_frame() ?
1318 * XDP core assumes that xdp frags are PAGE_SIZE in length, while in
1319 * virtio-net there are 2 points that do not match its requirements:
1320 * 1. The size of the prefilled buffer is not fixed before xdp is set.
1321 * 2. xdp_build_skb_from_frame() does more checks that we don't need,
1322 * like eth_type_trans() (which virtio-net does in receive_buf()).
1323 */
1324static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev,

--- 79 unchanged lines hidden (view full) ---

1404 shinfo->nr_frags = 0;
1405 shinfo->xdp_frags_size = 0;
1406 }
1407
1408 if (*num_buf > MAX_SKB_FRAGS + 1)
1409 return -EINVAL;
1410
1411 while (--*num_buf > 0) {
1129 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
1412 buf = virtnet_rq_get_buf(rq, &len, &ctx);
1130 if (unlikely(!buf)) {
1131 pr_debug("%s: rx error: %d buffers out of %d missing\n",
1132 dev->name, *num_buf,
1133 virtio16_to_cpu(vi->vdev, hdr->num_buffers));
1134 dev->stats.rx_length_errors++;
1413 if (unlikely(!buf)) {
1414 pr_debug("%s: rx error: %d buffers out of %d missing\n",
1415 dev->name, *num_buf,
1416 virtio16_to_cpu(vi->vdev, hdr->num_buffers));
1417 dev->stats.rx_length_errors++;
1135 return -EINVAL;
1418 goto err;
1136 }
1137
1138 stats->bytes += len;
1139 page = virt_to_head_page(buf);
1140 offset = buf - page_address(page);
1141
1142 truesize = mergeable_ctx_to_truesize(ctx);
1143 headroom = mergeable_ctx_to_headroom(ctx);
1144 tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1145 room = SKB_DATA_ALIGN(headroom + tailroom);
1146
1147 cur_frag_size = truesize;
1148 xdp_frags_truesz += cur_frag_size;
1149 if (unlikely(len > truesize - room || cur_frag_size > PAGE_SIZE)) {
1150 put_page(page);
1151 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
1152 dev->name, len, (unsigned long)(truesize - room));
1153 dev->stats.rx_length_errors++;
1419 }
1420
1421 stats->bytes += len;
1422 page = virt_to_head_page(buf);
1423 offset = buf - page_address(page);
1424
1425 truesize = mergeable_ctx_to_truesize(ctx);
1426 headroom = mergeable_ctx_to_headroom(ctx);
1427 tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1428 room = SKB_DATA_ALIGN(headroom + tailroom);
1429
1430 cur_frag_size = truesize;
1431 xdp_frags_truesz += cur_frag_size;
1432 if (unlikely(len > truesize - room || cur_frag_size > PAGE_SIZE)) {
1433 put_page(page);
1434 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
1435 dev->name, len, (unsigned long)(truesize - room));
1436 dev->stats.rx_length_errors++;
1154 return -EINVAL;
1437 goto err;
1155 }
1156
1157 frag = &shinfo->frags[shinfo->nr_frags++];
1438 }
1439
1440 frag = &shinfo->frags[shinfo->nr_frags++];
1158 __skb_frag_set_page(frag, page);
1159 skb_frag_off_set(frag, offset);
1160 skb_frag_size_set(frag, len);
1441 skb_frag_fill_page_desc(frag, page, offset, len);
1161 if (page_is_pfmemalloc(page))
1162 xdp_buff_set_frag_pfmemalloc(xdp);
1163
1164 shinfo->xdp_frags_size += len;
1165 }
1166
1167 *xdp_frags_truesize = xdp_frags_truesz;
1168 return 0;
1442 if (page_is_pfmemalloc(page))
1443 xdp_buff_set_frag_pfmemalloc(xdp);
1444
1445 shinfo->xdp_frags_size += len;
1446 }
1447
1448 *xdp_frags_truesize = xdp_frags_truesz;
1449 return 0;
1450
1451err:
1452 put_xdp_frags(xdp);
1453 return -EINVAL;
1169}
1170
1454}
1455
1456static void *mergeable_xdp_get_buf(struct virtnet_info *vi,
1457 struct receive_queue *rq,
1458 struct bpf_prog *xdp_prog,
1459 void *ctx,
1460 unsigned int *frame_sz,
1461 int *num_buf,
1462 struct page **page,
1463 int offset,
1464 unsigned int *len,
1465 struct virtio_net_hdr_mrg_rxbuf *hdr)
1466{
1467 unsigned int truesize = mergeable_ctx_to_truesize(ctx);
1468 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
1469 struct page *xdp_page;
1470 unsigned int xdp_room;
1471
1472 /* Transient failure which in theory could occur if
1473 * in-flight packets from before XDP was enabled reach
1474 * the receive path after XDP is loaded.
1475 */
1476 if (unlikely(hdr->hdr.gso_type))
1477 return NULL;
1478
1479 /* Now XDP core assumes frag size is PAGE_SIZE, but buffers
1480 * with headroom may add hole in truesize, which
1481 * make their length exceed PAGE_SIZE. So we disabled the
1482 * hole mechanism for xdp. See add_recvbuf_mergeable().
1483 */
1484 *frame_sz = truesize;
1485
1486 if (likely(headroom >= virtnet_get_headroom(vi) &&
1487 (*num_buf == 1 || xdp_prog->aux->xdp_has_frags))) {
1488 return page_address(*page) + offset;
1489 }
1490
1491 /* This happens when headroom is not enough because
1492 * of the buffer was prefilled before XDP is set.
1493 * This should only happen for the first several packets.
1494 * In fact, vq reset can be used here to help us clean up
1495 * the prefilled buffers, but many existing devices do not
1496 * support it, and we don't want to bother users who are
1497 * using xdp normally.
1498 */
1499 if (!xdp_prog->aux->xdp_has_frags) {
1500 /* linearize data for XDP */
1501 xdp_page = xdp_linearize_page(rq, num_buf,
1502 *page, offset,
1503 VIRTIO_XDP_HEADROOM,
1504 len);
1505 if (!xdp_page)
1506 return NULL;
1507 } else {
1508 xdp_room = SKB_DATA_ALIGN(VIRTIO_XDP_HEADROOM +
1509 sizeof(struct skb_shared_info));
1510 if (*len + xdp_room > PAGE_SIZE)
1511 return NULL;
1512
1513 xdp_page = alloc_page(GFP_ATOMIC);
1514 if (!xdp_page)
1515 return NULL;
1516
1517 memcpy(page_address(xdp_page) + VIRTIO_XDP_HEADROOM,
1518 page_address(*page) + offset, *len);
1519 }
1520
1521 *frame_sz = PAGE_SIZE;
1522
1523 put_page(*page);
1524
1525 *page = xdp_page;
1526
1527 return page_address(*page) + VIRTIO_XDP_HEADROOM;
1528}
1529
1530static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
1531 struct virtnet_info *vi,
1532 struct receive_queue *rq,
1533 struct bpf_prog *xdp_prog,
1534 void *buf,
1535 void *ctx,
1536 unsigned int len,
1537 unsigned int *xdp_xmit,
1538 struct virtnet_rq_stats *stats)
1539{
1540 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
1541 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
1542 struct page *page = virt_to_head_page(buf);
1543 int offset = buf - page_address(page);
1544 unsigned int xdp_frags_truesz = 0;
1545 struct sk_buff *head_skb;
1546 unsigned int frame_sz;
1547 struct xdp_buff xdp;
1548 void *data;
1549 u32 act;
1550 int err;
1551
1552 data = mergeable_xdp_get_buf(vi, rq, xdp_prog, ctx, &frame_sz, &num_buf, &page,
1553 offset, &len, hdr);
1554 if (unlikely(!data))
1555 goto err_xdp;
1556
1557 err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz,
1558 &num_buf, &xdp_frags_truesz, stats);
1559 if (unlikely(err))
1560 goto err_xdp;
1561
1562 act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
1563
1564 switch (act) {
1565 case XDP_PASS:
1566 head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
1567 if (unlikely(!head_skb))
1568 break;
1569 return head_skb;
1570
1571 case XDP_TX:
1572 case XDP_REDIRECT:
1573 return NULL;
1574
1575 default:
1576 break;
1577 }
1578
1579 put_xdp_frags(&xdp);
1580
1581err_xdp:
1582 put_page(page);
1583 mergeable_buf_free(rq, num_buf, dev, stats);
1584
1585 stats->xdp_drops++;
1586 stats->drops++;
1587 return NULL;
1588}
1589
1171static struct sk_buff *receive_mergeable(struct net_device *dev,
1172 struct virtnet_info *vi,
1173 struct receive_queue *rq,
1174 void *buf,
1175 void *ctx,
1176 unsigned int len,
1177 unsigned int *xdp_xmit,
1178 struct virtnet_rq_stats *stats)
1179{
1180 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
1181 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
1182 struct page *page = virt_to_head_page(buf);
1183 int offset = buf - page_address(page);
1184 struct sk_buff *head_skb, *curr_skb;
1590static struct sk_buff *receive_mergeable(struct net_device *dev,
1591 struct virtnet_info *vi,
1592 struct receive_queue *rq,
1593 void *buf,
1594 void *ctx,
1595 unsigned int len,
1596 unsigned int *xdp_xmit,
1597 struct virtnet_rq_stats *stats)
1598{
1599 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
1600 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
1601 struct page *page = virt_to_head_page(buf);
1602 int offset = buf - page_address(page);
1603 struct sk_buff *head_skb, *curr_skb;
1185 struct bpf_prog *xdp_prog;
1186 unsigned int truesize = mergeable_ctx_to_truesize(ctx);
1187 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
1188 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1189 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
1604 unsigned int truesize = mergeable_ctx_to_truesize(ctx);
1605 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
1606 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1607 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
1190 unsigned int frame_sz, xdp_room;
1191 int err;
1192
1193 head_skb = NULL;
1194 stats->bytes += len - vi->hdr_len;
1195
1196 if (unlikely(len > truesize - room)) {
1197 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
1198 dev->name, len, (unsigned long)(truesize - room));
1199 dev->stats.rx_length_errors++;
1200 goto err_skb;
1201 }
1202
1608
1609 head_skb = NULL;
1610 stats->bytes += len - vi->hdr_len;
1611
1612 if (unlikely(len > truesize - room)) {
1613 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
1614 dev->name, len, (unsigned long)(truesize - room));
1615 dev->stats.rx_length_errors++;
1616 goto err_skb;
1617 }
1618
1203 if (likely(!vi->xdp_enabled)) {
1204 xdp_prog = NULL;
1205 goto skip_xdp;
1206 }
1619 if (unlikely(vi->xdp_enabled)) {
1620 struct bpf_prog *xdp_prog;
1207
1621
1208 rcu_read_lock();
1209 xdp_prog = rcu_dereference(rq->xdp_prog);
1210 if (xdp_prog) {
1211 unsigned int xdp_frags_truesz = 0;
1212 struct skb_shared_info *shinfo;
1213 struct xdp_frame *xdpf;
1214 struct page *xdp_page;
1215 struct xdp_buff xdp;
1216 void *data;
1217 u32 act;
1218 int i;
1219
1220 /* Transient failure which in theory could occur if
1221 * in-flight packets from before XDP was enabled reach
1222 * the receive path after XDP is loaded.
1223 */
1224 if (unlikely(hdr->hdr.gso_type))
1225 goto err_xdp;
1226
1227 /* Now XDP core assumes frag size is PAGE_SIZE, but buffers
1228 * with headroom may add hole in truesize, which
1229 * make their length exceed PAGE_SIZE. So we disabled the
1230 * hole mechanism for xdp. See add_recvbuf_mergeable().
1231 */
1232 frame_sz = truesize;
1233
1234 /* This happens when headroom is not enough because
1235 * of the buffer was prefilled before XDP is set.
1236 * This should only happen for the first several packets.
1237 * In fact, vq reset can be used here to help us clean up
1238 * the prefilled buffers, but many existing devices do not
1239 * support it, and we don't want to bother users who are
1240 * using xdp normally.
1241 */
1242 if (!xdp_prog->aux->xdp_has_frags &&
1243 (num_buf > 1 || headroom < virtnet_get_headroom(vi))) {
1244 /* linearize data for XDP */
1245 xdp_page = xdp_linearize_page(rq, &num_buf,
1246 page, offset,
1247 VIRTIO_XDP_HEADROOM,
1248 &len);
1249 frame_sz = PAGE_SIZE;
1250
1251 if (!xdp_page)
1252 goto err_xdp;
1253 offset = VIRTIO_XDP_HEADROOM;
1254 } else if (unlikely(headroom < virtnet_get_headroom(vi))) {
1255 xdp_room = SKB_DATA_ALIGN(VIRTIO_XDP_HEADROOM +
1256 sizeof(struct skb_shared_info));
1257 if (len + xdp_room > PAGE_SIZE)
1258 goto err_xdp;
1259
1260 xdp_page = alloc_page(GFP_ATOMIC);
1261 if (!xdp_page)
1262 goto err_xdp;
1263
1264 memcpy(page_address(xdp_page) + VIRTIO_XDP_HEADROOM,
1265 page_address(page) + offset, len);
1266 frame_sz = PAGE_SIZE;
1267 offset = VIRTIO_XDP_HEADROOM;
1268 } else {
1269 xdp_page = page;
1270 }
1271
1272 data = page_address(xdp_page) + offset;
1273 err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz,
1274 &num_buf, &xdp_frags_truesz, stats);
1275 if (unlikely(err))
1276 goto err_xdp_frags;
1277
1278 act = bpf_prog_run_xdp(xdp_prog, &xdp);
1279 stats->xdp_packets++;
1280
1281 switch (act) {
1282 case XDP_PASS:
1283 head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
1284 if (unlikely(!head_skb))
1285 goto err_xdp_frags;
1286
1287 if (unlikely(xdp_page != page))
1288 put_page(page);
1622 rcu_read_lock();
1623 xdp_prog = rcu_dereference(rq->xdp_prog);
1624 if (xdp_prog) {
1625 head_skb = receive_mergeable_xdp(dev, vi, rq, xdp_prog, buf, ctx,
1626 len, xdp_xmit, stats);
1289 rcu_read_unlock();
1290 return head_skb;
1627 rcu_read_unlock();
1628 return head_skb;
1291 case XDP_TX:
1292 stats->xdp_tx++;
1293 xdpf = xdp_convert_buff_to_frame(&xdp);
1294 if (unlikely(!xdpf)) {
1295 netdev_dbg(dev, "convert buff to frame failed for xdp\n");
1296 goto err_xdp_frags;
1297 }
1298 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
1299 if (unlikely(!err)) {
1300 xdp_return_frame_rx_napi(xdpf);
1301 } else if (unlikely(err < 0)) {
1302 trace_xdp_exception(vi->dev, xdp_prog, act);
1303 goto err_xdp_frags;
1304 }
1305 *xdp_xmit |= VIRTIO_XDP_TX;
1306 if (unlikely(xdp_page != page))
1307 put_page(page);
1308 rcu_read_unlock();
1309 goto xdp_xmit;
1310 case XDP_REDIRECT:
1311 stats->xdp_redirects++;
1312 err = xdp_do_redirect(dev, &xdp, xdp_prog);
1313 if (err)
1314 goto err_xdp_frags;
1315 *xdp_xmit |= VIRTIO_XDP_REDIR;
1316 if (unlikely(xdp_page != page))
1317 put_page(page);
1318 rcu_read_unlock();
1319 goto xdp_xmit;
1320 default:
1321 bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act);
1322 fallthrough;
1323 case XDP_ABORTED:
1324 trace_xdp_exception(vi->dev, xdp_prog, act);
1325 fallthrough;
1326 case XDP_DROP:
1327 goto err_xdp_frags;
1328 }
1629 }
1329err_xdp_frags:
1330 if (unlikely(xdp_page != page))
1331 __free_pages(xdp_page, 0);
1332
1333 if (xdp_buff_has_frags(&xdp)) {
1334 shinfo = xdp_get_shared_info_from_buff(&xdp);
1335 for (i = 0; i < shinfo->nr_frags; i++) {
1336 xdp_page = skb_frag_page(&shinfo->frags[i]);
1337 put_page(xdp_page);
1338 }
1339 }
1340
1341 goto err_xdp;
1630 rcu_read_unlock();
1342 }
1631 }
1343 rcu_read_unlock();
1344
1632
1345skip_xdp:
1346 head_skb = page_to_skb(vi, rq, page, offset, len, truesize, headroom);
1347 curr_skb = head_skb;
1348
1349 if (unlikely(!curr_skb))
1350 goto err_skb;
1351 while (--num_buf) {
1352 int num_skb_frags;
1353
1633 head_skb = page_to_skb(vi, rq, page, offset, len, truesize, headroom);
1634 curr_skb = head_skb;
1635
1636 if (unlikely(!curr_skb))
1637 goto err_skb;
1638 while (--num_buf) {
1639 int num_skb_frags;
1640
1354 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
1641 buf = virtnet_rq_get_buf(rq, &len, &ctx);
1355 if (unlikely(!buf)) {
1356 pr_debug("%s: rx error: %d buffers out of %d missing\n",
1357 dev->name, num_buf,
1358 virtio16_to_cpu(vi->vdev,
1359 hdr->num_buffers));
1360 dev->stats.rx_length_errors++;
1361 goto err_buf;
1362 }

--- 40 unchanged lines hidden (view full) ---

1403 skb_add_rx_frag(curr_skb, num_skb_frags, page,
1404 offset, len, truesize);
1405 }
1406 }
1407
1408 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
1409 return head_skb;
1410
1642 if (unlikely(!buf)) {
1643 pr_debug("%s: rx error: %d buffers out of %d missing\n",
1644 dev->name, num_buf,
1645 virtio16_to_cpu(vi->vdev,
1646 hdr->num_buffers));
1647 dev->stats.rx_length_errors++;
1648 goto err_buf;
1649 }

--- 40 unchanged lines hidden (view full) ---

1690 skb_add_rx_frag(curr_skb, num_skb_frags, page,
1691 offset, len, truesize);
1692 }
1693 }
1694
1695 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
1696 return head_skb;
1697
1411err_xdp:
1412 rcu_read_unlock();
1413 stats->xdp_drops++;
1414err_skb:
1415 put_page(page);
1698err_skb:
1699 put_page(page);
1416 while (num_buf-- > 1) {
1417 buf = virtqueue_get_buf(rq->vq, &len);
1418 if (unlikely(!buf)) {
1419 pr_debug("%s: rx error: %d buffers missing\n",
1420 dev->name, num_buf);
1421 dev->stats.rx_length_errors++;
1422 break;
1423 }
1424 stats->bytes += len;
1425 page = virt_to_head_page(buf);
1426 put_page(page);
1427 }
1700 mergeable_buf_free(rq, num_buf, dev, stats);
1701
1428err_buf:
1429 stats->drops++;
1430 dev_kfree_skb(head_skb);
1702err_buf:
1703 stats->drops++;
1704 dev_kfree_skb(head_skb);
1431xdp_xmit:
1432 return NULL;
1433}
1434
1435static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
1436 struct sk_buff *skb)
1437{
1438 enum pkt_hash_types rss_hash_type;
1439

--- 79 unchanged lines hidden (view full) ---

1519/* Unlike mergeable buffers, all buffers are allocated to the
1520 * same size, except for the headroom. For this reason we do
1521 * not need to use mergeable_len_to_ctx here - it is enough
1522 * to store the headroom as the context ignoring the truesize.
1523 */
1524static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
1525 gfp_t gfp)
1526{
1705 return NULL;
1706}
1707
1708static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
1709 struct sk_buff *skb)
1710{
1711 enum pkt_hash_types rss_hash_type;
1712

--- 79 unchanged lines hidden (view full) ---

1792/* Unlike mergeable buffers, all buffers are allocated to the
1793 * same size, except for the headroom. For this reason we do
1794 * not need to use mergeable_len_to_ctx here - it is enough
1795 * to store the headroom as the context ignoring the truesize.
1796 */
1797static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
1798 gfp_t gfp)
1799{
1527 struct page_frag *alloc_frag = &rq->alloc_frag;
1528 char *buf;
1529 unsigned int xdp_headroom = virtnet_get_headroom(vi);
1530 void *ctx = (void *)(unsigned long)xdp_headroom;
1531 int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
1532 int err;
1533
1534 len = SKB_DATA_ALIGN(len) +
1535 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1800 char *buf;
1801 unsigned int xdp_headroom = virtnet_get_headroom(vi);
1802 void *ctx = (void *)(unsigned long)xdp_headroom;
1803 int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
1804 int err;
1805
1806 len = SKB_DATA_ALIGN(len) +
1807 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1536 if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
1808
1809 buf = virtnet_rq_alloc(rq, len, gfp);
1810 if (unlikely(!buf))
1537 return -ENOMEM;
1538
1811 return -ENOMEM;
1812
1539 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1540 get_page(alloc_frag->page);
1541 alloc_frag->offset += len;
1542 sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
1543 vi->hdr_len + GOOD_PACKET_LEN);
1813 virtnet_rq_init_one_sg(rq, buf + VIRTNET_RX_PAD + xdp_headroom,
1814 vi->hdr_len + GOOD_PACKET_LEN);
1815
1544 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1816 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1545 if (err < 0)
1817 if (err < 0) {
1818 if (rq->do_dma)
1819 virtnet_rq_unmap(rq, buf, 0);
1546 put_page(virt_to_head_page(buf));
1820 put_page(virt_to_head_page(buf));
1821 }
1822
1547 return err;
1548}
1549
1550static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
1551 gfp_t gfp)
1552{
1553 struct page *first, *list = NULL;
1554 char *p;

--- 60 unchanged lines hidden (view full) ---

1615
1616static int add_recvbuf_mergeable(struct virtnet_info *vi,
1617 struct receive_queue *rq, gfp_t gfp)
1618{
1619 struct page_frag *alloc_frag = &rq->alloc_frag;
1620 unsigned int headroom = virtnet_get_headroom(vi);
1621 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1622 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
1823 return err;
1824}
1825
1826static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
1827 gfp_t gfp)
1828{
1829 struct page *first, *list = NULL;
1830 char *p;

--- 60 unchanged lines hidden (view full) ---

1891
1892static int add_recvbuf_mergeable(struct virtnet_info *vi,
1893 struct receive_queue *rq, gfp_t gfp)
1894{
1895 struct page_frag *alloc_frag = &rq->alloc_frag;
1896 unsigned int headroom = virtnet_get_headroom(vi);
1897 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1898 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
1623 char *buf;
1899 unsigned int len, hole;
1624 void *ctx;
1900 void *ctx;
1901 char *buf;
1625 int err;
1902 int err;
1626 unsigned int len, hole;
1627
1628 /* Extra tailroom is needed to satisfy XDP's assumption. This
1629 * means rx frags coalescing won't work, but consider we've
1630 * disabled GSO for XDP, it won't be a big issue.
1631 */
1632 len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
1903
1904 /* Extra tailroom is needed to satisfy XDP's assumption. This
1905 * means rx frags coalescing won't work, but consider we've
1906 * disabled GSO for XDP, it won't be a big issue.
1907 */
1908 len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
1633 if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
1909
1910 buf = virtnet_rq_alloc(rq, len + room, gfp);
1911 if (unlikely(!buf))
1634 return -ENOMEM;
1635
1912 return -ENOMEM;
1913
1636 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1637 buf += headroom; /* advance address leaving hole at front of pkt */
1914 buf += headroom; /* advance address leaving hole at front of pkt */
1638 get_page(alloc_frag->page);
1639 alloc_frag->offset += len + room;
1640 hole = alloc_frag->size - alloc_frag->offset;
1641 if (hole < len + room) {
1642 /* To avoid internal fragmentation, if there is very likely not
1643 * enough space for another buffer, add the remaining space to
1644 * the current buffer.
1645 * XDP core assumes that frame_size of xdp_buff and the length
1646 * of the frag are PAGE_SIZE, so we disable the hole mechanism.
1647 */
1648 if (!headroom)
1649 len += hole;
1650 alloc_frag->offset += hole;
1651 }
1652
1915 hole = alloc_frag->size - alloc_frag->offset;
1916 if (hole < len + room) {
1917 /* To avoid internal fragmentation, if there is very likely not
1918 * enough space for another buffer, add the remaining space to
1919 * the current buffer.
1920 * XDP core assumes that frame_size of xdp_buff and the length
1921 * of the frag are PAGE_SIZE, so we disable the hole mechanism.
1922 */
1923 if (!headroom)
1924 len += hole;
1925 alloc_frag->offset += hole;
1926 }
1927
1653 sg_init_one(rq->sg, buf, len);
1928 virtnet_rq_init_one_sg(rq, buf, len);
1929
1654 ctx = mergeable_len_to_ctx(len + room, headroom);
1655 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1930 ctx = mergeable_len_to_ctx(len + room, headroom);
1931 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1656 if (err < 0)
1932 if (err < 0) {
1933 if (rq->do_dma)
1934 virtnet_rq_unmap(rq, buf, 0);
1657 put_page(virt_to_head_page(buf));
1935 put_page(virt_to_head_page(buf));
1936 }
1658
1659 return err;
1660}
1661
1662/*
1663 * Returns false if we couldn't fill entirely (OOM).
1664 *
1665 * Normally run in the receive path, but can also be run from ndo_open

--- 104 unchanged lines hidden (view full) ---

1770 unsigned int len;
1771 void *buf;
1772 int i;
1773
1774 if (!vi->big_packets || vi->mergeable_rx_bufs) {
1775 void *ctx;
1776
1777 while (stats.packets < budget &&
1937
1938 return err;
1939}
1940
1941/*
1942 * Returns false if we couldn't fill entirely (OOM).
1943 *
1944 * Normally run in the receive path, but can also be run from ndo_open

--- 104 unchanged lines hidden (view full) ---

2049 unsigned int len;
2050 void *buf;
2051 int i;
2052
2053 if (!vi->big_packets || vi->mergeable_rx_bufs) {
2054 void *ctx;
2055
2056 while (stats.packets < budget &&
1778 (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {
2057 (buf = virtnet_rq_get_buf(rq, &len, &ctx))) {
1779 receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats);
1780 stats.packets++;
1781 }
1782 } else {
1783 while (stats.packets < budget &&
2058 receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats);
2059 stats.packets++;
2060 }
2061 } else {
2062 while (stats.packets < budget &&
1784 (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
2063 (buf = virtnet_rq_get_buf(rq, &len, NULL)) != NULL) {
1785 receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats);
1786 stats.packets++;
1787 }
1788 }
1789
1790 if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) {
1791 if (!try_fill_recv(vi, rq, GFP_ATOMIC)) {
1792 spin_lock(&vi->refill_lock);

--- 854 unchanged lines hidden (view full) ---

2647 ? vi->rss_indir_table_size - 1 : 0;
2648 vi->ctrl->rss.unclassified_queue = 0;
2649
2650 for (; i < vi->rss_indir_table_size; ++i) {
2651 indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs);
2652 vi->ctrl->rss.indirection_table[i] = indir_val;
2653 }
2654
2064 receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats);
2065 stats.packets++;
2066 }
2067 }
2068
2069 if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) {
2070 if (!try_fill_recv(vi, rq, GFP_ATOMIC)) {
2071 spin_lock(&vi->refill_lock);

--- 854 unchanged lines hidden (view full) ---

2926 ? vi->rss_indir_table_size - 1 : 0;
2927 vi->ctrl->rss.unclassified_queue = 0;
2928
2929 for (; i < vi->rss_indir_table_size; ++i) {
2930 indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs);
2931 vi->ctrl->rss.indirection_table[i] = indir_val;
2932 }
2933
2655 vi->ctrl->rss.max_tx_vq = vi->curr_queue_pairs;
2934 vi->ctrl->rss.max_tx_vq = vi->has_rss ? vi->curr_queue_pairs : 0;
2656 vi->ctrl->rss.hash_key_length = vi->rss_key_size;
2657
2658 netdev_rss_key_fill(vi->ctrl->rss.key, vi->rss_key_size);
2659}
2660
2661static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info)
2662{
2663 info->data = 0;

--- 884 unchanged lines hidden (view full) ---

3548 _free_receive_bufs(vi);
3549 rtnl_unlock();
3550}
3551
3552static void free_receive_page_frags(struct virtnet_info *vi)
3553{
3554 int i;
3555 for (i = 0; i < vi->max_queue_pairs; i++)
2935 vi->ctrl->rss.hash_key_length = vi->rss_key_size;
2936
2937 netdev_rss_key_fill(vi->ctrl->rss.key, vi->rss_key_size);
2938}
2939
2940static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info)
2941{
2942 info->data = 0;

--- 884 unchanged lines hidden (view full) ---

3827 _free_receive_bufs(vi);
3828 rtnl_unlock();
3829}
3830
3831static void free_receive_page_frags(struct virtnet_info *vi)
3832{
3833 int i;
3834 for (i = 0; i < vi->max_queue_pairs; i++)
3556 if (vi->rq[i].alloc_frag.page)
3835 if (vi->rq[i].alloc_frag.page) {
3836 if (vi->rq[i].do_dma && vi->rq[i].last_dma)
3837 virtnet_rq_unmap(&vi->rq[i], vi->rq[i].last_dma, 0);
3557 put_page(vi->rq[i].alloc_frag.page);
3838 put_page(vi->rq[i].alloc_frag.page);
3839 }
3558}
3559
3560static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
3561{
3562 if (!is_xdp_frame(buf))
3563 dev_kfree_skb(buf);
3564 else
3565 xdp_return_frame(ptr_to_xdp(buf));

--- 20 unchanged lines hidden (view full) ---

3586 for (i = 0; i < vi->max_queue_pairs; i++) {
3587 struct virtqueue *vq = vi->sq[i].vq;
3588 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
3589 virtnet_sq_free_unused_buf(vq, buf);
3590 cond_resched();
3591 }
3592
3593 for (i = 0; i < vi->max_queue_pairs; i++) {
3840}
3841
3842static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
3843{
3844 if (!is_xdp_frame(buf))
3845 dev_kfree_skb(buf);
3846 else
3847 xdp_return_frame(ptr_to_xdp(buf));

--- 20 unchanged lines hidden (view full) ---

3868 for (i = 0; i < vi->max_queue_pairs; i++) {
3869 struct virtqueue *vq = vi->sq[i].vq;
3870 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
3871 virtnet_sq_free_unused_buf(vq, buf);
3872 cond_resched();
3873 }
3874
3875 for (i = 0; i < vi->max_queue_pairs; i++) {
3594 struct virtqueue *vq = vi->rq[i].vq;
3595 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
3596 virtnet_rq_free_unused_buf(vq, buf);
3876 struct receive_queue *rq = &vi->rq[i];
3877
3878 while ((buf = virtnet_rq_detach_unused_buf(rq)) != NULL)
3879 virtnet_rq_free_unused_buf(rq->vq, buf);
3597 cond_resched();
3598 }
3599}
3600
3601static void virtnet_del_vqs(struct virtnet_info *vi)
3602{
3603 struct virtio_device *vdev = vi->vdev;
3604

--- 157 unchanged lines hidden (view full) ---

3762 ret = virtnet_alloc_queues(vi);
3763 if (ret)
3764 goto err;
3765
3766 ret = virtnet_find_vqs(vi);
3767 if (ret)
3768 goto err_free;
3769
3880 cond_resched();
3881 }
3882}
3883
3884static void virtnet_del_vqs(struct virtnet_info *vi)
3885{
3886 struct virtio_device *vdev = vi->vdev;
3887

--- 157 unchanged lines hidden (view full) ---

4045 ret = virtnet_alloc_queues(vi);
4046 if (ret)
4047 goto err;
4048
4049 ret = virtnet_find_vqs(vi);
4050 if (ret)
4051 goto err_free;
4052
4053 virtnet_rq_set_premapped(vi);
4054
3770 cpus_read_lock();
3771 virtnet_set_affinity(vi);
3772 cpus_read_unlock();
3773
3774 return 0;
3775
3776err_free:
3777 virtnet_free_queues(vi);

--- 339 unchanged lines hidden (view full) ---

4117 if (err) {
4118 pr_debug("virtio_net: registering device failed\n");
4119 rtnl_unlock();
4120 goto free_failover;
4121 }
4122
4123 virtio_device_ready(vdev);
4124
4055 cpus_read_lock();
4056 virtnet_set_affinity(vi);
4057 cpus_read_unlock();
4058
4059 return 0;
4060
4061err_free:
4062 virtnet_free_queues(vi);

--- 339 unchanged lines hidden (view full) ---

4402 if (err) {
4403 pr_debug("virtio_net: registering device failed\n");
4404 rtnl_unlock();
4405 goto free_failover;
4406 }
4407
4408 virtio_device_ready(vdev);
4409
4410 _virtnet_set_queues(vi, vi->curr_queue_pairs);
4411
4125 /* a random MAC address has been assigned, notify the device.
4126 * We don't fail probe if VIRTIO_NET_F_CTRL_MAC_ADDR is not there
4127 * because many devices work fine without getting MAC explicitly
4128 */
4129 if (!virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
4130 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
4131 struct scatterlist sg;
4132

--- 10 unchanged lines hidden (view full) ---

4143 rtnl_unlock();
4144
4145 err = virtnet_cpu_notif_add(vi);
4146 if (err) {
4147 pr_debug("virtio_net: registering cpu notifier failed\n");
4148 goto free_unregister_netdev;
4149 }
4150
4412 /* a random MAC address has been assigned, notify the device.
4413 * We don't fail probe if VIRTIO_NET_F_CTRL_MAC_ADDR is not there
4414 * because many devices work fine without getting MAC explicitly
4415 */
4416 if (!virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
4417 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
4418 struct scatterlist sg;
4419

--- 10 unchanged lines hidden (view full) ---

4430 rtnl_unlock();
4431
4432 err = virtnet_cpu_notif_add(vi);
4433 if (err) {
4434 pr_debug("virtio_net: registering cpu notifier failed\n");
4435 goto free_unregister_netdev;
4436 }
4437
4151 virtnet_set_queues(vi, vi->curr_queue_pairs);
4152
4153 /* Assume link up if device can't report link status,
4154 otherwise get link status from config. */
4155 netif_carrier_off(dev);
4156 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
4157 schedule_work(&vi->config_work);
4158 } else {
4159 vi->status = VIRTIO_NET_S_LINK_UP;
4160 virtnet_update_settings(vi);

--- 177 unchanged lines hidden --- 		4438 /* Assume link up if device can't report link status,
4439 otherwise get link status from config. */
4440 netif_carrier_off(dev);
4441 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
4442 schedule_work(&vi->config_work);
4443 } else {
4444 vi->status = VIRTIO_NET_S_LINK_UP;
4445 virtnet_update_settings(vi);

--- 177 unchanged lines hidden ---