| virtio_net.c (d8673afbf51036ed1d72d9828d0d679035bb0d54) | virtio_net.c (981f14d42a7f1610292a1ef0f7cd00138fff361d) |
|---|---|
| 1// SPDX-License-Identifier: GPL-2.0-or-later 2/* A network driver using virtio. 3 * 4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation 5 */ 6//#define DEBUG 7#include <linux/netdevice.h> 8#include <linux/etherdevice.h> --- 709 unchanged lines hidden (view full) --- 718 * with large buffers with sufficient headroom - so it should affect 719 * at most queue size packets. 720 * Afterwards, the conditions to enable 721 * XDP should preclude the underlying device from sending packets 722 * across multiple buffers (num_buf > 1), and we make sure buffers 723 * have enough headroom. 724 */ 725static struct page *xdp_linearize_page(struct receive_queue *rq, | 1// SPDX-License-Identifier: GPL-2.0-or-later 2/* A network driver using virtio. 3 * 4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation 5 */ 6//#define DEBUG 7#include <linux/netdevice.h> 8#include <linux/etherdevice.h> --- 709 unchanged lines hidden (view full) --- 718 * with large buffers with sufficient headroom - so it should affect 719 * at most queue size packets. 720 * Afterwards, the conditions to enable 721 * XDP should preclude the underlying device from sending packets 722 * across multiple buffers (num_buf > 1), and we make sure buffers 723 * have enough headroom. 724 */ 725static struct page *xdp_linearize_page(struct receive_queue *rq, |
| 726 u16 *num_buf, | 726 int *num_buf, |
| 727 struct page *p, 728 int offset, 729 int page_off, 730 unsigned int *len) 731{ 732 struct page *page = alloc_page(GFP_ATOMIC); 733 734 if (!page) --- 83 unchanged lines hidden (view full) --- 818 u32 act; 819 820 if (unlikely(hdr->hdr.gso_type)) 821 goto err_xdp; 822 823 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) { 824 int offset = buf - page_address(page) + header_offset; 825 unsigned int tlen = len + vi->hdr_len; | 727 struct page *p, 728 int offset, 729 int page_off, 730 unsigned int *len) 731{ 732 struct page *page = alloc_page(GFP_ATOMIC); 733 734 if (!page) --- 83 unchanged lines hidden (view full) --- 818 u32 act; 819 820 if (unlikely(hdr->hdr.gso_type)) 821 goto err_xdp; 822 823 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) { 824 int offset = buf - page_address(page) + header_offset; 825 unsigned int tlen = len + vi->hdr_len; |
| 826 u16 num_buf = 1; | 826 int num_buf = 1; |
| 827 828 xdp_headroom = virtnet_get_headroom(vi); 829 header_offset = VIRTNET_RX_PAD + xdp_headroom; 830 headroom = vi->hdr_len + header_offset; 831 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 832 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 833 xdp_page = xdp_linearize_page(rq, &num_buf, page, 834 offset, header_offset, --- 156 unchanged lines hidden (view full) --- 991/* TODO: build xdp in big mode */ 992static int virtnet_build_xdp_buff_mrg(struct net_device *dev, 993 struct virtnet_info *vi, 994 struct receive_queue *rq, 995 struct xdp_buff *xdp, 996 void *buf, 997 unsigned int len, 998 unsigned int frame_sz, | 827 828 xdp_headroom = virtnet_get_headroom(vi); 829 header_offset = VIRTNET_RX_PAD + xdp_headroom; 830 headroom = vi->hdr_len + header_offset; 831 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 832 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 833 xdp_page = xdp_linearize_page(rq, &num_buf, page, 834 offset, header_offset, --- 156 unchanged lines hidden (view full) --- 991/* TODO: build xdp in big mode */ 992static int virtnet_build_xdp_buff_mrg(struct net_device *dev, 993 struct virtnet_info *vi, 994 struct receive_queue *rq, 995 struct xdp_buff *xdp, 996 void *buf, 997 unsigned int len, 998 unsigned int frame_sz, |
| 999 u16 *num_buf, | 999 int *num_buf, |
| 1000 unsigned int *xdp_frags_truesize, 1001 struct virtnet_rq_stats *stats) 1002{ 1003 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 1004 unsigned int headroom, tailroom, room; 1005 unsigned int truesize, cur_frag_size; 1006 struct skb_shared_info *shinfo; 1007 unsigned int xdp_frags_truesz = 0; 1008 struct page *page; 1009 skb_frag_t *frag; 1010 int offset; 1011 void *ctx; 1012 1013 xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq); 1014 xdp_prepare_buff(xdp, buf - VIRTIO_XDP_HEADROOM, 1015 VIRTIO_XDP_HEADROOM + vi->hdr_len, len - vi->hdr_len, true); 1016 | 1000 unsigned int *xdp_frags_truesize, 1001 struct virtnet_rq_stats *stats) 1002{ 1003 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 1004 unsigned int headroom, tailroom, room; 1005 unsigned int truesize, cur_frag_size; 1006 struct skb_shared_info *shinfo; 1007 unsigned int xdp_frags_truesz = 0; 1008 struct page *page; 1009 skb_frag_t *frag; 1010 int offset; 1011 void *ctx; 1012 1013 xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq); 1014 xdp_prepare_buff(xdp, buf - VIRTIO_XDP_HEADROOM, 1015 VIRTIO_XDP_HEADROOM + vi->hdr_len, len - vi->hdr_len, true); 1016 |
| 1017 if (!*num_buf) 1018 return 0; 1019 |
|
| 1017 if (*num_buf > 1) { 1018 /* If we want to build multi-buffer xdp, we need 1019 * to specify that the flags of xdp_buff have the 1020 * XDP_FLAGS_HAS_FRAG bit. 1021 */ 1022 if (!xdp_buff_has_frags(xdp)) 1023 xdp_buff_set_frags_flag(xdp); 1024 1025 shinfo = xdp_get_shared_info_from_buff(xdp); 1026 shinfo->nr_frags = 0; 1027 shinfo->xdp_frags_size = 0; 1028 } 1029 | 1020 if (*num_buf > 1) { 1021 /* If we want to build multi-buffer xdp, we need 1022 * to specify that the flags of xdp_buff have the 1023 * XDP_FLAGS_HAS_FRAG bit. 1024 */ 1025 if (!xdp_buff_has_frags(xdp)) 1026 xdp_buff_set_frags_flag(xdp); 1027 1028 shinfo = xdp_get_shared_info_from_buff(xdp); 1029 shinfo->nr_frags = 0; 1030 shinfo->xdp_frags_size = 0; 1031 } 1032 |
| 1030 if ((*num_buf - 1) > MAX_SKB_FRAGS) | 1033 if (*num_buf > MAX_SKB_FRAGS + 1) |
| 1031 return -EINVAL; 1032 | 1034 return -EINVAL; 1035 |
| 1033 while ((--*num_buf) >= 1) { | 1036 while (--*num_buf > 0) { |
| 1034 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx); 1035 if (unlikely(!buf)) { 1036 pr_debug("%s: rx error: %d buffers out of %d missing\n", 1037 dev->name, *num_buf, 1038 virtio16_to_cpu(vi->vdev, hdr->num_buffers)); 1039 dev->stats.rx_length_errors++; 1040 return -EINVAL; 1041 } --- 36 unchanged lines hidden (view full) --- 1078 struct receive_queue *rq, 1079 void *buf, 1080 void *ctx, 1081 unsigned int len, 1082 unsigned int *xdp_xmit, 1083 struct virtnet_rq_stats *stats) 1084{ 1085 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; | 1037 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx); 1038 if (unlikely(!buf)) { 1039 pr_debug("%s: rx error: %d buffers out of %d missing\n", 1040 dev->name, *num_buf, 1041 virtio16_to_cpu(vi->vdev, hdr->num_buffers)); 1042 dev->stats.rx_length_errors++; 1043 return -EINVAL; 1044 } --- 36 unchanged lines hidden (view full) --- 1081 struct receive_queue *rq, 1082 void *buf, 1083 void *ctx, 1084 unsigned int len, 1085 unsigned int *xdp_xmit, 1086 struct virtnet_rq_stats *stats) 1087{ 1088 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; |
| 1086 u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); | 1089 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); |
| 1087 struct page *page = virt_to_head_page(buf); 1088 int offset = buf - page_address(page); 1089 struct sk_buff *head_skb, *curr_skb; 1090 struct bpf_prog *xdp_prog; 1091 unsigned int truesize = mergeable_ctx_to_truesize(ctx); 1092 unsigned int headroom = mergeable_ctx_to_headroom(ctx); 1093 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 1094 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); --- 3180 unchanged lines hidden --- | 1090 struct page *page = virt_to_head_page(buf); 1091 int offset = buf - page_address(page); 1092 struct sk_buff *head_skb, *curr_skb; 1093 struct bpf_prog *xdp_prog; 1094 unsigned int truesize = mergeable_ctx_to_truesize(ctx); 1095 unsigned int headroom = mergeable_ctx_to_headroom(ctx); 1096 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 1097 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); --- 3180 unchanged lines hidden --- |