virtio_net.c (e377fcc8486d40867c6c217077ad0fa40977e060) virtio_net.c (d85b758f72b05a774045545f24d70980e3e9aac4)
1/* A network driver using virtio.
2 *
3 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.

--- 15 unchanged lines hidden (view full) ---

24#include <linux/virtio_net.h>
25#include <linux/bpf.h>
26#include <linux/bpf_trace.h>
27#include <linux/scatterlist.h>
28#include <linux/if_vlan.h>
29#include <linux/slab.h>
30#include <linux/cpu.h>
31#include <linux/average.h>
1/* A network driver using virtio.
2 *
3 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.

--- 15 unchanged lines hidden (view full) ---

24#include <linux/virtio_net.h>
25#include <linux/bpf.h>
26#include <linux/bpf_trace.h>
27#include <linux/scatterlist.h>
28#include <linux/if_vlan.h>
29#include <linux/slab.h>
30#include <linux/cpu.h>
31#include <linux/average.h>
32#include <net/route.h>
32
33static int napi_weight = NAPI_POLL_WEIGHT;
34module_param(napi_weight, int, 0444);
35
36static bool csum = true, gso = true;
37module_param(csum, bool, 0444);
38module_param(gso, bool, 0444);
39

--- 53 unchanged lines hidden (view full) ---

93 struct ewma_pkt_len mrg_avg_pkt_len;
94
95 /* Page frag for packet buffer allocation. */
96 struct page_frag alloc_frag;
97
98 /* RX: fragments + linear part + virtio header */
99 struct scatterlist sg[MAX_SKB_FRAGS + 2];
100
33
34static int napi_weight = NAPI_POLL_WEIGHT;
35module_param(napi_weight, int, 0444);
36
37static bool csum = true, gso = true;
38module_param(csum, bool, 0444);
39module_param(gso, bool, 0444);
40

--- 53 unchanged lines hidden (view full) ---

94 struct ewma_pkt_len mrg_avg_pkt_len;
95
96 /* Page frag for packet buffer allocation. */
97 struct page_frag alloc_frag;
98
99 /* RX: fragments + linear part + virtio header */
100 struct scatterlist sg[MAX_SKB_FRAGS + 2];
101
102 /* Min single buffer size for mergeable buffers case. */
103 unsigned int min_buf_len;
104
101 /* Name of this receive queue: input.$index */
102 char name[40];
103};
104
105struct virtnet_info {
106 struct virtio_device *vdev;
107 struct virtqueue *cvq;
108 struct net_device *dev;

--- 717 unchanged lines hidden (view full) ---

826 err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
827 first, gfp);
828 if (err < 0)
829 give_pages(rq, first);
830
831 return err;
832}
833
105 /* Name of this receive queue: input.$index */
106 char name[40];
107};
108
109struct virtnet_info {
110 struct virtio_device *vdev;
111 struct virtqueue *cvq;
112 struct net_device *dev;

--- 717 unchanged lines hidden (view full) ---

830 err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
831 first, gfp);
832 if (err < 0)
833 give_pages(rq, first);
834
835 return err;
836}
837
834static unsigned int get_mergeable_buf_len(struct ewma_pkt_len *avg_pkt_len)
838static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
839 struct ewma_pkt_len *avg_pkt_len)
835{
836 const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
837 unsigned int len;
838
839 len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
840{
841 const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
842 unsigned int len;
843
844 len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
840 GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
845 rq->min_buf_len - hdr_len, PAGE_SIZE - hdr_len);
841 return ALIGN(len, L1_CACHE_BYTES);
842}
843
844static int add_recvbuf_mergeable(struct virtnet_info *vi,
845 struct receive_queue *rq, gfp_t gfp)
846{
847 struct page_frag *alloc_frag = &rq->alloc_frag;
848 unsigned int headroom = virtnet_get_headroom(vi);
849 char *buf;
850 void *ctx;
851 int err;
852 unsigned int len, hole;
853
846 return ALIGN(len, L1_CACHE_BYTES);
847}
848
849static int add_recvbuf_mergeable(struct virtnet_info *vi,
850 struct receive_queue *rq, gfp_t gfp)
851{
852 struct page_frag *alloc_frag = &rq->alloc_frag;
853 unsigned int headroom = virtnet_get_headroom(vi);
854 char *buf;
855 void *ctx;
856 int err;
857 unsigned int len, hole;
858
854 len = get_mergeable_buf_len(&rq->mrg_avg_pkt_len);
859 len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len);
855 if (unlikely(!skb_page_frag_refill(len + headroom, alloc_frag, gfp)))
856 return -ENOMEM;
857
858 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
859 buf += headroom; /* advance address leaving hole at front of pkt */
860 ctx = (void *)(unsigned long)len;
861 get_page(alloc_frag->page);
862 alloc_frag->offset += len + headroom;

--- 1155 unchanged lines hidden (view full) ---

2018
2019 virtnet_clean_affinity(vi, -1);
2020
2021 vdev->config->del_vqs(vdev);
2022
2023 virtnet_free_queues(vi);
2024}
2025
860 if (unlikely(!skb_page_frag_refill(len + headroom, alloc_frag, gfp)))
861 return -ENOMEM;
862
863 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
864 buf += headroom; /* advance address leaving hole at front of pkt */
865 ctx = (void *)(unsigned long)len;
866 get_page(alloc_frag->page);
867 alloc_frag->offset += len + headroom;

--- 1155 unchanged lines hidden (view full) ---

2023
2024 virtnet_clean_affinity(vi, -1);
2025
2026 vdev->config->del_vqs(vdev);
2027
2028 virtnet_free_queues(vi);
2029}
2030
2031/* How large should a single buffer be so a queue full of these can fit at
2032 * least one full packet?
2033 * Logic below assumes the mergeable buffer header is used.
2034 */
2035static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
2036{
2037 const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2038 unsigned int rq_size = virtqueue_get_vring_size(vq);
2039 unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
2040 unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
2041 unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);
2042
2043 return max(min_buf_len, hdr_len);
2044}
2045
2026static int virtnet_find_vqs(struct virtnet_info *vi)
2027{
2028 vq_callback_t **callbacks;
2029 struct virtqueue **vqs;
2030 int ret = -ENOMEM;
2031 int i, total_vqs;
2032 const char **names;
2033 bool *ctx;

--- 49 unchanged lines hidden (view full) ---

2083 if (vi->has_cvq) {
2084 vi->cvq = vqs[total_vqs - 1];
2085 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
2086 vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2087 }
2088
2089 for (i = 0; i < vi->max_queue_pairs; i++) {
2090 vi->rq[i].vq = vqs[rxq2vq(i)];
2046static int virtnet_find_vqs(struct virtnet_info *vi)
2047{
2048 vq_callback_t **callbacks;
2049 struct virtqueue **vqs;
2050 int ret = -ENOMEM;
2051 int i, total_vqs;
2052 const char **names;
2053 bool *ctx;

--- 49 unchanged lines hidden (view full) ---

2103 if (vi->has_cvq) {
2104 vi->cvq = vqs[total_vqs - 1];
2105 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
2106 vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2107 }
2108
2109 for (i = 0; i < vi->max_queue_pairs; i++) {
2110 vi->rq[i].vq = vqs[rxq2vq(i)];
2111 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
2091 vi->sq[i].vq = vqs[txq2vq(i)];
2092 }
2093
2094 kfree(names);
2095 kfree(callbacks);
2096 kfree(vqs);
2097
2098 return 0;

--- 70 unchanged lines hidden (view full) ---

2169 struct rx_queue_attribute *attribute, char *buf)
2170{
2171 struct virtnet_info *vi = netdev_priv(queue->dev);
2172 unsigned int queue_index = get_netdev_rx_queue_index(queue);
2173 struct ewma_pkt_len *avg;
2174
2175 BUG_ON(queue_index >= vi->max_queue_pairs);
2176 avg = &vi->rq[queue_index].mrg_avg_pkt_len;
2112 vi->sq[i].vq = vqs[txq2vq(i)];
2113 }
2114
2115 kfree(names);
2116 kfree(callbacks);
2117 kfree(vqs);
2118
2119 return 0;

--- 70 unchanged lines hidden (view full) ---

2190 struct rx_queue_attribute *attribute, char *buf)
2191{
2192 struct virtnet_info *vi = netdev_priv(queue->dev);
2193 unsigned int queue_index = get_netdev_rx_queue_index(queue);
2194 struct ewma_pkt_len *avg;
2195
2196 BUG_ON(queue_index >= vi->max_queue_pairs);
2197 avg = &vi->rq[queue_index].mrg_avg_pkt_len;
2177 return sprintf(buf, "%u\n", get_mergeable_buf_len(avg));
2198 return sprintf(buf, "%u\n",
2199 get_mergeable_buf_len(&vi->rq[queue_index], avg));
2178}
2179
2180static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
2181 __ATTR_RO(mergeable_rx_buffer_size);
2182
2183static struct attribute *virtio_net_mrg_rx_attrs[] = {
2184 &mergeable_rx_buffer_size_attribute.attr,
2185 NULL

--- 430 unchanged lines hidden --- 		2200}
2201
2202static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
2203 __ATTR_RO(mergeable_rx_buffer_size);
2204
2205static struct attribute *virtio_net_mrg_rx_attrs[] = {
2206 &mergeable_rx_buffer_size_attribute.attr,
2207 NULL

--- 430 unchanged lines hidden ---