tun.c (e205ceeb253723f4e4bdce619844ed678ae48276) tun.c (c39e342a050a4425348e6fe7f75827c0a1a7ebc5)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * TUN - Universal TUN/TAP device driver.
4 * Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com>
5 *
6 * $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $
7 */
8

--- 122 unchanged lines hidden (view full) ---

131/* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal
132 * to max number of VCPUs in guest. */
133#define MAX_TAP_QUEUES 256
134#define MAX_TAP_FLOWS 4096
135
136#define TUN_FLOW_EXPIRE (3 * HZ)
137
138struct tun_pcpu_stats {
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * TUN - Universal TUN/TAP device driver.
4 * Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com>
5 *
6 * $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $
7 */
8

--- 122 unchanged lines hidden (view full) ---

131/* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal
132 * to max number of VCPUs in guest. */
133#define MAX_TAP_QUEUES 256
134#define MAX_TAP_FLOWS 4096
135
136#define TUN_FLOW_EXPIRE (3 * HZ)
137
138struct tun_pcpu_stats {
139 u64 rx_packets;
140 u64 rx_bytes;
141 u64 tx_packets;
142 u64 tx_bytes;
139 u64_stats_t rx_packets;
140 u64_stats_t rx_bytes;
141 u64_stats_t tx_packets;
142 u64_stats_t tx_bytes;
143 struct u64_stats_sync syncp;
144 u32 rx_dropped;
145 u32 tx_dropped;
146 u32 rx_frame_errors;
147};
148
149/* A tun_file connects an open character device to a tuntap netdevice. It
150 * also contains all socket related structures (except sock_fprog and tap_filter)

--- 157 unchanged lines hidden (view full) ---

308}
309
310static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
311 bool napi_en, bool napi_frags)
312{
313 tfile->napi_enabled = napi_en;
314 tfile->napi_frags_enabled = napi_en && napi_frags;
315 if (napi_en) {
143 struct u64_stats_sync syncp;
144 u32 rx_dropped;
145 u32 tx_dropped;
146 u32 rx_frame_errors;
147};
148
149/* A tun_file connects an open character device to a tuntap netdevice. It
150 * also contains all socket related structures (except sock_fprog and tap_filter)

--- 157 unchanged lines hidden (view full) ---

308}
309
310static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
311 bool napi_en, bool napi_frags)
312{
313 tfile->napi_enabled = napi_en;
314 tfile->napi_frags_enabled = napi_en && napi_frags;
315 if (napi_en) {
316 netif_napi_add(tun->dev, &tfile->napi, tun_napi_poll,
317 NAPI_POLL_WEIGHT);
316 netif_tx_napi_add(tun->dev, &tfile->napi, tun_napi_poll,
317 NAPI_POLL_WEIGHT);
318 napi_enable(&tfile->napi);
319 }
320}
321
322static void tun_napi_disable(struct tun_file *tfile)
323{
324 if (tfile->napi_enabled)
325 napi_disable(&tfile->napi);

--- 836 unchanged lines hidden (view full) ---

1162
1163 for_each_possible_cpu(i) {
1164 u64 rxpackets, rxbytes, txpackets, txbytes;
1165 unsigned int start;
1166
1167 p = per_cpu_ptr(tun->pcpu_stats, i);
1168 do {
1169 start = u64_stats_fetch_begin(&p->syncp);
318 napi_enable(&tfile->napi);
319 }
320}
321
322static void tun_napi_disable(struct tun_file *tfile)
323{
324 if (tfile->napi_enabled)
325 napi_disable(&tfile->napi);

--- 836 unchanged lines hidden (view full) ---

1162
1163 for_each_possible_cpu(i) {
1164 u64 rxpackets, rxbytes, txpackets, txbytes;
1165 unsigned int start;
1166
1167 p = per_cpu_ptr(tun->pcpu_stats, i);
1168 do {
1169 start = u64_stats_fetch_begin(&p->syncp);
1170 rxpackets = p->rx_packets;
1171 rxbytes = p->rx_bytes;
1172 txpackets = p->tx_packets;
1173 txbytes = p->tx_bytes;
1170 rxpackets = u64_stats_read(&p->rx_packets);
1171 rxbytes = u64_stats_read(&p->rx_bytes);
1172 txpackets = u64_stats_read(&p->tx_packets);
1173 txbytes = u64_stats_read(&p->tx_bytes);
1174 } while (u64_stats_fetch_retry(&p->syncp, start));
1175
1176 stats->rx_packets += rxpackets;
1177 stats->rx_bytes += rxbytes;
1178 stats->tx_packets += txpackets;
1179 stats->tx_bytes += txbytes;
1180
1181 /* u32 counters */

--- 811 unchanged lines hidden (view full) ---

1993 tun_rx_batched(tun, tfile, skb, more);
1994 } else {
1995 netif_rx_ni(skb);
1996 }
1997 rcu_read_unlock();
1998
1999 stats = get_cpu_ptr(tun->pcpu_stats);
2000 u64_stats_update_begin(&stats->syncp);
1174 } while (u64_stats_fetch_retry(&p->syncp, start));
1175
1176 stats->rx_packets += rxpackets;
1177 stats->rx_bytes += rxbytes;
1178 stats->tx_packets += txpackets;
1179 stats->tx_bytes += txbytes;
1180
1181 /* u32 counters */

--- 811 unchanged lines hidden (view full) ---

1993 tun_rx_batched(tun, tfile, skb, more);
1994 } else {
1995 netif_rx_ni(skb);
1996 }
1997 rcu_read_unlock();
1998
1999 stats = get_cpu_ptr(tun->pcpu_stats);
2000 u64_stats_update_begin(&stats->syncp);
2001 stats->rx_packets++;
2002 stats->rx_bytes += len;
2001 u64_stats_inc(&stats->rx_packets);
2002 u64_stats_add(&stats->rx_bytes, len);
2003 u64_stats_update_end(&stats->syncp);
2004 put_cpu_ptr(stats);
2005
2006 if (rxhash)
2007 tun_flow_update(tun, rxhash, tfile);
2008
2009 return total_len;
2010}

--- 36 unchanged lines hidden (view full) ---

2047 return -EFAULT;
2048 iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
2049 }
2050
2051 ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz;
2052
2053 stats = get_cpu_ptr(tun->pcpu_stats);
2054 u64_stats_update_begin(&stats->syncp);
2003 u64_stats_update_end(&stats->syncp);
2004 put_cpu_ptr(stats);
2005
2006 if (rxhash)
2007 tun_flow_update(tun, rxhash, tfile);
2008
2009 return total_len;
2010}

--- 36 unchanged lines hidden (view full) ---

2047 return -EFAULT;
2048 iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
2049 }
2050
2051 ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz;
2052
2053 stats = get_cpu_ptr(tun->pcpu_stats);
2054 u64_stats_update_begin(&stats->syncp);
2055 stats->tx_packets++;
2056 stats->tx_bytes += ret;
2055 u64_stats_inc(&stats->tx_packets);
2056 u64_stats_add(&stats->tx_bytes, ret);
2057 u64_stats_update_end(&stats->syncp);
2058 put_cpu_ptr(tun->pcpu_stats);
2059
2060 return ret;
2061}
2062
2063/* Put packet to the user space buffer */
2064static ssize_t tun_put_user(struct tun_struct *tun,

--- 77 unchanged lines hidden (view full) ---

2142 }
2143
2144 skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset);
2145
2146done:
2147 /* caller is in process context, */
2148 stats = get_cpu_ptr(tun->pcpu_stats);
2149 u64_stats_update_begin(&stats->syncp);
2057 u64_stats_update_end(&stats->syncp);
2058 put_cpu_ptr(tun->pcpu_stats);
2059
2060 return ret;
2061}
2062
2063/* Put packet to the user space buffer */
2064static ssize_t tun_put_user(struct tun_struct *tun,

--- 77 unchanged lines hidden (view full) ---

2142 }
2143
2144 skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset);
2145
2146done:
2147 /* caller is in process context, */
2148 stats = get_cpu_ptr(tun->pcpu_stats);
2149 u64_stats_update_begin(&stats->syncp);
2150 stats->tx_packets++;
2151 stats->tx_bytes += skb->len + vlan_hlen;
2150 u64_stats_inc(&stats->tx_packets);
2151 u64_stats_add(&stats->tx_bytes, skb->len + vlan_hlen);
2152 u64_stats_update_end(&stats->syncp);
2153 put_cpu_ptr(tun->pcpu_stats);
2154
2155 return total;
2156}
2157
2158static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
2159{

--- 125 unchanged lines hidden (view full) ---

2285 return 0;
2286}
2287
2288static void tun_free_netdev(struct net_device *dev)
2289{
2290 struct tun_struct *tun = netdev_priv(dev);
2291
2292 BUG_ON(!(list_empty(&tun->disabled)));
2152 u64_stats_update_end(&stats->syncp);
2153 put_cpu_ptr(tun->pcpu_stats);
2154
2155 return total;
2156}
2157
2158static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
2159{

--- 125 unchanged lines hidden (view full) ---

2285 return 0;
2286}
2287
2288static void tun_free_netdev(struct net_device *dev)
2289{
2290 struct tun_struct *tun = netdev_priv(dev);
2291
2292 BUG_ON(!(list_empty(&tun->disabled)));
2293
2293 free_percpu(tun->pcpu_stats);
2294 free_percpu(tun->pcpu_stats);
2295 /* We clear pcpu_stats so that tun_set_iff() can tell if
2296 * tun_free_netdev() has been called from register_netdevice().
2297 */
2298 tun->pcpu_stats = NULL;
2299
2294 tun_flow_uninit(tun);
2295 security_tun_dev_free_security(tun->security);
2296 __tun_set_ebpf(tun, &tun->steering_prog, NULL);
2297 __tun_set_ebpf(tun, &tun->filter_prog, NULL);
2298}
2299
2300static void tun_setup(struct net_device *dev)
2301{

--- 198 unchanged lines hidden (view full) ---

2500 skb_record_rx_queue(skb, tfile->queue_index);
2501 netif_receive_skb(skb);
2502
2503 /* No need for get_cpu_ptr() here since this function is
2504 * always called with bh disabled
2505 */
2506 stats = this_cpu_ptr(tun->pcpu_stats);
2507 u64_stats_update_begin(&stats->syncp);
2300 tun_flow_uninit(tun);
2301 security_tun_dev_free_security(tun->security);
2302 __tun_set_ebpf(tun, &tun->steering_prog, NULL);
2303 __tun_set_ebpf(tun, &tun->filter_prog, NULL);
2304}
2305
2306static void tun_setup(struct net_device *dev)
2307{

--- 198 unchanged lines hidden (view full) ---

2506 skb_record_rx_queue(skb, tfile->queue_index);
2507 netif_receive_skb(skb);
2508
2509 /* No need for get_cpu_ptr() here since this function is
2510 * always called with bh disabled
2511 */
2512 stats = this_cpu_ptr(tun->pcpu_stats);
2513 u64_stats_update_begin(&stats->syncp);
2508 stats->rx_packets++;
2509 stats->rx_bytes += datasize;
2514 u64_stats_inc(&stats->rx_packets);
2515 u64_stats_add(&stats->rx_bytes, datasize);
2510 u64_stats_update_end(&stats->syncp);
2511
2512 if (rxhash)
2513 tun_flow_update(tun, rxhash, tfile);
2514
2515out:
2516 return err;
2517}

--- 259 unchanged lines hidden (view full) ---

2777 name = ifr->ifr_name;
2778
2779 dev = alloc_netdev_mqs(sizeof(struct tun_struct), name,
2780 NET_NAME_UNKNOWN, tun_setup, queues,
2781 queues);
2782
2783 if (!dev)
2784 return -ENOMEM;
2516 u64_stats_update_end(&stats->syncp);
2517
2518 if (rxhash)
2519 tun_flow_update(tun, rxhash, tfile);
2520
2521out:
2522 return err;
2523}

--- 259 unchanged lines hidden (view full) ---

2783 name = ifr->ifr_name;
2784
2785 dev = alloc_netdev_mqs(sizeof(struct tun_struct), name,
2786 NET_NAME_UNKNOWN, tun_setup, queues,
2787 queues);
2788
2789 if (!dev)
2790 return -ENOMEM;
2785 err = dev_get_valid_name(net, dev, name);
2786 if (err < 0)
2787 goto err_free_dev;
2788
2789 dev_net_set(dev, net);
2790 dev->rtnl_link_ops = &tun_link_ops;
2791 dev->ifindex = tfile->ifindex;
2792 dev->sysfs_groups[0] = &tun_attr_group;
2793
2794 tun = netdev_priv(dev);
2795 tun->dev = dev;

--- 58 unchanged lines hidden (view full) ---

2854 if (netif_running(tun->dev))
2855 netif_tx_wake_all_queues(tun->dev);
2856
2857 strcpy(ifr->ifr_name, tun->dev->name);
2858 return 0;
2859
2860err_detach:
2861 tun_detach_all(dev);
2791
2792 dev_net_set(dev, net);
2793 dev->rtnl_link_ops = &tun_link_ops;
2794 dev->ifindex = tfile->ifindex;
2795 dev->sysfs_groups[0] = &tun_attr_group;
2796
2797 tun = netdev_priv(dev);
2798 tun->dev = dev;

--- 58 unchanged lines hidden (view full) ---

2857 if (netif_running(tun->dev))
2858 netif_tx_wake_all_queues(tun->dev);
2859
2860 strcpy(ifr->ifr_name, tun->dev->name);
2861 return 0;
2862
2863err_detach:
2864 tun_detach_all(dev);
2862 /* register_netdevice() already called tun_free_netdev() */
2863 goto err_free_dev;
2865 /* We are here because register_netdevice() has failed.
2866 * If register_netdevice() already called tun_free_netdev()
2867 * while dealing with the error, tun->pcpu_stats has been cleared.
2868 */
2869 if (!tun->pcpu_stats)
2870 goto err_free_dev;
2864
2865err_free_flow:
2866 tun_flow_uninit(tun);
2867 security_tun_dev_free_security(tun->security);
2868err_free_stat:
2869 free_percpu(tun->pcpu_stats);
2870err_free_dev:
2871 free_netdev(dev);

--- 868 unchanged lines hidden --- 		2871
2872err_free_flow:
2873 tun_flow_uninit(tun);
2874 security_tun_dev_free_security(tun->security);
2875err_free_stat:
2876 free_percpu(tun->pcpu_stats);
2877err_free_dev:
2878 free_netdev(dev);

--- 868 unchanged lines hidden ---