109c434b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e314dbdcSPavel Emelyanov /* 3e314dbdcSPavel Emelyanov * drivers/net/veth.c 4e314dbdcSPavel Emelyanov * 5e314dbdcSPavel Emelyanov * Copyright (C) 2007 OpenVZ http://openvz.org, SWsoft Inc 6e314dbdcSPavel Emelyanov * 7e314dbdcSPavel Emelyanov * Author: Pavel Emelianov <xemul@openvz.org> 8e314dbdcSPavel Emelyanov * Ethtool interface from: Eric W. Biederman <ebiederm@xmission.com> 9e314dbdcSPavel Emelyanov * 10e314dbdcSPavel Emelyanov */ 11e314dbdcSPavel Emelyanov 12e314dbdcSPavel Emelyanov #include <linux/netdevice.h> 135a0e3ad6STejun Heo #include <linux/slab.h> 14e314dbdcSPavel Emelyanov #include <linux/ethtool.h> 15e314dbdcSPavel Emelyanov #include <linux/etherdevice.h> 16cf05c700SEric Dumazet #include <linux/u64_stats_sync.h> 17e314dbdcSPavel Emelyanov 18f7b12606SJiri Pirko #include <net/rtnetlink.h> 19e314dbdcSPavel Emelyanov #include <net/dst.h> 20e314dbdcSPavel Emelyanov #include <net/xfrm.h> 21af87a3aaSToshiaki Makita #include <net/xdp.h> 22ecef969eSStephen Hemminger #include <linux/veth.h> 239d9779e7SPaul Gortmaker #include <linux/module.h> 24948d4f21SToshiaki Makita #include <linux/bpf.h> 25948d4f21SToshiaki Makita #include <linux/filter.h> 26948d4f21SToshiaki Makita #include <linux/ptr_ring.h> 27948d4f21SToshiaki Makita #include <linux/bpf_trace.h> 28aa4e689eSMichael Walle #include <linux/net_tstamp.h> 29e314dbdcSPavel Emelyanov 30e314dbdcSPavel Emelyanov #define DRV_NAME "veth" 31e314dbdcSPavel Emelyanov #define DRV_VERSION "1.0" 32e314dbdcSPavel Emelyanov 339fc8d518SToshiaki Makita #define VETH_XDP_FLAG BIT(0) 34948d4f21SToshiaki Makita #define VETH_RING_SIZE 256 35948d4f21SToshiaki Makita #define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN) 36948d4f21SToshiaki Makita 379cda7807SToshiaki Makita #define VETH_XDP_TX_BULK_SIZE 16 3865e6dcf7SLorenzo Bianconi #define VETH_XDP_BATCH 16 399cda7807SToshiaki Makita 4065780c56SLorenzo Bianconi struct veth_stats { 411c5b82e5SLorenzo Bianconi u64 rx_drops; 421c5b82e5SLorenzo Bianconi /* xdp */ 434195e54aSToshiaki Makita u64 xdp_packets; 444195e54aSToshiaki Makita u64 xdp_bytes; 451c5b82e5SLorenzo Bianconi u64 xdp_redirect; 464195e54aSToshiaki Makita u64 xdp_drops; 471c5b82e5SLorenzo Bianconi u64 xdp_tx; 489152cff0SLorenzo Bianconi u64 xdp_tx_err; 495fe6e567SLorenzo Bianconi u64 peer_tq_xdp_xmit; 505fe6e567SLorenzo Bianconi u64 peer_tq_xdp_xmit_err; 5165780c56SLorenzo Bianconi }; 5265780c56SLorenzo Bianconi 5365780c56SLorenzo Bianconi struct veth_rq_stats { 5465780c56SLorenzo Bianconi struct veth_stats vs; 554195e54aSToshiaki Makita struct u64_stats_sync syncp; 564195e54aSToshiaki Makita }; 574195e54aSToshiaki Makita 58638264dcSToshiaki Makita struct veth_rq { 59948d4f21SToshiaki Makita struct napi_struct xdp_napi; 60d3256efdSPaolo Abeni struct napi_struct __rcu *napi; /* points to xdp_napi when the latter is initialized */ 61948d4f21SToshiaki Makita struct net_device *dev; 62948d4f21SToshiaki Makita struct bpf_prog __rcu *xdp_prog; 63d1396004SToshiaki Makita struct xdp_mem_info xdp_mem; 644195e54aSToshiaki Makita struct veth_rq_stats stats; 65948d4f21SToshiaki Makita bool rx_notify_masked; 66948d4f21SToshiaki Makita struct ptr_ring xdp_ring; 67948d4f21SToshiaki Makita struct xdp_rxq_info xdp_rxq; 68e314dbdcSPavel Emelyanov }; 69e314dbdcSPavel Emelyanov 70638264dcSToshiaki Makita struct veth_priv { 71638264dcSToshiaki Makita struct net_device __rcu *peer; 72638264dcSToshiaki Makita atomic64_t dropped; 73638264dcSToshiaki Makita struct bpf_prog *_xdp_prog; 74638264dcSToshiaki Makita struct veth_rq *rq; 75638264dcSToshiaki Makita unsigned int requested_headroom; 76638264dcSToshiaki Makita }; 77638264dcSToshiaki Makita 789cda7807SToshiaki Makita struct veth_xdp_tx_bq { 799cda7807SToshiaki Makita struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE]; 809cda7807SToshiaki Makita unsigned int count; 819cda7807SToshiaki Makita }; 829cda7807SToshiaki Makita 83e314dbdcSPavel Emelyanov /* 84e314dbdcSPavel Emelyanov * ethtool interface 85e314dbdcSPavel Emelyanov */ 86e314dbdcSPavel Emelyanov 87d397b968SToshiaki Makita struct veth_q_stat_desc { 88d397b968SToshiaki Makita char desc[ETH_GSTRING_LEN]; 89d397b968SToshiaki Makita size_t offset; 90d397b968SToshiaki Makita }; 91d397b968SToshiaki Makita 9265780c56SLorenzo Bianconi #define VETH_RQ_STAT(m) offsetof(struct veth_stats, m) 93d397b968SToshiaki Makita 94d397b968SToshiaki Makita static const struct veth_q_stat_desc veth_rq_stats_desc[] = { 95d397b968SToshiaki Makita { "xdp_packets", VETH_RQ_STAT(xdp_packets) }, 96d397b968SToshiaki Makita { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) }, 975fe6e567SLorenzo Bianconi { "drops", VETH_RQ_STAT(rx_drops) }, 985fe6e567SLorenzo Bianconi { "xdp_redirect", VETH_RQ_STAT(xdp_redirect) }, 995fe6e567SLorenzo Bianconi { "xdp_drops", VETH_RQ_STAT(xdp_drops) }, 1005fe6e567SLorenzo Bianconi { "xdp_tx", VETH_RQ_STAT(xdp_tx) }, 1015fe6e567SLorenzo Bianconi { "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) }, 102d397b968SToshiaki Makita }; 103d397b968SToshiaki Makita 104d397b968SToshiaki Makita #define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc) 105d397b968SToshiaki Makita 1065fe6e567SLorenzo Bianconi static const struct veth_q_stat_desc veth_tq_stats_desc[] = { 1075fe6e567SLorenzo Bianconi { "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) }, 1085fe6e567SLorenzo Bianconi { "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) }, 1095fe6e567SLorenzo Bianconi }; 1105fe6e567SLorenzo Bianconi 1115fe6e567SLorenzo Bianconi #define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc) 1125fe6e567SLorenzo Bianconi 113e314dbdcSPavel Emelyanov static struct { 114e314dbdcSPavel Emelyanov const char string[ETH_GSTRING_LEN]; 115e314dbdcSPavel Emelyanov } ethtool_stats_keys[] = { 116e314dbdcSPavel Emelyanov { "peer_ifindex" }, 117e314dbdcSPavel Emelyanov }; 118e314dbdcSPavel Emelyanov 11956607b98SPhilippe Reynes static int veth_get_link_ksettings(struct net_device *dev, 12056607b98SPhilippe Reynes struct ethtool_link_ksettings *cmd) 121e314dbdcSPavel Emelyanov { 12256607b98SPhilippe Reynes cmd->base.speed = SPEED_10000; 12356607b98SPhilippe Reynes cmd->base.duplex = DUPLEX_FULL; 12456607b98SPhilippe Reynes cmd->base.port = PORT_TP; 12556607b98SPhilippe Reynes cmd->base.autoneg = AUTONEG_DISABLE; 126e314dbdcSPavel Emelyanov return 0; 127e314dbdcSPavel Emelyanov } 128e314dbdcSPavel Emelyanov 129e314dbdcSPavel Emelyanov static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 130e314dbdcSPavel Emelyanov { 13133a5ba14SRick Jones strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); 13233a5ba14SRick Jones strlcpy(info->version, DRV_VERSION, sizeof(info->version)); 133e314dbdcSPavel Emelyanov } 134e314dbdcSPavel Emelyanov 135e314dbdcSPavel Emelyanov static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf) 136e314dbdcSPavel Emelyanov { 137a0341b73STonghao Zhang u8 *p = buf; 138d397b968SToshiaki Makita int i, j; 139d397b968SToshiaki Makita 140e314dbdcSPavel Emelyanov switch(stringset) { 141e314dbdcSPavel Emelyanov case ETH_SS_STATS: 142d397b968SToshiaki Makita memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys)); 143d397b968SToshiaki Makita p += sizeof(ethtool_stats_keys); 144a0341b73STonghao Zhang for (i = 0; i < dev->real_num_rx_queues; i++) 145a0341b73STonghao Zhang for (j = 0; j < VETH_RQ_STATS_LEN; j++) 146a0341b73STonghao Zhang ethtool_sprintf(&p, "rx_queue_%u_%.18s", 147d397b968SToshiaki Makita i, veth_rq_stats_desc[j].desc); 148a0341b73STonghao Zhang 149a0341b73STonghao Zhang for (i = 0; i < dev->real_num_tx_queues; i++) 150a0341b73STonghao Zhang for (j = 0; j < VETH_TQ_STATS_LEN; j++) 151a0341b73STonghao Zhang ethtool_sprintf(&p, "tx_queue_%u_%.18s", 1525fe6e567SLorenzo Bianconi i, veth_tq_stats_desc[j].desc); 153e314dbdcSPavel Emelyanov break; 154e314dbdcSPavel Emelyanov } 155e314dbdcSPavel Emelyanov } 156e314dbdcSPavel Emelyanov 157b9f2c044SJeff Garzik static int veth_get_sset_count(struct net_device *dev, int sset) 158e314dbdcSPavel Emelyanov { 159b9f2c044SJeff Garzik switch (sset) { 160b9f2c044SJeff Garzik case ETH_SS_STATS: 161d397b968SToshiaki Makita return ARRAY_SIZE(ethtool_stats_keys) + 1625fe6e567SLorenzo Bianconi VETH_RQ_STATS_LEN * dev->real_num_rx_queues + 1635fe6e567SLorenzo Bianconi VETH_TQ_STATS_LEN * dev->real_num_tx_queues; 164b9f2c044SJeff Garzik default: 165b9f2c044SJeff Garzik return -EOPNOTSUPP; 166b9f2c044SJeff Garzik } 167e314dbdcSPavel Emelyanov } 168e314dbdcSPavel Emelyanov 169e314dbdcSPavel Emelyanov static void veth_get_ethtool_stats(struct net_device *dev, 170e314dbdcSPavel Emelyanov struct ethtool_stats *stats, u64 *data) 171e314dbdcSPavel Emelyanov { 1725fe6e567SLorenzo Bianconi struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 173d0e2c55eSEric Dumazet struct net_device *peer = rtnl_dereference(priv->peer); 174d397b968SToshiaki Makita int i, j, idx; 175e314dbdcSPavel Emelyanov 176d0e2c55eSEric Dumazet data[0] = peer ? peer->ifindex : 0; 177d397b968SToshiaki Makita idx = 1; 178d397b968SToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 179d397b968SToshiaki Makita const struct veth_rq_stats *rq_stats = &priv->rq[i].stats; 18065780c56SLorenzo Bianconi const void *stats_base = (void *)&rq_stats->vs; 181d397b968SToshiaki Makita unsigned int start; 182d397b968SToshiaki Makita size_t offset; 183d397b968SToshiaki Makita 184d397b968SToshiaki Makita do { 185d397b968SToshiaki Makita start = u64_stats_fetch_begin_irq(&rq_stats->syncp); 186d397b968SToshiaki Makita for (j = 0; j < VETH_RQ_STATS_LEN; j++) { 187d397b968SToshiaki Makita offset = veth_rq_stats_desc[j].offset; 188d397b968SToshiaki Makita data[idx + j] = *(u64 *)(stats_base + offset); 189d397b968SToshiaki Makita } 190d397b968SToshiaki Makita } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); 191d397b968SToshiaki Makita idx += VETH_RQ_STATS_LEN; 192d397b968SToshiaki Makita } 1935fe6e567SLorenzo Bianconi 1945fe6e567SLorenzo Bianconi if (!peer) 1955fe6e567SLorenzo Bianconi return; 1965fe6e567SLorenzo Bianconi 1975fe6e567SLorenzo Bianconi rcv_priv = netdev_priv(peer); 1985fe6e567SLorenzo Bianconi for (i = 0; i < peer->real_num_rx_queues; i++) { 1995fe6e567SLorenzo Bianconi const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats; 2005fe6e567SLorenzo Bianconi const void *base = (void *)&rq_stats->vs; 2015fe6e567SLorenzo Bianconi unsigned int start, tx_idx = idx; 2025fe6e567SLorenzo Bianconi size_t offset; 2035fe6e567SLorenzo Bianconi 2045fe6e567SLorenzo Bianconi tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; 2055fe6e567SLorenzo Bianconi do { 2065fe6e567SLorenzo Bianconi start = u64_stats_fetch_begin_irq(&rq_stats->syncp); 2075fe6e567SLorenzo Bianconi for (j = 0; j < VETH_TQ_STATS_LEN; j++) { 2085fe6e567SLorenzo Bianconi offset = veth_tq_stats_desc[j].offset; 2095fe6e567SLorenzo Bianconi data[tx_idx + j] += *(u64 *)(base + offset); 2105fe6e567SLorenzo Bianconi } 2115fe6e567SLorenzo Bianconi } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); 2125fe6e567SLorenzo Bianconi } 213e314dbdcSPavel Emelyanov } 214e314dbdcSPavel Emelyanov 21534829eecSMaciej Fijalkowski static void veth_get_channels(struct net_device *dev, 21634829eecSMaciej Fijalkowski struct ethtool_channels *channels) 21734829eecSMaciej Fijalkowski { 21834829eecSMaciej Fijalkowski channels->tx_count = dev->real_num_tx_queues; 21934829eecSMaciej Fijalkowski channels->rx_count = dev->real_num_rx_queues; 2204752eeb3SPaolo Abeni channels->max_tx = dev->num_tx_queues; 2214752eeb3SPaolo Abeni channels->max_rx = dev->num_rx_queues; 22234829eecSMaciej Fijalkowski } 22334829eecSMaciej Fijalkowski 2244752eeb3SPaolo Abeni static int veth_set_channels(struct net_device *dev, 2254752eeb3SPaolo Abeni struct ethtool_channels *ch); 2264752eeb3SPaolo Abeni 2270fc0b732SStephen Hemminger static const struct ethtool_ops veth_ethtool_ops = { 228e314dbdcSPavel Emelyanov .get_drvinfo = veth_get_drvinfo, 229e314dbdcSPavel Emelyanov .get_link = ethtool_op_get_link, 230e314dbdcSPavel Emelyanov .get_strings = veth_get_strings, 231b9f2c044SJeff Garzik .get_sset_count = veth_get_sset_count, 232e314dbdcSPavel Emelyanov .get_ethtool_stats = veth_get_ethtool_stats, 23356607b98SPhilippe Reynes .get_link_ksettings = veth_get_link_ksettings, 234056b21fbSJulian Wiedmann .get_ts_info = ethtool_op_get_ts_info, 23534829eecSMaciej Fijalkowski .get_channels = veth_get_channels, 2364752eeb3SPaolo Abeni .set_channels = veth_set_channels, 237e314dbdcSPavel Emelyanov }; 238e314dbdcSPavel Emelyanov 239948d4f21SToshiaki Makita /* general routines */ 240948d4f21SToshiaki Makita 2419fc8d518SToshiaki Makita static bool veth_is_xdp_frame(void *ptr) 2429fc8d518SToshiaki Makita { 2439fc8d518SToshiaki Makita return (unsigned long)ptr & VETH_XDP_FLAG; 2449fc8d518SToshiaki Makita } 2459fc8d518SToshiaki Makita 246defcffebSMaciej Żenczykowski static struct xdp_frame *veth_ptr_to_xdp(void *ptr) 2479fc8d518SToshiaki Makita { 2489fc8d518SToshiaki Makita return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG); 2499fc8d518SToshiaki Makita } 2509fc8d518SToshiaki Makita 251defcffebSMaciej Żenczykowski static void *veth_xdp_to_ptr(struct xdp_frame *xdp) 252af87a3aaSToshiaki Makita { 253defcffebSMaciej Żenczykowski return (void *)((unsigned long)xdp | VETH_XDP_FLAG); 254af87a3aaSToshiaki Makita } 255af87a3aaSToshiaki Makita 2569fc8d518SToshiaki Makita static void veth_ptr_free(void *ptr) 2579fc8d518SToshiaki Makita { 2589fc8d518SToshiaki Makita if (veth_is_xdp_frame(ptr)) 2599fc8d518SToshiaki Makita xdp_return_frame(veth_ptr_to_xdp(ptr)); 2609fc8d518SToshiaki Makita else 2619fc8d518SToshiaki Makita kfree_skb(ptr); 2629fc8d518SToshiaki Makita } 2639fc8d518SToshiaki Makita 264638264dcSToshiaki Makita static void __veth_xdp_flush(struct veth_rq *rq) 265948d4f21SToshiaki Makita { 266948d4f21SToshiaki Makita /* Write ptr_ring before reading rx_notify_masked */ 267948d4f21SToshiaki Makita smp_mb(); 26868468d8cSEric Dumazet if (!READ_ONCE(rq->rx_notify_masked) && 26968468d8cSEric Dumazet napi_schedule_prep(&rq->xdp_napi)) { 27068468d8cSEric Dumazet WRITE_ONCE(rq->rx_notify_masked, true); 27168468d8cSEric Dumazet __napi_schedule(&rq->xdp_napi); 272948d4f21SToshiaki Makita } 273948d4f21SToshiaki Makita } 274948d4f21SToshiaki Makita 275638264dcSToshiaki Makita static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb) 276948d4f21SToshiaki Makita { 277638264dcSToshiaki Makita if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) { 278948d4f21SToshiaki Makita dev_kfree_skb_any(skb); 279948d4f21SToshiaki Makita return NET_RX_DROP; 280948d4f21SToshiaki Makita } 281948d4f21SToshiaki Makita 282948d4f21SToshiaki Makita return NET_RX_SUCCESS; 283948d4f21SToshiaki Makita } 284948d4f21SToshiaki Makita 285638264dcSToshiaki Makita static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, 286638264dcSToshiaki Makita struct veth_rq *rq, bool xdp) 287e314dbdcSPavel Emelyanov { 288948d4f21SToshiaki Makita return __dev_forward_skb(dev, skb) ?: xdp ? 289638264dcSToshiaki Makita veth_xdp_rx(rq, skb) : 290baebdf48SSebastian Andrzej Siewior __netif_rx(skb); 291948d4f21SToshiaki Makita } 292948d4f21SToshiaki Makita 29347e550e0SPaolo Abeni /* return true if the specified skb has chances of GRO aggregation 29447e550e0SPaolo Abeni * Don't strive for accuracy, but try to avoid GRO overhead in the most 29547e550e0SPaolo Abeni * common scenarios. 29647e550e0SPaolo Abeni * When XDP is enabled, all traffic is considered eligible, as the xmit 29747e550e0SPaolo Abeni * device has TSO off. 29847e550e0SPaolo Abeni * When TSO is enabled on the xmit device, we are likely interested only 29947e550e0SPaolo Abeni * in UDP aggregation, explicitly check for that if the skb is suspected 30047e550e0SPaolo Abeni * - the sock_wfree destructor is used by UDP, ICMP and XDP sockets - 30147e550e0SPaolo Abeni * to belong to locally generated UDP traffic. 30247e550e0SPaolo Abeni */ 30347e550e0SPaolo Abeni static bool veth_skb_is_eligible_for_gro(const struct net_device *dev, 30447e550e0SPaolo Abeni const struct net_device *rcv, 30547e550e0SPaolo Abeni const struct sk_buff *skb) 30647e550e0SPaolo Abeni { 30747e550e0SPaolo Abeni return !(dev->features & NETIF_F_ALL_TSO) || 30847e550e0SPaolo Abeni (skb->destructor == sock_wfree && 30947e550e0SPaolo Abeni rcv->features & (NETIF_F_GRO_FRAGLIST | NETIF_F_GRO_UDP_FWD)); 31047e550e0SPaolo Abeni } 31147e550e0SPaolo Abeni 312948d4f21SToshiaki Makita static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) 313948d4f21SToshiaki Makita { 314948d4f21SToshiaki Makita struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 315638264dcSToshiaki Makita struct veth_rq *rq = NULL; 316d0e2c55eSEric Dumazet struct net_device *rcv; 3172681128fSEric Dumazet int length = skb->len; 318d3256efdSPaolo Abeni bool use_napi = false; 319638264dcSToshiaki Makita int rxq; 320e314dbdcSPavel Emelyanov 321d0e2c55eSEric Dumazet rcu_read_lock(); 322d0e2c55eSEric Dumazet rcv = rcu_dereference(priv->peer); 323d0e2c55eSEric Dumazet if (unlikely(!rcv)) { 324d0e2c55eSEric Dumazet kfree_skb(skb); 325d0e2c55eSEric Dumazet goto drop; 326d0e2c55eSEric Dumazet } 327e314dbdcSPavel Emelyanov 328948d4f21SToshiaki Makita rcv_priv = netdev_priv(rcv); 329638264dcSToshiaki Makita rxq = skb_get_queue_mapping(skb); 330638264dcSToshiaki Makita if (rxq < rcv->real_num_rx_queues) { 331638264dcSToshiaki Makita rq = &rcv_priv->rq[rxq]; 332d3256efdSPaolo Abeni 333d3256efdSPaolo Abeni /* The napi pointer is available when an XDP program is 334d3256efdSPaolo Abeni * attached or when GRO is enabled 33547e550e0SPaolo Abeni * Don't bother with napi/GRO if the skb can't be aggregated 336d3256efdSPaolo Abeni */ 33747e550e0SPaolo Abeni use_napi = rcu_access_pointer(rq->napi) && 33847e550e0SPaolo Abeni veth_skb_is_eligible_for_gro(dev, rcv, skb); 339638264dcSToshiaki Makita } 340948d4f21SToshiaki Makita 341aa4e689eSMichael Walle skb_tx_timestamp(skb); 342d3256efdSPaolo Abeni if (likely(veth_forward_skb(rcv, skb, rq, use_napi) == NET_RX_SUCCESS)) { 343d3256efdSPaolo Abeni if (!use_napi) 344b4fba476SEric Dumazet dev_lstats_add(dev, length); 3452681128fSEric Dumazet } else { 346d0e2c55eSEric Dumazet drop: 3472681128fSEric Dumazet atomic64_inc(&priv->dropped); 3482681128fSEric Dumazet } 349948d4f21SToshiaki Makita 350d3256efdSPaolo Abeni if (use_napi) 351638264dcSToshiaki Makita __veth_xdp_flush(rq); 352948d4f21SToshiaki Makita 353d0e2c55eSEric Dumazet rcu_read_unlock(); 354948d4f21SToshiaki Makita 3556ed10654SPatrick McHardy return NETDEV_TX_OK; 356e314dbdcSPavel Emelyanov } 357e314dbdcSPavel Emelyanov 358b4fba476SEric Dumazet static u64 veth_stats_tx(struct net_device *dev, u64 *packets, u64 *bytes) 359e314dbdcSPavel Emelyanov { 360cf05c700SEric Dumazet struct veth_priv *priv = netdev_priv(dev); 36111687a10SDavid S. Miller 362b4fba476SEric Dumazet dev_lstats_read(dev, packets, bytes); 3632681128fSEric Dumazet return atomic64_read(&priv->dropped); 3642681128fSEric Dumazet } 3652681128fSEric Dumazet 36665780c56SLorenzo Bianconi static void veth_stats_rx(struct veth_stats *result, struct net_device *dev) 3674195e54aSToshiaki Makita { 3684195e54aSToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 3694195e54aSToshiaki Makita int i; 3704195e54aSToshiaki Makita 3715fe6e567SLorenzo Bianconi result->peer_tq_xdp_xmit_err = 0; 3724195e54aSToshiaki Makita result->xdp_packets = 0; 373d99a7c2fSLorenzo Bianconi result->xdp_tx_err = 0; 3744195e54aSToshiaki Makita result->xdp_bytes = 0; 37566fe4a07SLorenzo Bianconi result->rx_drops = 0; 3764195e54aSToshiaki Makita for (i = 0; i < dev->num_rx_queues; i++) { 3775fe6e567SLorenzo Bianconi u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err; 3784195e54aSToshiaki Makita struct veth_rq_stats *stats = &priv->rq[i].stats; 3794195e54aSToshiaki Makita unsigned int start; 3804195e54aSToshiaki Makita 3814195e54aSToshiaki Makita do { 3824195e54aSToshiaki Makita start = u64_stats_fetch_begin_irq(&stats->syncp); 3835fe6e567SLorenzo Bianconi peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err; 384d99a7c2fSLorenzo Bianconi xdp_tx_err = stats->vs.xdp_tx_err; 38565780c56SLorenzo Bianconi packets = stats->vs.xdp_packets; 38665780c56SLorenzo Bianconi bytes = stats->vs.xdp_bytes; 38766fe4a07SLorenzo Bianconi drops = stats->vs.rx_drops; 3884195e54aSToshiaki Makita } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); 3895fe6e567SLorenzo Bianconi result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err; 390d99a7c2fSLorenzo Bianconi result->xdp_tx_err += xdp_tx_err; 3914195e54aSToshiaki Makita result->xdp_packets += packets; 3924195e54aSToshiaki Makita result->xdp_bytes += bytes; 39366fe4a07SLorenzo Bianconi result->rx_drops += drops; 3944195e54aSToshiaki Makita } 3954195e54aSToshiaki Makita } 3964195e54aSToshiaki Makita 397bc1f4470Sstephen hemminger static void veth_get_stats64(struct net_device *dev, 3982681128fSEric Dumazet struct rtnl_link_stats64 *tot) 3992681128fSEric Dumazet { 4002681128fSEric Dumazet struct veth_priv *priv = netdev_priv(dev); 401d0e2c55eSEric Dumazet struct net_device *peer; 40265780c56SLorenzo Bianconi struct veth_stats rx; 403b4fba476SEric Dumazet u64 packets, bytes; 4042681128fSEric Dumazet 405b4fba476SEric Dumazet tot->tx_dropped = veth_stats_tx(dev, &packets, &bytes); 406b4fba476SEric Dumazet tot->tx_bytes = bytes; 407b4fba476SEric Dumazet tot->tx_packets = packets; 4084195e54aSToshiaki Makita 4094195e54aSToshiaki Makita veth_stats_rx(&rx, dev); 4105fe6e567SLorenzo Bianconi tot->tx_dropped += rx.xdp_tx_err; 4115fe6e567SLorenzo Bianconi tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err; 4124195e54aSToshiaki Makita tot->rx_bytes = rx.xdp_bytes; 4134195e54aSToshiaki Makita tot->rx_packets = rx.xdp_packets; 4142681128fSEric Dumazet 415d0e2c55eSEric Dumazet rcu_read_lock(); 416d0e2c55eSEric Dumazet peer = rcu_dereference(priv->peer); 417d0e2c55eSEric Dumazet if (peer) { 418e25d5dbcSJiang Lidong veth_stats_tx(peer, &packets, &bytes); 419b4fba476SEric Dumazet tot->rx_bytes += bytes; 420b4fba476SEric Dumazet tot->rx_packets += packets; 4214195e54aSToshiaki Makita 4224195e54aSToshiaki Makita veth_stats_rx(&rx, peer); 4235fe6e567SLorenzo Bianconi tot->tx_dropped += rx.peer_tq_xdp_xmit_err; 4245fe6e567SLorenzo Bianconi tot->rx_dropped += rx.xdp_tx_err; 4254195e54aSToshiaki Makita tot->tx_bytes += rx.xdp_bytes; 4264195e54aSToshiaki Makita tot->tx_packets += rx.xdp_packets; 427d0e2c55eSEric Dumazet } 428d0e2c55eSEric Dumazet rcu_read_unlock(); 429e314dbdcSPavel Emelyanov } 430e314dbdcSPavel Emelyanov 4315c70ef85SGao feng /* fake multicast ability */ 4325c70ef85SGao feng static void veth_set_multicast_list(struct net_device *dev) 4335c70ef85SGao feng { 4345c70ef85SGao feng } 4355c70ef85SGao feng 436948d4f21SToshiaki Makita static struct sk_buff *veth_build_skb(void *head, int headroom, int len, 437948d4f21SToshiaki Makita int buflen) 438948d4f21SToshiaki Makita { 439948d4f21SToshiaki Makita struct sk_buff *skb; 440948d4f21SToshiaki Makita 441948d4f21SToshiaki Makita skb = build_skb(head, buflen); 442948d4f21SToshiaki Makita if (!skb) 443948d4f21SToshiaki Makita return NULL; 444948d4f21SToshiaki Makita 445948d4f21SToshiaki Makita skb_reserve(skb, headroom); 446948d4f21SToshiaki Makita skb_put(skb, len); 447948d4f21SToshiaki Makita 448948d4f21SToshiaki Makita return skb; 449948d4f21SToshiaki Makita } 450948d4f21SToshiaki Makita 451638264dcSToshiaki Makita static int veth_select_rxq(struct net_device *dev) 452638264dcSToshiaki Makita { 453638264dcSToshiaki Makita return smp_processor_id() % dev->real_num_rx_queues; 454638264dcSToshiaki Makita } 455638264dcSToshiaki Makita 4569aa1206eSDaniel Borkmann static struct net_device *veth_peer_dev(struct net_device *dev) 4579aa1206eSDaniel Borkmann { 4589aa1206eSDaniel Borkmann struct veth_priv *priv = netdev_priv(dev); 4599aa1206eSDaniel Borkmann 4609aa1206eSDaniel Borkmann /* Callers must be under RCU read side. */ 4619aa1206eSDaniel Borkmann return rcu_dereference(priv->peer); 4629aa1206eSDaniel Borkmann } 4639aa1206eSDaniel Borkmann 464af87a3aaSToshiaki Makita static int veth_xdp_xmit(struct net_device *dev, int n, 4659152cff0SLorenzo Bianconi struct xdp_frame **frames, 4669152cff0SLorenzo Bianconi u32 flags, bool ndo_xmit) 467af87a3aaSToshiaki Makita { 468af87a3aaSToshiaki Makita struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 469fdc13979SLorenzo Bianconi int i, ret = -ENXIO, nxmit = 0; 470af87a3aaSToshiaki Makita struct net_device *rcv; 4715fe6e567SLorenzo Bianconi unsigned int max_len; 472638264dcSToshiaki Makita struct veth_rq *rq; 473af87a3aaSToshiaki Makita 4745fe6e567SLorenzo Bianconi if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 475d99a7c2fSLorenzo Bianconi return -EINVAL; 476af87a3aaSToshiaki Makita 4775fe6e567SLorenzo Bianconi rcu_read_lock(); 478af87a3aaSToshiaki Makita rcv = rcu_dereference(priv->peer); 4795fe6e567SLorenzo Bianconi if (unlikely(!rcv)) 4805fe6e567SLorenzo Bianconi goto out; 481af87a3aaSToshiaki Makita 482af87a3aaSToshiaki Makita rcv_priv = netdev_priv(rcv); 4835fe6e567SLorenzo Bianconi rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 4840e672f30SToke Høiland-Jørgensen /* The napi pointer is set if NAPI is enabled, which ensures that 4850e672f30SToke Høiland-Jørgensen * xdp_ring is initialized on receive side and the peer device is up. 486af87a3aaSToshiaki Makita */ 4870e672f30SToke Høiland-Jørgensen if (!rcu_access_pointer(rq->napi)) 4885fe6e567SLorenzo Bianconi goto out; 489af87a3aaSToshiaki Makita 490af87a3aaSToshiaki Makita max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN; 491af87a3aaSToshiaki Makita 492638264dcSToshiaki Makita spin_lock(&rq->xdp_ring.producer_lock); 493af87a3aaSToshiaki Makita for (i = 0; i < n; i++) { 494af87a3aaSToshiaki Makita struct xdp_frame *frame = frames[i]; 495af87a3aaSToshiaki Makita void *ptr = veth_xdp_to_ptr(frame); 496af87a3aaSToshiaki Makita 497*5142239aSLorenzo Bianconi if (unlikely(xdp_get_frame_len(frame) > max_len || 498fdc13979SLorenzo Bianconi __ptr_ring_produce(&rq->xdp_ring, ptr))) 499fdc13979SLorenzo Bianconi break; 500fdc13979SLorenzo Bianconi nxmit++; 501af87a3aaSToshiaki Makita } 502638264dcSToshiaki Makita spin_unlock(&rq->xdp_ring.producer_lock); 503af87a3aaSToshiaki Makita 504af87a3aaSToshiaki Makita if (flags & XDP_XMIT_FLUSH) 505638264dcSToshiaki Makita __veth_xdp_flush(rq); 506af87a3aaSToshiaki Makita 507fdc13979SLorenzo Bianconi ret = nxmit; 5089152cff0SLorenzo Bianconi if (ndo_xmit) { 5095fe6e567SLorenzo Bianconi u64_stats_update_begin(&rq->stats.syncp); 510fdc13979SLorenzo Bianconi rq->stats.vs.peer_tq_xdp_xmit += nxmit; 511fdc13979SLorenzo Bianconi rq->stats.vs.peer_tq_xdp_xmit_err += n - nxmit; 5129152cff0SLorenzo Bianconi u64_stats_update_end(&rq->stats.syncp); 5135fe6e567SLorenzo Bianconi } 5149152cff0SLorenzo Bianconi 5155fe6e567SLorenzo Bianconi out: 516b23bfa56SJohn Fastabend rcu_read_unlock(); 5172131479dSToshiaki Makita 5182131479dSToshiaki Makita return ret; 519af87a3aaSToshiaki Makita } 520af87a3aaSToshiaki Makita 5219152cff0SLorenzo Bianconi static int veth_ndo_xdp_xmit(struct net_device *dev, int n, 5229152cff0SLorenzo Bianconi struct xdp_frame **frames, u32 flags) 5239152cff0SLorenzo Bianconi { 5245fe6e567SLorenzo Bianconi int err; 5255fe6e567SLorenzo Bianconi 5265fe6e567SLorenzo Bianconi err = veth_xdp_xmit(dev, n, frames, flags, true); 5275fe6e567SLorenzo Bianconi if (err < 0) { 5285fe6e567SLorenzo Bianconi struct veth_priv *priv = netdev_priv(dev); 5295fe6e567SLorenzo Bianconi 5305fe6e567SLorenzo Bianconi atomic64_add(n, &priv->dropped); 5315fe6e567SLorenzo Bianconi } 5325fe6e567SLorenzo Bianconi 5335fe6e567SLorenzo Bianconi return err; 5349152cff0SLorenzo Bianconi } 5359152cff0SLorenzo Bianconi 536bd32aa1fSLorenzo Bianconi static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 5379cda7807SToshiaki Makita { 538fdc13979SLorenzo Bianconi int sent, i, err = 0, drops; 5399cda7807SToshiaki Makita 540bd32aa1fSLorenzo Bianconi sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); 5419cda7807SToshiaki Makita if (sent < 0) { 5429cda7807SToshiaki Makita err = sent; 5439cda7807SToshiaki Makita sent = 0; 5449cda7807SToshiaki Makita } 545fdc13979SLorenzo Bianconi 546fdc13979SLorenzo Bianconi for (i = sent; unlikely(i < bq->count); i++) 547fdc13979SLorenzo Bianconi xdp_return_frame(bq->q[i]); 548fdc13979SLorenzo Bianconi 549fdc13979SLorenzo Bianconi drops = bq->count - sent; 550fdc13979SLorenzo Bianconi trace_xdp_bulk_tx(rq->dev, sent, drops, err); 5519cda7807SToshiaki Makita 5525fe6e567SLorenzo Bianconi u64_stats_update_begin(&rq->stats.syncp); 5535fe6e567SLorenzo Bianconi rq->stats.vs.xdp_tx += sent; 554fdc13979SLorenzo Bianconi rq->stats.vs.xdp_tx_err += drops; 5555fe6e567SLorenzo Bianconi u64_stats_update_end(&rq->stats.syncp); 5565fe6e567SLorenzo Bianconi 5579cda7807SToshiaki Makita bq->count = 0; 5589cda7807SToshiaki Makita } 5599cda7807SToshiaki Makita 560bd32aa1fSLorenzo Bianconi static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 561d1396004SToshiaki Makita { 562bd32aa1fSLorenzo Bianconi struct veth_priv *rcv_priv, *priv = netdev_priv(rq->dev); 563d1396004SToshiaki Makita struct net_device *rcv; 564bd32aa1fSLorenzo Bianconi struct veth_rq *rcv_rq; 565d1396004SToshiaki Makita 566d1396004SToshiaki Makita rcu_read_lock(); 567bd32aa1fSLorenzo Bianconi veth_xdp_flush_bq(rq, bq); 568d1396004SToshiaki Makita rcv = rcu_dereference(priv->peer); 569d1396004SToshiaki Makita if (unlikely(!rcv)) 570d1396004SToshiaki Makita goto out; 571d1396004SToshiaki Makita 572d1396004SToshiaki Makita rcv_priv = netdev_priv(rcv); 573bd32aa1fSLorenzo Bianconi rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 574d1396004SToshiaki Makita /* xdp_ring is initialized on receive side? */ 575bd32aa1fSLorenzo Bianconi if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog))) 576d1396004SToshiaki Makita goto out; 577d1396004SToshiaki Makita 578bd32aa1fSLorenzo Bianconi __veth_xdp_flush(rcv_rq); 579d1396004SToshiaki Makita out: 580d1396004SToshiaki Makita rcu_read_unlock(); 581d1396004SToshiaki Makita } 582d1396004SToshiaki Makita 583bd32aa1fSLorenzo Bianconi static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp, 5849cda7807SToshiaki Makita struct veth_xdp_tx_bq *bq) 585d1396004SToshiaki Makita { 5861b698fa5SLorenzo Bianconi struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); 587d1396004SToshiaki Makita 588d1396004SToshiaki Makita if (unlikely(!frame)) 589d1396004SToshiaki Makita return -EOVERFLOW; 590d1396004SToshiaki Makita 5919cda7807SToshiaki Makita if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE)) 592bd32aa1fSLorenzo Bianconi veth_xdp_flush_bq(rq, bq); 5939cda7807SToshiaki Makita 5949cda7807SToshiaki Makita bq->q[bq->count++] = frame; 5959cda7807SToshiaki Makita 5969cda7807SToshiaki Makita return 0; 597d1396004SToshiaki Makita } 598d1396004SToshiaki Makita 59965e6dcf7SLorenzo Bianconi static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq, 600d1396004SToshiaki Makita struct xdp_frame *frame, 6011c5b82e5SLorenzo Bianconi struct veth_xdp_tx_bq *bq, 6021c5b82e5SLorenzo Bianconi struct veth_stats *stats) 6039fc8d518SToshiaki Makita { 604d1396004SToshiaki Makita struct xdp_frame orig_frame; 6059fc8d518SToshiaki Makita struct bpf_prog *xdp_prog; 6069fc8d518SToshiaki Makita 6079fc8d518SToshiaki Makita rcu_read_lock(); 608638264dcSToshiaki Makita xdp_prog = rcu_dereference(rq->xdp_prog); 6099fc8d518SToshiaki Makita if (likely(xdp_prog)) { 6109fc8d518SToshiaki Makita struct xdp_buff xdp; 6119fc8d518SToshiaki Makita u32 act; 6129fc8d518SToshiaki Makita 613fc379872SLorenzo Bianconi xdp_convert_frame_to_buff(frame, &xdp); 614638264dcSToshiaki Makita xdp.rxq = &rq->xdp_rxq; 6159fc8d518SToshiaki Makita 6169fc8d518SToshiaki Makita act = bpf_prog_run_xdp(xdp_prog, &xdp); 6179fc8d518SToshiaki Makita 6189fc8d518SToshiaki Makita switch (act) { 6199fc8d518SToshiaki Makita case XDP_PASS: 62089f479f0SLorenzo Bianconi if (xdp_update_frame_from_buff(&xdp, frame)) 62189f479f0SLorenzo Bianconi goto err_xdp; 6229fc8d518SToshiaki Makita break; 623d1396004SToshiaki Makita case XDP_TX: 624d1396004SToshiaki Makita orig_frame = *frame; 625d1396004SToshiaki Makita xdp.rxq->mem = frame->mem; 626bd32aa1fSLorenzo Bianconi if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) { 627638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 628d1396004SToshiaki Makita frame = &orig_frame; 6291c5b82e5SLorenzo Bianconi stats->rx_drops++; 630d1396004SToshiaki Makita goto err_xdp; 631d1396004SToshiaki Makita } 6321c5b82e5SLorenzo Bianconi stats->xdp_tx++; 633d1396004SToshiaki Makita rcu_read_unlock(); 634d1396004SToshiaki Makita goto xdp_xmit; 635d1396004SToshiaki Makita case XDP_REDIRECT: 636d1396004SToshiaki Makita orig_frame = *frame; 637d1396004SToshiaki Makita xdp.rxq->mem = frame->mem; 638638264dcSToshiaki Makita if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { 639d1396004SToshiaki Makita frame = &orig_frame; 6401c5b82e5SLorenzo Bianconi stats->rx_drops++; 641d1396004SToshiaki Makita goto err_xdp; 642d1396004SToshiaki Makita } 6431c5b82e5SLorenzo Bianconi stats->xdp_redirect++; 644d1396004SToshiaki Makita rcu_read_unlock(); 645d1396004SToshiaki Makita goto xdp_xmit; 6469fc8d518SToshiaki Makita default: 647c8064e5bSPaolo Abeni bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); 648df561f66SGustavo A. R. Silva fallthrough; 6499fc8d518SToshiaki Makita case XDP_ABORTED: 650638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 651df561f66SGustavo A. R. Silva fallthrough; 6529fc8d518SToshiaki Makita case XDP_DROP: 6531c5b82e5SLorenzo Bianconi stats->xdp_drops++; 6549fc8d518SToshiaki Makita goto err_xdp; 6559fc8d518SToshiaki Makita } 6569fc8d518SToshiaki Makita } 6579fc8d518SToshiaki Makita rcu_read_unlock(); 6589fc8d518SToshiaki Makita 65965e6dcf7SLorenzo Bianconi return frame; 6609fc8d518SToshiaki Makita err_xdp: 6619fc8d518SToshiaki Makita rcu_read_unlock(); 6629fc8d518SToshiaki Makita xdp_return_frame(frame); 663d1396004SToshiaki Makita xdp_xmit: 6649fc8d518SToshiaki Makita return NULL; 6659fc8d518SToshiaki Makita } 6669fc8d518SToshiaki Makita 66765e6dcf7SLorenzo Bianconi /* frames array contains VETH_XDP_BATCH at most */ 66865e6dcf7SLorenzo Bianconi static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames, 66965e6dcf7SLorenzo Bianconi int n_xdpf, struct veth_xdp_tx_bq *bq, 67065e6dcf7SLorenzo Bianconi struct veth_stats *stats) 67165e6dcf7SLorenzo Bianconi { 67265e6dcf7SLorenzo Bianconi void *skbs[VETH_XDP_BATCH]; 67365e6dcf7SLorenzo Bianconi int i; 67465e6dcf7SLorenzo Bianconi 67565e6dcf7SLorenzo Bianconi if (xdp_alloc_skb_bulk(skbs, n_xdpf, 67665e6dcf7SLorenzo Bianconi GFP_ATOMIC | __GFP_ZERO) < 0) { 67765e6dcf7SLorenzo Bianconi for (i = 0; i < n_xdpf; i++) 67865e6dcf7SLorenzo Bianconi xdp_return_frame(frames[i]); 67965e6dcf7SLorenzo Bianconi stats->rx_drops += n_xdpf; 68065e6dcf7SLorenzo Bianconi 68165e6dcf7SLorenzo Bianconi return; 68265e6dcf7SLorenzo Bianconi } 68365e6dcf7SLorenzo Bianconi 68465e6dcf7SLorenzo Bianconi for (i = 0; i < n_xdpf; i++) { 68565e6dcf7SLorenzo Bianconi struct sk_buff *skb = skbs[i]; 68665e6dcf7SLorenzo Bianconi 68765e6dcf7SLorenzo Bianconi skb = __xdp_build_skb_from_frame(frames[i], skb, 68865e6dcf7SLorenzo Bianconi rq->dev); 68965e6dcf7SLorenzo Bianconi if (!skb) { 69065e6dcf7SLorenzo Bianconi xdp_return_frame(frames[i]); 69165e6dcf7SLorenzo Bianconi stats->rx_drops++; 69265e6dcf7SLorenzo Bianconi continue; 69365e6dcf7SLorenzo Bianconi } 69465e6dcf7SLorenzo Bianconi napi_gro_receive(&rq->xdp_napi, skb); 69565e6dcf7SLorenzo Bianconi } 69665e6dcf7SLorenzo Bianconi } 69765e6dcf7SLorenzo Bianconi 6981c5b82e5SLorenzo Bianconi static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, 6991c5b82e5SLorenzo Bianconi struct sk_buff *skb, 7001c5b82e5SLorenzo Bianconi struct veth_xdp_tx_bq *bq, 7011c5b82e5SLorenzo Bianconi struct veth_stats *stats) 702948d4f21SToshiaki Makita { 70343b5169dSLorenzo Bianconi u32 pktlen, headroom, act, metalen, frame_sz; 704948d4f21SToshiaki Makita void *orig_data, *orig_data_end; 705948d4f21SToshiaki Makita struct bpf_prog *xdp_prog; 706948d4f21SToshiaki Makita int mac_len, delta, off; 707948d4f21SToshiaki Makita struct xdp_buff xdp; 708948d4f21SToshiaki Makita 709d504fff0SPaolo Abeni skb_prepare_for_gro(skb); 7104bf9ffa0SToshiaki Makita 711948d4f21SToshiaki Makita rcu_read_lock(); 712638264dcSToshiaki Makita xdp_prog = rcu_dereference(rq->xdp_prog); 713948d4f21SToshiaki Makita if (unlikely(!xdp_prog)) { 714948d4f21SToshiaki Makita rcu_read_unlock(); 715948d4f21SToshiaki Makita goto out; 716948d4f21SToshiaki Makita } 717948d4f21SToshiaki Makita 718948d4f21SToshiaki Makita mac_len = skb->data - skb_mac_header(skb); 719948d4f21SToshiaki Makita pktlen = skb->len + mac_len; 720948d4f21SToshiaki Makita headroom = skb_headroom(skb) - mac_len; 721948d4f21SToshiaki Makita 722948d4f21SToshiaki Makita if (skb_shared(skb) || skb_head_is_locked(skb) || 723948d4f21SToshiaki Makita skb_is_nonlinear(skb) || headroom < XDP_PACKET_HEADROOM) { 724948d4f21SToshiaki Makita struct sk_buff *nskb; 725948d4f21SToshiaki Makita int size, head_off; 726948d4f21SToshiaki Makita void *head, *start; 727948d4f21SToshiaki Makita struct page *page; 728948d4f21SToshiaki Makita 729948d4f21SToshiaki Makita size = SKB_DATA_ALIGN(VETH_XDP_HEADROOM + pktlen) + 730948d4f21SToshiaki Makita SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 731948d4f21SToshiaki Makita if (size > PAGE_SIZE) 732948d4f21SToshiaki Makita goto drop; 733948d4f21SToshiaki Makita 734948d4f21SToshiaki Makita page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); 735948d4f21SToshiaki Makita if (!page) 736948d4f21SToshiaki Makita goto drop; 737948d4f21SToshiaki Makita 738948d4f21SToshiaki Makita head = page_address(page); 739948d4f21SToshiaki Makita start = head + VETH_XDP_HEADROOM; 740948d4f21SToshiaki Makita if (skb_copy_bits(skb, -mac_len, start, pktlen)) { 741948d4f21SToshiaki Makita page_frag_free(head); 742948d4f21SToshiaki Makita goto drop; 743948d4f21SToshiaki Makita } 744948d4f21SToshiaki Makita 74545a9e6d8SJesper Dangaard Brouer nskb = veth_build_skb(head, VETH_XDP_HEADROOM + mac_len, 74645a9e6d8SJesper Dangaard Brouer skb->len, PAGE_SIZE); 747948d4f21SToshiaki Makita if (!nskb) { 748948d4f21SToshiaki Makita page_frag_free(head); 749948d4f21SToshiaki Makita goto drop; 750948d4f21SToshiaki Makita } 751948d4f21SToshiaki Makita 752948d4f21SToshiaki Makita skb_copy_header(nskb, skb); 753948d4f21SToshiaki Makita head_off = skb_headroom(nskb) - skb_headroom(skb); 754948d4f21SToshiaki Makita skb_headers_offset_update(nskb, head_off); 755948d4f21SToshiaki Makita consume_skb(skb); 756948d4f21SToshiaki Makita skb = nskb; 757948d4f21SToshiaki Makita } 758948d4f21SToshiaki Makita 75945a9e6d8SJesper Dangaard Brouer /* SKB "head" area always have tailroom for skb_shared_info */ 760be9df4afSLorenzo Bianconi frame_sz = skb_end_pointer(skb) - skb->head; 76143b5169dSLorenzo Bianconi frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 76243b5169dSLorenzo Bianconi xdp_init_buff(&xdp, frame_sz, &rq->xdp_rxq); 763be9df4afSLorenzo Bianconi xdp_prepare_buff(&xdp, skb->head, skb->mac_header, pktlen, true); 76445a9e6d8SJesper Dangaard Brouer 765948d4f21SToshiaki Makita orig_data = xdp.data; 766948d4f21SToshiaki Makita orig_data_end = xdp.data_end; 767948d4f21SToshiaki Makita 768948d4f21SToshiaki Makita act = bpf_prog_run_xdp(xdp_prog, &xdp); 769948d4f21SToshiaki Makita 770948d4f21SToshiaki Makita switch (act) { 771948d4f21SToshiaki Makita case XDP_PASS: 772948d4f21SToshiaki Makita break; 773d1396004SToshiaki Makita case XDP_TX: 774d1396004SToshiaki Makita get_page(virt_to_page(xdp.data)); 775d1396004SToshiaki Makita consume_skb(skb); 776638264dcSToshiaki Makita xdp.rxq->mem = rq->xdp_mem; 777bd32aa1fSLorenzo Bianconi if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) { 778638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 7791c5b82e5SLorenzo Bianconi stats->rx_drops++; 780d1396004SToshiaki Makita goto err_xdp; 781d1396004SToshiaki Makita } 7821c5b82e5SLorenzo Bianconi stats->xdp_tx++; 783d1396004SToshiaki Makita rcu_read_unlock(); 784d1396004SToshiaki Makita goto xdp_xmit; 785d1396004SToshiaki Makita case XDP_REDIRECT: 786d1396004SToshiaki Makita get_page(virt_to_page(xdp.data)); 787d1396004SToshiaki Makita consume_skb(skb); 788638264dcSToshiaki Makita xdp.rxq->mem = rq->xdp_mem; 7891c5b82e5SLorenzo Bianconi if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { 7901c5b82e5SLorenzo Bianconi stats->rx_drops++; 791d1396004SToshiaki Makita goto err_xdp; 7921c5b82e5SLorenzo Bianconi } 7931c5b82e5SLorenzo Bianconi stats->xdp_redirect++; 794d1396004SToshiaki Makita rcu_read_unlock(); 795d1396004SToshiaki Makita goto xdp_xmit; 796948d4f21SToshiaki Makita default: 797c8064e5bSPaolo Abeni bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); 798df561f66SGustavo A. R. Silva fallthrough; 799948d4f21SToshiaki Makita case XDP_ABORTED: 800638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 801df561f66SGustavo A. R. Silva fallthrough; 802948d4f21SToshiaki Makita case XDP_DROP: 8031c5b82e5SLorenzo Bianconi stats->xdp_drops++; 8041c5b82e5SLorenzo Bianconi goto xdp_drop; 805948d4f21SToshiaki Makita } 806948d4f21SToshiaki Makita rcu_read_unlock(); 807948d4f21SToshiaki Makita 80845a9e6d8SJesper Dangaard Brouer /* check if bpf_xdp_adjust_head was used */ 809948d4f21SToshiaki Makita delta = orig_data - xdp.data; 810948d4f21SToshiaki Makita off = mac_len + delta; 811948d4f21SToshiaki Makita if (off > 0) 812948d4f21SToshiaki Makita __skb_push(skb, off); 813948d4f21SToshiaki Makita else if (off < 0) 814948d4f21SToshiaki Makita __skb_pull(skb, -off); 815948d4f21SToshiaki Makita skb->mac_header -= delta; 81645a9e6d8SJesper Dangaard Brouer 81745a9e6d8SJesper Dangaard Brouer /* check if bpf_xdp_adjust_tail was used */ 818948d4f21SToshiaki Makita off = xdp.data_end - orig_data_end; 819948d4f21SToshiaki Makita if (off != 0) 82045a9e6d8SJesper Dangaard Brouer __skb_put(skb, off); /* positive on grow, negative on shrink */ 821638264dcSToshiaki Makita skb->protocol = eth_type_trans(skb, rq->dev); 822948d4f21SToshiaki Makita 823948d4f21SToshiaki Makita metalen = xdp.data - xdp.data_meta; 824948d4f21SToshiaki Makita if (metalen) 825948d4f21SToshiaki Makita skb_metadata_set(skb, metalen); 826948d4f21SToshiaki Makita out: 827948d4f21SToshiaki Makita return skb; 828948d4f21SToshiaki Makita drop: 8291c5b82e5SLorenzo Bianconi stats->rx_drops++; 8301c5b82e5SLorenzo Bianconi xdp_drop: 831948d4f21SToshiaki Makita rcu_read_unlock(); 832948d4f21SToshiaki Makita kfree_skb(skb); 833948d4f21SToshiaki Makita return NULL; 834d1396004SToshiaki Makita err_xdp: 835d1396004SToshiaki Makita rcu_read_unlock(); 836d1396004SToshiaki Makita page_frag_free(xdp.data); 837d1396004SToshiaki Makita xdp_xmit: 838d1396004SToshiaki Makita return NULL; 839948d4f21SToshiaki Makita } 840948d4f21SToshiaki Makita 8411c5b82e5SLorenzo Bianconi static int veth_xdp_rcv(struct veth_rq *rq, int budget, 8421c5b82e5SLorenzo Bianconi struct veth_xdp_tx_bq *bq, 8431c5b82e5SLorenzo Bianconi struct veth_stats *stats) 844948d4f21SToshiaki Makita { 84565e6dcf7SLorenzo Bianconi int i, done = 0, n_xdpf = 0; 84665e6dcf7SLorenzo Bianconi void *xdpf[VETH_XDP_BATCH]; 847948d4f21SToshiaki Makita 848948d4f21SToshiaki Makita for (i = 0; i < budget; i++) { 849638264dcSToshiaki Makita void *ptr = __ptr_ring_consume(&rq->xdp_ring); 850948d4f21SToshiaki Makita 8519fc8d518SToshiaki Makita if (!ptr) 852948d4f21SToshiaki Makita break; 853948d4f21SToshiaki Makita 854d1396004SToshiaki Makita if (veth_is_xdp_frame(ptr)) { 85565e6dcf7SLorenzo Bianconi /* ndo_xdp_xmit */ 8564195e54aSToshiaki Makita struct xdp_frame *frame = veth_ptr_to_xdp(ptr); 8574195e54aSToshiaki Makita 858*5142239aSLorenzo Bianconi stats->xdp_bytes += xdp_get_frame_len(frame); 85965e6dcf7SLorenzo Bianconi frame = veth_xdp_rcv_one(rq, frame, bq, stats); 86065e6dcf7SLorenzo Bianconi if (frame) { 86165e6dcf7SLorenzo Bianconi /* XDP_PASS */ 86265e6dcf7SLorenzo Bianconi xdpf[n_xdpf++] = frame; 86365e6dcf7SLorenzo Bianconi if (n_xdpf == VETH_XDP_BATCH) { 86465e6dcf7SLorenzo Bianconi veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, 86565e6dcf7SLorenzo Bianconi bq, stats); 86665e6dcf7SLorenzo Bianconi n_xdpf = 0; 86765e6dcf7SLorenzo Bianconi } 86865e6dcf7SLorenzo Bianconi } 869d1396004SToshiaki Makita } else { 87065e6dcf7SLorenzo Bianconi /* ndo_start_xmit */ 87165e6dcf7SLorenzo Bianconi struct sk_buff *skb = ptr; 87265e6dcf7SLorenzo Bianconi 8731c5b82e5SLorenzo Bianconi stats->xdp_bytes += skb->len; 8741c5b82e5SLorenzo Bianconi skb = veth_xdp_rcv_skb(rq, skb, bq, stats); 8759695b7deSPaolo Abeni if (skb) { 8769695b7deSPaolo Abeni if (skb_shared(skb) || skb_unclone(skb, GFP_ATOMIC)) 8779695b7deSPaolo Abeni netif_receive_skb(skb); 8789695b7deSPaolo Abeni else 879638264dcSToshiaki Makita napi_gro_receive(&rq->xdp_napi, skb); 88065e6dcf7SLorenzo Bianconi } 8819695b7deSPaolo Abeni } 882948d4f21SToshiaki Makita done++; 883948d4f21SToshiaki Makita } 884948d4f21SToshiaki Makita 88565e6dcf7SLorenzo Bianconi if (n_xdpf) 88665e6dcf7SLorenzo Bianconi veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, bq, stats); 88765e6dcf7SLorenzo Bianconi 8884195e54aSToshiaki Makita u64_stats_update_begin(&rq->stats.syncp); 8899152cff0SLorenzo Bianconi rq->stats.vs.xdp_redirect += stats->xdp_redirect; 8901c5b82e5SLorenzo Bianconi rq->stats.vs.xdp_bytes += stats->xdp_bytes; 89166fe4a07SLorenzo Bianconi rq->stats.vs.xdp_drops += stats->xdp_drops; 89266fe4a07SLorenzo Bianconi rq->stats.vs.rx_drops += stats->rx_drops; 89365780c56SLorenzo Bianconi rq->stats.vs.xdp_packets += done; 8944195e54aSToshiaki Makita u64_stats_update_end(&rq->stats.syncp); 8954195e54aSToshiaki Makita 896948d4f21SToshiaki Makita return done; 897948d4f21SToshiaki Makita } 898948d4f21SToshiaki Makita 899948d4f21SToshiaki Makita static int veth_poll(struct napi_struct *napi, int budget) 900948d4f21SToshiaki Makita { 901638264dcSToshiaki Makita struct veth_rq *rq = 902638264dcSToshiaki Makita container_of(napi, struct veth_rq, xdp_napi); 9031c5b82e5SLorenzo Bianconi struct veth_stats stats = {}; 9049cda7807SToshiaki Makita struct veth_xdp_tx_bq bq; 905948d4f21SToshiaki Makita int done; 906948d4f21SToshiaki Makita 9079cda7807SToshiaki Makita bq.count = 0; 9089cda7807SToshiaki Makita 909d1396004SToshiaki Makita xdp_set_return_frame_no_direct(); 9101c5b82e5SLorenzo Bianconi done = veth_xdp_rcv(rq, budget, &bq, &stats); 911948d4f21SToshiaki Makita 912948d4f21SToshiaki Makita if (done < budget && napi_complete_done(napi, done)) { 913948d4f21SToshiaki Makita /* Write rx_notify_masked before reading ptr_ring */ 914638264dcSToshiaki Makita smp_store_mb(rq->rx_notify_masked, false); 915638264dcSToshiaki Makita if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) { 91668468d8cSEric Dumazet if (napi_schedule_prep(&rq->xdp_napi)) { 91768468d8cSEric Dumazet WRITE_ONCE(rq->rx_notify_masked, true); 91868468d8cSEric Dumazet __napi_schedule(&rq->xdp_napi); 91968468d8cSEric Dumazet } 920948d4f21SToshiaki Makita } 921948d4f21SToshiaki Makita } 922948d4f21SToshiaki Makita 9231c5b82e5SLorenzo Bianconi if (stats.xdp_tx > 0) 924bd32aa1fSLorenzo Bianconi veth_xdp_flush(rq, &bq); 9251c5b82e5SLorenzo Bianconi if (stats.xdp_redirect > 0) 9261d233886SToke Høiland-Jørgensen xdp_do_flush(); 927d1396004SToshiaki Makita xdp_clear_return_frame_no_direct(); 928d1396004SToshiaki Makita 929948d4f21SToshiaki Makita return done; 930948d4f21SToshiaki Makita } 931948d4f21SToshiaki Makita 932dedd53c5SPaolo Abeni static int __veth_napi_enable_range(struct net_device *dev, int start, int end) 933948d4f21SToshiaki Makita { 934948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 935638264dcSToshiaki Makita int err, i; 936948d4f21SToshiaki Makita 937dedd53c5SPaolo Abeni for (i = start; i < end; i++) { 938638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 939638264dcSToshiaki Makita 940638264dcSToshiaki Makita err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL); 941948d4f21SToshiaki Makita if (err) 942638264dcSToshiaki Makita goto err_xdp_ring; 943638264dcSToshiaki Makita } 944948d4f21SToshiaki Makita 945dedd53c5SPaolo Abeni for (i = start; i < end; i++) { 946638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 947638264dcSToshiaki Makita 948638264dcSToshiaki Makita napi_enable(&rq->xdp_napi); 949d3256efdSPaolo Abeni rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi); 950638264dcSToshiaki Makita } 951948d4f21SToshiaki Makita 952948d4f21SToshiaki Makita return 0; 953dedd53c5SPaolo Abeni 954638264dcSToshiaki Makita err_xdp_ring: 955dedd53c5SPaolo Abeni for (i--; i >= start; i--) 956638264dcSToshiaki Makita ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free); 957638264dcSToshiaki Makita 958638264dcSToshiaki Makita return err; 959948d4f21SToshiaki Makita } 960948d4f21SToshiaki Makita 961dedd53c5SPaolo Abeni static int __veth_napi_enable(struct net_device *dev) 962dedd53c5SPaolo Abeni { 963dedd53c5SPaolo Abeni return __veth_napi_enable_range(dev, 0, dev->real_num_rx_queues); 964dedd53c5SPaolo Abeni } 965dedd53c5SPaolo Abeni 966dedd53c5SPaolo Abeni static void veth_napi_del_range(struct net_device *dev, int start, int end) 967948d4f21SToshiaki Makita { 968948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 969638264dcSToshiaki Makita int i; 970948d4f21SToshiaki Makita 971dedd53c5SPaolo Abeni for (i = start; i < end; i++) { 972638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 973638264dcSToshiaki Makita 974d3256efdSPaolo Abeni rcu_assign_pointer(priv->rq[i].napi, NULL); 975638264dcSToshiaki Makita napi_disable(&rq->xdp_napi); 9765198d545SJakub Kicinski __netif_napi_del(&rq->xdp_napi); 977638264dcSToshiaki Makita } 978638264dcSToshiaki Makita synchronize_net(); 979638264dcSToshiaki Makita 980dedd53c5SPaolo Abeni for (i = start; i < end; i++) { 981638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 982638264dcSToshiaki Makita 983638264dcSToshiaki Makita rq->rx_notify_masked = false; 984638264dcSToshiaki Makita ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free); 985638264dcSToshiaki Makita } 986948d4f21SToshiaki Makita } 987948d4f21SToshiaki Makita 988dedd53c5SPaolo Abeni static void veth_napi_del(struct net_device *dev) 989dedd53c5SPaolo Abeni { 990dedd53c5SPaolo Abeni veth_napi_del_range(dev, 0, dev->real_num_rx_queues); 991dedd53c5SPaolo Abeni } 992dedd53c5SPaolo Abeni 993d3256efdSPaolo Abeni static bool veth_gro_requested(const struct net_device *dev) 994d3256efdSPaolo Abeni { 995d3256efdSPaolo Abeni return !!(dev->wanted_features & NETIF_F_GRO); 996d3256efdSPaolo Abeni } 997d3256efdSPaolo Abeni 998dedd53c5SPaolo Abeni static int veth_enable_xdp_range(struct net_device *dev, int start, int end, 999dedd53c5SPaolo Abeni bool napi_already_on) 1000948d4f21SToshiaki Makita { 1001948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 1002638264dcSToshiaki Makita int err, i; 1003948d4f21SToshiaki Makita 1004dedd53c5SPaolo Abeni for (i = start; i < end; i++) { 1005638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 1006948d4f21SToshiaki Makita 1007d3256efdSPaolo Abeni if (!napi_already_on) 1008b02e5a0eSBjörn Töpel netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); 1009b02e5a0eSBjörn Töpel err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id); 1010948d4f21SToshiaki Makita if (err < 0) 1011638264dcSToshiaki Makita goto err_rxq_reg; 1012638264dcSToshiaki Makita 1013638264dcSToshiaki Makita err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 1014638264dcSToshiaki Makita MEM_TYPE_PAGE_SHARED, 1015638264dcSToshiaki Makita NULL); 1016638264dcSToshiaki Makita if (err < 0) 1017638264dcSToshiaki Makita goto err_reg_mem; 1018638264dcSToshiaki Makita 1019638264dcSToshiaki Makita /* Save original mem info as it can be overwritten */ 1020638264dcSToshiaki Makita rq->xdp_mem = rq->xdp_rxq.mem; 1021638264dcSToshiaki Makita } 1022dedd53c5SPaolo Abeni return 0; 1023dedd53c5SPaolo Abeni 1024dedd53c5SPaolo Abeni err_reg_mem: 1025dedd53c5SPaolo Abeni xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); 1026dedd53c5SPaolo Abeni err_rxq_reg: 1027dedd53c5SPaolo Abeni for (i--; i >= start; i--) { 1028dedd53c5SPaolo Abeni struct veth_rq *rq = &priv->rq[i]; 1029dedd53c5SPaolo Abeni 1030dedd53c5SPaolo Abeni xdp_rxq_info_unreg(&rq->xdp_rxq); 1031dedd53c5SPaolo Abeni if (!napi_already_on) 1032dedd53c5SPaolo Abeni netif_napi_del(&rq->xdp_napi); 1033dedd53c5SPaolo Abeni } 1034dedd53c5SPaolo Abeni 1035dedd53c5SPaolo Abeni return err; 1036dedd53c5SPaolo Abeni } 1037dedd53c5SPaolo Abeni 1038dedd53c5SPaolo Abeni static void veth_disable_xdp_range(struct net_device *dev, int start, int end, 1039dedd53c5SPaolo Abeni bool delete_napi) 1040dedd53c5SPaolo Abeni { 1041dedd53c5SPaolo Abeni struct veth_priv *priv = netdev_priv(dev); 1042dedd53c5SPaolo Abeni int i; 1043dedd53c5SPaolo Abeni 1044dedd53c5SPaolo Abeni for (i = start; i < end; i++) { 1045dedd53c5SPaolo Abeni struct veth_rq *rq = &priv->rq[i]; 1046dedd53c5SPaolo Abeni 1047dedd53c5SPaolo Abeni rq->xdp_rxq.mem = rq->xdp_mem; 1048dedd53c5SPaolo Abeni xdp_rxq_info_unreg(&rq->xdp_rxq); 1049dedd53c5SPaolo Abeni 1050dedd53c5SPaolo Abeni if (delete_napi) 1051dedd53c5SPaolo Abeni netif_napi_del(&rq->xdp_napi); 1052dedd53c5SPaolo Abeni } 1053dedd53c5SPaolo Abeni } 1054dedd53c5SPaolo Abeni 1055dedd53c5SPaolo Abeni static int veth_enable_xdp(struct net_device *dev) 1056dedd53c5SPaolo Abeni { 1057dedd53c5SPaolo Abeni bool napi_already_on = veth_gro_requested(dev) && (dev->flags & IFF_UP); 1058dedd53c5SPaolo Abeni struct veth_priv *priv = netdev_priv(dev); 1059dedd53c5SPaolo Abeni int err, i; 1060dedd53c5SPaolo Abeni 1061dedd53c5SPaolo Abeni if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) { 1062dedd53c5SPaolo Abeni err = veth_enable_xdp_range(dev, 0, dev->real_num_rx_queues, napi_already_on); 1063dedd53c5SPaolo Abeni if (err) 1064dedd53c5SPaolo Abeni return err; 1065948d4f21SToshiaki Makita 1066d3256efdSPaolo Abeni if (!napi_already_on) { 1067d3256efdSPaolo Abeni err = __veth_napi_enable(dev); 1068dedd53c5SPaolo Abeni if (err) { 1069dedd53c5SPaolo Abeni veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, true); 1070dedd53c5SPaolo Abeni return err; 1071dedd53c5SPaolo Abeni } 1072d3256efdSPaolo Abeni 1073d3256efdSPaolo Abeni if (!veth_gro_requested(dev)) { 1074d3256efdSPaolo Abeni /* user-space did not require GRO, but adding XDP 1075d3256efdSPaolo Abeni * is supposed to get GRO working 1076d3256efdSPaolo Abeni */ 1077d3256efdSPaolo Abeni dev->features |= NETIF_F_GRO; 1078d3256efdSPaolo Abeni netdev_features_change(dev); 1079d3256efdSPaolo Abeni } 1080d3256efdSPaolo Abeni } 1081948d4f21SToshiaki Makita } 1082948d4f21SToshiaki Makita 1083d3256efdSPaolo Abeni for (i = 0; i < dev->real_num_rx_queues; i++) { 1084638264dcSToshiaki Makita rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog); 1085d3256efdSPaolo Abeni rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi); 1086d3256efdSPaolo Abeni } 1087948d4f21SToshiaki Makita 1088948d4f21SToshiaki Makita return 0; 1089948d4f21SToshiaki Makita } 1090948d4f21SToshiaki Makita 1091948d4f21SToshiaki Makita static void veth_disable_xdp(struct net_device *dev) 1092948d4f21SToshiaki Makita { 1093948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 1094638264dcSToshiaki Makita int i; 1095948d4f21SToshiaki Makita 1096638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) 1097638264dcSToshiaki Makita rcu_assign_pointer(priv->rq[i].xdp_prog, NULL); 1098d3256efdSPaolo Abeni 1099d3256efdSPaolo Abeni if (!netif_running(dev) || !veth_gro_requested(dev)) { 1100948d4f21SToshiaki Makita veth_napi_del(dev); 1101d3256efdSPaolo Abeni 1102d3256efdSPaolo Abeni /* if user-space did not require GRO, since adding XDP 1103d3256efdSPaolo Abeni * enabled it, clear it now 1104d3256efdSPaolo Abeni */ 1105d3256efdSPaolo Abeni if (!veth_gro_requested(dev) && netif_running(dev)) { 1106d3256efdSPaolo Abeni dev->features &= ~NETIF_F_GRO; 1107d3256efdSPaolo Abeni netdev_features_change(dev); 1108d3256efdSPaolo Abeni } 1109d3256efdSPaolo Abeni } 1110d3256efdSPaolo Abeni 1111dedd53c5SPaolo Abeni veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, false); 1112948d4f21SToshiaki Makita } 1113948d4f21SToshiaki Makita 1114dedd53c5SPaolo Abeni static int veth_napi_enable_range(struct net_device *dev, int start, int end) 1115d3256efdSPaolo Abeni { 1116d3256efdSPaolo Abeni struct veth_priv *priv = netdev_priv(dev); 1117d3256efdSPaolo Abeni int err, i; 1118d3256efdSPaolo Abeni 1119dedd53c5SPaolo Abeni for (i = start; i < end; i++) { 1120d3256efdSPaolo Abeni struct veth_rq *rq = &priv->rq[i]; 1121d3256efdSPaolo Abeni 1122d3256efdSPaolo Abeni netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); 1123d3256efdSPaolo Abeni } 1124d3256efdSPaolo Abeni 1125dedd53c5SPaolo Abeni err = __veth_napi_enable_range(dev, start, end); 1126d3256efdSPaolo Abeni if (err) { 1127dedd53c5SPaolo Abeni for (i = start; i < end; i++) { 1128d3256efdSPaolo Abeni struct veth_rq *rq = &priv->rq[i]; 1129d3256efdSPaolo Abeni 1130d3256efdSPaolo Abeni netif_napi_del(&rq->xdp_napi); 1131d3256efdSPaolo Abeni } 1132d3256efdSPaolo Abeni return err; 1133d3256efdSPaolo Abeni } 1134d3256efdSPaolo Abeni return err; 1135d3256efdSPaolo Abeni } 1136d3256efdSPaolo Abeni 1137dedd53c5SPaolo Abeni static int veth_napi_enable(struct net_device *dev) 1138dedd53c5SPaolo Abeni { 1139dedd53c5SPaolo Abeni return veth_napi_enable_range(dev, 0, dev->real_num_rx_queues); 1140dedd53c5SPaolo Abeni } 1141dedd53c5SPaolo Abeni 11424752eeb3SPaolo Abeni static void veth_disable_range_safe(struct net_device *dev, int start, int end) 11434752eeb3SPaolo Abeni { 11444752eeb3SPaolo Abeni struct veth_priv *priv = netdev_priv(dev); 11454752eeb3SPaolo Abeni 11464752eeb3SPaolo Abeni if (start >= end) 11474752eeb3SPaolo Abeni return; 11484752eeb3SPaolo Abeni 11494752eeb3SPaolo Abeni if (priv->_xdp_prog) { 11504752eeb3SPaolo Abeni veth_napi_del_range(dev, start, end); 11514752eeb3SPaolo Abeni veth_disable_xdp_range(dev, start, end, false); 11524752eeb3SPaolo Abeni } else if (veth_gro_requested(dev)) { 11534752eeb3SPaolo Abeni veth_napi_del_range(dev, start, end); 11544752eeb3SPaolo Abeni } 11554752eeb3SPaolo Abeni } 11564752eeb3SPaolo Abeni 11574752eeb3SPaolo Abeni static int veth_enable_range_safe(struct net_device *dev, int start, int end) 11584752eeb3SPaolo Abeni { 11594752eeb3SPaolo Abeni struct veth_priv *priv = netdev_priv(dev); 11604752eeb3SPaolo Abeni int err; 11614752eeb3SPaolo Abeni 11624752eeb3SPaolo Abeni if (start >= end) 11634752eeb3SPaolo Abeni return 0; 11644752eeb3SPaolo Abeni 11654752eeb3SPaolo Abeni if (priv->_xdp_prog) { 11664752eeb3SPaolo Abeni /* these channels are freshly initialized, napi is not on there even 11674752eeb3SPaolo Abeni * when GRO is requeste 11684752eeb3SPaolo Abeni */ 11694752eeb3SPaolo Abeni err = veth_enable_xdp_range(dev, start, end, false); 11704752eeb3SPaolo Abeni if (err) 11714752eeb3SPaolo Abeni return err; 11724752eeb3SPaolo Abeni 11734752eeb3SPaolo Abeni err = __veth_napi_enable_range(dev, start, end); 11744752eeb3SPaolo Abeni if (err) { 11754752eeb3SPaolo Abeni /* on error always delete the newly added napis */ 11764752eeb3SPaolo Abeni veth_disable_xdp_range(dev, start, end, true); 11774752eeb3SPaolo Abeni return err; 11784752eeb3SPaolo Abeni } 11794752eeb3SPaolo Abeni } else if (veth_gro_requested(dev)) { 11804752eeb3SPaolo Abeni return veth_napi_enable_range(dev, start, end); 11814752eeb3SPaolo Abeni } 11824752eeb3SPaolo Abeni return 0; 11834752eeb3SPaolo Abeni } 11844752eeb3SPaolo Abeni 11854752eeb3SPaolo Abeni static int veth_set_channels(struct net_device *dev, 11864752eeb3SPaolo Abeni struct ethtool_channels *ch) 11874752eeb3SPaolo Abeni { 11884752eeb3SPaolo Abeni struct veth_priv *priv = netdev_priv(dev); 11894752eeb3SPaolo Abeni unsigned int old_rx_count, new_rx_count; 11904752eeb3SPaolo Abeni struct veth_priv *peer_priv; 11914752eeb3SPaolo Abeni struct net_device *peer; 11924752eeb3SPaolo Abeni int err; 11934752eeb3SPaolo Abeni 11944752eeb3SPaolo Abeni /* sanity check. Upper bounds are already enforced by the caller */ 11954752eeb3SPaolo Abeni if (!ch->rx_count || !ch->tx_count) 11964752eeb3SPaolo Abeni return -EINVAL; 11974752eeb3SPaolo Abeni 11984752eeb3SPaolo Abeni /* avoid braking XDP, if that is enabled */ 11994752eeb3SPaolo Abeni peer = rtnl_dereference(priv->peer); 12004752eeb3SPaolo Abeni peer_priv = peer ? netdev_priv(peer) : NULL; 12014752eeb3SPaolo Abeni if (priv->_xdp_prog && peer && ch->rx_count < peer->real_num_tx_queues) 12024752eeb3SPaolo Abeni return -EINVAL; 12034752eeb3SPaolo Abeni 12044752eeb3SPaolo Abeni if (peer && peer_priv && peer_priv->_xdp_prog && ch->tx_count > peer->real_num_rx_queues) 12054752eeb3SPaolo Abeni return -EINVAL; 12064752eeb3SPaolo Abeni 12074752eeb3SPaolo Abeni old_rx_count = dev->real_num_rx_queues; 12084752eeb3SPaolo Abeni new_rx_count = ch->rx_count; 12094752eeb3SPaolo Abeni if (netif_running(dev)) { 12104752eeb3SPaolo Abeni /* turn device off */ 12114752eeb3SPaolo Abeni netif_carrier_off(dev); 12124752eeb3SPaolo Abeni if (peer) 12134752eeb3SPaolo Abeni netif_carrier_off(peer); 12144752eeb3SPaolo Abeni 12154752eeb3SPaolo Abeni /* try to allocate new resurces, as needed*/ 12164752eeb3SPaolo Abeni err = veth_enable_range_safe(dev, old_rx_count, new_rx_count); 12174752eeb3SPaolo Abeni if (err) 12184752eeb3SPaolo Abeni goto out; 12194752eeb3SPaolo Abeni } 12204752eeb3SPaolo Abeni 12214752eeb3SPaolo Abeni err = netif_set_real_num_rx_queues(dev, ch->rx_count); 12224752eeb3SPaolo Abeni if (err) 12234752eeb3SPaolo Abeni goto revert; 12244752eeb3SPaolo Abeni 12254752eeb3SPaolo Abeni err = netif_set_real_num_tx_queues(dev, ch->tx_count); 12264752eeb3SPaolo Abeni if (err) { 12274752eeb3SPaolo Abeni int err2 = netif_set_real_num_rx_queues(dev, old_rx_count); 12284752eeb3SPaolo Abeni 12294752eeb3SPaolo Abeni /* this error condition could happen only if rx and tx change 12304752eeb3SPaolo Abeni * in opposite directions (e.g. tx nr raises, rx nr decreases) 12314752eeb3SPaolo Abeni * and we can't do anything to fully restore the original 12324752eeb3SPaolo Abeni * status 12334752eeb3SPaolo Abeni */ 12344752eeb3SPaolo Abeni if (err2) 12354752eeb3SPaolo Abeni pr_warn("Can't restore rx queues config %d -> %d %d", 12364752eeb3SPaolo Abeni new_rx_count, old_rx_count, err2); 12374752eeb3SPaolo Abeni else 12384752eeb3SPaolo Abeni goto revert; 12394752eeb3SPaolo Abeni } 12404752eeb3SPaolo Abeni 12414752eeb3SPaolo Abeni out: 12424752eeb3SPaolo Abeni if (netif_running(dev)) { 12434752eeb3SPaolo Abeni /* note that we need to swap the arguments WRT the enable part 12444752eeb3SPaolo Abeni * to identify the range we have to disable 12454752eeb3SPaolo Abeni */ 12464752eeb3SPaolo Abeni veth_disable_range_safe(dev, new_rx_count, old_rx_count); 12474752eeb3SPaolo Abeni netif_carrier_on(dev); 12484752eeb3SPaolo Abeni if (peer) 12494752eeb3SPaolo Abeni netif_carrier_on(peer); 12504752eeb3SPaolo Abeni } 12514752eeb3SPaolo Abeni return err; 12524752eeb3SPaolo Abeni 12534752eeb3SPaolo Abeni revert: 12544752eeb3SPaolo Abeni new_rx_count = old_rx_count; 12554752eeb3SPaolo Abeni old_rx_count = ch->rx_count; 12564752eeb3SPaolo Abeni goto out; 12574752eeb3SPaolo Abeni } 12584752eeb3SPaolo Abeni 1259e314dbdcSPavel Emelyanov static int veth_open(struct net_device *dev) 1260e314dbdcSPavel Emelyanov { 1261d0e2c55eSEric Dumazet struct veth_priv *priv = netdev_priv(dev); 1262d0e2c55eSEric Dumazet struct net_device *peer = rtnl_dereference(priv->peer); 1263948d4f21SToshiaki Makita int err; 1264e314dbdcSPavel Emelyanov 1265d0e2c55eSEric Dumazet if (!peer) 1266e314dbdcSPavel Emelyanov return -ENOTCONN; 1267e314dbdcSPavel Emelyanov 1268948d4f21SToshiaki Makita if (priv->_xdp_prog) { 1269948d4f21SToshiaki Makita err = veth_enable_xdp(dev); 1270948d4f21SToshiaki Makita if (err) 1271948d4f21SToshiaki Makita return err; 1272d3256efdSPaolo Abeni } else if (veth_gro_requested(dev)) { 1273d3256efdSPaolo Abeni err = veth_napi_enable(dev); 1274d3256efdSPaolo Abeni if (err) 1275d3256efdSPaolo Abeni return err; 1276948d4f21SToshiaki Makita } 1277948d4f21SToshiaki Makita 1278d0e2c55eSEric Dumazet if (peer->flags & IFF_UP) { 1279e314dbdcSPavel Emelyanov netif_carrier_on(dev); 1280d0e2c55eSEric Dumazet netif_carrier_on(peer); 1281e314dbdcSPavel Emelyanov } 1282948d4f21SToshiaki Makita 1283e314dbdcSPavel Emelyanov return 0; 1284e314dbdcSPavel Emelyanov } 1285e314dbdcSPavel Emelyanov 12862cf48a10SEric W. Biederman static int veth_close(struct net_device *dev) 12872cf48a10SEric W. Biederman { 12882cf48a10SEric W. Biederman struct veth_priv *priv = netdev_priv(dev); 12892efd32eeSEric Dumazet struct net_device *peer = rtnl_dereference(priv->peer); 12902cf48a10SEric W. Biederman 12912cf48a10SEric W. Biederman netif_carrier_off(dev); 12922efd32eeSEric Dumazet if (peer) 12932efd32eeSEric Dumazet netif_carrier_off(peer); 12942cf48a10SEric W. Biederman 1295948d4f21SToshiaki Makita if (priv->_xdp_prog) 1296948d4f21SToshiaki Makita veth_disable_xdp(dev); 1297d3256efdSPaolo Abeni else if (veth_gro_requested(dev)) 1298d3256efdSPaolo Abeni veth_napi_del(dev); 1299948d4f21SToshiaki Makita 13002cf48a10SEric W. Biederman return 0; 13012cf48a10SEric W. Biederman } 13022cf48a10SEric W. Biederman 130391572088SJarod Wilson static int is_valid_veth_mtu(int mtu) 130438d40815SEric Biederman { 130591572088SJarod Wilson return mtu >= ETH_MIN_MTU && mtu <= ETH_MAX_MTU; 130638d40815SEric Biederman } 130738d40815SEric Biederman 13087797b93bSToshiaki Makita static int veth_alloc_queues(struct net_device *dev) 13097797b93bSToshiaki Makita { 13107797b93bSToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 13117797b93bSToshiaki Makita int i; 13127797b93bSToshiaki Makita 13137797b93bSToshiaki Makita priv->rq = kcalloc(dev->num_rx_queues, sizeof(*priv->rq), GFP_KERNEL); 13147797b93bSToshiaki Makita if (!priv->rq) 13157797b93bSToshiaki Makita return -ENOMEM; 13167797b93bSToshiaki Makita 13174195e54aSToshiaki Makita for (i = 0; i < dev->num_rx_queues; i++) { 13187797b93bSToshiaki Makita priv->rq[i].dev = dev; 13194195e54aSToshiaki Makita u64_stats_init(&priv->rq[i].stats.syncp); 13204195e54aSToshiaki Makita } 13217797b93bSToshiaki Makita 13227797b93bSToshiaki Makita return 0; 13237797b93bSToshiaki Makita } 13247797b93bSToshiaki Makita 13257797b93bSToshiaki Makita static void veth_free_queues(struct net_device *dev) 13267797b93bSToshiaki Makita { 13277797b93bSToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 13287797b93bSToshiaki Makita 13297797b93bSToshiaki Makita kfree(priv->rq); 13307797b93bSToshiaki Makita } 13317797b93bSToshiaki Makita 1332e314dbdcSPavel Emelyanov static int veth_dev_init(struct net_device *dev) 1333e314dbdcSPavel Emelyanov { 13347797b93bSToshiaki Makita int err; 13357797b93bSToshiaki Makita 133614d73416SLi RongQing dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); 133714d73416SLi RongQing if (!dev->lstats) 1338e314dbdcSPavel Emelyanov return -ENOMEM; 13397797b93bSToshiaki Makita 13407797b93bSToshiaki Makita err = veth_alloc_queues(dev); 13417797b93bSToshiaki Makita if (err) { 134214d73416SLi RongQing free_percpu(dev->lstats); 13437797b93bSToshiaki Makita return err; 13447797b93bSToshiaki Makita } 13457797b93bSToshiaki Makita 1346e314dbdcSPavel Emelyanov return 0; 1347e314dbdcSPavel Emelyanov } 1348e314dbdcSPavel Emelyanov 134911687a10SDavid S. Miller static void veth_dev_free(struct net_device *dev) 135011687a10SDavid S. Miller { 13517797b93bSToshiaki Makita veth_free_queues(dev); 135214d73416SLi RongQing free_percpu(dev->lstats); 135311687a10SDavid S. Miller } 135411687a10SDavid S. Miller 1355bb446c19SWANG Cong #ifdef CONFIG_NET_POLL_CONTROLLER 1356bb446c19SWANG Cong static void veth_poll_controller(struct net_device *dev) 1357bb446c19SWANG Cong { 1358bb446c19SWANG Cong /* veth only receives frames when its peer sends one 1359948d4f21SToshiaki Makita * Since it has nothing to do with disabling irqs, we are guaranteed 1360bb446c19SWANG Cong * never to have pending data when we poll for it so 1361bb446c19SWANG Cong * there is nothing to do here. 1362bb446c19SWANG Cong * 1363bb446c19SWANG Cong * We need this though so netpoll recognizes us as an interface that 1364bb446c19SWANG Cong * supports polling, which enables bridge devices in virt setups to 1365bb446c19SWANG Cong * still use netconsole 1366bb446c19SWANG Cong */ 1367bb446c19SWANG Cong } 1368bb446c19SWANG Cong #endif /* CONFIG_NET_POLL_CONTROLLER */ 1369bb446c19SWANG Cong 1370a45253bfSNicolas Dichtel static int veth_get_iflink(const struct net_device *dev) 1371a45253bfSNicolas Dichtel { 1372a45253bfSNicolas Dichtel struct veth_priv *priv = netdev_priv(dev); 1373a45253bfSNicolas Dichtel struct net_device *peer; 1374a45253bfSNicolas Dichtel int iflink; 1375a45253bfSNicolas Dichtel 1376a45253bfSNicolas Dichtel rcu_read_lock(); 1377a45253bfSNicolas Dichtel peer = rcu_dereference(priv->peer); 1378a45253bfSNicolas Dichtel iflink = peer ? peer->ifindex : 0; 1379a45253bfSNicolas Dichtel rcu_read_unlock(); 1380a45253bfSNicolas Dichtel 1381a45253bfSNicolas Dichtel return iflink; 1382a45253bfSNicolas Dichtel } 1383a45253bfSNicolas Dichtel 1384dc224822SToshiaki Makita static netdev_features_t veth_fix_features(struct net_device *dev, 1385dc224822SToshiaki Makita netdev_features_t features) 1386dc224822SToshiaki Makita { 1387dc224822SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 1388dc224822SToshiaki Makita struct net_device *peer; 1389dc224822SToshiaki Makita 1390dc224822SToshiaki Makita peer = rtnl_dereference(priv->peer); 1391dc224822SToshiaki Makita if (peer) { 1392dc224822SToshiaki Makita struct veth_priv *peer_priv = netdev_priv(peer); 1393dc224822SToshiaki Makita 1394dc224822SToshiaki Makita if (peer_priv->_xdp_prog) 1395dc224822SToshiaki Makita features &= ~NETIF_F_GSO_SOFTWARE; 1396dc224822SToshiaki Makita } 1397d3256efdSPaolo Abeni if (priv->_xdp_prog) 1398d3256efdSPaolo Abeni features |= NETIF_F_GRO; 1399dc224822SToshiaki Makita 1400dc224822SToshiaki Makita return features; 1401dc224822SToshiaki Makita } 1402dc224822SToshiaki Makita 1403d3256efdSPaolo Abeni static int veth_set_features(struct net_device *dev, 1404d3256efdSPaolo Abeni netdev_features_t features) 1405d3256efdSPaolo Abeni { 1406d3256efdSPaolo Abeni netdev_features_t changed = features ^ dev->features; 1407d3256efdSPaolo Abeni struct veth_priv *priv = netdev_priv(dev); 1408d3256efdSPaolo Abeni int err; 1409d3256efdSPaolo Abeni 1410d3256efdSPaolo Abeni if (!(changed & NETIF_F_GRO) || !(dev->flags & IFF_UP) || priv->_xdp_prog) 1411d3256efdSPaolo Abeni return 0; 1412d3256efdSPaolo Abeni 1413d3256efdSPaolo Abeni if (features & NETIF_F_GRO) { 1414d3256efdSPaolo Abeni err = veth_napi_enable(dev); 1415d3256efdSPaolo Abeni if (err) 1416d3256efdSPaolo Abeni return err; 1417d3256efdSPaolo Abeni } else { 1418d3256efdSPaolo Abeni veth_napi_del(dev); 1419d3256efdSPaolo Abeni } 1420d3256efdSPaolo Abeni return 0; 1421d3256efdSPaolo Abeni } 1422d3256efdSPaolo Abeni 1423163e5292SPaolo Abeni static void veth_set_rx_headroom(struct net_device *dev, int new_hr) 1424163e5292SPaolo Abeni { 1425163e5292SPaolo Abeni struct veth_priv *peer_priv, *priv = netdev_priv(dev); 1426163e5292SPaolo Abeni struct net_device *peer; 1427163e5292SPaolo Abeni 1428163e5292SPaolo Abeni if (new_hr < 0) 1429163e5292SPaolo Abeni new_hr = 0; 1430163e5292SPaolo Abeni 1431163e5292SPaolo Abeni rcu_read_lock(); 1432163e5292SPaolo Abeni peer = rcu_dereference(priv->peer); 1433163e5292SPaolo Abeni if (unlikely(!peer)) 1434163e5292SPaolo Abeni goto out; 1435163e5292SPaolo Abeni 1436163e5292SPaolo Abeni peer_priv = netdev_priv(peer); 1437163e5292SPaolo Abeni priv->requested_headroom = new_hr; 1438163e5292SPaolo Abeni new_hr = max(priv->requested_headroom, peer_priv->requested_headroom); 1439163e5292SPaolo Abeni dev->needed_headroom = new_hr; 1440163e5292SPaolo Abeni peer->needed_headroom = new_hr; 1441163e5292SPaolo Abeni 1442163e5292SPaolo Abeni out: 1443163e5292SPaolo Abeni rcu_read_unlock(); 1444163e5292SPaolo Abeni } 1445163e5292SPaolo Abeni 1446948d4f21SToshiaki Makita static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1447948d4f21SToshiaki Makita struct netlink_ext_ack *extack) 1448948d4f21SToshiaki Makita { 1449948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 1450948d4f21SToshiaki Makita struct bpf_prog *old_prog; 1451948d4f21SToshiaki Makita struct net_device *peer; 1452dc224822SToshiaki Makita unsigned int max_mtu; 1453948d4f21SToshiaki Makita int err; 1454948d4f21SToshiaki Makita 1455948d4f21SToshiaki Makita old_prog = priv->_xdp_prog; 1456948d4f21SToshiaki Makita priv->_xdp_prog = prog; 1457948d4f21SToshiaki Makita peer = rtnl_dereference(priv->peer); 1458948d4f21SToshiaki Makita 1459948d4f21SToshiaki Makita if (prog) { 1460948d4f21SToshiaki Makita if (!peer) { 1461948d4f21SToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached"); 1462948d4f21SToshiaki Makita err = -ENOTCONN; 1463948d4f21SToshiaki Makita goto err; 1464948d4f21SToshiaki Makita } 1465948d4f21SToshiaki Makita 1466dc224822SToshiaki Makita max_mtu = PAGE_SIZE - VETH_XDP_HEADROOM - 1467dc224822SToshiaki Makita peer->hard_header_len - 1468dc224822SToshiaki Makita SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1469dc224822SToshiaki Makita if (peer->mtu > max_mtu) { 1470dc224822SToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP"); 1471dc224822SToshiaki Makita err = -ERANGE; 1472dc224822SToshiaki Makita goto err; 1473dc224822SToshiaki Makita } 1474dc224822SToshiaki Makita 1475638264dcSToshiaki Makita if (dev->real_num_rx_queues < peer->real_num_tx_queues) { 1476638264dcSToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues"); 1477638264dcSToshiaki Makita err = -ENOSPC; 1478638264dcSToshiaki Makita goto err; 1479638264dcSToshiaki Makita } 1480638264dcSToshiaki Makita 1481948d4f21SToshiaki Makita if (dev->flags & IFF_UP) { 1482948d4f21SToshiaki Makita err = veth_enable_xdp(dev); 1483948d4f21SToshiaki Makita if (err) { 1484948d4f21SToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed"); 1485948d4f21SToshiaki Makita goto err; 1486948d4f21SToshiaki Makita } 1487948d4f21SToshiaki Makita } 1488dc224822SToshiaki Makita 1489dc224822SToshiaki Makita if (!old_prog) { 1490dc224822SToshiaki Makita peer->hw_features &= ~NETIF_F_GSO_SOFTWARE; 1491dc224822SToshiaki Makita peer->max_mtu = max_mtu; 1492dc224822SToshiaki Makita } 1493948d4f21SToshiaki Makita } 1494948d4f21SToshiaki Makita 1495948d4f21SToshiaki Makita if (old_prog) { 1496dc224822SToshiaki Makita if (!prog) { 1497dc224822SToshiaki Makita if (dev->flags & IFF_UP) 1498948d4f21SToshiaki Makita veth_disable_xdp(dev); 1499dc224822SToshiaki Makita 1500dc224822SToshiaki Makita if (peer) { 1501dc224822SToshiaki Makita peer->hw_features |= NETIF_F_GSO_SOFTWARE; 1502dc224822SToshiaki Makita peer->max_mtu = ETH_MAX_MTU; 1503dc224822SToshiaki Makita } 1504dc224822SToshiaki Makita } 1505948d4f21SToshiaki Makita bpf_prog_put(old_prog); 1506948d4f21SToshiaki Makita } 1507948d4f21SToshiaki Makita 1508dc224822SToshiaki Makita if ((!!old_prog ^ !!prog) && peer) 1509dc224822SToshiaki Makita netdev_update_features(peer); 1510dc224822SToshiaki Makita 1511948d4f21SToshiaki Makita return 0; 1512948d4f21SToshiaki Makita err: 1513948d4f21SToshiaki Makita priv->_xdp_prog = old_prog; 1514948d4f21SToshiaki Makita 1515948d4f21SToshiaki Makita return err; 1516948d4f21SToshiaki Makita } 1517948d4f21SToshiaki Makita 1518948d4f21SToshiaki Makita static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1519948d4f21SToshiaki Makita { 1520948d4f21SToshiaki Makita switch (xdp->command) { 1521948d4f21SToshiaki Makita case XDP_SETUP_PROG: 1522948d4f21SToshiaki Makita return veth_xdp_set(dev, xdp->prog, xdp->extack); 1523948d4f21SToshiaki Makita default: 1524948d4f21SToshiaki Makita return -EINVAL; 1525948d4f21SToshiaki Makita } 1526948d4f21SToshiaki Makita } 1527948d4f21SToshiaki Makita 15284456e7bdSStephen Hemminger static const struct net_device_ops veth_netdev_ops = { 15294456e7bdSStephen Hemminger .ndo_init = veth_dev_init, 15304456e7bdSStephen Hemminger .ndo_open = veth_open, 15312cf48a10SEric W. Biederman .ndo_stop = veth_close, 153200829823SStephen Hemminger .ndo_start_xmit = veth_xmit, 15336311cc44Sstephen hemminger .ndo_get_stats64 = veth_get_stats64, 15345c70ef85SGao feng .ndo_set_rx_mode = veth_set_multicast_list, 1535ee923623SDaniel Lezcano .ndo_set_mac_address = eth_mac_addr, 1536bb446c19SWANG Cong #ifdef CONFIG_NET_POLL_CONTROLLER 1537bb446c19SWANG Cong .ndo_poll_controller = veth_poll_controller, 1538bb446c19SWANG Cong #endif 1539a45253bfSNicolas Dichtel .ndo_get_iflink = veth_get_iflink, 1540dc224822SToshiaki Makita .ndo_fix_features = veth_fix_features, 1541d3256efdSPaolo Abeni .ndo_set_features = veth_set_features, 15421a04a821SToshiaki Makita .ndo_features_check = passthru_features_check, 1543163e5292SPaolo Abeni .ndo_set_rx_headroom = veth_set_rx_headroom, 1544948d4f21SToshiaki Makita .ndo_bpf = veth_xdp, 15459152cff0SLorenzo Bianconi .ndo_xdp_xmit = veth_ndo_xdp_xmit, 15469aa1206eSDaniel Borkmann .ndo_get_peer_dev = veth_peer_dev, 15474456e7bdSStephen Hemminger }; 15484456e7bdSStephen Hemminger 1549732912d7SAlexander Duyck #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ 1550c80fafbbSXin Long NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \ 1551732912d7SAlexander Duyck NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \ 155228d2b136SPatrick McHardy NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \ 155328d2b136SPatrick McHardy NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX ) 15548093315aSEric Dumazet 1555e314dbdcSPavel Emelyanov static void veth_setup(struct net_device *dev) 1556e314dbdcSPavel Emelyanov { 1557e314dbdcSPavel Emelyanov ether_setup(dev); 1558e314dbdcSPavel Emelyanov 1559550fd08cSNeil Horman dev->priv_flags &= ~IFF_TX_SKB_SHARING; 156023ea5a96SHannes Frederic Sowa dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 156102f01ec1SPhil Sutter dev->priv_flags |= IFF_NO_QUEUE; 1562163e5292SPaolo Abeni dev->priv_flags |= IFF_PHONY_HEADROOM; 1563550fd08cSNeil Horman 15644456e7bdSStephen Hemminger dev->netdev_ops = &veth_netdev_ops; 1565e314dbdcSPavel Emelyanov dev->ethtool_ops = &veth_ethtool_ops; 1566e314dbdcSPavel Emelyanov dev->features |= NETIF_F_LLTX; 15678093315aSEric Dumazet dev->features |= VETH_FEATURES; 15688d0d21f4SToshiaki Makita dev->vlan_features = dev->features & 15693f8c707bSVlad Yasevich ~(NETIF_F_HW_VLAN_CTAG_TX | 15703f8c707bSVlad Yasevich NETIF_F_HW_VLAN_STAG_TX | 15713f8c707bSVlad Yasevich NETIF_F_HW_VLAN_CTAG_RX | 15723f8c707bSVlad Yasevich NETIF_F_HW_VLAN_STAG_RX); 1573cf124db5SDavid S. Miller dev->needs_free_netdev = true; 1574cf124db5SDavid S. Miller dev->priv_destructor = veth_dev_free; 157591572088SJarod Wilson dev->max_mtu = ETH_MAX_MTU; 1576a2c725faSMichał Mirosław 15778093315aSEric Dumazet dev->hw_features = VETH_FEATURES; 157882d81898SEric Dumazet dev->hw_enc_features = VETH_FEATURES; 1579607fca9aSDavid Ahern dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 1580e314dbdcSPavel Emelyanov } 1581e314dbdcSPavel Emelyanov 1582e314dbdcSPavel Emelyanov /* 1583e314dbdcSPavel Emelyanov * netlink interface 1584e314dbdcSPavel Emelyanov */ 1585e314dbdcSPavel Emelyanov 1586a8b8a889SMatthias Schiffer static int veth_validate(struct nlattr *tb[], struct nlattr *data[], 1587a8b8a889SMatthias Schiffer struct netlink_ext_ack *extack) 1588e314dbdcSPavel Emelyanov { 1589e314dbdcSPavel Emelyanov if (tb[IFLA_ADDRESS]) { 1590e314dbdcSPavel Emelyanov if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) 1591e314dbdcSPavel Emelyanov return -EINVAL; 1592e314dbdcSPavel Emelyanov if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) 1593e314dbdcSPavel Emelyanov return -EADDRNOTAVAIL; 1594e314dbdcSPavel Emelyanov } 159538d40815SEric Biederman if (tb[IFLA_MTU]) { 159638d40815SEric Biederman if (!is_valid_veth_mtu(nla_get_u32(tb[IFLA_MTU]))) 159738d40815SEric Biederman return -EINVAL; 159838d40815SEric Biederman } 1599e314dbdcSPavel Emelyanov return 0; 1600e314dbdcSPavel Emelyanov } 1601e314dbdcSPavel Emelyanov 1602e314dbdcSPavel Emelyanov static struct rtnl_link_ops veth_link_ops; 1603e314dbdcSPavel Emelyanov 1604d3256efdSPaolo Abeni static void veth_disable_gro(struct net_device *dev) 1605d3256efdSPaolo Abeni { 1606d3256efdSPaolo Abeni dev->features &= ~NETIF_F_GRO; 1607d3256efdSPaolo Abeni dev->wanted_features &= ~NETIF_F_GRO; 1608d3256efdSPaolo Abeni netdev_update_features(dev); 1609d3256efdSPaolo Abeni } 1610d3256efdSPaolo Abeni 16119d3684c2SPaolo Abeni static int veth_init_queues(struct net_device *dev, struct nlattr *tb[]) 16129d3684c2SPaolo Abeni { 16139d3684c2SPaolo Abeni int err; 16149d3684c2SPaolo Abeni 16159d3684c2SPaolo Abeni if (!tb[IFLA_NUM_TX_QUEUES] && dev->num_tx_queues > 1) { 16169d3684c2SPaolo Abeni err = netif_set_real_num_tx_queues(dev, 1); 16179d3684c2SPaolo Abeni if (err) 16189d3684c2SPaolo Abeni return err; 16199d3684c2SPaolo Abeni } 16209d3684c2SPaolo Abeni if (!tb[IFLA_NUM_RX_QUEUES] && dev->num_rx_queues > 1) { 16219d3684c2SPaolo Abeni err = netif_set_real_num_rx_queues(dev, 1); 16229d3684c2SPaolo Abeni if (err) 16239d3684c2SPaolo Abeni return err; 16249d3684c2SPaolo Abeni } 16259d3684c2SPaolo Abeni return 0; 16269d3684c2SPaolo Abeni } 16279d3684c2SPaolo Abeni 162881adee47SEric W. Biederman static int veth_newlink(struct net *src_net, struct net_device *dev, 16297a3f4a18SMatthias Schiffer struct nlattr *tb[], struct nlattr *data[], 16307a3f4a18SMatthias Schiffer struct netlink_ext_ack *extack) 1631e314dbdcSPavel Emelyanov { 16327797b93bSToshiaki Makita int err; 1633e314dbdcSPavel Emelyanov struct net_device *peer; 1634e314dbdcSPavel Emelyanov struct veth_priv *priv; 1635e314dbdcSPavel Emelyanov char ifname[IFNAMSIZ]; 1636e314dbdcSPavel Emelyanov struct nlattr *peer_tb[IFLA_MAX + 1], **tbp; 16375517750fSTom Gundersen unsigned char name_assign_type; 16383729d502SPatrick McHardy struct ifinfomsg *ifmp; 163981adee47SEric W. Biederman struct net *net; 1640e314dbdcSPavel Emelyanov 1641e314dbdcSPavel Emelyanov /* 1642e314dbdcSPavel Emelyanov * create and register peer first 1643e314dbdcSPavel Emelyanov */ 1644e314dbdcSPavel Emelyanov if (data != NULL && data[VETH_INFO_PEER] != NULL) { 1645e314dbdcSPavel Emelyanov struct nlattr *nla_peer; 1646e314dbdcSPavel Emelyanov 1647e314dbdcSPavel Emelyanov nla_peer = data[VETH_INFO_PEER]; 16483729d502SPatrick McHardy ifmp = nla_data(nla_peer); 1649f7b12606SJiri Pirko err = rtnl_nla_parse_ifla(peer_tb, 1650e314dbdcSPavel Emelyanov nla_data(nla_peer) + sizeof(struct ifinfomsg), 1651fceb6435SJohannes Berg nla_len(nla_peer) - sizeof(struct ifinfomsg), 1652fceb6435SJohannes Berg NULL); 1653e314dbdcSPavel Emelyanov if (err < 0) 1654e314dbdcSPavel Emelyanov return err; 1655e314dbdcSPavel Emelyanov 1656a8b8a889SMatthias Schiffer err = veth_validate(peer_tb, NULL, extack); 1657e314dbdcSPavel Emelyanov if (err < 0) 1658e314dbdcSPavel Emelyanov return err; 1659e314dbdcSPavel Emelyanov 1660e314dbdcSPavel Emelyanov tbp = peer_tb; 16613729d502SPatrick McHardy } else { 16623729d502SPatrick McHardy ifmp = NULL; 1663e314dbdcSPavel Emelyanov tbp = tb; 16643729d502SPatrick McHardy } 1665e314dbdcSPavel Emelyanov 1666191cdb38SSerhey Popovych if (ifmp && tbp[IFLA_IFNAME]) { 1667872f6903SFrancis Laniel nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 16685517750fSTom Gundersen name_assign_type = NET_NAME_USER; 16695517750fSTom Gundersen } else { 1670e314dbdcSPavel Emelyanov snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d"); 16715517750fSTom Gundersen name_assign_type = NET_NAME_ENUM; 16725517750fSTom Gundersen } 1673e314dbdcSPavel Emelyanov 167481adee47SEric W. Biederman net = rtnl_link_get_net(src_net, tbp); 167581adee47SEric W. Biederman if (IS_ERR(net)) 167681adee47SEric W. Biederman return PTR_ERR(net); 167781adee47SEric W. Biederman 16785517750fSTom Gundersen peer = rtnl_create_link(net, ifname, name_assign_type, 1679d0522f1cSDavid Ahern &veth_link_ops, tbp, extack); 168081adee47SEric W. Biederman if (IS_ERR(peer)) { 168181adee47SEric W. Biederman put_net(net); 1682e314dbdcSPavel Emelyanov return PTR_ERR(peer); 168381adee47SEric W. Biederman } 1684e314dbdcSPavel Emelyanov 1685191cdb38SSerhey Popovych if (!ifmp || !tbp[IFLA_ADDRESS]) 1686f2cedb63SDanny Kukawka eth_hw_addr_random(peer); 1687e314dbdcSPavel Emelyanov 1688e6f8f1a7SPavel Emelyanov if (ifmp && (dev->ifindex != 0)) 1689e6f8f1a7SPavel Emelyanov peer->ifindex = ifmp->ifi_index; 1690e6f8f1a7SPavel Emelyanov 16914b66d216SEric Dumazet netif_set_gso_max_size(peer, dev->gso_max_size); 16926d872df3SEric Dumazet netif_set_gso_max_segs(peer, dev->gso_max_segs); 169372d24955SStephen Hemminger 1694e314dbdcSPavel Emelyanov err = register_netdevice(peer); 169581adee47SEric W. Biederman put_net(net); 169681adee47SEric W. Biederman net = NULL; 1697e314dbdcSPavel Emelyanov if (err < 0) 1698e314dbdcSPavel Emelyanov goto err_register_peer; 1699e314dbdcSPavel Emelyanov 1700d3256efdSPaolo Abeni /* keep GRO disabled by default to be consistent with the established 1701d3256efdSPaolo Abeni * veth behavior 1702d3256efdSPaolo Abeni */ 1703d3256efdSPaolo Abeni veth_disable_gro(peer); 1704e314dbdcSPavel Emelyanov netif_carrier_off(peer); 1705e314dbdcSPavel Emelyanov 17063729d502SPatrick McHardy err = rtnl_configure_link(peer, ifmp); 17073729d502SPatrick McHardy if (err < 0) 17083729d502SPatrick McHardy goto err_configure_peer; 17093729d502SPatrick McHardy 1710e314dbdcSPavel Emelyanov /* 1711e314dbdcSPavel Emelyanov * register dev last 1712e314dbdcSPavel Emelyanov * 1713e314dbdcSPavel Emelyanov * note, that since we've registered new device the dev's name 1714e314dbdcSPavel Emelyanov * should be re-allocated 1715e314dbdcSPavel Emelyanov */ 1716e314dbdcSPavel Emelyanov 1717e314dbdcSPavel Emelyanov if (tb[IFLA_ADDRESS] == NULL) 1718f2cedb63SDanny Kukawka eth_hw_addr_random(dev); 1719e314dbdcSPavel Emelyanov 17206c8c4446SJiri Pirko if (tb[IFLA_IFNAME]) 1721872f6903SFrancis Laniel nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 17226c8c4446SJiri Pirko else 17236c8c4446SJiri Pirko snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d"); 17246c8c4446SJiri Pirko 1725e314dbdcSPavel Emelyanov err = register_netdevice(dev); 1726e314dbdcSPavel Emelyanov if (err < 0) 1727e314dbdcSPavel Emelyanov goto err_register_dev; 1728e314dbdcSPavel Emelyanov 1729e314dbdcSPavel Emelyanov netif_carrier_off(dev); 1730e314dbdcSPavel Emelyanov 1731e314dbdcSPavel Emelyanov /* 1732e314dbdcSPavel Emelyanov * tie the deviced together 1733e314dbdcSPavel Emelyanov */ 1734e314dbdcSPavel Emelyanov 1735e314dbdcSPavel Emelyanov priv = netdev_priv(dev); 1736d0e2c55eSEric Dumazet rcu_assign_pointer(priv->peer, peer); 17379d3684c2SPaolo Abeni err = veth_init_queues(dev, tb); 17389d3684c2SPaolo Abeni if (err) 17399d3684c2SPaolo Abeni goto err_queues; 1740e314dbdcSPavel Emelyanov 1741e314dbdcSPavel Emelyanov priv = netdev_priv(peer); 1742d0e2c55eSEric Dumazet rcu_assign_pointer(priv->peer, dev); 17439d3684c2SPaolo Abeni err = veth_init_queues(peer, tb); 17449d3684c2SPaolo Abeni if (err) 17459d3684c2SPaolo Abeni goto err_queues; 1746948d4f21SToshiaki Makita 1747d3256efdSPaolo Abeni veth_disable_gro(dev); 1748e314dbdcSPavel Emelyanov return 0; 1749e314dbdcSPavel Emelyanov 17509d3684c2SPaolo Abeni err_queues: 17519d3684c2SPaolo Abeni unregister_netdevice(dev); 1752e314dbdcSPavel Emelyanov err_register_dev: 1753e314dbdcSPavel Emelyanov /* nothing to do */ 17543729d502SPatrick McHardy err_configure_peer: 1755e314dbdcSPavel Emelyanov unregister_netdevice(peer); 1756e314dbdcSPavel Emelyanov return err; 1757e314dbdcSPavel Emelyanov 1758e314dbdcSPavel Emelyanov err_register_peer: 1759e314dbdcSPavel Emelyanov free_netdev(peer); 1760e314dbdcSPavel Emelyanov return err; 1761e314dbdcSPavel Emelyanov } 1762e314dbdcSPavel Emelyanov 176323289a37SEric Dumazet static void veth_dellink(struct net_device *dev, struct list_head *head) 1764e314dbdcSPavel Emelyanov { 1765e314dbdcSPavel Emelyanov struct veth_priv *priv; 1766e314dbdcSPavel Emelyanov struct net_device *peer; 1767e314dbdcSPavel Emelyanov 1768e314dbdcSPavel Emelyanov priv = netdev_priv(dev); 1769d0e2c55eSEric Dumazet peer = rtnl_dereference(priv->peer); 1770d0e2c55eSEric Dumazet 1771d0e2c55eSEric Dumazet /* Note : dellink() is called from default_device_exit_batch(), 1772d0e2c55eSEric Dumazet * before a rcu_synchronize() point. The devices are guaranteed 1773d0e2c55eSEric Dumazet * not being freed before one RCU grace period. 1774d0e2c55eSEric Dumazet */ 1775d0e2c55eSEric Dumazet RCU_INIT_POINTER(priv->peer, NULL); 1776f45a5c26SEric Dumazet unregister_netdevice_queue(dev, head); 1777d0e2c55eSEric Dumazet 1778f45a5c26SEric Dumazet if (peer) { 1779d0e2c55eSEric Dumazet priv = netdev_priv(peer); 1780d0e2c55eSEric Dumazet RCU_INIT_POINTER(priv->peer, NULL); 178124540535SEric Dumazet unregister_netdevice_queue(peer, head); 1782e314dbdcSPavel Emelyanov } 1783f45a5c26SEric Dumazet } 1784e314dbdcSPavel Emelyanov 178523711438SThomas Graf static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = { 178623711438SThomas Graf [VETH_INFO_PEER] = { .len = sizeof(struct ifinfomsg) }, 178723711438SThomas Graf }; 1788e314dbdcSPavel Emelyanov 1789e5f4e7b9SNicolas Dichtel static struct net *veth_get_link_net(const struct net_device *dev) 1790e5f4e7b9SNicolas Dichtel { 1791e5f4e7b9SNicolas Dichtel struct veth_priv *priv = netdev_priv(dev); 1792e5f4e7b9SNicolas Dichtel struct net_device *peer = rtnl_dereference(priv->peer); 1793e5f4e7b9SNicolas Dichtel 1794e5f4e7b9SNicolas Dichtel return peer ? dev_net(peer) : dev_net(dev); 1795e5f4e7b9SNicolas Dichtel } 1796e5f4e7b9SNicolas Dichtel 17979d3684c2SPaolo Abeni static unsigned int veth_get_num_queues(void) 17989d3684c2SPaolo Abeni { 17999d3684c2SPaolo Abeni /* enforce the same queue limit as rtnl_create_link */ 18009d3684c2SPaolo Abeni int queues = num_possible_cpus(); 18019d3684c2SPaolo Abeni 18029d3684c2SPaolo Abeni if (queues > 4096) 18039d3684c2SPaolo Abeni queues = 4096; 18049d3684c2SPaolo Abeni return queues; 18059d3684c2SPaolo Abeni } 18069d3684c2SPaolo Abeni 1807e314dbdcSPavel Emelyanov static struct rtnl_link_ops veth_link_ops = { 1808e314dbdcSPavel Emelyanov .kind = DRV_NAME, 1809e314dbdcSPavel Emelyanov .priv_size = sizeof(struct veth_priv), 1810e314dbdcSPavel Emelyanov .setup = veth_setup, 1811e314dbdcSPavel Emelyanov .validate = veth_validate, 1812e314dbdcSPavel Emelyanov .newlink = veth_newlink, 1813e314dbdcSPavel Emelyanov .dellink = veth_dellink, 1814e314dbdcSPavel Emelyanov .policy = veth_policy, 1815e314dbdcSPavel Emelyanov .maxtype = VETH_INFO_MAX, 1816e5f4e7b9SNicolas Dichtel .get_link_net = veth_get_link_net, 18179d3684c2SPaolo Abeni .get_num_tx_queues = veth_get_num_queues, 18189d3684c2SPaolo Abeni .get_num_rx_queues = veth_get_num_queues, 1819e314dbdcSPavel Emelyanov }; 1820e314dbdcSPavel Emelyanov 1821e314dbdcSPavel Emelyanov /* 1822e314dbdcSPavel Emelyanov * init/fini 1823e314dbdcSPavel Emelyanov */ 1824e314dbdcSPavel Emelyanov 1825e314dbdcSPavel Emelyanov static __init int veth_init(void) 1826e314dbdcSPavel Emelyanov { 1827e314dbdcSPavel Emelyanov return rtnl_link_register(&veth_link_ops); 1828e314dbdcSPavel Emelyanov } 1829e314dbdcSPavel Emelyanov 1830e314dbdcSPavel Emelyanov static __exit void veth_exit(void) 1831e314dbdcSPavel Emelyanov { 183268365458SPatrick McHardy rtnl_link_unregister(&veth_link_ops); 1833e314dbdcSPavel Emelyanov } 1834e314dbdcSPavel Emelyanov 1835e314dbdcSPavel Emelyanov module_init(veth_init); 1836e314dbdcSPavel Emelyanov module_exit(veth_exit); 1837e314dbdcSPavel Emelyanov 1838e314dbdcSPavel Emelyanov MODULE_DESCRIPTION("Virtual Ethernet Tunnel"); 1839e314dbdcSPavel Emelyanov MODULE_LICENSE("GPL v2"); 1840e314dbdcSPavel Emelyanov MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1841