109c434b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e314dbdcSPavel Emelyanov /* 3e314dbdcSPavel Emelyanov * drivers/net/veth.c 4e314dbdcSPavel Emelyanov * 5e314dbdcSPavel Emelyanov * Copyright (C) 2007 OpenVZ http://openvz.org, SWsoft Inc 6e314dbdcSPavel Emelyanov * 7e314dbdcSPavel Emelyanov * Author: Pavel Emelianov <xemul@openvz.org> 8e314dbdcSPavel Emelyanov * Ethtool interface from: Eric W. Biederman <ebiederm@xmission.com> 9e314dbdcSPavel Emelyanov * 10e314dbdcSPavel Emelyanov */ 11e314dbdcSPavel Emelyanov 12e314dbdcSPavel Emelyanov #include <linux/netdevice.h> 135a0e3ad6STejun Heo #include <linux/slab.h> 14e314dbdcSPavel Emelyanov #include <linux/ethtool.h> 15e314dbdcSPavel Emelyanov #include <linux/etherdevice.h> 16cf05c700SEric Dumazet #include <linux/u64_stats_sync.h> 17e314dbdcSPavel Emelyanov 18f7b12606SJiri Pirko #include <net/rtnetlink.h> 19e314dbdcSPavel Emelyanov #include <net/dst.h> 20e314dbdcSPavel Emelyanov #include <net/xfrm.h> 21af87a3aaSToshiaki Makita #include <net/xdp.h> 22ecef969eSStephen Hemminger #include <linux/veth.h> 239d9779e7SPaul Gortmaker #include <linux/module.h> 24948d4f21SToshiaki Makita #include <linux/bpf.h> 25948d4f21SToshiaki Makita #include <linux/filter.h> 26948d4f21SToshiaki Makita #include <linux/ptr_ring.h> 27948d4f21SToshiaki Makita #include <linux/bpf_trace.h> 28aa4e689eSMichael Walle #include <linux/net_tstamp.h> 29e314dbdcSPavel Emelyanov 30e314dbdcSPavel Emelyanov #define DRV_NAME "veth" 31e314dbdcSPavel Emelyanov #define DRV_VERSION "1.0" 32e314dbdcSPavel Emelyanov 339fc8d518SToshiaki Makita #define VETH_XDP_FLAG BIT(0) 34948d4f21SToshiaki Makita #define VETH_RING_SIZE 256 35948d4f21SToshiaki Makita #define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN) 36948d4f21SToshiaki Makita 379cda7807SToshiaki Makita #define VETH_XDP_TX_BULK_SIZE 16 3865e6dcf7SLorenzo Bianconi #define VETH_XDP_BATCH 16 399cda7807SToshiaki Makita 4065780c56SLorenzo Bianconi struct veth_stats { 411c5b82e5SLorenzo Bianconi u64 rx_drops; 421c5b82e5SLorenzo Bianconi /* xdp */ 434195e54aSToshiaki Makita u64 xdp_packets; 444195e54aSToshiaki Makita u64 xdp_bytes; 451c5b82e5SLorenzo Bianconi u64 xdp_redirect; 464195e54aSToshiaki Makita u64 xdp_drops; 471c5b82e5SLorenzo Bianconi u64 xdp_tx; 489152cff0SLorenzo Bianconi u64 xdp_tx_err; 495fe6e567SLorenzo Bianconi u64 peer_tq_xdp_xmit; 505fe6e567SLorenzo Bianconi u64 peer_tq_xdp_xmit_err; 5165780c56SLorenzo Bianconi }; 5265780c56SLorenzo Bianconi 5365780c56SLorenzo Bianconi struct veth_rq_stats { 5465780c56SLorenzo Bianconi struct veth_stats vs; 554195e54aSToshiaki Makita struct u64_stats_sync syncp; 564195e54aSToshiaki Makita }; 574195e54aSToshiaki Makita 58638264dcSToshiaki Makita struct veth_rq { 59948d4f21SToshiaki Makita struct napi_struct xdp_napi; 60948d4f21SToshiaki Makita struct net_device *dev; 61948d4f21SToshiaki Makita struct bpf_prog __rcu *xdp_prog; 62d1396004SToshiaki Makita struct xdp_mem_info xdp_mem; 634195e54aSToshiaki Makita struct veth_rq_stats stats; 64948d4f21SToshiaki Makita bool rx_notify_masked; 65948d4f21SToshiaki Makita struct ptr_ring xdp_ring; 66948d4f21SToshiaki Makita struct xdp_rxq_info xdp_rxq; 67e314dbdcSPavel Emelyanov }; 68e314dbdcSPavel Emelyanov 69638264dcSToshiaki Makita struct veth_priv { 70638264dcSToshiaki Makita struct net_device __rcu *peer; 71638264dcSToshiaki Makita atomic64_t dropped; 72638264dcSToshiaki Makita struct bpf_prog *_xdp_prog; 73638264dcSToshiaki Makita struct veth_rq *rq; 74638264dcSToshiaki Makita unsigned int requested_headroom; 75638264dcSToshiaki Makita }; 76638264dcSToshiaki Makita 779cda7807SToshiaki Makita struct veth_xdp_tx_bq { 789cda7807SToshiaki Makita struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE]; 799cda7807SToshiaki Makita unsigned int count; 809cda7807SToshiaki Makita }; 819cda7807SToshiaki Makita 82e314dbdcSPavel Emelyanov /* 83e314dbdcSPavel Emelyanov * ethtool interface 84e314dbdcSPavel Emelyanov */ 85e314dbdcSPavel Emelyanov 86d397b968SToshiaki Makita struct veth_q_stat_desc { 87d397b968SToshiaki Makita char desc[ETH_GSTRING_LEN]; 88d397b968SToshiaki Makita size_t offset; 89d397b968SToshiaki Makita }; 90d397b968SToshiaki Makita 9165780c56SLorenzo Bianconi #define VETH_RQ_STAT(m) offsetof(struct veth_stats, m) 92d397b968SToshiaki Makita 93d397b968SToshiaki Makita static const struct veth_q_stat_desc veth_rq_stats_desc[] = { 94d397b968SToshiaki Makita { "xdp_packets", VETH_RQ_STAT(xdp_packets) }, 95d397b968SToshiaki Makita { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) }, 965fe6e567SLorenzo Bianconi { "drops", VETH_RQ_STAT(rx_drops) }, 975fe6e567SLorenzo Bianconi { "xdp_redirect", VETH_RQ_STAT(xdp_redirect) }, 985fe6e567SLorenzo Bianconi { "xdp_drops", VETH_RQ_STAT(xdp_drops) }, 995fe6e567SLorenzo Bianconi { "xdp_tx", VETH_RQ_STAT(xdp_tx) }, 1005fe6e567SLorenzo Bianconi { "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) }, 101d397b968SToshiaki Makita }; 102d397b968SToshiaki Makita 103d397b968SToshiaki Makita #define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc) 104d397b968SToshiaki Makita 1055fe6e567SLorenzo Bianconi static const struct veth_q_stat_desc veth_tq_stats_desc[] = { 1065fe6e567SLorenzo Bianconi { "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) }, 1075fe6e567SLorenzo Bianconi { "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) }, 1085fe6e567SLorenzo Bianconi }; 1095fe6e567SLorenzo Bianconi 1105fe6e567SLorenzo Bianconi #define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc) 1115fe6e567SLorenzo Bianconi 112e314dbdcSPavel Emelyanov static struct { 113e314dbdcSPavel Emelyanov const char string[ETH_GSTRING_LEN]; 114e314dbdcSPavel Emelyanov } ethtool_stats_keys[] = { 115e314dbdcSPavel Emelyanov { "peer_ifindex" }, 116e314dbdcSPavel Emelyanov }; 117e314dbdcSPavel Emelyanov 11856607b98SPhilippe Reynes static int veth_get_link_ksettings(struct net_device *dev, 11956607b98SPhilippe Reynes struct ethtool_link_ksettings *cmd) 120e314dbdcSPavel Emelyanov { 12156607b98SPhilippe Reynes cmd->base.speed = SPEED_10000; 12256607b98SPhilippe Reynes cmd->base.duplex = DUPLEX_FULL; 12356607b98SPhilippe Reynes cmd->base.port = PORT_TP; 12456607b98SPhilippe Reynes cmd->base.autoneg = AUTONEG_DISABLE; 125e314dbdcSPavel Emelyanov return 0; 126e314dbdcSPavel Emelyanov } 127e314dbdcSPavel Emelyanov 128e314dbdcSPavel Emelyanov static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 129e314dbdcSPavel Emelyanov { 13033a5ba14SRick Jones strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); 13133a5ba14SRick Jones strlcpy(info->version, DRV_VERSION, sizeof(info->version)); 132e314dbdcSPavel Emelyanov } 133e314dbdcSPavel Emelyanov 134e314dbdcSPavel Emelyanov static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf) 135e314dbdcSPavel Emelyanov { 136d397b968SToshiaki Makita char *p = (char *)buf; 137d397b968SToshiaki Makita int i, j; 138d397b968SToshiaki Makita 139e314dbdcSPavel Emelyanov switch(stringset) { 140e314dbdcSPavel Emelyanov case ETH_SS_STATS: 141d397b968SToshiaki Makita memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys)); 142d397b968SToshiaki Makita p += sizeof(ethtool_stats_keys); 143d397b968SToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 144d397b968SToshiaki Makita for (j = 0; j < VETH_RQ_STATS_LEN; j++) { 145abdf47aaSFlorian Fainelli snprintf(p, ETH_GSTRING_LEN, 1469152cff0SLorenzo Bianconi "rx_queue_%u_%.18s", 147d397b968SToshiaki Makita i, veth_rq_stats_desc[j].desc); 148d397b968SToshiaki Makita p += ETH_GSTRING_LEN; 149d397b968SToshiaki Makita } 150d397b968SToshiaki Makita } 1515fe6e567SLorenzo Bianconi for (i = 0; i < dev->real_num_tx_queues; i++) { 1525fe6e567SLorenzo Bianconi for (j = 0; j < VETH_TQ_STATS_LEN; j++) { 1535fe6e567SLorenzo Bianconi snprintf(p, ETH_GSTRING_LEN, 1545fe6e567SLorenzo Bianconi "tx_queue_%u_%.18s", 1555fe6e567SLorenzo Bianconi i, veth_tq_stats_desc[j].desc); 1565fe6e567SLorenzo Bianconi p += ETH_GSTRING_LEN; 1575fe6e567SLorenzo Bianconi } 1585fe6e567SLorenzo Bianconi } 159e314dbdcSPavel Emelyanov break; 160e314dbdcSPavel Emelyanov } 161e314dbdcSPavel Emelyanov } 162e314dbdcSPavel Emelyanov 163b9f2c044SJeff Garzik static int veth_get_sset_count(struct net_device *dev, int sset) 164e314dbdcSPavel Emelyanov { 165b9f2c044SJeff Garzik switch (sset) { 166b9f2c044SJeff Garzik case ETH_SS_STATS: 167d397b968SToshiaki Makita return ARRAY_SIZE(ethtool_stats_keys) + 1685fe6e567SLorenzo Bianconi VETH_RQ_STATS_LEN * dev->real_num_rx_queues + 1695fe6e567SLorenzo Bianconi VETH_TQ_STATS_LEN * dev->real_num_tx_queues; 170b9f2c044SJeff Garzik default: 171b9f2c044SJeff Garzik return -EOPNOTSUPP; 172b9f2c044SJeff Garzik } 173e314dbdcSPavel Emelyanov } 174e314dbdcSPavel Emelyanov 175e314dbdcSPavel Emelyanov static void veth_get_ethtool_stats(struct net_device *dev, 176e314dbdcSPavel Emelyanov struct ethtool_stats *stats, u64 *data) 177e314dbdcSPavel Emelyanov { 1785fe6e567SLorenzo Bianconi struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 179d0e2c55eSEric Dumazet struct net_device *peer = rtnl_dereference(priv->peer); 180d397b968SToshiaki Makita int i, j, idx; 181e314dbdcSPavel Emelyanov 182d0e2c55eSEric Dumazet data[0] = peer ? peer->ifindex : 0; 183d397b968SToshiaki Makita idx = 1; 184d397b968SToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 185d397b968SToshiaki Makita const struct veth_rq_stats *rq_stats = &priv->rq[i].stats; 18665780c56SLorenzo Bianconi const void *stats_base = (void *)&rq_stats->vs; 187d397b968SToshiaki Makita unsigned int start; 188d397b968SToshiaki Makita size_t offset; 189d397b968SToshiaki Makita 190d397b968SToshiaki Makita do { 191d397b968SToshiaki Makita start = u64_stats_fetch_begin_irq(&rq_stats->syncp); 192d397b968SToshiaki Makita for (j = 0; j < VETH_RQ_STATS_LEN; j++) { 193d397b968SToshiaki Makita offset = veth_rq_stats_desc[j].offset; 194d397b968SToshiaki Makita data[idx + j] = *(u64 *)(stats_base + offset); 195d397b968SToshiaki Makita } 196d397b968SToshiaki Makita } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); 197d397b968SToshiaki Makita idx += VETH_RQ_STATS_LEN; 198d397b968SToshiaki Makita } 1995fe6e567SLorenzo Bianconi 2005fe6e567SLorenzo Bianconi if (!peer) 2015fe6e567SLorenzo Bianconi return; 2025fe6e567SLorenzo Bianconi 2035fe6e567SLorenzo Bianconi rcv_priv = netdev_priv(peer); 2045fe6e567SLorenzo Bianconi for (i = 0; i < peer->real_num_rx_queues; i++) { 2055fe6e567SLorenzo Bianconi const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats; 2065fe6e567SLorenzo Bianconi const void *base = (void *)&rq_stats->vs; 2075fe6e567SLorenzo Bianconi unsigned int start, tx_idx = idx; 2085fe6e567SLorenzo Bianconi size_t offset; 2095fe6e567SLorenzo Bianconi 2105fe6e567SLorenzo Bianconi tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; 2115fe6e567SLorenzo Bianconi do { 2125fe6e567SLorenzo Bianconi start = u64_stats_fetch_begin_irq(&rq_stats->syncp); 2135fe6e567SLorenzo Bianconi for (j = 0; j < VETH_TQ_STATS_LEN; j++) { 2145fe6e567SLorenzo Bianconi offset = veth_tq_stats_desc[j].offset; 2155fe6e567SLorenzo Bianconi data[tx_idx + j] += *(u64 *)(base + offset); 2165fe6e567SLorenzo Bianconi } 2175fe6e567SLorenzo Bianconi } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); 2185fe6e567SLorenzo Bianconi } 219e314dbdcSPavel Emelyanov } 220e314dbdcSPavel Emelyanov 2210fc0b732SStephen Hemminger static const struct ethtool_ops veth_ethtool_ops = { 222e314dbdcSPavel Emelyanov .get_drvinfo = veth_get_drvinfo, 223e314dbdcSPavel Emelyanov .get_link = ethtool_op_get_link, 224e314dbdcSPavel Emelyanov .get_strings = veth_get_strings, 225b9f2c044SJeff Garzik .get_sset_count = veth_get_sset_count, 226e314dbdcSPavel Emelyanov .get_ethtool_stats = veth_get_ethtool_stats, 22756607b98SPhilippe Reynes .get_link_ksettings = veth_get_link_ksettings, 228056b21fbSJulian Wiedmann .get_ts_info = ethtool_op_get_ts_info, 229e314dbdcSPavel Emelyanov }; 230e314dbdcSPavel Emelyanov 231948d4f21SToshiaki Makita /* general routines */ 232948d4f21SToshiaki Makita 2339fc8d518SToshiaki Makita static bool veth_is_xdp_frame(void *ptr) 2349fc8d518SToshiaki Makita { 2359fc8d518SToshiaki Makita return (unsigned long)ptr & VETH_XDP_FLAG; 2369fc8d518SToshiaki Makita } 2379fc8d518SToshiaki Makita 238defcffebSMaciej Żenczykowski static struct xdp_frame *veth_ptr_to_xdp(void *ptr) 2399fc8d518SToshiaki Makita { 2409fc8d518SToshiaki Makita return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG); 2419fc8d518SToshiaki Makita } 2429fc8d518SToshiaki Makita 243defcffebSMaciej Żenczykowski static void *veth_xdp_to_ptr(struct xdp_frame *xdp) 244af87a3aaSToshiaki Makita { 245defcffebSMaciej Żenczykowski return (void *)((unsigned long)xdp | VETH_XDP_FLAG); 246af87a3aaSToshiaki Makita } 247af87a3aaSToshiaki Makita 2489fc8d518SToshiaki Makita static void veth_ptr_free(void *ptr) 2499fc8d518SToshiaki Makita { 2509fc8d518SToshiaki Makita if (veth_is_xdp_frame(ptr)) 2519fc8d518SToshiaki Makita xdp_return_frame(veth_ptr_to_xdp(ptr)); 2529fc8d518SToshiaki Makita else 2539fc8d518SToshiaki Makita kfree_skb(ptr); 2549fc8d518SToshiaki Makita } 2559fc8d518SToshiaki Makita 256638264dcSToshiaki Makita static void __veth_xdp_flush(struct veth_rq *rq) 257948d4f21SToshiaki Makita { 258948d4f21SToshiaki Makita /* Write ptr_ring before reading rx_notify_masked */ 259948d4f21SToshiaki Makita smp_mb(); 260638264dcSToshiaki Makita if (!rq->rx_notify_masked) { 261638264dcSToshiaki Makita rq->rx_notify_masked = true; 262638264dcSToshiaki Makita napi_schedule(&rq->xdp_napi); 263948d4f21SToshiaki Makita } 264948d4f21SToshiaki Makita } 265948d4f21SToshiaki Makita 266638264dcSToshiaki Makita static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb) 267948d4f21SToshiaki Makita { 268638264dcSToshiaki Makita if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) { 269948d4f21SToshiaki Makita dev_kfree_skb_any(skb); 270948d4f21SToshiaki Makita return NET_RX_DROP; 271948d4f21SToshiaki Makita } 272948d4f21SToshiaki Makita 273948d4f21SToshiaki Makita return NET_RX_SUCCESS; 274948d4f21SToshiaki Makita } 275948d4f21SToshiaki Makita 276638264dcSToshiaki Makita static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, 277638264dcSToshiaki Makita struct veth_rq *rq, bool xdp) 278e314dbdcSPavel Emelyanov { 279948d4f21SToshiaki Makita return __dev_forward_skb(dev, skb) ?: xdp ? 280638264dcSToshiaki Makita veth_xdp_rx(rq, skb) : 281948d4f21SToshiaki Makita netif_rx(skb); 282948d4f21SToshiaki Makita } 283948d4f21SToshiaki Makita 284948d4f21SToshiaki Makita static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) 285948d4f21SToshiaki Makita { 286948d4f21SToshiaki Makita struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 287638264dcSToshiaki Makita struct veth_rq *rq = NULL; 288d0e2c55eSEric Dumazet struct net_device *rcv; 2892681128fSEric Dumazet int length = skb->len; 290948d4f21SToshiaki Makita bool rcv_xdp = false; 291638264dcSToshiaki Makita int rxq; 292e314dbdcSPavel Emelyanov 293d0e2c55eSEric Dumazet rcu_read_lock(); 294d0e2c55eSEric Dumazet rcv = rcu_dereference(priv->peer); 295d0e2c55eSEric Dumazet if (unlikely(!rcv)) { 296d0e2c55eSEric Dumazet kfree_skb(skb); 297d0e2c55eSEric Dumazet goto drop; 298d0e2c55eSEric Dumazet } 299e314dbdcSPavel Emelyanov 300948d4f21SToshiaki Makita rcv_priv = netdev_priv(rcv); 301638264dcSToshiaki Makita rxq = skb_get_queue_mapping(skb); 302638264dcSToshiaki Makita if (rxq < rcv->real_num_rx_queues) { 303638264dcSToshiaki Makita rq = &rcv_priv->rq[rxq]; 304638264dcSToshiaki Makita rcv_xdp = rcu_access_pointer(rq->xdp_prog); 305638264dcSToshiaki Makita if (rcv_xdp) 306638264dcSToshiaki Makita skb_record_rx_queue(skb, rxq); 307638264dcSToshiaki Makita } 308948d4f21SToshiaki Makita 309aa4e689eSMichael Walle skb_tx_timestamp(skb); 310638264dcSToshiaki Makita if (likely(veth_forward_skb(rcv, skb, rq, rcv_xdp) == NET_RX_SUCCESS)) { 311b4fba476SEric Dumazet if (!rcv_xdp) 312b4fba476SEric Dumazet dev_lstats_add(dev, length); 3132681128fSEric Dumazet } else { 314d0e2c55eSEric Dumazet drop: 3152681128fSEric Dumazet atomic64_inc(&priv->dropped); 3162681128fSEric Dumazet } 317948d4f21SToshiaki Makita 318948d4f21SToshiaki Makita if (rcv_xdp) 319638264dcSToshiaki Makita __veth_xdp_flush(rq); 320948d4f21SToshiaki Makita 321d0e2c55eSEric Dumazet rcu_read_unlock(); 322948d4f21SToshiaki Makita 3236ed10654SPatrick McHardy return NETDEV_TX_OK; 324e314dbdcSPavel Emelyanov } 325e314dbdcSPavel Emelyanov 326b4fba476SEric Dumazet static u64 veth_stats_tx(struct net_device *dev, u64 *packets, u64 *bytes) 327e314dbdcSPavel Emelyanov { 328cf05c700SEric Dumazet struct veth_priv *priv = netdev_priv(dev); 32911687a10SDavid S. Miller 330b4fba476SEric Dumazet dev_lstats_read(dev, packets, bytes); 3312681128fSEric Dumazet return atomic64_read(&priv->dropped); 3322681128fSEric Dumazet } 3332681128fSEric Dumazet 33465780c56SLorenzo Bianconi static void veth_stats_rx(struct veth_stats *result, struct net_device *dev) 3354195e54aSToshiaki Makita { 3364195e54aSToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 3374195e54aSToshiaki Makita int i; 3384195e54aSToshiaki Makita 3395fe6e567SLorenzo Bianconi result->peer_tq_xdp_xmit_err = 0; 3404195e54aSToshiaki Makita result->xdp_packets = 0; 341d99a7c2fSLorenzo Bianconi result->xdp_tx_err = 0; 3424195e54aSToshiaki Makita result->xdp_bytes = 0; 34366fe4a07SLorenzo Bianconi result->rx_drops = 0; 3444195e54aSToshiaki Makita for (i = 0; i < dev->num_rx_queues; i++) { 3455fe6e567SLorenzo Bianconi u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err; 3464195e54aSToshiaki Makita struct veth_rq_stats *stats = &priv->rq[i].stats; 3474195e54aSToshiaki Makita unsigned int start; 3484195e54aSToshiaki Makita 3494195e54aSToshiaki Makita do { 3504195e54aSToshiaki Makita start = u64_stats_fetch_begin_irq(&stats->syncp); 3515fe6e567SLorenzo Bianconi peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err; 352d99a7c2fSLorenzo Bianconi xdp_tx_err = stats->vs.xdp_tx_err; 35365780c56SLorenzo Bianconi packets = stats->vs.xdp_packets; 35465780c56SLorenzo Bianconi bytes = stats->vs.xdp_bytes; 35566fe4a07SLorenzo Bianconi drops = stats->vs.rx_drops; 3564195e54aSToshiaki Makita } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); 3575fe6e567SLorenzo Bianconi result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err; 358d99a7c2fSLorenzo Bianconi result->xdp_tx_err += xdp_tx_err; 3594195e54aSToshiaki Makita result->xdp_packets += packets; 3604195e54aSToshiaki Makita result->xdp_bytes += bytes; 36166fe4a07SLorenzo Bianconi result->rx_drops += drops; 3624195e54aSToshiaki Makita } 3634195e54aSToshiaki Makita } 3644195e54aSToshiaki Makita 365bc1f4470Sstephen hemminger static void veth_get_stats64(struct net_device *dev, 3662681128fSEric Dumazet struct rtnl_link_stats64 *tot) 3672681128fSEric Dumazet { 3682681128fSEric Dumazet struct veth_priv *priv = netdev_priv(dev); 369d0e2c55eSEric Dumazet struct net_device *peer; 37065780c56SLorenzo Bianconi struct veth_stats rx; 371b4fba476SEric Dumazet u64 packets, bytes; 3722681128fSEric Dumazet 373b4fba476SEric Dumazet tot->tx_dropped = veth_stats_tx(dev, &packets, &bytes); 374b4fba476SEric Dumazet tot->tx_bytes = bytes; 375b4fba476SEric Dumazet tot->tx_packets = packets; 3764195e54aSToshiaki Makita 3774195e54aSToshiaki Makita veth_stats_rx(&rx, dev); 3785fe6e567SLorenzo Bianconi tot->tx_dropped += rx.xdp_tx_err; 3795fe6e567SLorenzo Bianconi tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err; 3804195e54aSToshiaki Makita tot->rx_bytes = rx.xdp_bytes; 3814195e54aSToshiaki Makita tot->rx_packets = rx.xdp_packets; 3822681128fSEric Dumazet 383d0e2c55eSEric Dumazet rcu_read_lock(); 384d0e2c55eSEric Dumazet peer = rcu_dereference(priv->peer); 385d0e2c55eSEric Dumazet if (peer) { 386e25d5dbcSJiang Lidong veth_stats_tx(peer, &packets, &bytes); 387b4fba476SEric Dumazet tot->rx_bytes += bytes; 388b4fba476SEric Dumazet tot->rx_packets += packets; 3894195e54aSToshiaki Makita 3904195e54aSToshiaki Makita veth_stats_rx(&rx, peer); 3915fe6e567SLorenzo Bianconi tot->tx_dropped += rx.peer_tq_xdp_xmit_err; 3925fe6e567SLorenzo Bianconi tot->rx_dropped += rx.xdp_tx_err; 3934195e54aSToshiaki Makita tot->tx_bytes += rx.xdp_bytes; 3944195e54aSToshiaki Makita tot->tx_packets += rx.xdp_packets; 395d0e2c55eSEric Dumazet } 396d0e2c55eSEric Dumazet rcu_read_unlock(); 397e314dbdcSPavel Emelyanov } 398e314dbdcSPavel Emelyanov 3995c70ef85SGao feng /* fake multicast ability */ 4005c70ef85SGao feng static void veth_set_multicast_list(struct net_device *dev) 4015c70ef85SGao feng { 4025c70ef85SGao feng } 4035c70ef85SGao feng 404948d4f21SToshiaki Makita static struct sk_buff *veth_build_skb(void *head, int headroom, int len, 405948d4f21SToshiaki Makita int buflen) 406948d4f21SToshiaki Makita { 407948d4f21SToshiaki Makita struct sk_buff *skb; 408948d4f21SToshiaki Makita 409948d4f21SToshiaki Makita skb = build_skb(head, buflen); 410948d4f21SToshiaki Makita if (!skb) 411948d4f21SToshiaki Makita return NULL; 412948d4f21SToshiaki Makita 413948d4f21SToshiaki Makita skb_reserve(skb, headroom); 414948d4f21SToshiaki Makita skb_put(skb, len); 415948d4f21SToshiaki Makita 416948d4f21SToshiaki Makita return skb; 417948d4f21SToshiaki Makita } 418948d4f21SToshiaki Makita 419638264dcSToshiaki Makita static int veth_select_rxq(struct net_device *dev) 420638264dcSToshiaki Makita { 421638264dcSToshiaki Makita return smp_processor_id() % dev->real_num_rx_queues; 422638264dcSToshiaki Makita } 423638264dcSToshiaki Makita 4249aa1206eSDaniel Borkmann static struct net_device *veth_peer_dev(struct net_device *dev) 4259aa1206eSDaniel Borkmann { 4269aa1206eSDaniel Borkmann struct veth_priv *priv = netdev_priv(dev); 4279aa1206eSDaniel Borkmann 4289aa1206eSDaniel Borkmann /* Callers must be under RCU read side. */ 4299aa1206eSDaniel Borkmann return rcu_dereference(priv->peer); 4309aa1206eSDaniel Borkmann } 4319aa1206eSDaniel Borkmann 432af87a3aaSToshiaki Makita static int veth_xdp_xmit(struct net_device *dev, int n, 4339152cff0SLorenzo Bianconi struct xdp_frame **frames, 4349152cff0SLorenzo Bianconi u32 flags, bool ndo_xmit) 435af87a3aaSToshiaki Makita { 436af87a3aaSToshiaki Makita struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 437*fdc13979SLorenzo Bianconi int i, ret = -ENXIO, nxmit = 0; 438af87a3aaSToshiaki Makita struct net_device *rcv; 4395fe6e567SLorenzo Bianconi unsigned int max_len; 440638264dcSToshiaki Makita struct veth_rq *rq; 441af87a3aaSToshiaki Makita 4425fe6e567SLorenzo Bianconi if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 443d99a7c2fSLorenzo Bianconi return -EINVAL; 444af87a3aaSToshiaki Makita 4455fe6e567SLorenzo Bianconi rcu_read_lock(); 446af87a3aaSToshiaki Makita rcv = rcu_dereference(priv->peer); 4475fe6e567SLorenzo Bianconi if (unlikely(!rcv)) 4485fe6e567SLorenzo Bianconi goto out; 449af87a3aaSToshiaki Makita 450af87a3aaSToshiaki Makita rcv_priv = netdev_priv(rcv); 4515fe6e567SLorenzo Bianconi rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 452af87a3aaSToshiaki Makita /* Non-NULL xdp_prog ensures that xdp_ring is initialized on receive 453af87a3aaSToshiaki Makita * side. This means an XDP program is loaded on the peer and the peer 454af87a3aaSToshiaki Makita * device is up. 455af87a3aaSToshiaki Makita */ 4565fe6e567SLorenzo Bianconi if (!rcu_access_pointer(rq->xdp_prog)) 4575fe6e567SLorenzo Bianconi goto out; 458af87a3aaSToshiaki Makita 459af87a3aaSToshiaki Makita max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN; 460af87a3aaSToshiaki Makita 461638264dcSToshiaki Makita spin_lock(&rq->xdp_ring.producer_lock); 462af87a3aaSToshiaki Makita for (i = 0; i < n; i++) { 463af87a3aaSToshiaki Makita struct xdp_frame *frame = frames[i]; 464af87a3aaSToshiaki Makita void *ptr = veth_xdp_to_ptr(frame); 465af87a3aaSToshiaki Makita 466af87a3aaSToshiaki Makita if (unlikely(frame->len > max_len || 467*fdc13979SLorenzo Bianconi __ptr_ring_produce(&rq->xdp_ring, ptr))) 468*fdc13979SLorenzo Bianconi break; 469*fdc13979SLorenzo Bianconi nxmit++; 470af87a3aaSToshiaki Makita } 471638264dcSToshiaki Makita spin_unlock(&rq->xdp_ring.producer_lock); 472af87a3aaSToshiaki Makita 473af87a3aaSToshiaki Makita if (flags & XDP_XMIT_FLUSH) 474638264dcSToshiaki Makita __veth_xdp_flush(rq); 475af87a3aaSToshiaki Makita 476*fdc13979SLorenzo Bianconi ret = nxmit; 4779152cff0SLorenzo Bianconi if (ndo_xmit) { 4785fe6e567SLorenzo Bianconi u64_stats_update_begin(&rq->stats.syncp); 479*fdc13979SLorenzo Bianconi rq->stats.vs.peer_tq_xdp_xmit += nxmit; 480*fdc13979SLorenzo Bianconi rq->stats.vs.peer_tq_xdp_xmit_err += n - nxmit; 4819152cff0SLorenzo Bianconi u64_stats_update_end(&rq->stats.syncp); 4825fe6e567SLorenzo Bianconi } 4839152cff0SLorenzo Bianconi 4845fe6e567SLorenzo Bianconi out: 485b23bfa56SJohn Fastabend rcu_read_unlock(); 4862131479dSToshiaki Makita 4872131479dSToshiaki Makita return ret; 488af87a3aaSToshiaki Makita } 489af87a3aaSToshiaki Makita 4909152cff0SLorenzo Bianconi static int veth_ndo_xdp_xmit(struct net_device *dev, int n, 4919152cff0SLorenzo Bianconi struct xdp_frame **frames, u32 flags) 4929152cff0SLorenzo Bianconi { 4935fe6e567SLorenzo Bianconi int err; 4945fe6e567SLorenzo Bianconi 4955fe6e567SLorenzo Bianconi err = veth_xdp_xmit(dev, n, frames, flags, true); 4965fe6e567SLorenzo Bianconi if (err < 0) { 4975fe6e567SLorenzo Bianconi struct veth_priv *priv = netdev_priv(dev); 4985fe6e567SLorenzo Bianconi 4995fe6e567SLorenzo Bianconi atomic64_add(n, &priv->dropped); 5005fe6e567SLorenzo Bianconi } 5015fe6e567SLorenzo Bianconi 5025fe6e567SLorenzo Bianconi return err; 5039152cff0SLorenzo Bianconi } 5049152cff0SLorenzo Bianconi 505bd32aa1fSLorenzo Bianconi static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 5069cda7807SToshiaki Makita { 507*fdc13979SLorenzo Bianconi int sent, i, err = 0, drops; 5089cda7807SToshiaki Makita 509bd32aa1fSLorenzo Bianconi sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); 5109cda7807SToshiaki Makita if (sent < 0) { 5119cda7807SToshiaki Makita err = sent; 5129cda7807SToshiaki Makita sent = 0; 5139cda7807SToshiaki Makita } 514*fdc13979SLorenzo Bianconi 515*fdc13979SLorenzo Bianconi for (i = sent; unlikely(i < bq->count); i++) 516*fdc13979SLorenzo Bianconi xdp_return_frame(bq->q[i]); 517*fdc13979SLorenzo Bianconi 518*fdc13979SLorenzo Bianconi drops = bq->count - sent; 519*fdc13979SLorenzo Bianconi trace_xdp_bulk_tx(rq->dev, sent, drops, err); 5209cda7807SToshiaki Makita 5215fe6e567SLorenzo Bianconi u64_stats_update_begin(&rq->stats.syncp); 5225fe6e567SLorenzo Bianconi rq->stats.vs.xdp_tx += sent; 523*fdc13979SLorenzo Bianconi rq->stats.vs.xdp_tx_err += drops; 5245fe6e567SLorenzo Bianconi u64_stats_update_end(&rq->stats.syncp); 5255fe6e567SLorenzo Bianconi 5269cda7807SToshiaki Makita bq->count = 0; 5279cda7807SToshiaki Makita } 5289cda7807SToshiaki Makita 529bd32aa1fSLorenzo Bianconi static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 530d1396004SToshiaki Makita { 531bd32aa1fSLorenzo Bianconi struct veth_priv *rcv_priv, *priv = netdev_priv(rq->dev); 532d1396004SToshiaki Makita struct net_device *rcv; 533bd32aa1fSLorenzo Bianconi struct veth_rq *rcv_rq; 534d1396004SToshiaki Makita 535d1396004SToshiaki Makita rcu_read_lock(); 536bd32aa1fSLorenzo Bianconi veth_xdp_flush_bq(rq, bq); 537d1396004SToshiaki Makita rcv = rcu_dereference(priv->peer); 538d1396004SToshiaki Makita if (unlikely(!rcv)) 539d1396004SToshiaki Makita goto out; 540d1396004SToshiaki Makita 541d1396004SToshiaki Makita rcv_priv = netdev_priv(rcv); 542bd32aa1fSLorenzo Bianconi rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 543d1396004SToshiaki Makita /* xdp_ring is initialized on receive side? */ 544bd32aa1fSLorenzo Bianconi if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog))) 545d1396004SToshiaki Makita goto out; 546d1396004SToshiaki Makita 547bd32aa1fSLorenzo Bianconi __veth_xdp_flush(rcv_rq); 548d1396004SToshiaki Makita out: 549d1396004SToshiaki Makita rcu_read_unlock(); 550d1396004SToshiaki Makita } 551d1396004SToshiaki Makita 552bd32aa1fSLorenzo Bianconi static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp, 5539cda7807SToshiaki Makita struct veth_xdp_tx_bq *bq) 554d1396004SToshiaki Makita { 5551b698fa5SLorenzo Bianconi struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); 556d1396004SToshiaki Makita 557d1396004SToshiaki Makita if (unlikely(!frame)) 558d1396004SToshiaki Makita return -EOVERFLOW; 559d1396004SToshiaki Makita 5609cda7807SToshiaki Makita if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE)) 561bd32aa1fSLorenzo Bianconi veth_xdp_flush_bq(rq, bq); 5629cda7807SToshiaki Makita 5639cda7807SToshiaki Makita bq->q[bq->count++] = frame; 5649cda7807SToshiaki Makita 5659cda7807SToshiaki Makita return 0; 566d1396004SToshiaki Makita } 567d1396004SToshiaki Makita 56865e6dcf7SLorenzo Bianconi static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq, 569d1396004SToshiaki Makita struct xdp_frame *frame, 5701c5b82e5SLorenzo Bianconi struct veth_xdp_tx_bq *bq, 5711c5b82e5SLorenzo Bianconi struct veth_stats *stats) 5729fc8d518SToshiaki Makita { 573d1396004SToshiaki Makita struct xdp_frame orig_frame; 5749fc8d518SToshiaki Makita struct bpf_prog *xdp_prog; 5759fc8d518SToshiaki Makita 5769fc8d518SToshiaki Makita rcu_read_lock(); 577638264dcSToshiaki Makita xdp_prog = rcu_dereference(rq->xdp_prog); 5789fc8d518SToshiaki Makita if (likely(xdp_prog)) { 5799fc8d518SToshiaki Makita struct xdp_buff xdp; 5809fc8d518SToshiaki Makita u32 act; 5819fc8d518SToshiaki Makita 582fc379872SLorenzo Bianconi xdp_convert_frame_to_buff(frame, &xdp); 583638264dcSToshiaki Makita xdp.rxq = &rq->xdp_rxq; 5849fc8d518SToshiaki Makita 5859fc8d518SToshiaki Makita act = bpf_prog_run_xdp(xdp_prog, &xdp); 5869fc8d518SToshiaki Makita 5879fc8d518SToshiaki Makita switch (act) { 5889fc8d518SToshiaki Makita case XDP_PASS: 58989f479f0SLorenzo Bianconi if (xdp_update_frame_from_buff(&xdp, frame)) 59089f479f0SLorenzo Bianconi goto err_xdp; 5919fc8d518SToshiaki Makita break; 592d1396004SToshiaki Makita case XDP_TX: 593d1396004SToshiaki Makita orig_frame = *frame; 594d1396004SToshiaki Makita xdp.rxq->mem = frame->mem; 595bd32aa1fSLorenzo Bianconi if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) { 596638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 597d1396004SToshiaki Makita frame = &orig_frame; 5981c5b82e5SLorenzo Bianconi stats->rx_drops++; 599d1396004SToshiaki Makita goto err_xdp; 600d1396004SToshiaki Makita } 6011c5b82e5SLorenzo Bianconi stats->xdp_tx++; 602d1396004SToshiaki Makita rcu_read_unlock(); 603d1396004SToshiaki Makita goto xdp_xmit; 604d1396004SToshiaki Makita case XDP_REDIRECT: 605d1396004SToshiaki Makita orig_frame = *frame; 606d1396004SToshiaki Makita xdp.rxq->mem = frame->mem; 607638264dcSToshiaki Makita if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { 608d1396004SToshiaki Makita frame = &orig_frame; 6091c5b82e5SLorenzo Bianconi stats->rx_drops++; 610d1396004SToshiaki Makita goto err_xdp; 611d1396004SToshiaki Makita } 6121c5b82e5SLorenzo Bianconi stats->xdp_redirect++; 613d1396004SToshiaki Makita rcu_read_unlock(); 614d1396004SToshiaki Makita goto xdp_xmit; 6159fc8d518SToshiaki Makita default: 6169fc8d518SToshiaki Makita bpf_warn_invalid_xdp_action(act); 617df561f66SGustavo A. R. Silva fallthrough; 6189fc8d518SToshiaki Makita case XDP_ABORTED: 619638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 620df561f66SGustavo A. R. Silva fallthrough; 6219fc8d518SToshiaki Makita case XDP_DROP: 6221c5b82e5SLorenzo Bianconi stats->xdp_drops++; 6239fc8d518SToshiaki Makita goto err_xdp; 6249fc8d518SToshiaki Makita } 6259fc8d518SToshiaki Makita } 6269fc8d518SToshiaki Makita rcu_read_unlock(); 6279fc8d518SToshiaki Makita 62865e6dcf7SLorenzo Bianconi return frame; 6299fc8d518SToshiaki Makita err_xdp: 6309fc8d518SToshiaki Makita rcu_read_unlock(); 6319fc8d518SToshiaki Makita xdp_return_frame(frame); 632d1396004SToshiaki Makita xdp_xmit: 6339fc8d518SToshiaki Makita return NULL; 6349fc8d518SToshiaki Makita } 6359fc8d518SToshiaki Makita 63665e6dcf7SLorenzo Bianconi /* frames array contains VETH_XDP_BATCH at most */ 63765e6dcf7SLorenzo Bianconi static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames, 63865e6dcf7SLorenzo Bianconi int n_xdpf, struct veth_xdp_tx_bq *bq, 63965e6dcf7SLorenzo Bianconi struct veth_stats *stats) 64065e6dcf7SLorenzo Bianconi { 64165e6dcf7SLorenzo Bianconi void *skbs[VETH_XDP_BATCH]; 64265e6dcf7SLorenzo Bianconi int i; 64365e6dcf7SLorenzo Bianconi 64465e6dcf7SLorenzo Bianconi if (xdp_alloc_skb_bulk(skbs, n_xdpf, 64565e6dcf7SLorenzo Bianconi GFP_ATOMIC | __GFP_ZERO) < 0) { 64665e6dcf7SLorenzo Bianconi for (i = 0; i < n_xdpf; i++) 64765e6dcf7SLorenzo Bianconi xdp_return_frame(frames[i]); 64865e6dcf7SLorenzo Bianconi stats->rx_drops += n_xdpf; 64965e6dcf7SLorenzo Bianconi 65065e6dcf7SLorenzo Bianconi return; 65165e6dcf7SLorenzo Bianconi } 65265e6dcf7SLorenzo Bianconi 65365e6dcf7SLorenzo Bianconi for (i = 0; i < n_xdpf; i++) { 65465e6dcf7SLorenzo Bianconi struct sk_buff *skb = skbs[i]; 65565e6dcf7SLorenzo Bianconi 65665e6dcf7SLorenzo Bianconi skb = __xdp_build_skb_from_frame(frames[i], skb, 65765e6dcf7SLorenzo Bianconi rq->dev); 65865e6dcf7SLorenzo Bianconi if (!skb) { 65965e6dcf7SLorenzo Bianconi xdp_return_frame(frames[i]); 66065e6dcf7SLorenzo Bianconi stats->rx_drops++; 66165e6dcf7SLorenzo Bianconi continue; 66265e6dcf7SLorenzo Bianconi } 66365e6dcf7SLorenzo Bianconi napi_gro_receive(&rq->xdp_napi, skb); 66465e6dcf7SLorenzo Bianconi } 66565e6dcf7SLorenzo Bianconi } 66665e6dcf7SLorenzo Bianconi 6671c5b82e5SLorenzo Bianconi static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, 6681c5b82e5SLorenzo Bianconi struct sk_buff *skb, 6691c5b82e5SLorenzo Bianconi struct veth_xdp_tx_bq *bq, 6701c5b82e5SLorenzo Bianconi struct veth_stats *stats) 671948d4f21SToshiaki Makita { 67243b5169dSLorenzo Bianconi u32 pktlen, headroom, act, metalen, frame_sz; 673948d4f21SToshiaki Makita void *orig_data, *orig_data_end; 674948d4f21SToshiaki Makita struct bpf_prog *xdp_prog; 675948d4f21SToshiaki Makita int mac_len, delta, off; 676948d4f21SToshiaki Makita struct xdp_buff xdp; 677948d4f21SToshiaki Makita 6784bf9ffa0SToshiaki Makita skb_orphan(skb); 6794bf9ffa0SToshiaki Makita 680948d4f21SToshiaki Makita rcu_read_lock(); 681638264dcSToshiaki Makita xdp_prog = rcu_dereference(rq->xdp_prog); 682948d4f21SToshiaki Makita if (unlikely(!xdp_prog)) { 683948d4f21SToshiaki Makita rcu_read_unlock(); 684948d4f21SToshiaki Makita goto out; 685948d4f21SToshiaki Makita } 686948d4f21SToshiaki Makita 687948d4f21SToshiaki Makita mac_len = skb->data - skb_mac_header(skb); 688948d4f21SToshiaki Makita pktlen = skb->len + mac_len; 689948d4f21SToshiaki Makita headroom = skb_headroom(skb) - mac_len; 690948d4f21SToshiaki Makita 691948d4f21SToshiaki Makita if (skb_shared(skb) || skb_head_is_locked(skb) || 692948d4f21SToshiaki Makita skb_is_nonlinear(skb) || headroom < XDP_PACKET_HEADROOM) { 693948d4f21SToshiaki Makita struct sk_buff *nskb; 694948d4f21SToshiaki Makita int size, head_off; 695948d4f21SToshiaki Makita void *head, *start; 696948d4f21SToshiaki Makita struct page *page; 697948d4f21SToshiaki Makita 698948d4f21SToshiaki Makita size = SKB_DATA_ALIGN(VETH_XDP_HEADROOM + pktlen) + 699948d4f21SToshiaki Makita SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 700948d4f21SToshiaki Makita if (size > PAGE_SIZE) 701948d4f21SToshiaki Makita goto drop; 702948d4f21SToshiaki Makita 703948d4f21SToshiaki Makita page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); 704948d4f21SToshiaki Makita if (!page) 705948d4f21SToshiaki Makita goto drop; 706948d4f21SToshiaki Makita 707948d4f21SToshiaki Makita head = page_address(page); 708948d4f21SToshiaki Makita start = head + VETH_XDP_HEADROOM; 709948d4f21SToshiaki Makita if (skb_copy_bits(skb, -mac_len, start, pktlen)) { 710948d4f21SToshiaki Makita page_frag_free(head); 711948d4f21SToshiaki Makita goto drop; 712948d4f21SToshiaki Makita } 713948d4f21SToshiaki Makita 71445a9e6d8SJesper Dangaard Brouer nskb = veth_build_skb(head, VETH_XDP_HEADROOM + mac_len, 71545a9e6d8SJesper Dangaard Brouer skb->len, PAGE_SIZE); 716948d4f21SToshiaki Makita if (!nskb) { 717948d4f21SToshiaki Makita page_frag_free(head); 718948d4f21SToshiaki Makita goto drop; 719948d4f21SToshiaki Makita } 720948d4f21SToshiaki Makita 721948d4f21SToshiaki Makita skb_copy_header(nskb, skb); 722948d4f21SToshiaki Makita head_off = skb_headroom(nskb) - skb_headroom(skb); 723948d4f21SToshiaki Makita skb_headers_offset_update(nskb, head_off); 724948d4f21SToshiaki Makita consume_skb(skb); 725948d4f21SToshiaki Makita skb = nskb; 726948d4f21SToshiaki Makita } 727948d4f21SToshiaki Makita 72845a9e6d8SJesper Dangaard Brouer /* SKB "head" area always have tailroom for skb_shared_info */ 729be9df4afSLorenzo Bianconi frame_sz = skb_end_pointer(skb) - skb->head; 73043b5169dSLorenzo Bianconi frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 73143b5169dSLorenzo Bianconi xdp_init_buff(&xdp, frame_sz, &rq->xdp_rxq); 732be9df4afSLorenzo Bianconi xdp_prepare_buff(&xdp, skb->head, skb->mac_header, pktlen, true); 73345a9e6d8SJesper Dangaard Brouer 734948d4f21SToshiaki Makita orig_data = xdp.data; 735948d4f21SToshiaki Makita orig_data_end = xdp.data_end; 736948d4f21SToshiaki Makita 737948d4f21SToshiaki Makita act = bpf_prog_run_xdp(xdp_prog, &xdp); 738948d4f21SToshiaki Makita 739948d4f21SToshiaki Makita switch (act) { 740948d4f21SToshiaki Makita case XDP_PASS: 741948d4f21SToshiaki Makita break; 742d1396004SToshiaki Makita case XDP_TX: 743d1396004SToshiaki Makita get_page(virt_to_page(xdp.data)); 744d1396004SToshiaki Makita consume_skb(skb); 745638264dcSToshiaki Makita xdp.rxq->mem = rq->xdp_mem; 746bd32aa1fSLorenzo Bianconi if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) { 747638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 7481c5b82e5SLorenzo Bianconi stats->rx_drops++; 749d1396004SToshiaki Makita goto err_xdp; 750d1396004SToshiaki Makita } 7511c5b82e5SLorenzo Bianconi stats->xdp_tx++; 752d1396004SToshiaki Makita rcu_read_unlock(); 753d1396004SToshiaki Makita goto xdp_xmit; 754d1396004SToshiaki Makita case XDP_REDIRECT: 755d1396004SToshiaki Makita get_page(virt_to_page(xdp.data)); 756d1396004SToshiaki Makita consume_skb(skb); 757638264dcSToshiaki Makita xdp.rxq->mem = rq->xdp_mem; 7581c5b82e5SLorenzo Bianconi if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { 7591c5b82e5SLorenzo Bianconi stats->rx_drops++; 760d1396004SToshiaki Makita goto err_xdp; 7611c5b82e5SLorenzo Bianconi } 7621c5b82e5SLorenzo Bianconi stats->xdp_redirect++; 763d1396004SToshiaki Makita rcu_read_unlock(); 764d1396004SToshiaki Makita goto xdp_xmit; 765948d4f21SToshiaki Makita default: 766948d4f21SToshiaki Makita bpf_warn_invalid_xdp_action(act); 767df561f66SGustavo A. R. Silva fallthrough; 768948d4f21SToshiaki Makita case XDP_ABORTED: 769638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 770df561f66SGustavo A. R. Silva fallthrough; 771948d4f21SToshiaki Makita case XDP_DROP: 7721c5b82e5SLorenzo Bianconi stats->xdp_drops++; 7731c5b82e5SLorenzo Bianconi goto xdp_drop; 774948d4f21SToshiaki Makita } 775948d4f21SToshiaki Makita rcu_read_unlock(); 776948d4f21SToshiaki Makita 77745a9e6d8SJesper Dangaard Brouer /* check if bpf_xdp_adjust_head was used */ 778948d4f21SToshiaki Makita delta = orig_data - xdp.data; 779948d4f21SToshiaki Makita off = mac_len + delta; 780948d4f21SToshiaki Makita if (off > 0) 781948d4f21SToshiaki Makita __skb_push(skb, off); 782948d4f21SToshiaki Makita else if (off < 0) 783948d4f21SToshiaki Makita __skb_pull(skb, -off); 784948d4f21SToshiaki Makita skb->mac_header -= delta; 78545a9e6d8SJesper Dangaard Brouer 78645a9e6d8SJesper Dangaard Brouer /* check if bpf_xdp_adjust_tail was used */ 787948d4f21SToshiaki Makita off = xdp.data_end - orig_data_end; 788948d4f21SToshiaki Makita if (off != 0) 78945a9e6d8SJesper Dangaard Brouer __skb_put(skb, off); /* positive on grow, negative on shrink */ 790638264dcSToshiaki Makita skb->protocol = eth_type_trans(skb, rq->dev); 791948d4f21SToshiaki Makita 792948d4f21SToshiaki Makita metalen = xdp.data - xdp.data_meta; 793948d4f21SToshiaki Makita if (metalen) 794948d4f21SToshiaki Makita skb_metadata_set(skb, metalen); 795948d4f21SToshiaki Makita out: 796948d4f21SToshiaki Makita return skb; 797948d4f21SToshiaki Makita drop: 7981c5b82e5SLorenzo Bianconi stats->rx_drops++; 7991c5b82e5SLorenzo Bianconi xdp_drop: 800948d4f21SToshiaki Makita rcu_read_unlock(); 801948d4f21SToshiaki Makita kfree_skb(skb); 802948d4f21SToshiaki Makita return NULL; 803d1396004SToshiaki Makita err_xdp: 804d1396004SToshiaki Makita rcu_read_unlock(); 805d1396004SToshiaki Makita page_frag_free(xdp.data); 806d1396004SToshiaki Makita xdp_xmit: 807d1396004SToshiaki Makita return NULL; 808948d4f21SToshiaki Makita } 809948d4f21SToshiaki Makita 8101c5b82e5SLorenzo Bianconi static int veth_xdp_rcv(struct veth_rq *rq, int budget, 8111c5b82e5SLorenzo Bianconi struct veth_xdp_tx_bq *bq, 8121c5b82e5SLorenzo Bianconi struct veth_stats *stats) 813948d4f21SToshiaki Makita { 81465e6dcf7SLorenzo Bianconi int i, done = 0, n_xdpf = 0; 81565e6dcf7SLorenzo Bianconi void *xdpf[VETH_XDP_BATCH]; 816948d4f21SToshiaki Makita 817948d4f21SToshiaki Makita for (i = 0; i < budget; i++) { 818638264dcSToshiaki Makita void *ptr = __ptr_ring_consume(&rq->xdp_ring); 819948d4f21SToshiaki Makita 8209fc8d518SToshiaki Makita if (!ptr) 821948d4f21SToshiaki Makita break; 822948d4f21SToshiaki Makita 823d1396004SToshiaki Makita if (veth_is_xdp_frame(ptr)) { 82465e6dcf7SLorenzo Bianconi /* ndo_xdp_xmit */ 8254195e54aSToshiaki Makita struct xdp_frame *frame = veth_ptr_to_xdp(ptr); 8264195e54aSToshiaki Makita 8271c5b82e5SLorenzo Bianconi stats->xdp_bytes += frame->len; 82865e6dcf7SLorenzo Bianconi frame = veth_xdp_rcv_one(rq, frame, bq, stats); 82965e6dcf7SLorenzo Bianconi if (frame) { 83065e6dcf7SLorenzo Bianconi /* XDP_PASS */ 83165e6dcf7SLorenzo Bianconi xdpf[n_xdpf++] = frame; 83265e6dcf7SLorenzo Bianconi if (n_xdpf == VETH_XDP_BATCH) { 83365e6dcf7SLorenzo Bianconi veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, 83465e6dcf7SLorenzo Bianconi bq, stats); 83565e6dcf7SLorenzo Bianconi n_xdpf = 0; 83665e6dcf7SLorenzo Bianconi } 83765e6dcf7SLorenzo Bianconi } 838d1396004SToshiaki Makita } else { 83965e6dcf7SLorenzo Bianconi /* ndo_start_xmit */ 84065e6dcf7SLorenzo Bianconi struct sk_buff *skb = ptr; 84165e6dcf7SLorenzo Bianconi 8421c5b82e5SLorenzo Bianconi stats->xdp_bytes += skb->len; 8431c5b82e5SLorenzo Bianconi skb = veth_xdp_rcv_skb(rq, skb, bq, stats); 844948d4f21SToshiaki Makita if (skb) 845638264dcSToshiaki Makita napi_gro_receive(&rq->xdp_napi, skb); 84665e6dcf7SLorenzo Bianconi } 847948d4f21SToshiaki Makita done++; 848948d4f21SToshiaki Makita } 849948d4f21SToshiaki Makita 85065e6dcf7SLorenzo Bianconi if (n_xdpf) 85165e6dcf7SLorenzo Bianconi veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, bq, stats); 85265e6dcf7SLorenzo Bianconi 8534195e54aSToshiaki Makita u64_stats_update_begin(&rq->stats.syncp); 8549152cff0SLorenzo Bianconi rq->stats.vs.xdp_redirect += stats->xdp_redirect; 8551c5b82e5SLorenzo Bianconi rq->stats.vs.xdp_bytes += stats->xdp_bytes; 85666fe4a07SLorenzo Bianconi rq->stats.vs.xdp_drops += stats->xdp_drops; 85766fe4a07SLorenzo Bianconi rq->stats.vs.rx_drops += stats->rx_drops; 85865780c56SLorenzo Bianconi rq->stats.vs.xdp_packets += done; 8594195e54aSToshiaki Makita u64_stats_update_end(&rq->stats.syncp); 8604195e54aSToshiaki Makita 861948d4f21SToshiaki Makita return done; 862948d4f21SToshiaki Makita } 863948d4f21SToshiaki Makita 864948d4f21SToshiaki Makita static int veth_poll(struct napi_struct *napi, int budget) 865948d4f21SToshiaki Makita { 866638264dcSToshiaki Makita struct veth_rq *rq = 867638264dcSToshiaki Makita container_of(napi, struct veth_rq, xdp_napi); 8681c5b82e5SLorenzo Bianconi struct veth_stats stats = {}; 8699cda7807SToshiaki Makita struct veth_xdp_tx_bq bq; 870948d4f21SToshiaki Makita int done; 871948d4f21SToshiaki Makita 8729cda7807SToshiaki Makita bq.count = 0; 8739cda7807SToshiaki Makita 874d1396004SToshiaki Makita xdp_set_return_frame_no_direct(); 8751c5b82e5SLorenzo Bianconi done = veth_xdp_rcv(rq, budget, &bq, &stats); 876948d4f21SToshiaki Makita 877948d4f21SToshiaki Makita if (done < budget && napi_complete_done(napi, done)) { 878948d4f21SToshiaki Makita /* Write rx_notify_masked before reading ptr_ring */ 879638264dcSToshiaki Makita smp_store_mb(rq->rx_notify_masked, false); 880638264dcSToshiaki Makita if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) { 881638264dcSToshiaki Makita rq->rx_notify_masked = true; 882638264dcSToshiaki Makita napi_schedule(&rq->xdp_napi); 883948d4f21SToshiaki Makita } 884948d4f21SToshiaki Makita } 885948d4f21SToshiaki Makita 8861c5b82e5SLorenzo Bianconi if (stats.xdp_tx > 0) 887bd32aa1fSLorenzo Bianconi veth_xdp_flush(rq, &bq); 8881c5b82e5SLorenzo Bianconi if (stats.xdp_redirect > 0) 8891d233886SToke Høiland-Jørgensen xdp_do_flush(); 890d1396004SToshiaki Makita xdp_clear_return_frame_no_direct(); 891d1396004SToshiaki Makita 892948d4f21SToshiaki Makita return done; 893948d4f21SToshiaki Makita } 894948d4f21SToshiaki Makita 895948d4f21SToshiaki Makita static int veth_napi_add(struct net_device *dev) 896948d4f21SToshiaki Makita { 897948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 898638264dcSToshiaki Makita int err, i; 899948d4f21SToshiaki Makita 900638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 901638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 902638264dcSToshiaki Makita 903638264dcSToshiaki Makita err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL); 904948d4f21SToshiaki Makita if (err) 905638264dcSToshiaki Makita goto err_xdp_ring; 906638264dcSToshiaki Makita } 907948d4f21SToshiaki Makita 908638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 909638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 910638264dcSToshiaki Makita 911638264dcSToshiaki Makita napi_enable(&rq->xdp_napi); 912638264dcSToshiaki Makita } 913948d4f21SToshiaki Makita 914948d4f21SToshiaki Makita return 0; 915638264dcSToshiaki Makita err_xdp_ring: 916638264dcSToshiaki Makita for (i--; i >= 0; i--) 917638264dcSToshiaki Makita ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free); 918638264dcSToshiaki Makita 919638264dcSToshiaki Makita return err; 920948d4f21SToshiaki Makita } 921948d4f21SToshiaki Makita 922948d4f21SToshiaki Makita static void veth_napi_del(struct net_device *dev) 923948d4f21SToshiaki Makita { 924948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 925638264dcSToshiaki Makita int i; 926948d4f21SToshiaki Makita 927638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 928638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 929638264dcSToshiaki Makita 930638264dcSToshiaki Makita napi_disable(&rq->xdp_napi); 9315198d545SJakub Kicinski __netif_napi_del(&rq->xdp_napi); 932638264dcSToshiaki Makita } 933638264dcSToshiaki Makita synchronize_net(); 934638264dcSToshiaki Makita 935638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 936638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 937638264dcSToshiaki Makita 938638264dcSToshiaki Makita rq->rx_notify_masked = false; 939638264dcSToshiaki Makita ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free); 940638264dcSToshiaki Makita } 941948d4f21SToshiaki Makita } 942948d4f21SToshiaki Makita 943948d4f21SToshiaki Makita static int veth_enable_xdp(struct net_device *dev) 944948d4f21SToshiaki Makita { 945948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 946638264dcSToshiaki Makita int err, i; 947948d4f21SToshiaki Makita 948638264dcSToshiaki Makita if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) { 949638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 950638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 951948d4f21SToshiaki Makita 952b02e5a0eSBjörn Töpel netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); 953b02e5a0eSBjörn Töpel err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id); 954948d4f21SToshiaki Makita if (err < 0) 955638264dcSToshiaki Makita goto err_rxq_reg; 956638264dcSToshiaki Makita 957638264dcSToshiaki Makita err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 958638264dcSToshiaki Makita MEM_TYPE_PAGE_SHARED, 959638264dcSToshiaki Makita NULL); 960638264dcSToshiaki Makita if (err < 0) 961638264dcSToshiaki Makita goto err_reg_mem; 962638264dcSToshiaki Makita 963638264dcSToshiaki Makita /* Save original mem info as it can be overwritten */ 964638264dcSToshiaki Makita rq->xdp_mem = rq->xdp_rxq.mem; 965638264dcSToshiaki Makita } 966948d4f21SToshiaki Makita 967948d4f21SToshiaki Makita err = veth_napi_add(dev); 968948d4f21SToshiaki Makita if (err) 969638264dcSToshiaki Makita goto err_rxq_reg; 970948d4f21SToshiaki Makita } 971948d4f21SToshiaki Makita 972638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) 973638264dcSToshiaki Makita rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog); 974948d4f21SToshiaki Makita 975948d4f21SToshiaki Makita return 0; 976638264dcSToshiaki Makita err_reg_mem: 977638264dcSToshiaki Makita xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); 978638264dcSToshiaki Makita err_rxq_reg: 979b02e5a0eSBjörn Töpel for (i--; i >= 0; i--) { 980b02e5a0eSBjörn Töpel struct veth_rq *rq = &priv->rq[i]; 981b02e5a0eSBjörn Töpel 982b02e5a0eSBjörn Töpel xdp_rxq_info_unreg(&rq->xdp_rxq); 983b02e5a0eSBjörn Töpel netif_napi_del(&rq->xdp_napi); 984b02e5a0eSBjörn Töpel } 985948d4f21SToshiaki Makita 986948d4f21SToshiaki Makita return err; 987948d4f21SToshiaki Makita } 988948d4f21SToshiaki Makita 989948d4f21SToshiaki Makita static void veth_disable_xdp(struct net_device *dev) 990948d4f21SToshiaki Makita { 991948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 992638264dcSToshiaki Makita int i; 993948d4f21SToshiaki Makita 994638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) 995638264dcSToshiaki Makita rcu_assign_pointer(priv->rq[i].xdp_prog, NULL); 996948d4f21SToshiaki Makita veth_napi_del(dev); 997638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 998638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 999638264dcSToshiaki Makita 1000638264dcSToshiaki Makita rq->xdp_rxq.mem = rq->xdp_mem; 1001638264dcSToshiaki Makita xdp_rxq_info_unreg(&rq->xdp_rxq); 1002638264dcSToshiaki Makita } 1003948d4f21SToshiaki Makita } 1004948d4f21SToshiaki Makita 1005e314dbdcSPavel Emelyanov static int veth_open(struct net_device *dev) 1006e314dbdcSPavel Emelyanov { 1007d0e2c55eSEric Dumazet struct veth_priv *priv = netdev_priv(dev); 1008d0e2c55eSEric Dumazet struct net_device *peer = rtnl_dereference(priv->peer); 1009948d4f21SToshiaki Makita int err; 1010e314dbdcSPavel Emelyanov 1011d0e2c55eSEric Dumazet if (!peer) 1012e314dbdcSPavel Emelyanov return -ENOTCONN; 1013e314dbdcSPavel Emelyanov 1014948d4f21SToshiaki Makita if (priv->_xdp_prog) { 1015948d4f21SToshiaki Makita err = veth_enable_xdp(dev); 1016948d4f21SToshiaki Makita if (err) 1017948d4f21SToshiaki Makita return err; 1018948d4f21SToshiaki Makita } 1019948d4f21SToshiaki Makita 1020d0e2c55eSEric Dumazet if (peer->flags & IFF_UP) { 1021e314dbdcSPavel Emelyanov netif_carrier_on(dev); 1022d0e2c55eSEric Dumazet netif_carrier_on(peer); 1023e314dbdcSPavel Emelyanov } 1024948d4f21SToshiaki Makita 1025e314dbdcSPavel Emelyanov return 0; 1026e314dbdcSPavel Emelyanov } 1027e314dbdcSPavel Emelyanov 10282cf48a10SEric W. Biederman static int veth_close(struct net_device *dev) 10292cf48a10SEric W. Biederman { 10302cf48a10SEric W. Biederman struct veth_priv *priv = netdev_priv(dev); 10312efd32eeSEric Dumazet struct net_device *peer = rtnl_dereference(priv->peer); 10322cf48a10SEric W. Biederman 10332cf48a10SEric W. Biederman netif_carrier_off(dev); 10342efd32eeSEric Dumazet if (peer) 10352efd32eeSEric Dumazet netif_carrier_off(peer); 10362cf48a10SEric W. Biederman 1037948d4f21SToshiaki Makita if (priv->_xdp_prog) 1038948d4f21SToshiaki Makita veth_disable_xdp(dev); 1039948d4f21SToshiaki Makita 10402cf48a10SEric W. Biederman return 0; 10412cf48a10SEric W. Biederman } 10422cf48a10SEric W. Biederman 104391572088SJarod Wilson static int is_valid_veth_mtu(int mtu) 104438d40815SEric Biederman { 104591572088SJarod Wilson return mtu >= ETH_MIN_MTU && mtu <= ETH_MAX_MTU; 104638d40815SEric Biederman } 104738d40815SEric Biederman 10487797b93bSToshiaki Makita static int veth_alloc_queues(struct net_device *dev) 10497797b93bSToshiaki Makita { 10507797b93bSToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 10517797b93bSToshiaki Makita int i; 10527797b93bSToshiaki Makita 10537797b93bSToshiaki Makita priv->rq = kcalloc(dev->num_rx_queues, sizeof(*priv->rq), GFP_KERNEL); 10547797b93bSToshiaki Makita if (!priv->rq) 10557797b93bSToshiaki Makita return -ENOMEM; 10567797b93bSToshiaki Makita 10574195e54aSToshiaki Makita for (i = 0; i < dev->num_rx_queues; i++) { 10587797b93bSToshiaki Makita priv->rq[i].dev = dev; 10594195e54aSToshiaki Makita u64_stats_init(&priv->rq[i].stats.syncp); 10604195e54aSToshiaki Makita } 10617797b93bSToshiaki Makita 10627797b93bSToshiaki Makita return 0; 10637797b93bSToshiaki Makita } 10647797b93bSToshiaki Makita 10657797b93bSToshiaki Makita static void veth_free_queues(struct net_device *dev) 10667797b93bSToshiaki Makita { 10677797b93bSToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 10687797b93bSToshiaki Makita 10697797b93bSToshiaki Makita kfree(priv->rq); 10707797b93bSToshiaki Makita } 10717797b93bSToshiaki Makita 1072e314dbdcSPavel Emelyanov static int veth_dev_init(struct net_device *dev) 1073e314dbdcSPavel Emelyanov { 10747797b93bSToshiaki Makita int err; 10757797b93bSToshiaki Makita 107614d73416SLi RongQing dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); 107714d73416SLi RongQing if (!dev->lstats) 1078e314dbdcSPavel Emelyanov return -ENOMEM; 10797797b93bSToshiaki Makita 10807797b93bSToshiaki Makita err = veth_alloc_queues(dev); 10817797b93bSToshiaki Makita if (err) { 108214d73416SLi RongQing free_percpu(dev->lstats); 10837797b93bSToshiaki Makita return err; 10847797b93bSToshiaki Makita } 10857797b93bSToshiaki Makita 1086e314dbdcSPavel Emelyanov return 0; 1087e314dbdcSPavel Emelyanov } 1088e314dbdcSPavel Emelyanov 108911687a10SDavid S. Miller static void veth_dev_free(struct net_device *dev) 109011687a10SDavid S. Miller { 10917797b93bSToshiaki Makita veth_free_queues(dev); 109214d73416SLi RongQing free_percpu(dev->lstats); 109311687a10SDavid S. Miller } 109411687a10SDavid S. Miller 1095bb446c19SWANG Cong #ifdef CONFIG_NET_POLL_CONTROLLER 1096bb446c19SWANG Cong static void veth_poll_controller(struct net_device *dev) 1097bb446c19SWANG Cong { 1098bb446c19SWANG Cong /* veth only receives frames when its peer sends one 1099948d4f21SToshiaki Makita * Since it has nothing to do with disabling irqs, we are guaranteed 1100bb446c19SWANG Cong * never to have pending data when we poll for it so 1101bb446c19SWANG Cong * there is nothing to do here. 1102bb446c19SWANG Cong * 1103bb446c19SWANG Cong * We need this though so netpoll recognizes us as an interface that 1104bb446c19SWANG Cong * supports polling, which enables bridge devices in virt setups to 1105bb446c19SWANG Cong * still use netconsole 1106bb446c19SWANG Cong */ 1107bb446c19SWANG Cong } 1108bb446c19SWANG Cong #endif /* CONFIG_NET_POLL_CONTROLLER */ 1109bb446c19SWANG Cong 1110a45253bfSNicolas Dichtel static int veth_get_iflink(const struct net_device *dev) 1111a45253bfSNicolas Dichtel { 1112a45253bfSNicolas Dichtel struct veth_priv *priv = netdev_priv(dev); 1113a45253bfSNicolas Dichtel struct net_device *peer; 1114a45253bfSNicolas Dichtel int iflink; 1115a45253bfSNicolas Dichtel 1116a45253bfSNicolas Dichtel rcu_read_lock(); 1117a45253bfSNicolas Dichtel peer = rcu_dereference(priv->peer); 1118a45253bfSNicolas Dichtel iflink = peer ? peer->ifindex : 0; 1119a45253bfSNicolas Dichtel rcu_read_unlock(); 1120a45253bfSNicolas Dichtel 1121a45253bfSNicolas Dichtel return iflink; 1122a45253bfSNicolas Dichtel } 1123a45253bfSNicolas Dichtel 1124dc224822SToshiaki Makita static netdev_features_t veth_fix_features(struct net_device *dev, 1125dc224822SToshiaki Makita netdev_features_t features) 1126dc224822SToshiaki Makita { 1127dc224822SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 1128dc224822SToshiaki Makita struct net_device *peer; 1129dc224822SToshiaki Makita 1130dc224822SToshiaki Makita peer = rtnl_dereference(priv->peer); 1131dc224822SToshiaki Makita if (peer) { 1132dc224822SToshiaki Makita struct veth_priv *peer_priv = netdev_priv(peer); 1133dc224822SToshiaki Makita 1134dc224822SToshiaki Makita if (peer_priv->_xdp_prog) 1135dc224822SToshiaki Makita features &= ~NETIF_F_GSO_SOFTWARE; 1136dc224822SToshiaki Makita } 1137dc224822SToshiaki Makita 1138dc224822SToshiaki Makita return features; 1139dc224822SToshiaki Makita } 1140dc224822SToshiaki Makita 1141163e5292SPaolo Abeni static void veth_set_rx_headroom(struct net_device *dev, int new_hr) 1142163e5292SPaolo Abeni { 1143163e5292SPaolo Abeni struct veth_priv *peer_priv, *priv = netdev_priv(dev); 1144163e5292SPaolo Abeni struct net_device *peer; 1145163e5292SPaolo Abeni 1146163e5292SPaolo Abeni if (new_hr < 0) 1147163e5292SPaolo Abeni new_hr = 0; 1148163e5292SPaolo Abeni 1149163e5292SPaolo Abeni rcu_read_lock(); 1150163e5292SPaolo Abeni peer = rcu_dereference(priv->peer); 1151163e5292SPaolo Abeni if (unlikely(!peer)) 1152163e5292SPaolo Abeni goto out; 1153163e5292SPaolo Abeni 1154163e5292SPaolo Abeni peer_priv = netdev_priv(peer); 1155163e5292SPaolo Abeni priv->requested_headroom = new_hr; 1156163e5292SPaolo Abeni new_hr = max(priv->requested_headroom, peer_priv->requested_headroom); 1157163e5292SPaolo Abeni dev->needed_headroom = new_hr; 1158163e5292SPaolo Abeni peer->needed_headroom = new_hr; 1159163e5292SPaolo Abeni 1160163e5292SPaolo Abeni out: 1161163e5292SPaolo Abeni rcu_read_unlock(); 1162163e5292SPaolo Abeni } 1163163e5292SPaolo Abeni 1164948d4f21SToshiaki Makita static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1165948d4f21SToshiaki Makita struct netlink_ext_ack *extack) 1166948d4f21SToshiaki Makita { 1167948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 1168948d4f21SToshiaki Makita struct bpf_prog *old_prog; 1169948d4f21SToshiaki Makita struct net_device *peer; 1170dc224822SToshiaki Makita unsigned int max_mtu; 1171948d4f21SToshiaki Makita int err; 1172948d4f21SToshiaki Makita 1173948d4f21SToshiaki Makita old_prog = priv->_xdp_prog; 1174948d4f21SToshiaki Makita priv->_xdp_prog = prog; 1175948d4f21SToshiaki Makita peer = rtnl_dereference(priv->peer); 1176948d4f21SToshiaki Makita 1177948d4f21SToshiaki Makita if (prog) { 1178948d4f21SToshiaki Makita if (!peer) { 1179948d4f21SToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached"); 1180948d4f21SToshiaki Makita err = -ENOTCONN; 1181948d4f21SToshiaki Makita goto err; 1182948d4f21SToshiaki Makita } 1183948d4f21SToshiaki Makita 1184dc224822SToshiaki Makita max_mtu = PAGE_SIZE - VETH_XDP_HEADROOM - 1185dc224822SToshiaki Makita peer->hard_header_len - 1186dc224822SToshiaki Makita SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1187dc224822SToshiaki Makita if (peer->mtu > max_mtu) { 1188dc224822SToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP"); 1189dc224822SToshiaki Makita err = -ERANGE; 1190dc224822SToshiaki Makita goto err; 1191dc224822SToshiaki Makita } 1192dc224822SToshiaki Makita 1193638264dcSToshiaki Makita if (dev->real_num_rx_queues < peer->real_num_tx_queues) { 1194638264dcSToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues"); 1195638264dcSToshiaki Makita err = -ENOSPC; 1196638264dcSToshiaki Makita goto err; 1197638264dcSToshiaki Makita } 1198638264dcSToshiaki Makita 1199948d4f21SToshiaki Makita if (dev->flags & IFF_UP) { 1200948d4f21SToshiaki Makita err = veth_enable_xdp(dev); 1201948d4f21SToshiaki Makita if (err) { 1202948d4f21SToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed"); 1203948d4f21SToshiaki Makita goto err; 1204948d4f21SToshiaki Makita } 1205948d4f21SToshiaki Makita } 1206dc224822SToshiaki Makita 1207dc224822SToshiaki Makita if (!old_prog) { 1208dc224822SToshiaki Makita peer->hw_features &= ~NETIF_F_GSO_SOFTWARE; 1209dc224822SToshiaki Makita peer->max_mtu = max_mtu; 1210dc224822SToshiaki Makita } 1211948d4f21SToshiaki Makita } 1212948d4f21SToshiaki Makita 1213948d4f21SToshiaki Makita if (old_prog) { 1214dc224822SToshiaki Makita if (!prog) { 1215dc224822SToshiaki Makita if (dev->flags & IFF_UP) 1216948d4f21SToshiaki Makita veth_disable_xdp(dev); 1217dc224822SToshiaki Makita 1218dc224822SToshiaki Makita if (peer) { 1219dc224822SToshiaki Makita peer->hw_features |= NETIF_F_GSO_SOFTWARE; 1220dc224822SToshiaki Makita peer->max_mtu = ETH_MAX_MTU; 1221dc224822SToshiaki Makita } 1222dc224822SToshiaki Makita } 1223948d4f21SToshiaki Makita bpf_prog_put(old_prog); 1224948d4f21SToshiaki Makita } 1225948d4f21SToshiaki Makita 1226dc224822SToshiaki Makita if ((!!old_prog ^ !!prog) && peer) 1227dc224822SToshiaki Makita netdev_update_features(peer); 1228dc224822SToshiaki Makita 1229948d4f21SToshiaki Makita return 0; 1230948d4f21SToshiaki Makita err: 1231948d4f21SToshiaki Makita priv->_xdp_prog = old_prog; 1232948d4f21SToshiaki Makita 1233948d4f21SToshiaki Makita return err; 1234948d4f21SToshiaki Makita } 1235948d4f21SToshiaki Makita 1236948d4f21SToshiaki Makita static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1237948d4f21SToshiaki Makita { 1238948d4f21SToshiaki Makita switch (xdp->command) { 1239948d4f21SToshiaki Makita case XDP_SETUP_PROG: 1240948d4f21SToshiaki Makita return veth_xdp_set(dev, xdp->prog, xdp->extack); 1241948d4f21SToshiaki Makita default: 1242948d4f21SToshiaki Makita return -EINVAL; 1243948d4f21SToshiaki Makita } 1244948d4f21SToshiaki Makita } 1245948d4f21SToshiaki Makita 12464456e7bdSStephen Hemminger static const struct net_device_ops veth_netdev_ops = { 12474456e7bdSStephen Hemminger .ndo_init = veth_dev_init, 12484456e7bdSStephen Hemminger .ndo_open = veth_open, 12492cf48a10SEric W. Biederman .ndo_stop = veth_close, 125000829823SStephen Hemminger .ndo_start_xmit = veth_xmit, 12516311cc44Sstephen hemminger .ndo_get_stats64 = veth_get_stats64, 12525c70ef85SGao feng .ndo_set_rx_mode = veth_set_multicast_list, 1253ee923623SDaniel Lezcano .ndo_set_mac_address = eth_mac_addr, 1254bb446c19SWANG Cong #ifdef CONFIG_NET_POLL_CONTROLLER 1255bb446c19SWANG Cong .ndo_poll_controller = veth_poll_controller, 1256bb446c19SWANG Cong #endif 1257a45253bfSNicolas Dichtel .ndo_get_iflink = veth_get_iflink, 1258dc224822SToshiaki Makita .ndo_fix_features = veth_fix_features, 12591a04a821SToshiaki Makita .ndo_features_check = passthru_features_check, 1260163e5292SPaolo Abeni .ndo_set_rx_headroom = veth_set_rx_headroom, 1261948d4f21SToshiaki Makita .ndo_bpf = veth_xdp, 12629152cff0SLorenzo Bianconi .ndo_xdp_xmit = veth_ndo_xdp_xmit, 12639aa1206eSDaniel Borkmann .ndo_get_peer_dev = veth_peer_dev, 12644456e7bdSStephen Hemminger }; 12654456e7bdSStephen Hemminger 1266732912d7SAlexander Duyck #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ 1267c80fafbbSXin Long NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \ 1268732912d7SAlexander Duyck NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \ 126928d2b136SPatrick McHardy NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \ 127028d2b136SPatrick McHardy NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX ) 12718093315aSEric Dumazet 1272e314dbdcSPavel Emelyanov static void veth_setup(struct net_device *dev) 1273e314dbdcSPavel Emelyanov { 1274e314dbdcSPavel Emelyanov ether_setup(dev); 1275e314dbdcSPavel Emelyanov 1276550fd08cSNeil Horman dev->priv_flags &= ~IFF_TX_SKB_SHARING; 127723ea5a96SHannes Frederic Sowa dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 127802f01ec1SPhil Sutter dev->priv_flags |= IFF_NO_QUEUE; 1279163e5292SPaolo Abeni dev->priv_flags |= IFF_PHONY_HEADROOM; 1280550fd08cSNeil Horman 12814456e7bdSStephen Hemminger dev->netdev_ops = &veth_netdev_ops; 1282e314dbdcSPavel Emelyanov dev->ethtool_ops = &veth_ethtool_ops; 1283e314dbdcSPavel Emelyanov dev->features |= NETIF_F_LLTX; 12848093315aSEric Dumazet dev->features |= VETH_FEATURES; 12858d0d21f4SToshiaki Makita dev->vlan_features = dev->features & 12863f8c707bSVlad Yasevich ~(NETIF_F_HW_VLAN_CTAG_TX | 12873f8c707bSVlad Yasevich NETIF_F_HW_VLAN_STAG_TX | 12883f8c707bSVlad Yasevich NETIF_F_HW_VLAN_CTAG_RX | 12893f8c707bSVlad Yasevich NETIF_F_HW_VLAN_STAG_RX); 1290cf124db5SDavid S. Miller dev->needs_free_netdev = true; 1291cf124db5SDavid S. Miller dev->priv_destructor = veth_dev_free; 129291572088SJarod Wilson dev->max_mtu = ETH_MAX_MTU; 1293a2c725faSMichał Mirosław 12948093315aSEric Dumazet dev->hw_features = VETH_FEATURES; 129582d81898SEric Dumazet dev->hw_enc_features = VETH_FEATURES; 1296607fca9aSDavid Ahern dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 1297e314dbdcSPavel Emelyanov } 1298e314dbdcSPavel Emelyanov 1299e314dbdcSPavel Emelyanov /* 1300e314dbdcSPavel Emelyanov * netlink interface 1301e314dbdcSPavel Emelyanov */ 1302e314dbdcSPavel Emelyanov 1303a8b8a889SMatthias Schiffer static int veth_validate(struct nlattr *tb[], struct nlattr *data[], 1304a8b8a889SMatthias Schiffer struct netlink_ext_ack *extack) 1305e314dbdcSPavel Emelyanov { 1306e314dbdcSPavel Emelyanov if (tb[IFLA_ADDRESS]) { 1307e314dbdcSPavel Emelyanov if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) 1308e314dbdcSPavel Emelyanov return -EINVAL; 1309e314dbdcSPavel Emelyanov if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) 1310e314dbdcSPavel Emelyanov return -EADDRNOTAVAIL; 1311e314dbdcSPavel Emelyanov } 131238d40815SEric Biederman if (tb[IFLA_MTU]) { 131338d40815SEric Biederman if (!is_valid_veth_mtu(nla_get_u32(tb[IFLA_MTU]))) 131438d40815SEric Biederman return -EINVAL; 131538d40815SEric Biederman } 1316e314dbdcSPavel Emelyanov return 0; 1317e314dbdcSPavel Emelyanov } 1318e314dbdcSPavel Emelyanov 1319e314dbdcSPavel Emelyanov static struct rtnl_link_ops veth_link_ops; 1320e314dbdcSPavel Emelyanov 132181adee47SEric W. Biederman static int veth_newlink(struct net *src_net, struct net_device *dev, 13227a3f4a18SMatthias Schiffer struct nlattr *tb[], struct nlattr *data[], 13237a3f4a18SMatthias Schiffer struct netlink_ext_ack *extack) 1324e314dbdcSPavel Emelyanov { 13257797b93bSToshiaki Makita int err; 1326e314dbdcSPavel Emelyanov struct net_device *peer; 1327e314dbdcSPavel Emelyanov struct veth_priv *priv; 1328e314dbdcSPavel Emelyanov char ifname[IFNAMSIZ]; 1329e314dbdcSPavel Emelyanov struct nlattr *peer_tb[IFLA_MAX + 1], **tbp; 13305517750fSTom Gundersen unsigned char name_assign_type; 13313729d502SPatrick McHardy struct ifinfomsg *ifmp; 133281adee47SEric W. Biederman struct net *net; 1333e314dbdcSPavel Emelyanov 1334e314dbdcSPavel Emelyanov /* 1335e314dbdcSPavel Emelyanov * create and register peer first 1336e314dbdcSPavel Emelyanov */ 1337e314dbdcSPavel Emelyanov if (data != NULL && data[VETH_INFO_PEER] != NULL) { 1338e314dbdcSPavel Emelyanov struct nlattr *nla_peer; 1339e314dbdcSPavel Emelyanov 1340e314dbdcSPavel Emelyanov nla_peer = data[VETH_INFO_PEER]; 13413729d502SPatrick McHardy ifmp = nla_data(nla_peer); 1342f7b12606SJiri Pirko err = rtnl_nla_parse_ifla(peer_tb, 1343e314dbdcSPavel Emelyanov nla_data(nla_peer) + sizeof(struct ifinfomsg), 1344fceb6435SJohannes Berg nla_len(nla_peer) - sizeof(struct ifinfomsg), 1345fceb6435SJohannes Berg NULL); 1346e314dbdcSPavel Emelyanov if (err < 0) 1347e314dbdcSPavel Emelyanov return err; 1348e314dbdcSPavel Emelyanov 1349a8b8a889SMatthias Schiffer err = veth_validate(peer_tb, NULL, extack); 1350e314dbdcSPavel Emelyanov if (err < 0) 1351e314dbdcSPavel Emelyanov return err; 1352e314dbdcSPavel Emelyanov 1353e314dbdcSPavel Emelyanov tbp = peer_tb; 13543729d502SPatrick McHardy } else { 13553729d502SPatrick McHardy ifmp = NULL; 1356e314dbdcSPavel Emelyanov tbp = tb; 13573729d502SPatrick McHardy } 1358e314dbdcSPavel Emelyanov 1359191cdb38SSerhey Popovych if (ifmp && tbp[IFLA_IFNAME]) { 1360872f6903SFrancis Laniel nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 13615517750fSTom Gundersen name_assign_type = NET_NAME_USER; 13625517750fSTom Gundersen } else { 1363e314dbdcSPavel Emelyanov snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d"); 13645517750fSTom Gundersen name_assign_type = NET_NAME_ENUM; 13655517750fSTom Gundersen } 1366e314dbdcSPavel Emelyanov 136781adee47SEric W. Biederman net = rtnl_link_get_net(src_net, tbp); 136881adee47SEric W. Biederman if (IS_ERR(net)) 136981adee47SEric W. Biederman return PTR_ERR(net); 137081adee47SEric W. Biederman 13715517750fSTom Gundersen peer = rtnl_create_link(net, ifname, name_assign_type, 1372d0522f1cSDavid Ahern &veth_link_ops, tbp, extack); 137381adee47SEric W. Biederman if (IS_ERR(peer)) { 137481adee47SEric W. Biederman put_net(net); 1375e314dbdcSPavel Emelyanov return PTR_ERR(peer); 137681adee47SEric W. Biederman } 1377e314dbdcSPavel Emelyanov 1378191cdb38SSerhey Popovych if (!ifmp || !tbp[IFLA_ADDRESS]) 1379f2cedb63SDanny Kukawka eth_hw_addr_random(peer); 1380e314dbdcSPavel Emelyanov 1381e6f8f1a7SPavel Emelyanov if (ifmp && (dev->ifindex != 0)) 1382e6f8f1a7SPavel Emelyanov peer->ifindex = ifmp->ifi_index; 1383e6f8f1a7SPavel Emelyanov 138472d24955SStephen Hemminger peer->gso_max_size = dev->gso_max_size; 138572d24955SStephen Hemminger peer->gso_max_segs = dev->gso_max_segs; 138672d24955SStephen Hemminger 1387e314dbdcSPavel Emelyanov err = register_netdevice(peer); 138881adee47SEric W. Biederman put_net(net); 138981adee47SEric W. Biederman net = NULL; 1390e314dbdcSPavel Emelyanov if (err < 0) 1391e314dbdcSPavel Emelyanov goto err_register_peer; 1392e314dbdcSPavel Emelyanov 1393e314dbdcSPavel Emelyanov netif_carrier_off(peer); 1394e314dbdcSPavel Emelyanov 13953729d502SPatrick McHardy err = rtnl_configure_link(peer, ifmp); 13963729d502SPatrick McHardy if (err < 0) 13973729d502SPatrick McHardy goto err_configure_peer; 13983729d502SPatrick McHardy 1399e314dbdcSPavel Emelyanov /* 1400e314dbdcSPavel Emelyanov * register dev last 1401e314dbdcSPavel Emelyanov * 1402e314dbdcSPavel Emelyanov * note, that since we've registered new device the dev's name 1403e314dbdcSPavel Emelyanov * should be re-allocated 1404e314dbdcSPavel Emelyanov */ 1405e314dbdcSPavel Emelyanov 1406e314dbdcSPavel Emelyanov if (tb[IFLA_ADDRESS] == NULL) 1407f2cedb63SDanny Kukawka eth_hw_addr_random(dev); 1408e314dbdcSPavel Emelyanov 14096c8c4446SJiri Pirko if (tb[IFLA_IFNAME]) 1410872f6903SFrancis Laniel nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 14116c8c4446SJiri Pirko else 14126c8c4446SJiri Pirko snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d"); 14136c8c4446SJiri Pirko 1414e314dbdcSPavel Emelyanov err = register_netdevice(dev); 1415e314dbdcSPavel Emelyanov if (err < 0) 1416e314dbdcSPavel Emelyanov goto err_register_dev; 1417e314dbdcSPavel Emelyanov 1418e314dbdcSPavel Emelyanov netif_carrier_off(dev); 1419e314dbdcSPavel Emelyanov 1420e314dbdcSPavel Emelyanov /* 1421e314dbdcSPavel Emelyanov * tie the deviced together 1422e314dbdcSPavel Emelyanov */ 1423e314dbdcSPavel Emelyanov 1424e314dbdcSPavel Emelyanov priv = netdev_priv(dev); 1425d0e2c55eSEric Dumazet rcu_assign_pointer(priv->peer, peer); 1426e314dbdcSPavel Emelyanov 1427e314dbdcSPavel Emelyanov priv = netdev_priv(peer); 1428d0e2c55eSEric Dumazet rcu_assign_pointer(priv->peer, dev); 1429948d4f21SToshiaki Makita 1430e314dbdcSPavel Emelyanov return 0; 1431e314dbdcSPavel Emelyanov 1432e314dbdcSPavel Emelyanov err_register_dev: 1433e314dbdcSPavel Emelyanov /* nothing to do */ 14343729d502SPatrick McHardy err_configure_peer: 1435e314dbdcSPavel Emelyanov unregister_netdevice(peer); 1436e314dbdcSPavel Emelyanov return err; 1437e314dbdcSPavel Emelyanov 1438e314dbdcSPavel Emelyanov err_register_peer: 1439e314dbdcSPavel Emelyanov free_netdev(peer); 1440e314dbdcSPavel Emelyanov return err; 1441e314dbdcSPavel Emelyanov } 1442e314dbdcSPavel Emelyanov 144323289a37SEric Dumazet static void veth_dellink(struct net_device *dev, struct list_head *head) 1444e314dbdcSPavel Emelyanov { 1445e314dbdcSPavel Emelyanov struct veth_priv *priv; 1446e314dbdcSPavel Emelyanov struct net_device *peer; 1447e314dbdcSPavel Emelyanov 1448e314dbdcSPavel Emelyanov priv = netdev_priv(dev); 1449d0e2c55eSEric Dumazet peer = rtnl_dereference(priv->peer); 1450d0e2c55eSEric Dumazet 1451d0e2c55eSEric Dumazet /* Note : dellink() is called from default_device_exit_batch(), 1452d0e2c55eSEric Dumazet * before a rcu_synchronize() point. The devices are guaranteed 1453d0e2c55eSEric Dumazet * not being freed before one RCU grace period. 1454d0e2c55eSEric Dumazet */ 1455d0e2c55eSEric Dumazet RCU_INIT_POINTER(priv->peer, NULL); 1456f45a5c26SEric Dumazet unregister_netdevice_queue(dev, head); 1457d0e2c55eSEric Dumazet 1458f45a5c26SEric Dumazet if (peer) { 1459d0e2c55eSEric Dumazet priv = netdev_priv(peer); 1460d0e2c55eSEric Dumazet RCU_INIT_POINTER(priv->peer, NULL); 146124540535SEric Dumazet unregister_netdevice_queue(peer, head); 1462e314dbdcSPavel Emelyanov } 1463f45a5c26SEric Dumazet } 1464e314dbdcSPavel Emelyanov 146523711438SThomas Graf static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = { 146623711438SThomas Graf [VETH_INFO_PEER] = { .len = sizeof(struct ifinfomsg) }, 146723711438SThomas Graf }; 1468e314dbdcSPavel Emelyanov 1469e5f4e7b9SNicolas Dichtel static struct net *veth_get_link_net(const struct net_device *dev) 1470e5f4e7b9SNicolas Dichtel { 1471e5f4e7b9SNicolas Dichtel struct veth_priv *priv = netdev_priv(dev); 1472e5f4e7b9SNicolas Dichtel struct net_device *peer = rtnl_dereference(priv->peer); 1473e5f4e7b9SNicolas Dichtel 1474e5f4e7b9SNicolas Dichtel return peer ? dev_net(peer) : dev_net(dev); 1475e5f4e7b9SNicolas Dichtel } 1476e5f4e7b9SNicolas Dichtel 1477e314dbdcSPavel Emelyanov static struct rtnl_link_ops veth_link_ops = { 1478e314dbdcSPavel Emelyanov .kind = DRV_NAME, 1479e314dbdcSPavel Emelyanov .priv_size = sizeof(struct veth_priv), 1480e314dbdcSPavel Emelyanov .setup = veth_setup, 1481e314dbdcSPavel Emelyanov .validate = veth_validate, 1482e314dbdcSPavel Emelyanov .newlink = veth_newlink, 1483e314dbdcSPavel Emelyanov .dellink = veth_dellink, 1484e314dbdcSPavel Emelyanov .policy = veth_policy, 1485e314dbdcSPavel Emelyanov .maxtype = VETH_INFO_MAX, 1486e5f4e7b9SNicolas Dichtel .get_link_net = veth_get_link_net, 1487e314dbdcSPavel Emelyanov }; 1488e314dbdcSPavel Emelyanov 1489e314dbdcSPavel Emelyanov /* 1490e314dbdcSPavel Emelyanov * init/fini 1491e314dbdcSPavel Emelyanov */ 1492e314dbdcSPavel Emelyanov 1493e314dbdcSPavel Emelyanov static __init int veth_init(void) 1494e314dbdcSPavel Emelyanov { 1495e314dbdcSPavel Emelyanov return rtnl_link_register(&veth_link_ops); 1496e314dbdcSPavel Emelyanov } 1497e314dbdcSPavel Emelyanov 1498e314dbdcSPavel Emelyanov static __exit void veth_exit(void) 1499e314dbdcSPavel Emelyanov { 150068365458SPatrick McHardy rtnl_link_unregister(&veth_link_ops); 1501e314dbdcSPavel Emelyanov } 1502e314dbdcSPavel Emelyanov 1503e314dbdcSPavel Emelyanov module_init(veth_init); 1504e314dbdcSPavel Emelyanov module_exit(veth_exit); 1505e314dbdcSPavel Emelyanov 1506e314dbdcSPavel Emelyanov MODULE_DESCRIPTION("Virtual Ethernet Tunnel"); 1507e314dbdcSPavel Emelyanov MODULE_LICENSE("GPL v2"); 1508e314dbdcSPavel Emelyanov MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1509