109c434b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e314dbdcSPavel Emelyanov /* 3e314dbdcSPavel Emelyanov * drivers/net/veth.c 4e314dbdcSPavel Emelyanov * 5e314dbdcSPavel Emelyanov * Copyright (C) 2007 OpenVZ http://openvz.org, SWsoft Inc 6e314dbdcSPavel Emelyanov * 7e314dbdcSPavel Emelyanov * Author: Pavel Emelianov <xemul@openvz.org> 8e314dbdcSPavel Emelyanov * Ethtool interface from: Eric W. Biederman <ebiederm@xmission.com> 9e314dbdcSPavel Emelyanov * 10e314dbdcSPavel Emelyanov */ 11e314dbdcSPavel Emelyanov 12e314dbdcSPavel Emelyanov #include <linux/netdevice.h> 135a0e3ad6STejun Heo #include <linux/slab.h> 14e314dbdcSPavel Emelyanov #include <linux/ethtool.h> 15e314dbdcSPavel Emelyanov #include <linux/etherdevice.h> 16cf05c700SEric Dumazet #include <linux/u64_stats_sync.h> 17e314dbdcSPavel Emelyanov 18f7b12606SJiri Pirko #include <net/rtnetlink.h> 19e314dbdcSPavel Emelyanov #include <net/dst.h> 20e314dbdcSPavel Emelyanov #include <net/xfrm.h> 21af87a3aaSToshiaki Makita #include <net/xdp.h> 22ecef969eSStephen Hemminger #include <linux/veth.h> 239d9779e7SPaul Gortmaker #include <linux/module.h> 24948d4f21SToshiaki Makita #include <linux/bpf.h> 25948d4f21SToshiaki Makita #include <linux/filter.h> 26948d4f21SToshiaki Makita #include <linux/ptr_ring.h> 27948d4f21SToshiaki Makita #include <linux/bpf_trace.h> 28aa4e689eSMichael Walle #include <linux/net_tstamp.h> 29e314dbdcSPavel Emelyanov 30e314dbdcSPavel Emelyanov #define DRV_NAME "veth" 31e314dbdcSPavel Emelyanov #define DRV_VERSION "1.0" 32e314dbdcSPavel Emelyanov 339fc8d518SToshiaki Makita #define VETH_XDP_FLAG BIT(0) 34948d4f21SToshiaki Makita #define VETH_RING_SIZE 256 35948d4f21SToshiaki Makita #define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN) 36948d4f21SToshiaki Makita 379cda7807SToshiaki Makita #define VETH_XDP_TX_BULK_SIZE 16 389cda7807SToshiaki Makita 3965780c56SLorenzo Bianconi struct veth_stats { 401c5b82e5SLorenzo Bianconi u64 rx_drops; 411c5b82e5SLorenzo Bianconi /* xdp */ 424195e54aSToshiaki Makita u64 xdp_packets; 434195e54aSToshiaki Makita u64 xdp_bytes; 441c5b82e5SLorenzo Bianconi u64 xdp_redirect; 454195e54aSToshiaki Makita u64 xdp_drops; 461c5b82e5SLorenzo Bianconi u64 xdp_tx; 479152cff0SLorenzo Bianconi u64 xdp_tx_err; 485fe6e567SLorenzo Bianconi u64 peer_tq_xdp_xmit; 495fe6e567SLorenzo Bianconi u64 peer_tq_xdp_xmit_err; 5065780c56SLorenzo Bianconi }; 5165780c56SLorenzo Bianconi 5265780c56SLorenzo Bianconi struct veth_rq_stats { 5365780c56SLorenzo Bianconi struct veth_stats vs; 544195e54aSToshiaki Makita struct u64_stats_sync syncp; 554195e54aSToshiaki Makita }; 564195e54aSToshiaki Makita 57638264dcSToshiaki Makita struct veth_rq { 58948d4f21SToshiaki Makita struct napi_struct xdp_napi; 59948d4f21SToshiaki Makita struct net_device *dev; 60948d4f21SToshiaki Makita struct bpf_prog __rcu *xdp_prog; 61d1396004SToshiaki Makita struct xdp_mem_info xdp_mem; 624195e54aSToshiaki Makita struct veth_rq_stats stats; 63948d4f21SToshiaki Makita bool rx_notify_masked; 64948d4f21SToshiaki Makita struct ptr_ring xdp_ring; 65948d4f21SToshiaki Makita struct xdp_rxq_info xdp_rxq; 66e314dbdcSPavel Emelyanov }; 67e314dbdcSPavel Emelyanov 68638264dcSToshiaki Makita struct veth_priv { 69638264dcSToshiaki Makita struct net_device __rcu *peer; 70638264dcSToshiaki Makita atomic64_t dropped; 71638264dcSToshiaki Makita struct bpf_prog *_xdp_prog; 72638264dcSToshiaki Makita struct veth_rq *rq; 73638264dcSToshiaki Makita unsigned int requested_headroom; 74638264dcSToshiaki Makita }; 75638264dcSToshiaki Makita 769cda7807SToshiaki Makita struct veth_xdp_tx_bq { 779cda7807SToshiaki Makita struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE]; 789cda7807SToshiaki Makita unsigned int count; 799cda7807SToshiaki Makita }; 809cda7807SToshiaki Makita 81e314dbdcSPavel Emelyanov /* 82e314dbdcSPavel Emelyanov * ethtool interface 83e314dbdcSPavel Emelyanov */ 84e314dbdcSPavel Emelyanov 85d397b968SToshiaki Makita struct veth_q_stat_desc { 86d397b968SToshiaki Makita char desc[ETH_GSTRING_LEN]; 87d397b968SToshiaki Makita size_t offset; 88d397b968SToshiaki Makita }; 89d397b968SToshiaki Makita 9065780c56SLorenzo Bianconi #define VETH_RQ_STAT(m) offsetof(struct veth_stats, m) 91d397b968SToshiaki Makita 92d397b968SToshiaki Makita static const struct veth_q_stat_desc veth_rq_stats_desc[] = { 93d397b968SToshiaki Makita { "xdp_packets", VETH_RQ_STAT(xdp_packets) }, 94d397b968SToshiaki Makita { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) }, 955fe6e567SLorenzo Bianconi { "drops", VETH_RQ_STAT(rx_drops) }, 965fe6e567SLorenzo Bianconi { "xdp_redirect", VETH_RQ_STAT(xdp_redirect) }, 975fe6e567SLorenzo Bianconi { "xdp_drops", VETH_RQ_STAT(xdp_drops) }, 985fe6e567SLorenzo Bianconi { "xdp_tx", VETH_RQ_STAT(xdp_tx) }, 995fe6e567SLorenzo Bianconi { "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) }, 100d397b968SToshiaki Makita }; 101d397b968SToshiaki Makita 102d397b968SToshiaki Makita #define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc) 103d397b968SToshiaki Makita 1045fe6e567SLorenzo Bianconi static const struct veth_q_stat_desc veth_tq_stats_desc[] = { 1055fe6e567SLorenzo Bianconi { "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) }, 1065fe6e567SLorenzo Bianconi { "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) }, 1075fe6e567SLorenzo Bianconi }; 1085fe6e567SLorenzo Bianconi 1095fe6e567SLorenzo Bianconi #define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc) 1105fe6e567SLorenzo Bianconi 111e314dbdcSPavel Emelyanov static struct { 112e314dbdcSPavel Emelyanov const char string[ETH_GSTRING_LEN]; 113e314dbdcSPavel Emelyanov } ethtool_stats_keys[] = { 114e314dbdcSPavel Emelyanov { "peer_ifindex" }, 115e314dbdcSPavel Emelyanov }; 116e314dbdcSPavel Emelyanov 11756607b98SPhilippe Reynes static int veth_get_link_ksettings(struct net_device *dev, 11856607b98SPhilippe Reynes struct ethtool_link_ksettings *cmd) 119e314dbdcSPavel Emelyanov { 12056607b98SPhilippe Reynes cmd->base.speed = SPEED_10000; 12156607b98SPhilippe Reynes cmd->base.duplex = DUPLEX_FULL; 12256607b98SPhilippe Reynes cmd->base.port = PORT_TP; 12356607b98SPhilippe Reynes cmd->base.autoneg = AUTONEG_DISABLE; 124e314dbdcSPavel Emelyanov return 0; 125e314dbdcSPavel Emelyanov } 126e314dbdcSPavel Emelyanov 127e314dbdcSPavel Emelyanov static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 128e314dbdcSPavel Emelyanov { 12933a5ba14SRick Jones strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); 13033a5ba14SRick Jones strlcpy(info->version, DRV_VERSION, sizeof(info->version)); 131e314dbdcSPavel Emelyanov } 132e314dbdcSPavel Emelyanov 133e314dbdcSPavel Emelyanov static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf) 134e314dbdcSPavel Emelyanov { 135d397b968SToshiaki Makita char *p = (char *)buf; 136d397b968SToshiaki Makita int i, j; 137d397b968SToshiaki Makita 138e314dbdcSPavel Emelyanov switch(stringset) { 139e314dbdcSPavel Emelyanov case ETH_SS_STATS: 140d397b968SToshiaki Makita memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys)); 141d397b968SToshiaki Makita p += sizeof(ethtool_stats_keys); 142d397b968SToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 143d397b968SToshiaki Makita for (j = 0; j < VETH_RQ_STATS_LEN; j++) { 144abdf47aaSFlorian Fainelli snprintf(p, ETH_GSTRING_LEN, 1459152cff0SLorenzo Bianconi "rx_queue_%u_%.18s", 146d397b968SToshiaki Makita i, veth_rq_stats_desc[j].desc); 147d397b968SToshiaki Makita p += ETH_GSTRING_LEN; 148d397b968SToshiaki Makita } 149d397b968SToshiaki Makita } 1505fe6e567SLorenzo Bianconi for (i = 0; i < dev->real_num_tx_queues; i++) { 1515fe6e567SLorenzo Bianconi for (j = 0; j < VETH_TQ_STATS_LEN; j++) { 1525fe6e567SLorenzo Bianconi snprintf(p, ETH_GSTRING_LEN, 1535fe6e567SLorenzo Bianconi "tx_queue_%u_%.18s", 1545fe6e567SLorenzo Bianconi i, veth_tq_stats_desc[j].desc); 1555fe6e567SLorenzo Bianconi p += ETH_GSTRING_LEN; 1565fe6e567SLorenzo Bianconi } 1575fe6e567SLorenzo Bianconi } 158e314dbdcSPavel Emelyanov break; 159e314dbdcSPavel Emelyanov } 160e314dbdcSPavel Emelyanov } 161e314dbdcSPavel Emelyanov 162b9f2c044SJeff Garzik static int veth_get_sset_count(struct net_device *dev, int sset) 163e314dbdcSPavel Emelyanov { 164b9f2c044SJeff Garzik switch (sset) { 165b9f2c044SJeff Garzik case ETH_SS_STATS: 166d397b968SToshiaki Makita return ARRAY_SIZE(ethtool_stats_keys) + 1675fe6e567SLorenzo Bianconi VETH_RQ_STATS_LEN * dev->real_num_rx_queues + 1685fe6e567SLorenzo Bianconi VETH_TQ_STATS_LEN * dev->real_num_tx_queues; 169b9f2c044SJeff Garzik default: 170b9f2c044SJeff Garzik return -EOPNOTSUPP; 171b9f2c044SJeff Garzik } 172e314dbdcSPavel Emelyanov } 173e314dbdcSPavel Emelyanov 174e314dbdcSPavel Emelyanov static void veth_get_ethtool_stats(struct net_device *dev, 175e314dbdcSPavel Emelyanov struct ethtool_stats *stats, u64 *data) 176e314dbdcSPavel Emelyanov { 1775fe6e567SLorenzo Bianconi struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 178d0e2c55eSEric Dumazet struct net_device *peer = rtnl_dereference(priv->peer); 179d397b968SToshiaki Makita int i, j, idx; 180e314dbdcSPavel Emelyanov 181d0e2c55eSEric Dumazet data[0] = peer ? peer->ifindex : 0; 182d397b968SToshiaki Makita idx = 1; 183d397b968SToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 184d397b968SToshiaki Makita const struct veth_rq_stats *rq_stats = &priv->rq[i].stats; 18565780c56SLorenzo Bianconi const void *stats_base = (void *)&rq_stats->vs; 186d397b968SToshiaki Makita unsigned int start; 187d397b968SToshiaki Makita size_t offset; 188d397b968SToshiaki Makita 189d397b968SToshiaki Makita do { 190d397b968SToshiaki Makita start = u64_stats_fetch_begin_irq(&rq_stats->syncp); 191d397b968SToshiaki Makita for (j = 0; j < VETH_RQ_STATS_LEN; j++) { 192d397b968SToshiaki Makita offset = veth_rq_stats_desc[j].offset; 193d397b968SToshiaki Makita data[idx + j] = *(u64 *)(stats_base + offset); 194d397b968SToshiaki Makita } 195d397b968SToshiaki Makita } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); 196d397b968SToshiaki Makita idx += VETH_RQ_STATS_LEN; 197d397b968SToshiaki Makita } 1985fe6e567SLorenzo Bianconi 1995fe6e567SLorenzo Bianconi if (!peer) 2005fe6e567SLorenzo Bianconi return; 2015fe6e567SLorenzo Bianconi 2025fe6e567SLorenzo Bianconi rcv_priv = netdev_priv(peer); 2035fe6e567SLorenzo Bianconi for (i = 0; i < peer->real_num_rx_queues; i++) { 2045fe6e567SLorenzo Bianconi const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats; 2055fe6e567SLorenzo Bianconi const void *base = (void *)&rq_stats->vs; 2065fe6e567SLorenzo Bianconi unsigned int start, tx_idx = idx; 2075fe6e567SLorenzo Bianconi size_t offset; 2085fe6e567SLorenzo Bianconi 2095fe6e567SLorenzo Bianconi tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; 2105fe6e567SLorenzo Bianconi do { 2115fe6e567SLorenzo Bianconi start = u64_stats_fetch_begin_irq(&rq_stats->syncp); 2125fe6e567SLorenzo Bianconi for (j = 0; j < VETH_TQ_STATS_LEN; j++) { 2135fe6e567SLorenzo Bianconi offset = veth_tq_stats_desc[j].offset; 2145fe6e567SLorenzo Bianconi data[tx_idx + j] += *(u64 *)(base + offset); 2155fe6e567SLorenzo Bianconi } 2165fe6e567SLorenzo Bianconi } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); 2175fe6e567SLorenzo Bianconi } 218e314dbdcSPavel Emelyanov } 219e314dbdcSPavel Emelyanov 2200fc0b732SStephen Hemminger static const struct ethtool_ops veth_ethtool_ops = { 221e314dbdcSPavel Emelyanov .get_drvinfo = veth_get_drvinfo, 222e314dbdcSPavel Emelyanov .get_link = ethtool_op_get_link, 223e314dbdcSPavel Emelyanov .get_strings = veth_get_strings, 224b9f2c044SJeff Garzik .get_sset_count = veth_get_sset_count, 225e314dbdcSPavel Emelyanov .get_ethtool_stats = veth_get_ethtool_stats, 22656607b98SPhilippe Reynes .get_link_ksettings = veth_get_link_ksettings, 227056b21fbSJulian Wiedmann .get_ts_info = ethtool_op_get_ts_info, 228e314dbdcSPavel Emelyanov }; 229e314dbdcSPavel Emelyanov 230948d4f21SToshiaki Makita /* general routines */ 231948d4f21SToshiaki Makita 2329fc8d518SToshiaki Makita static bool veth_is_xdp_frame(void *ptr) 2339fc8d518SToshiaki Makita { 2349fc8d518SToshiaki Makita return (unsigned long)ptr & VETH_XDP_FLAG; 2359fc8d518SToshiaki Makita } 2369fc8d518SToshiaki Makita 2379fc8d518SToshiaki Makita static void *veth_ptr_to_xdp(void *ptr) 2389fc8d518SToshiaki Makita { 2399fc8d518SToshiaki Makita return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG); 2409fc8d518SToshiaki Makita } 2419fc8d518SToshiaki Makita 242af87a3aaSToshiaki Makita static void *veth_xdp_to_ptr(void *ptr) 243af87a3aaSToshiaki Makita { 244af87a3aaSToshiaki Makita return (void *)((unsigned long)ptr | VETH_XDP_FLAG); 245af87a3aaSToshiaki Makita } 246af87a3aaSToshiaki Makita 2479fc8d518SToshiaki Makita static void veth_ptr_free(void *ptr) 2489fc8d518SToshiaki Makita { 2499fc8d518SToshiaki Makita if (veth_is_xdp_frame(ptr)) 2509fc8d518SToshiaki Makita xdp_return_frame(veth_ptr_to_xdp(ptr)); 2519fc8d518SToshiaki Makita else 2529fc8d518SToshiaki Makita kfree_skb(ptr); 2539fc8d518SToshiaki Makita } 2549fc8d518SToshiaki Makita 255638264dcSToshiaki Makita static void __veth_xdp_flush(struct veth_rq *rq) 256948d4f21SToshiaki Makita { 257948d4f21SToshiaki Makita /* Write ptr_ring before reading rx_notify_masked */ 258948d4f21SToshiaki Makita smp_mb(); 259638264dcSToshiaki Makita if (!rq->rx_notify_masked) { 260638264dcSToshiaki Makita rq->rx_notify_masked = true; 261638264dcSToshiaki Makita napi_schedule(&rq->xdp_napi); 262948d4f21SToshiaki Makita } 263948d4f21SToshiaki Makita } 264948d4f21SToshiaki Makita 265638264dcSToshiaki Makita static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb) 266948d4f21SToshiaki Makita { 267638264dcSToshiaki Makita if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) { 268948d4f21SToshiaki Makita dev_kfree_skb_any(skb); 269948d4f21SToshiaki Makita return NET_RX_DROP; 270948d4f21SToshiaki Makita } 271948d4f21SToshiaki Makita 272948d4f21SToshiaki Makita return NET_RX_SUCCESS; 273948d4f21SToshiaki Makita } 274948d4f21SToshiaki Makita 275638264dcSToshiaki Makita static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, 276638264dcSToshiaki Makita struct veth_rq *rq, bool xdp) 277e314dbdcSPavel Emelyanov { 278948d4f21SToshiaki Makita return __dev_forward_skb(dev, skb) ?: xdp ? 279638264dcSToshiaki Makita veth_xdp_rx(rq, skb) : 280948d4f21SToshiaki Makita netif_rx(skb); 281948d4f21SToshiaki Makita } 282948d4f21SToshiaki Makita 283948d4f21SToshiaki Makita static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) 284948d4f21SToshiaki Makita { 285948d4f21SToshiaki Makita struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 286638264dcSToshiaki Makita struct veth_rq *rq = NULL; 287d0e2c55eSEric Dumazet struct net_device *rcv; 2882681128fSEric Dumazet int length = skb->len; 289948d4f21SToshiaki Makita bool rcv_xdp = false; 290638264dcSToshiaki Makita int rxq; 291e314dbdcSPavel Emelyanov 292d0e2c55eSEric Dumazet rcu_read_lock(); 293d0e2c55eSEric Dumazet rcv = rcu_dereference(priv->peer); 294d0e2c55eSEric Dumazet if (unlikely(!rcv)) { 295d0e2c55eSEric Dumazet kfree_skb(skb); 296d0e2c55eSEric Dumazet goto drop; 297d0e2c55eSEric Dumazet } 298e314dbdcSPavel Emelyanov 299948d4f21SToshiaki Makita rcv_priv = netdev_priv(rcv); 300638264dcSToshiaki Makita rxq = skb_get_queue_mapping(skb); 301638264dcSToshiaki Makita if (rxq < rcv->real_num_rx_queues) { 302638264dcSToshiaki Makita rq = &rcv_priv->rq[rxq]; 303638264dcSToshiaki Makita rcv_xdp = rcu_access_pointer(rq->xdp_prog); 304638264dcSToshiaki Makita if (rcv_xdp) 305638264dcSToshiaki Makita skb_record_rx_queue(skb, rxq); 306638264dcSToshiaki Makita } 307948d4f21SToshiaki Makita 308aa4e689eSMichael Walle skb_tx_timestamp(skb); 309638264dcSToshiaki Makita if (likely(veth_forward_skb(rcv, skb, rq, rcv_xdp) == NET_RX_SUCCESS)) { 310b4fba476SEric Dumazet if (!rcv_xdp) 311b4fba476SEric Dumazet dev_lstats_add(dev, length); 3122681128fSEric Dumazet } else { 313d0e2c55eSEric Dumazet drop: 3142681128fSEric Dumazet atomic64_inc(&priv->dropped); 3152681128fSEric Dumazet } 316948d4f21SToshiaki Makita 317948d4f21SToshiaki Makita if (rcv_xdp) 318638264dcSToshiaki Makita __veth_xdp_flush(rq); 319948d4f21SToshiaki Makita 320d0e2c55eSEric Dumazet rcu_read_unlock(); 321948d4f21SToshiaki Makita 3226ed10654SPatrick McHardy return NETDEV_TX_OK; 323e314dbdcSPavel Emelyanov } 324e314dbdcSPavel Emelyanov 325b4fba476SEric Dumazet static u64 veth_stats_tx(struct net_device *dev, u64 *packets, u64 *bytes) 326e314dbdcSPavel Emelyanov { 327cf05c700SEric Dumazet struct veth_priv *priv = netdev_priv(dev); 32811687a10SDavid S. Miller 329b4fba476SEric Dumazet dev_lstats_read(dev, packets, bytes); 3302681128fSEric Dumazet return atomic64_read(&priv->dropped); 3312681128fSEric Dumazet } 3322681128fSEric Dumazet 33365780c56SLorenzo Bianconi static void veth_stats_rx(struct veth_stats *result, struct net_device *dev) 3344195e54aSToshiaki Makita { 3354195e54aSToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 3364195e54aSToshiaki Makita int i; 3374195e54aSToshiaki Makita 3385fe6e567SLorenzo Bianconi result->peer_tq_xdp_xmit_err = 0; 3394195e54aSToshiaki Makita result->xdp_packets = 0; 340d99a7c2fSLorenzo Bianconi result->xdp_tx_err = 0; 3414195e54aSToshiaki Makita result->xdp_bytes = 0; 34266fe4a07SLorenzo Bianconi result->rx_drops = 0; 3434195e54aSToshiaki Makita for (i = 0; i < dev->num_rx_queues; i++) { 3445fe6e567SLorenzo Bianconi u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err; 3454195e54aSToshiaki Makita struct veth_rq_stats *stats = &priv->rq[i].stats; 3464195e54aSToshiaki Makita unsigned int start; 3474195e54aSToshiaki Makita 3484195e54aSToshiaki Makita do { 3494195e54aSToshiaki Makita start = u64_stats_fetch_begin_irq(&stats->syncp); 3505fe6e567SLorenzo Bianconi peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err; 351d99a7c2fSLorenzo Bianconi xdp_tx_err = stats->vs.xdp_tx_err; 35265780c56SLorenzo Bianconi packets = stats->vs.xdp_packets; 35365780c56SLorenzo Bianconi bytes = stats->vs.xdp_bytes; 35466fe4a07SLorenzo Bianconi drops = stats->vs.rx_drops; 3554195e54aSToshiaki Makita } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); 3565fe6e567SLorenzo Bianconi result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err; 357d99a7c2fSLorenzo Bianconi result->xdp_tx_err += xdp_tx_err; 3584195e54aSToshiaki Makita result->xdp_packets += packets; 3594195e54aSToshiaki Makita result->xdp_bytes += bytes; 36066fe4a07SLorenzo Bianconi result->rx_drops += drops; 3614195e54aSToshiaki Makita } 3624195e54aSToshiaki Makita } 3634195e54aSToshiaki Makita 364bc1f4470Sstephen hemminger static void veth_get_stats64(struct net_device *dev, 3652681128fSEric Dumazet struct rtnl_link_stats64 *tot) 3662681128fSEric Dumazet { 3672681128fSEric Dumazet struct veth_priv *priv = netdev_priv(dev); 368d0e2c55eSEric Dumazet struct net_device *peer; 36965780c56SLorenzo Bianconi struct veth_stats rx; 370b4fba476SEric Dumazet u64 packets, bytes; 3712681128fSEric Dumazet 372b4fba476SEric Dumazet tot->tx_dropped = veth_stats_tx(dev, &packets, &bytes); 373b4fba476SEric Dumazet tot->tx_bytes = bytes; 374b4fba476SEric Dumazet tot->tx_packets = packets; 3754195e54aSToshiaki Makita 3764195e54aSToshiaki Makita veth_stats_rx(&rx, dev); 3775fe6e567SLorenzo Bianconi tot->tx_dropped += rx.xdp_tx_err; 3785fe6e567SLorenzo Bianconi tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err; 3794195e54aSToshiaki Makita tot->rx_bytes = rx.xdp_bytes; 3804195e54aSToshiaki Makita tot->rx_packets = rx.xdp_packets; 3812681128fSEric Dumazet 382d0e2c55eSEric Dumazet rcu_read_lock(); 383d0e2c55eSEric Dumazet peer = rcu_dereference(priv->peer); 384d0e2c55eSEric Dumazet if (peer) { 385e25d5dbcSJiang Lidong veth_stats_tx(peer, &packets, &bytes); 386b4fba476SEric Dumazet tot->rx_bytes += bytes; 387b4fba476SEric Dumazet tot->rx_packets += packets; 3884195e54aSToshiaki Makita 3894195e54aSToshiaki Makita veth_stats_rx(&rx, peer); 3905fe6e567SLorenzo Bianconi tot->tx_dropped += rx.peer_tq_xdp_xmit_err; 3915fe6e567SLorenzo Bianconi tot->rx_dropped += rx.xdp_tx_err; 3924195e54aSToshiaki Makita tot->tx_bytes += rx.xdp_bytes; 3934195e54aSToshiaki Makita tot->tx_packets += rx.xdp_packets; 394d0e2c55eSEric Dumazet } 395d0e2c55eSEric Dumazet rcu_read_unlock(); 396e314dbdcSPavel Emelyanov } 397e314dbdcSPavel Emelyanov 3985c70ef85SGao feng /* fake multicast ability */ 3995c70ef85SGao feng static void veth_set_multicast_list(struct net_device *dev) 4005c70ef85SGao feng { 4015c70ef85SGao feng } 4025c70ef85SGao feng 403948d4f21SToshiaki Makita static struct sk_buff *veth_build_skb(void *head, int headroom, int len, 404948d4f21SToshiaki Makita int buflen) 405948d4f21SToshiaki Makita { 406948d4f21SToshiaki Makita struct sk_buff *skb; 407948d4f21SToshiaki Makita 408948d4f21SToshiaki Makita if (!buflen) { 409948d4f21SToshiaki Makita buflen = SKB_DATA_ALIGN(headroom + len) + 410948d4f21SToshiaki Makita SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 411948d4f21SToshiaki Makita } 412948d4f21SToshiaki Makita skb = build_skb(head, buflen); 413948d4f21SToshiaki Makita if (!skb) 414948d4f21SToshiaki Makita return NULL; 415948d4f21SToshiaki Makita 416948d4f21SToshiaki Makita skb_reserve(skb, headroom); 417948d4f21SToshiaki Makita skb_put(skb, len); 418948d4f21SToshiaki Makita 419948d4f21SToshiaki Makita return skb; 420948d4f21SToshiaki Makita } 421948d4f21SToshiaki Makita 422638264dcSToshiaki Makita static int veth_select_rxq(struct net_device *dev) 423638264dcSToshiaki Makita { 424638264dcSToshiaki Makita return smp_processor_id() % dev->real_num_rx_queues; 425638264dcSToshiaki Makita } 426638264dcSToshiaki Makita 427af87a3aaSToshiaki Makita static int veth_xdp_xmit(struct net_device *dev, int n, 4289152cff0SLorenzo Bianconi struct xdp_frame **frames, 4299152cff0SLorenzo Bianconi u32 flags, bool ndo_xmit) 430af87a3aaSToshiaki Makita { 431af87a3aaSToshiaki Makita struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 4325fe6e567SLorenzo Bianconi int i, ret = -ENXIO, drops = 0; 433af87a3aaSToshiaki Makita struct net_device *rcv; 4345fe6e567SLorenzo Bianconi unsigned int max_len; 435638264dcSToshiaki Makita struct veth_rq *rq; 436af87a3aaSToshiaki Makita 4375fe6e567SLorenzo Bianconi if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 438d99a7c2fSLorenzo Bianconi return -EINVAL; 439af87a3aaSToshiaki Makita 4405fe6e567SLorenzo Bianconi rcu_read_lock(); 441af87a3aaSToshiaki Makita rcv = rcu_dereference(priv->peer); 4425fe6e567SLorenzo Bianconi if (unlikely(!rcv)) 4435fe6e567SLorenzo Bianconi goto out; 444af87a3aaSToshiaki Makita 445af87a3aaSToshiaki Makita rcv_priv = netdev_priv(rcv); 4465fe6e567SLorenzo Bianconi rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 447af87a3aaSToshiaki Makita /* Non-NULL xdp_prog ensures that xdp_ring is initialized on receive 448af87a3aaSToshiaki Makita * side. This means an XDP program is loaded on the peer and the peer 449af87a3aaSToshiaki Makita * device is up. 450af87a3aaSToshiaki Makita */ 4515fe6e567SLorenzo Bianconi if (!rcu_access_pointer(rq->xdp_prog)) 4525fe6e567SLorenzo Bianconi goto out; 453af87a3aaSToshiaki Makita 454af87a3aaSToshiaki Makita max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN; 455af87a3aaSToshiaki Makita 456638264dcSToshiaki Makita spin_lock(&rq->xdp_ring.producer_lock); 457af87a3aaSToshiaki Makita for (i = 0; i < n; i++) { 458af87a3aaSToshiaki Makita struct xdp_frame *frame = frames[i]; 459af87a3aaSToshiaki Makita void *ptr = veth_xdp_to_ptr(frame); 460af87a3aaSToshiaki Makita 461af87a3aaSToshiaki Makita if (unlikely(frame->len > max_len || 462638264dcSToshiaki Makita __ptr_ring_produce(&rq->xdp_ring, ptr))) { 463af87a3aaSToshiaki Makita xdp_return_frame_rx_napi(frame); 464af87a3aaSToshiaki Makita drops++; 465af87a3aaSToshiaki Makita } 466af87a3aaSToshiaki Makita } 467638264dcSToshiaki Makita spin_unlock(&rq->xdp_ring.producer_lock); 468af87a3aaSToshiaki Makita 469af87a3aaSToshiaki Makita if (flags & XDP_XMIT_FLUSH) 470638264dcSToshiaki Makita __veth_xdp_flush(rq); 471af87a3aaSToshiaki Makita 472d99a7c2fSLorenzo Bianconi ret = n - drops; 4739152cff0SLorenzo Bianconi if (ndo_xmit) { 4745fe6e567SLorenzo Bianconi u64_stats_update_begin(&rq->stats.syncp); 4755fe6e567SLorenzo Bianconi rq->stats.vs.peer_tq_xdp_xmit += n - drops; 4765fe6e567SLorenzo Bianconi rq->stats.vs.peer_tq_xdp_xmit_err += drops; 4779152cff0SLorenzo Bianconi u64_stats_update_end(&rq->stats.syncp); 4785fe6e567SLorenzo Bianconi } 4799152cff0SLorenzo Bianconi 4805fe6e567SLorenzo Bianconi out: 481b23bfa56SJohn Fastabend rcu_read_unlock(); 4822131479dSToshiaki Makita 4832131479dSToshiaki Makita return ret; 484af87a3aaSToshiaki Makita } 485af87a3aaSToshiaki Makita 4869152cff0SLorenzo Bianconi static int veth_ndo_xdp_xmit(struct net_device *dev, int n, 4879152cff0SLorenzo Bianconi struct xdp_frame **frames, u32 flags) 4889152cff0SLorenzo Bianconi { 4895fe6e567SLorenzo Bianconi int err; 4905fe6e567SLorenzo Bianconi 4915fe6e567SLorenzo Bianconi err = veth_xdp_xmit(dev, n, frames, flags, true); 4925fe6e567SLorenzo Bianconi if (err < 0) { 4935fe6e567SLorenzo Bianconi struct veth_priv *priv = netdev_priv(dev); 4945fe6e567SLorenzo Bianconi 4955fe6e567SLorenzo Bianconi atomic64_add(n, &priv->dropped); 4965fe6e567SLorenzo Bianconi } 4975fe6e567SLorenzo Bianconi 4985fe6e567SLorenzo Bianconi return err; 4999152cff0SLorenzo Bianconi } 5009152cff0SLorenzo Bianconi 501bd32aa1fSLorenzo Bianconi static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 5029cda7807SToshiaki Makita { 5039cda7807SToshiaki Makita int sent, i, err = 0; 5049cda7807SToshiaki Makita 505bd32aa1fSLorenzo Bianconi sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); 5069cda7807SToshiaki Makita if (sent < 0) { 5079cda7807SToshiaki Makita err = sent; 5089cda7807SToshiaki Makita sent = 0; 5099cda7807SToshiaki Makita for (i = 0; i < bq->count; i++) 5109cda7807SToshiaki Makita xdp_return_frame(bq->q[i]); 5119cda7807SToshiaki Makita } 512bd32aa1fSLorenzo Bianconi trace_xdp_bulk_tx(rq->dev, sent, bq->count - sent, err); 5139cda7807SToshiaki Makita 5145fe6e567SLorenzo Bianconi u64_stats_update_begin(&rq->stats.syncp); 5155fe6e567SLorenzo Bianconi rq->stats.vs.xdp_tx += sent; 5165fe6e567SLorenzo Bianconi rq->stats.vs.xdp_tx_err += bq->count - sent; 5175fe6e567SLorenzo Bianconi u64_stats_update_end(&rq->stats.syncp); 5185fe6e567SLorenzo Bianconi 5199cda7807SToshiaki Makita bq->count = 0; 5209cda7807SToshiaki Makita } 5219cda7807SToshiaki Makita 522bd32aa1fSLorenzo Bianconi static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 523d1396004SToshiaki Makita { 524bd32aa1fSLorenzo Bianconi struct veth_priv *rcv_priv, *priv = netdev_priv(rq->dev); 525d1396004SToshiaki Makita struct net_device *rcv; 526bd32aa1fSLorenzo Bianconi struct veth_rq *rcv_rq; 527d1396004SToshiaki Makita 528d1396004SToshiaki Makita rcu_read_lock(); 529bd32aa1fSLorenzo Bianconi veth_xdp_flush_bq(rq, bq); 530d1396004SToshiaki Makita rcv = rcu_dereference(priv->peer); 531d1396004SToshiaki Makita if (unlikely(!rcv)) 532d1396004SToshiaki Makita goto out; 533d1396004SToshiaki Makita 534d1396004SToshiaki Makita rcv_priv = netdev_priv(rcv); 535bd32aa1fSLorenzo Bianconi rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 536d1396004SToshiaki Makita /* xdp_ring is initialized on receive side? */ 537bd32aa1fSLorenzo Bianconi if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog))) 538d1396004SToshiaki Makita goto out; 539d1396004SToshiaki Makita 540bd32aa1fSLorenzo Bianconi __veth_xdp_flush(rcv_rq); 541d1396004SToshiaki Makita out: 542d1396004SToshiaki Makita rcu_read_unlock(); 543d1396004SToshiaki Makita } 544d1396004SToshiaki Makita 545bd32aa1fSLorenzo Bianconi static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp, 5469cda7807SToshiaki Makita struct veth_xdp_tx_bq *bq) 547d1396004SToshiaki Makita { 548d1396004SToshiaki Makita struct xdp_frame *frame = convert_to_xdp_frame(xdp); 549d1396004SToshiaki Makita 550d1396004SToshiaki Makita if (unlikely(!frame)) 551d1396004SToshiaki Makita return -EOVERFLOW; 552d1396004SToshiaki Makita 5539cda7807SToshiaki Makita if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE)) 554bd32aa1fSLorenzo Bianconi veth_xdp_flush_bq(rq, bq); 5559cda7807SToshiaki Makita 5569cda7807SToshiaki Makita bq->q[bq->count++] = frame; 5579cda7807SToshiaki Makita 5589cda7807SToshiaki Makita return 0; 559d1396004SToshiaki Makita } 560d1396004SToshiaki Makita 561638264dcSToshiaki Makita static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq, 562d1396004SToshiaki Makita struct xdp_frame *frame, 5631c5b82e5SLorenzo Bianconi struct veth_xdp_tx_bq *bq, 5641c5b82e5SLorenzo Bianconi struct veth_stats *stats) 5659fc8d518SToshiaki Makita { 5669fc8d518SToshiaki Makita void *hard_start = frame->data - frame->headroom; 5679fc8d518SToshiaki Makita void *head = hard_start - sizeof(struct xdp_frame); 5689fc8d518SToshiaki Makita int len = frame->len, delta = 0; 569d1396004SToshiaki Makita struct xdp_frame orig_frame; 5709fc8d518SToshiaki Makita struct bpf_prog *xdp_prog; 5719fc8d518SToshiaki Makita unsigned int headroom; 5729fc8d518SToshiaki Makita struct sk_buff *skb; 5739fc8d518SToshiaki Makita 5749fc8d518SToshiaki Makita rcu_read_lock(); 575638264dcSToshiaki Makita xdp_prog = rcu_dereference(rq->xdp_prog); 5769fc8d518SToshiaki Makita if (likely(xdp_prog)) { 5779fc8d518SToshiaki Makita struct xdp_buff xdp; 5789fc8d518SToshiaki Makita u32 act; 5799fc8d518SToshiaki Makita 5809fc8d518SToshiaki Makita xdp.data_hard_start = hard_start; 5819fc8d518SToshiaki Makita xdp.data = frame->data; 5829fc8d518SToshiaki Makita xdp.data_end = frame->data + frame->len; 5839fc8d518SToshiaki Makita xdp.data_meta = frame->data - frame->metasize; 584638264dcSToshiaki Makita xdp.rxq = &rq->xdp_rxq; 5859fc8d518SToshiaki Makita 5869fc8d518SToshiaki Makita act = bpf_prog_run_xdp(xdp_prog, &xdp); 5879fc8d518SToshiaki Makita 5889fc8d518SToshiaki Makita switch (act) { 5899fc8d518SToshiaki Makita case XDP_PASS: 5909fc8d518SToshiaki Makita delta = frame->data - xdp.data; 5919fc8d518SToshiaki Makita len = xdp.data_end - xdp.data; 5929fc8d518SToshiaki Makita break; 593d1396004SToshiaki Makita case XDP_TX: 594d1396004SToshiaki Makita orig_frame = *frame; 595d1396004SToshiaki Makita xdp.data_hard_start = head; 596d1396004SToshiaki Makita xdp.rxq->mem = frame->mem; 597bd32aa1fSLorenzo Bianconi if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) { 598638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 599d1396004SToshiaki Makita frame = &orig_frame; 6001c5b82e5SLorenzo Bianconi stats->rx_drops++; 601d1396004SToshiaki Makita goto err_xdp; 602d1396004SToshiaki Makita } 6031c5b82e5SLorenzo Bianconi stats->xdp_tx++; 604d1396004SToshiaki Makita rcu_read_unlock(); 605d1396004SToshiaki Makita goto xdp_xmit; 606d1396004SToshiaki Makita case XDP_REDIRECT: 607d1396004SToshiaki Makita orig_frame = *frame; 608d1396004SToshiaki Makita xdp.data_hard_start = head; 609d1396004SToshiaki Makita xdp.rxq->mem = frame->mem; 610638264dcSToshiaki Makita if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { 611d1396004SToshiaki Makita frame = &orig_frame; 6121c5b82e5SLorenzo Bianconi stats->rx_drops++; 613d1396004SToshiaki Makita goto err_xdp; 614d1396004SToshiaki Makita } 6151c5b82e5SLorenzo Bianconi stats->xdp_redirect++; 616d1396004SToshiaki Makita rcu_read_unlock(); 617d1396004SToshiaki Makita goto xdp_xmit; 6189fc8d518SToshiaki Makita default: 6199fc8d518SToshiaki Makita bpf_warn_invalid_xdp_action(act); 620a9b6d9efSGustavo A. R. Silva /* fall through */ 6219fc8d518SToshiaki Makita case XDP_ABORTED: 622638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 623a9b6d9efSGustavo A. R. Silva /* fall through */ 6249fc8d518SToshiaki Makita case XDP_DROP: 6251c5b82e5SLorenzo Bianconi stats->xdp_drops++; 6269fc8d518SToshiaki Makita goto err_xdp; 6279fc8d518SToshiaki Makita } 6289fc8d518SToshiaki Makita } 6299fc8d518SToshiaki Makita rcu_read_unlock(); 6309fc8d518SToshiaki Makita 6319fc8d518SToshiaki Makita headroom = sizeof(struct xdp_frame) + frame->headroom - delta; 6329fc8d518SToshiaki Makita skb = veth_build_skb(head, headroom, len, 0); 6339fc8d518SToshiaki Makita if (!skb) { 6349fc8d518SToshiaki Makita xdp_return_frame(frame); 6351c5b82e5SLorenzo Bianconi stats->rx_drops++; 6369fc8d518SToshiaki Makita goto err; 6379fc8d518SToshiaki Makita } 6389fc8d518SToshiaki Makita 639cbf33510SJesper Dangaard Brouer xdp_release_frame(frame); 6409fc8d518SToshiaki Makita xdp_scrub_frame(frame); 641638264dcSToshiaki Makita skb->protocol = eth_type_trans(skb, rq->dev); 6429fc8d518SToshiaki Makita err: 6439fc8d518SToshiaki Makita return skb; 6449fc8d518SToshiaki Makita err_xdp: 6459fc8d518SToshiaki Makita rcu_read_unlock(); 6469fc8d518SToshiaki Makita xdp_return_frame(frame); 647d1396004SToshiaki Makita xdp_xmit: 6489fc8d518SToshiaki Makita return NULL; 6499fc8d518SToshiaki Makita } 6509fc8d518SToshiaki Makita 6511c5b82e5SLorenzo Bianconi static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, 6521c5b82e5SLorenzo Bianconi struct sk_buff *skb, 6531c5b82e5SLorenzo Bianconi struct veth_xdp_tx_bq *bq, 6541c5b82e5SLorenzo Bianconi struct veth_stats *stats) 655948d4f21SToshiaki Makita { 656948d4f21SToshiaki Makita u32 pktlen, headroom, act, metalen; 657948d4f21SToshiaki Makita void *orig_data, *orig_data_end; 658948d4f21SToshiaki Makita struct bpf_prog *xdp_prog; 659948d4f21SToshiaki Makita int mac_len, delta, off; 660948d4f21SToshiaki Makita struct xdp_buff xdp; 661948d4f21SToshiaki Makita 6624bf9ffa0SToshiaki Makita skb_orphan(skb); 6634bf9ffa0SToshiaki Makita 664948d4f21SToshiaki Makita rcu_read_lock(); 665638264dcSToshiaki Makita xdp_prog = rcu_dereference(rq->xdp_prog); 666948d4f21SToshiaki Makita if (unlikely(!xdp_prog)) { 667948d4f21SToshiaki Makita rcu_read_unlock(); 668948d4f21SToshiaki Makita goto out; 669948d4f21SToshiaki Makita } 670948d4f21SToshiaki Makita 671948d4f21SToshiaki Makita mac_len = skb->data - skb_mac_header(skb); 672948d4f21SToshiaki Makita pktlen = skb->len + mac_len; 673948d4f21SToshiaki Makita headroom = skb_headroom(skb) - mac_len; 674948d4f21SToshiaki Makita 675948d4f21SToshiaki Makita if (skb_shared(skb) || skb_head_is_locked(skb) || 676948d4f21SToshiaki Makita skb_is_nonlinear(skb) || headroom < XDP_PACKET_HEADROOM) { 677948d4f21SToshiaki Makita struct sk_buff *nskb; 678948d4f21SToshiaki Makita int size, head_off; 679948d4f21SToshiaki Makita void *head, *start; 680948d4f21SToshiaki Makita struct page *page; 681948d4f21SToshiaki Makita 682948d4f21SToshiaki Makita size = SKB_DATA_ALIGN(VETH_XDP_HEADROOM + pktlen) + 683948d4f21SToshiaki Makita SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 684948d4f21SToshiaki Makita if (size > PAGE_SIZE) 685948d4f21SToshiaki Makita goto drop; 686948d4f21SToshiaki Makita 687948d4f21SToshiaki Makita page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); 688948d4f21SToshiaki Makita if (!page) 689948d4f21SToshiaki Makita goto drop; 690948d4f21SToshiaki Makita 691948d4f21SToshiaki Makita head = page_address(page); 692948d4f21SToshiaki Makita start = head + VETH_XDP_HEADROOM; 693948d4f21SToshiaki Makita if (skb_copy_bits(skb, -mac_len, start, pktlen)) { 694948d4f21SToshiaki Makita page_frag_free(head); 695948d4f21SToshiaki Makita goto drop; 696948d4f21SToshiaki Makita } 697948d4f21SToshiaki Makita 698948d4f21SToshiaki Makita nskb = veth_build_skb(head, 699948d4f21SToshiaki Makita VETH_XDP_HEADROOM + mac_len, skb->len, 700948d4f21SToshiaki Makita PAGE_SIZE); 701948d4f21SToshiaki Makita if (!nskb) { 702948d4f21SToshiaki Makita page_frag_free(head); 703948d4f21SToshiaki Makita goto drop; 704948d4f21SToshiaki Makita } 705948d4f21SToshiaki Makita 706948d4f21SToshiaki Makita skb_copy_header(nskb, skb); 707948d4f21SToshiaki Makita head_off = skb_headroom(nskb) - skb_headroom(skb); 708948d4f21SToshiaki Makita skb_headers_offset_update(nskb, head_off); 709948d4f21SToshiaki Makita consume_skb(skb); 710948d4f21SToshiaki Makita skb = nskb; 711948d4f21SToshiaki Makita } 712948d4f21SToshiaki Makita 713948d4f21SToshiaki Makita xdp.data_hard_start = skb->head; 714948d4f21SToshiaki Makita xdp.data = skb_mac_header(skb); 715948d4f21SToshiaki Makita xdp.data_end = xdp.data + pktlen; 716948d4f21SToshiaki Makita xdp.data_meta = xdp.data; 717638264dcSToshiaki Makita xdp.rxq = &rq->xdp_rxq; 718948d4f21SToshiaki Makita orig_data = xdp.data; 719948d4f21SToshiaki Makita orig_data_end = xdp.data_end; 720948d4f21SToshiaki Makita 721948d4f21SToshiaki Makita act = bpf_prog_run_xdp(xdp_prog, &xdp); 722948d4f21SToshiaki Makita 723948d4f21SToshiaki Makita switch (act) { 724948d4f21SToshiaki Makita case XDP_PASS: 725948d4f21SToshiaki Makita break; 726d1396004SToshiaki Makita case XDP_TX: 727d1396004SToshiaki Makita get_page(virt_to_page(xdp.data)); 728d1396004SToshiaki Makita consume_skb(skb); 729638264dcSToshiaki Makita xdp.rxq->mem = rq->xdp_mem; 730bd32aa1fSLorenzo Bianconi if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) { 731638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 7321c5b82e5SLorenzo Bianconi stats->rx_drops++; 733d1396004SToshiaki Makita goto err_xdp; 734d1396004SToshiaki Makita } 7351c5b82e5SLorenzo Bianconi stats->xdp_tx++; 736d1396004SToshiaki Makita rcu_read_unlock(); 737d1396004SToshiaki Makita goto xdp_xmit; 738d1396004SToshiaki Makita case XDP_REDIRECT: 739d1396004SToshiaki Makita get_page(virt_to_page(xdp.data)); 740d1396004SToshiaki Makita consume_skb(skb); 741638264dcSToshiaki Makita xdp.rxq->mem = rq->xdp_mem; 7421c5b82e5SLorenzo Bianconi if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { 7431c5b82e5SLorenzo Bianconi stats->rx_drops++; 744d1396004SToshiaki Makita goto err_xdp; 7451c5b82e5SLorenzo Bianconi } 7461c5b82e5SLorenzo Bianconi stats->xdp_redirect++; 747d1396004SToshiaki Makita rcu_read_unlock(); 748d1396004SToshiaki Makita goto xdp_xmit; 749948d4f21SToshiaki Makita default: 750948d4f21SToshiaki Makita bpf_warn_invalid_xdp_action(act); 751a9b6d9efSGustavo A. R. Silva /* fall through */ 752948d4f21SToshiaki Makita case XDP_ABORTED: 753638264dcSToshiaki Makita trace_xdp_exception(rq->dev, xdp_prog, act); 754a9b6d9efSGustavo A. R. Silva /* fall through */ 755948d4f21SToshiaki Makita case XDP_DROP: 7561c5b82e5SLorenzo Bianconi stats->xdp_drops++; 7571c5b82e5SLorenzo Bianconi goto xdp_drop; 758948d4f21SToshiaki Makita } 759948d4f21SToshiaki Makita rcu_read_unlock(); 760948d4f21SToshiaki Makita 761948d4f21SToshiaki Makita delta = orig_data - xdp.data; 762948d4f21SToshiaki Makita off = mac_len + delta; 763948d4f21SToshiaki Makita if (off > 0) 764948d4f21SToshiaki Makita __skb_push(skb, off); 765948d4f21SToshiaki Makita else if (off < 0) 766948d4f21SToshiaki Makita __skb_pull(skb, -off); 767948d4f21SToshiaki Makita skb->mac_header -= delta; 768948d4f21SToshiaki Makita off = xdp.data_end - orig_data_end; 769948d4f21SToshiaki Makita if (off != 0) 770948d4f21SToshiaki Makita __skb_put(skb, off); 771638264dcSToshiaki Makita skb->protocol = eth_type_trans(skb, rq->dev); 772948d4f21SToshiaki Makita 773948d4f21SToshiaki Makita metalen = xdp.data - xdp.data_meta; 774948d4f21SToshiaki Makita if (metalen) 775948d4f21SToshiaki Makita skb_metadata_set(skb, metalen); 776948d4f21SToshiaki Makita out: 777948d4f21SToshiaki Makita return skb; 778948d4f21SToshiaki Makita drop: 7791c5b82e5SLorenzo Bianconi stats->rx_drops++; 7801c5b82e5SLorenzo Bianconi xdp_drop: 781948d4f21SToshiaki Makita rcu_read_unlock(); 782948d4f21SToshiaki Makita kfree_skb(skb); 783948d4f21SToshiaki Makita return NULL; 784d1396004SToshiaki Makita err_xdp: 785d1396004SToshiaki Makita rcu_read_unlock(); 786d1396004SToshiaki Makita page_frag_free(xdp.data); 787d1396004SToshiaki Makita xdp_xmit: 788d1396004SToshiaki Makita return NULL; 789948d4f21SToshiaki Makita } 790948d4f21SToshiaki Makita 7911c5b82e5SLorenzo Bianconi static int veth_xdp_rcv(struct veth_rq *rq, int budget, 7921c5b82e5SLorenzo Bianconi struct veth_xdp_tx_bq *bq, 7931c5b82e5SLorenzo Bianconi struct veth_stats *stats) 794948d4f21SToshiaki Makita { 7951c5b82e5SLorenzo Bianconi int i, done = 0; 796948d4f21SToshiaki Makita 797948d4f21SToshiaki Makita for (i = 0; i < budget; i++) { 798638264dcSToshiaki Makita void *ptr = __ptr_ring_consume(&rq->xdp_ring); 7999fc8d518SToshiaki Makita struct sk_buff *skb; 800948d4f21SToshiaki Makita 8019fc8d518SToshiaki Makita if (!ptr) 802948d4f21SToshiaki Makita break; 803948d4f21SToshiaki Makita 804d1396004SToshiaki Makita if (veth_is_xdp_frame(ptr)) { 8054195e54aSToshiaki Makita struct xdp_frame *frame = veth_ptr_to_xdp(ptr); 8064195e54aSToshiaki Makita 8071c5b82e5SLorenzo Bianconi stats->xdp_bytes += frame->len; 8081c5b82e5SLorenzo Bianconi skb = veth_xdp_rcv_one(rq, frame, bq, stats); 809d1396004SToshiaki Makita } else { 8104195e54aSToshiaki Makita skb = ptr; 8111c5b82e5SLorenzo Bianconi stats->xdp_bytes += skb->len; 8121c5b82e5SLorenzo Bianconi skb = veth_xdp_rcv_skb(rq, skb, bq, stats); 813d1396004SToshiaki Makita } 814948d4f21SToshiaki Makita 815948d4f21SToshiaki Makita if (skb) 816638264dcSToshiaki Makita napi_gro_receive(&rq->xdp_napi, skb); 817948d4f21SToshiaki Makita 818948d4f21SToshiaki Makita done++; 819948d4f21SToshiaki Makita } 820948d4f21SToshiaki Makita 8214195e54aSToshiaki Makita u64_stats_update_begin(&rq->stats.syncp); 8229152cff0SLorenzo Bianconi rq->stats.vs.xdp_redirect += stats->xdp_redirect; 8231c5b82e5SLorenzo Bianconi rq->stats.vs.xdp_bytes += stats->xdp_bytes; 82466fe4a07SLorenzo Bianconi rq->stats.vs.xdp_drops += stats->xdp_drops; 82566fe4a07SLorenzo Bianconi rq->stats.vs.rx_drops += stats->rx_drops; 82665780c56SLorenzo Bianconi rq->stats.vs.xdp_packets += done; 8274195e54aSToshiaki Makita u64_stats_update_end(&rq->stats.syncp); 8284195e54aSToshiaki Makita 829948d4f21SToshiaki Makita return done; 830948d4f21SToshiaki Makita } 831948d4f21SToshiaki Makita 832948d4f21SToshiaki Makita static int veth_poll(struct napi_struct *napi, int budget) 833948d4f21SToshiaki Makita { 834638264dcSToshiaki Makita struct veth_rq *rq = 835638264dcSToshiaki Makita container_of(napi, struct veth_rq, xdp_napi); 8361c5b82e5SLorenzo Bianconi struct veth_stats stats = {}; 8379cda7807SToshiaki Makita struct veth_xdp_tx_bq bq; 838948d4f21SToshiaki Makita int done; 839948d4f21SToshiaki Makita 8409cda7807SToshiaki Makita bq.count = 0; 8419cda7807SToshiaki Makita 842d1396004SToshiaki Makita xdp_set_return_frame_no_direct(); 8431c5b82e5SLorenzo Bianconi done = veth_xdp_rcv(rq, budget, &bq, &stats); 844948d4f21SToshiaki Makita 845948d4f21SToshiaki Makita if (done < budget && napi_complete_done(napi, done)) { 846948d4f21SToshiaki Makita /* Write rx_notify_masked before reading ptr_ring */ 847638264dcSToshiaki Makita smp_store_mb(rq->rx_notify_masked, false); 848638264dcSToshiaki Makita if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) { 849638264dcSToshiaki Makita rq->rx_notify_masked = true; 850638264dcSToshiaki Makita napi_schedule(&rq->xdp_napi); 851948d4f21SToshiaki Makita } 852948d4f21SToshiaki Makita } 853948d4f21SToshiaki Makita 8541c5b82e5SLorenzo Bianconi if (stats.xdp_tx > 0) 855bd32aa1fSLorenzo Bianconi veth_xdp_flush(rq, &bq); 8561c5b82e5SLorenzo Bianconi if (stats.xdp_redirect > 0) 8571d233886SToke Høiland-Jørgensen xdp_do_flush(); 858d1396004SToshiaki Makita xdp_clear_return_frame_no_direct(); 859d1396004SToshiaki Makita 860948d4f21SToshiaki Makita return done; 861948d4f21SToshiaki Makita } 862948d4f21SToshiaki Makita 863948d4f21SToshiaki Makita static int veth_napi_add(struct net_device *dev) 864948d4f21SToshiaki Makita { 865948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 866638264dcSToshiaki Makita int err, i; 867948d4f21SToshiaki Makita 868638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 869638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 870638264dcSToshiaki Makita 871638264dcSToshiaki Makita err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL); 872948d4f21SToshiaki Makita if (err) 873638264dcSToshiaki Makita goto err_xdp_ring; 874638264dcSToshiaki Makita } 875948d4f21SToshiaki Makita 876638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 877638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 878638264dcSToshiaki Makita 879638264dcSToshiaki Makita netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); 880638264dcSToshiaki Makita napi_enable(&rq->xdp_napi); 881638264dcSToshiaki Makita } 882948d4f21SToshiaki Makita 883948d4f21SToshiaki Makita return 0; 884638264dcSToshiaki Makita err_xdp_ring: 885638264dcSToshiaki Makita for (i--; i >= 0; i--) 886638264dcSToshiaki Makita ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free); 887638264dcSToshiaki Makita 888638264dcSToshiaki Makita return err; 889948d4f21SToshiaki Makita } 890948d4f21SToshiaki Makita 891948d4f21SToshiaki Makita static void veth_napi_del(struct net_device *dev) 892948d4f21SToshiaki Makita { 893948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 894638264dcSToshiaki Makita int i; 895948d4f21SToshiaki Makita 896638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 897638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 898638264dcSToshiaki Makita 899638264dcSToshiaki Makita napi_disable(&rq->xdp_napi); 900638264dcSToshiaki Makita napi_hash_del(&rq->xdp_napi); 901638264dcSToshiaki Makita } 902638264dcSToshiaki Makita synchronize_net(); 903638264dcSToshiaki Makita 904638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 905638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 906638264dcSToshiaki Makita 907638264dcSToshiaki Makita netif_napi_del(&rq->xdp_napi); 908638264dcSToshiaki Makita rq->rx_notify_masked = false; 909638264dcSToshiaki Makita ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free); 910638264dcSToshiaki Makita } 911948d4f21SToshiaki Makita } 912948d4f21SToshiaki Makita 913948d4f21SToshiaki Makita static int veth_enable_xdp(struct net_device *dev) 914948d4f21SToshiaki Makita { 915948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 916638264dcSToshiaki Makita int err, i; 917948d4f21SToshiaki Makita 918638264dcSToshiaki Makita if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) { 919638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 920638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 921948d4f21SToshiaki Makita 922638264dcSToshiaki Makita err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i); 923948d4f21SToshiaki Makita if (err < 0) 924638264dcSToshiaki Makita goto err_rxq_reg; 925638264dcSToshiaki Makita 926638264dcSToshiaki Makita err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 927638264dcSToshiaki Makita MEM_TYPE_PAGE_SHARED, 928638264dcSToshiaki Makita NULL); 929638264dcSToshiaki Makita if (err < 0) 930638264dcSToshiaki Makita goto err_reg_mem; 931638264dcSToshiaki Makita 932638264dcSToshiaki Makita /* Save original mem info as it can be overwritten */ 933638264dcSToshiaki Makita rq->xdp_mem = rq->xdp_rxq.mem; 934638264dcSToshiaki Makita } 935948d4f21SToshiaki Makita 936948d4f21SToshiaki Makita err = veth_napi_add(dev); 937948d4f21SToshiaki Makita if (err) 938638264dcSToshiaki Makita goto err_rxq_reg; 939948d4f21SToshiaki Makita } 940948d4f21SToshiaki Makita 941638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) 942638264dcSToshiaki Makita rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog); 943948d4f21SToshiaki Makita 944948d4f21SToshiaki Makita return 0; 945638264dcSToshiaki Makita err_reg_mem: 946638264dcSToshiaki Makita xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); 947638264dcSToshiaki Makita err_rxq_reg: 948638264dcSToshiaki Makita for (i--; i >= 0; i--) 949638264dcSToshiaki Makita xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); 950948d4f21SToshiaki Makita 951948d4f21SToshiaki Makita return err; 952948d4f21SToshiaki Makita } 953948d4f21SToshiaki Makita 954948d4f21SToshiaki Makita static void veth_disable_xdp(struct net_device *dev) 955948d4f21SToshiaki Makita { 956948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 957638264dcSToshiaki Makita int i; 958948d4f21SToshiaki Makita 959638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) 960638264dcSToshiaki Makita rcu_assign_pointer(priv->rq[i].xdp_prog, NULL); 961948d4f21SToshiaki Makita veth_napi_del(dev); 962638264dcSToshiaki Makita for (i = 0; i < dev->real_num_rx_queues; i++) { 963638264dcSToshiaki Makita struct veth_rq *rq = &priv->rq[i]; 964638264dcSToshiaki Makita 965638264dcSToshiaki Makita rq->xdp_rxq.mem = rq->xdp_mem; 966638264dcSToshiaki Makita xdp_rxq_info_unreg(&rq->xdp_rxq); 967638264dcSToshiaki Makita } 968948d4f21SToshiaki Makita } 969948d4f21SToshiaki Makita 970e314dbdcSPavel Emelyanov static int veth_open(struct net_device *dev) 971e314dbdcSPavel Emelyanov { 972d0e2c55eSEric Dumazet struct veth_priv *priv = netdev_priv(dev); 973d0e2c55eSEric Dumazet struct net_device *peer = rtnl_dereference(priv->peer); 974948d4f21SToshiaki Makita int err; 975e314dbdcSPavel Emelyanov 976d0e2c55eSEric Dumazet if (!peer) 977e314dbdcSPavel Emelyanov return -ENOTCONN; 978e314dbdcSPavel Emelyanov 979948d4f21SToshiaki Makita if (priv->_xdp_prog) { 980948d4f21SToshiaki Makita err = veth_enable_xdp(dev); 981948d4f21SToshiaki Makita if (err) 982948d4f21SToshiaki Makita return err; 983948d4f21SToshiaki Makita } 984948d4f21SToshiaki Makita 985d0e2c55eSEric Dumazet if (peer->flags & IFF_UP) { 986e314dbdcSPavel Emelyanov netif_carrier_on(dev); 987d0e2c55eSEric Dumazet netif_carrier_on(peer); 988e314dbdcSPavel Emelyanov } 989948d4f21SToshiaki Makita 990e314dbdcSPavel Emelyanov return 0; 991e314dbdcSPavel Emelyanov } 992e314dbdcSPavel Emelyanov 9932cf48a10SEric W. Biederman static int veth_close(struct net_device *dev) 9942cf48a10SEric W. Biederman { 9952cf48a10SEric W. Biederman struct veth_priv *priv = netdev_priv(dev); 9962efd32eeSEric Dumazet struct net_device *peer = rtnl_dereference(priv->peer); 9972cf48a10SEric W. Biederman 9982cf48a10SEric W. Biederman netif_carrier_off(dev); 9992efd32eeSEric Dumazet if (peer) 10002efd32eeSEric Dumazet netif_carrier_off(peer); 10012cf48a10SEric W. Biederman 1002948d4f21SToshiaki Makita if (priv->_xdp_prog) 1003948d4f21SToshiaki Makita veth_disable_xdp(dev); 1004948d4f21SToshiaki Makita 10052cf48a10SEric W. Biederman return 0; 10062cf48a10SEric W. Biederman } 10072cf48a10SEric W. Biederman 100891572088SJarod Wilson static int is_valid_veth_mtu(int mtu) 100938d40815SEric Biederman { 101091572088SJarod Wilson return mtu >= ETH_MIN_MTU && mtu <= ETH_MAX_MTU; 101138d40815SEric Biederman } 101238d40815SEric Biederman 10137797b93bSToshiaki Makita static int veth_alloc_queues(struct net_device *dev) 10147797b93bSToshiaki Makita { 10157797b93bSToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 10167797b93bSToshiaki Makita int i; 10177797b93bSToshiaki Makita 10187797b93bSToshiaki Makita priv->rq = kcalloc(dev->num_rx_queues, sizeof(*priv->rq), GFP_KERNEL); 10197797b93bSToshiaki Makita if (!priv->rq) 10207797b93bSToshiaki Makita return -ENOMEM; 10217797b93bSToshiaki Makita 10224195e54aSToshiaki Makita for (i = 0; i < dev->num_rx_queues; i++) { 10237797b93bSToshiaki Makita priv->rq[i].dev = dev; 10244195e54aSToshiaki Makita u64_stats_init(&priv->rq[i].stats.syncp); 10254195e54aSToshiaki Makita } 10267797b93bSToshiaki Makita 10277797b93bSToshiaki Makita return 0; 10287797b93bSToshiaki Makita } 10297797b93bSToshiaki Makita 10307797b93bSToshiaki Makita static void veth_free_queues(struct net_device *dev) 10317797b93bSToshiaki Makita { 10327797b93bSToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 10337797b93bSToshiaki Makita 10347797b93bSToshiaki Makita kfree(priv->rq); 10357797b93bSToshiaki Makita } 10367797b93bSToshiaki Makita 1037e314dbdcSPavel Emelyanov static int veth_dev_init(struct net_device *dev) 1038e314dbdcSPavel Emelyanov { 10397797b93bSToshiaki Makita int err; 10407797b93bSToshiaki Makita 104114d73416SLi RongQing dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); 104214d73416SLi RongQing if (!dev->lstats) 1043e314dbdcSPavel Emelyanov return -ENOMEM; 10447797b93bSToshiaki Makita 10457797b93bSToshiaki Makita err = veth_alloc_queues(dev); 10467797b93bSToshiaki Makita if (err) { 104714d73416SLi RongQing free_percpu(dev->lstats); 10487797b93bSToshiaki Makita return err; 10497797b93bSToshiaki Makita } 10507797b93bSToshiaki Makita 1051e314dbdcSPavel Emelyanov return 0; 1052e314dbdcSPavel Emelyanov } 1053e314dbdcSPavel Emelyanov 105411687a10SDavid S. Miller static void veth_dev_free(struct net_device *dev) 105511687a10SDavid S. Miller { 10567797b93bSToshiaki Makita veth_free_queues(dev); 105714d73416SLi RongQing free_percpu(dev->lstats); 105811687a10SDavid S. Miller } 105911687a10SDavid S. Miller 1060bb446c19SWANG Cong #ifdef CONFIG_NET_POLL_CONTROLLER 1061bb446c19SWANG Cong static void veth_poll_controller(struct net_device *dev) 1062bb446c19SWANG Cong { 1063bb446c19SWANG Cong /* veth only receives frames when its peer sends one 1064948d4f21SToshiaki Makita * Since it has nothing to do with disabling irqs, we are guaranteed 1065bb446c19SWANG Cong * never to have pending data when we poll for it so 1066bb446c19SWANG Cong * there is nothing to do here. 1067bb446c19SWANG Cong * 1068bb446c19SWANG Cong * We need this though so netpoll recognizes us as an interface that 1069bb446c19SWANG Cong * supports polling, which enables bridge devices in virt setups to 1070bb446c19SWANG Cong * still use netconsole 1071bb446c19SWANG Cong */ 1072bb446c19SWANG Cong } 1073bb446c19SWANG Cong #endif /* CONFIG_NET_POLL_CONTROLLER */ 1074bb446c19SWANG Cong 1075a45253bfSNicolas Dichtel static int veth_get_iflink(const struct net_device *dev) 1076a45253bfSNicolas Dichtel { 1077a45253bfSNicolas Dichtel struct veth_priv *priv = netdev_priv(dev); 1078a45253bfSNicolas Dichtel struct net_device *peer; 1079a45253bfSNicolas Dichtel int iflink; 1080a45253bfSNicolas Dichtel 1081a45253bfSNicolas Dichtel rcu_read_lock(); 1082a45253bfSNicolas Dichtel peer = rcu_dereference(priv->peer); 1083a45253bfSNicolas Dichtel iflink = peer ? peer->ifindex : 0; 1084a45253bfSNicolas Dichtel rcu_read_unlock(); 1085a45253bfSNicolas Dichtel 1086a45253bfSNicolas Dichtel return iflink; 1087a45253bfSNicolas Dichtel } 1088a45253bfSNicolas Dichtel 1089dc224822SToshiaki Makita static netdev_features_t veth_fix_features(struct net_device *dev, 1090dc224822SToshiaki Makita netdev_features_t features) 1091dc224822SToshiaki Makita { 1092dc224822SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 1093dc224822SToshiaki Makita struct net_device *peer; 1094dc224822SToshiaki Makita 1095dc224822SToshiaki Makita peer = rtnl_dereference(priv->peer); 1096dc224822SToshiaki Makita if (peer) { 1097dc224822SToshiaki Makita struct veth_priv *peer_priv = netdev_priv(peer); 1098dc224822SToshiaki Makita 1099dc224822SToshiaki Makita if (peer_priv->_xdp_prog) 1100dc224822SToshiaki Makita features &= ~NETIF_F_GSO_SOFTWARE; 1101dc224822SToshiaki Makita } 1102dc224822SToshiaki Makita 1103dc224822SToshiaki Makita return features; 1104dc224822SToshiaki Makita } 1105dc224822SToshiaki Makita 1106163e5292SPaolo Abeni static void veth_set_rx_headroom(struct net_device *dev, int new_hr) 1107163e5292SPaolo Abeni { 1108163e5292SPaolo Abeni struct veth_priv *peer_priv, *priv = netdev_priv(dev); 1109163e5292SPaolo Abeni struct net_device *peer; 1110163e5292SPaolo Abeni 1111163e5292SPaolo Abeni if (new_hr < 0) 1112163e5292SPaolo Abeni new_hr = 0; 1113163e5292SPaolo Abeni 1114163e5292SPaolo Abeni rcu_read_lock(); 1115163e5292SPaolo Abeni peer = rcu_dereference(priv->peer); 1116163e5292SPaolo Abeni if (unlikely(!peer)) 1117163e5292SPaolo Abeni goto out; 1118163e5292SPaolo Abeni 1119163e5292SPaolo Abeni peer_priv = netdev_priv(peer); 1120163e5292SPaolo Abeni priv->requested_headroom = new_hr; 1121163e5292SPaolo Abeni new_hr = max(priv->requested_headroom, peer_priv->requested_headroom); 1122163e5292SPaolo Abeni dev->needed_headroom = new_hr; 1123163e5292SPaolo Abeni peer->needed_headroom = new_hr; 1124163e5292SPaolo Abeni 1125163e5292SPaolo Abeni out: 1126163e5292SPaolo Abeni rcu_read_unlock(); 1127163e5292SPaolo Abeni } 1128163e5292SPaolo Abeni 1129948d4f21SToshiaki Makita static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1130948d4f21SToshiaki Makita struct netlink_ext_ack *extack) 1131948d4f21SToshiaki Makita { 1132948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 1133948d4f21SToshiaki Makita struct bpf_prog *old_prog; 1134948d4f21SToshiaki Makita struct net_device *peer; 1135dc224822SToshiaki Makita unsigned int max_mtu; 1136948d4f21SToshiaki Makita int err; 1137948d4f21SToshiaki Makita 1138948d4f21SToshiaki Makita old_prog = priv->_xdp_prog; 1139948d4f21SToshiaki Makita priv->_xdp_prog = prog; 1140948d4f21SToshiaki Makita peer = rtnl_dereference(priv->peer); 1141948d4f21SToshiaki Makita 1142948d4f21SToshiaki Makita if (prog) { 1143948d4f21SToshiaki Makita if (!peer) { 1144948d4f21SToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached"); 1145948d4f21SToshiaki Makita err = -ENOTCONN; 1146948d4f21SToshiaki Makita goto err; 1147948d4f21SToshiaki Makita } 1148948d4f21SToshiaki Makita 1149dc224822SToshiaki Makita max_mtu = PAGE_SIZE - VETH_XDP_HEADROOM - 1150dc224822SToshiaki Makita peer->hard_header_len - 1151dc224822SToshiaki Makita SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1152dc224822SToshiaki Makita if (peer->mtu > max_mtu) { 1153dc224822SToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP"); 1154dc224822SToshiaki Makita err = -ERANGE; 1155dc224822SToshiaki Makita goto err; 1156dc224822SToshiaki Makita } 1157dc224822SToshiaki Makita 1158638264dcSToshiaki Makita if (dev->real_num_rx_queues < peer->real_num_tx_queues) { 1159638264dcSToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues"); 1160638264dcSToshiaki Makita err = -ENOSPC; 1161638264dcSToshiaki Makita goto err; 1162638264dcSToshiaki Makita } 1163638264dcSToshiaki Makita 1164948d4f21SToshiaki Makita if (dev->flags & IFF_UP) { 1165948d4f21SToshiaki Makita err = veth_enable_xdp(dev); 1166948d4f21SToshiaki Makita if (err) { 1167948d4f21SToshiaki Makita NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed"); 1168948d4f21SToshiaki Makita goto err; 1169948d4f21SToshiaki Makita } 1170948d4f21SToshiaki Makita } 1171dc224822SToshiaki Makita 1172dc224822SToshiaki Makita if (!old_prog) { 1173dc224822SToshiaki Makita peer->hw_features &= ~NETIF_F_GSO_SOFTWARE; 1174dc224822SToshiaki Makita peer->max_mtu = max_mtu; 1175dc224822SToshiaki Makita } 1176948d4f21SToshiaki Makita } 1177948d4f21SToshiaki Makita 1178948d4f21SToshiaki Makita if (old_prog) { 1179dc224822SToshiaki Makita if (!prog) { 1180dc224822SToshiaki Makita if (dev->flags & IFF_UP) 1181948d4f21SToshiaki Makita veth_disable_xdp(dev); 1182dc224822SToshiaki Makita 1183dc224822SToshiaki Makita if (peer) { 1184dc224822SToshiaki Makita peer->hw_features |= NETIF_F_GSO_SOFTWARE; 1185dc224822SToshiaki Makita peer->max_mtu = ETH_MAX_MTU; 1186dc224822SToshiaki Makita } 1187dc224822SToshiaki Makita } 1188948d4f21SToshiaki Makita bpf_prog_put(old_prog); 1189948d4f21SToshiaki Makita } 1190948d4f21SToshiaki Makita 1191dc224822SToshiaki Makita if ((!!old_prog ^ !!prog) && peer) 1192dc224822SToshiaki Makita netdev_update_features(peer); 1193dc224822SToshiaki Makita 1194948d4f21SToshiaki Makita return 0; 1195948d4f21SToshiaki Makita err: 1196948d4f21SToshiaki Makita priv->_xdp_prog = old_prog; 1197948d4f21SToshiaki Makita 1198948d4f21SToshiaki Makita return err; 1199948d4f21SToshiaki Makita } 1200948d4f21SToshiaki Makita 1201948d4f21SToshiaki Makita static u32 veth_xdp_query(struct net_device *dev) 1202948d4f21SToshiaki Makita { 1203948d4f21SToshiaki Makita struct veth_priv *priv = netdev_priv(dev); 1204948d4f21SToshiaki Makita const struct bpf_prog *xdp_prog; 1205948d4f21SToshiaki Makita 1206948d4f21SToshiaki Makita xdp_prog = priv->_xdp_prog; 1207948d4f21SToshiaki Makita if (xdp_prog) 1208948d4f21SToshiaki Makita return xdp_prog->aux->id; 1209948d4f21SToshiaki Makita 1210948d4f21SToshiaki Makita return 0; 1211948d4f21SToshiaki Makita } 1212948d4f21SToshiaki Makita 1213948d4f21SToshiaki Makita static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1214948d4f21SToshiaki Makita { 1215948d4f21SToshiaki Makita switch (xdp->command) { 1216948d4f21SToshiaki Makita case XDP_SETUP_PROG: 1217948d4f21SToshiaki Makita return veth_xdp_set(dev, xdp->prog, xdp->extack); 1218948d4f21SToshiaki Makita case XDP_QUERY_PROG: 1219948d4f21SToshiaki Makita xdp->prog_id = veth_xdp_query(dev); 1220948d4f21SToshiaki Makita return 0; 1221948d4f21SToshiaki Makita default: 1222948d4f21SToshiaki Makita return -EINVAL; 1223948d4f21SToshiaki Makita } 1224948d4f21SToshiaki Makita } 1225948d4f21SToshiaki Makita 12264456e7bdSStephen Hemminger static const struct net_device_ops veth_netdev_ops = { 12274456e7bdSStephen Hemminger .ndo_init = veth_dev_init, 12284456e7bdSStephen Hemminger .ndo_open = veth_open, 12292cf48a10SEric W. Biederman .ndo_stop = veth_close, 123000829823SStephen Hemminger .ndo_start_xmit = veth_xmit, 12316311cc44Sstephen hemminger .ndo_get_stats64 = veth_get_stats64, 12325c70ef85SGao feng .ndo_set_rx_mode = veth_set_multicast_list, 1233ee923623SDaniel Lezcano .ndo_set_mac_address = eth_mac_addr, 1234bb446c19SWANG Cong #ifdef CONFIG_NET_POLL_CONTROLLER 1235bb446c19SWANG Cong .ndo_poll_controller = veth_poll_controller, 1236bb446c19SWANG Cong #endif 1237a45253bfSNicolas Dichtel .ndo_get_iflink = veth_get_iflink, 1238dc224822SToshiaki Makita .ndo_fix_features = veth_fix_features, 12391a04a821SToshiaki Makita .ndo_features_check = passthru_features_check, 1240163e5292SPaolo Abeni .ndo_set_rx_headroom = veth_set_rx_headroom, 1241948d4f21SToshiaki Makita .ndo_bpf = veth_xdp, 12429152cff0SLorenzo Bianconi .ndo_xdp_xmit = veth_ndo_xdp_xmit, 12434456e7bdSStephen Hemminger }; 12444456e7bdSStephen Hemminger 1245732912d7SAlexander Duyck #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ 1246c80fafbbSXin Long NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \ 1247732912d7SAlexander Duyck NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \ 124828d2b136SPatrick McHardy NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \ 124928d2b136SPatrick McHardy NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX ) 12508093315aSEric Dumazet 1251e314dbdcSPavel Emelyanov static void veth_setup(struct net_device *dev) 1252e314dbdcSPavel Emelyanov { 1253e314dbdcSPavel Emelyanov ether_setup(dev); 1254e314dbdcSPavel Emelyanov 1255550fd08cSNeil Horman dev->priv_flags &= ~IFF_TX_SKB_SHARING; 125623ea5a96SHannes Frederic Sowa dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 125702f01ec1SPhil Sutter dev->priv_flags |= IFF_NO_QUEUE; 1258163e5292SPaolo Abeni dev->priv_flags |= IFF_PHONY_HEADROOM; 1259550fd08cSNeil Horman 12604456e7bdSStephen Hemminger dev->netdev_ops = &veth_netdev_ops; 1261e314dbdcSPavel Emelyanov dev->ethtool_ops = &veth_ethtool_ops; 1262e314dbdcSPavel Emelyanov dev->features |= NETIF_F_LLTX; 12638093315aSEric Dumazet dev->features |= VETH_FEATURES; 12648d0d21f4SToshiaki Makita dev->vlan_features = dev->features & 12653f8c707bSVlad Yasevich ~(NETIF_F_HW_VLAN_CTAG_TX | 12663f8c707bSVlad Yasevich NETIF_F_HW_VLAN_STAG_TX | 12673f8c707bSVlad Yasevich NETIF_F_HW_VLAN_CTAG_RX | 12683f8c707bSVlad Yasevich NETIF_F_HW_VLAN_STAG_RX); 1269cf124db5SDavid S. Miller dev->needs_free_netdev = true; 1270cf124db5SDavid S. Miller dev->priv_destructor = veth_dev_free; 127191572088SJarod Wilson dev->max_mtu = ETH_MAX_MTU; 1272a2c725faSMichał Mirosław 12738093315aSEric Dumazet dev->hw_features = VETH_FEATURES; 127482d81898SEric Dumazet dev->hw_enc_features = VETH_FEATURES; 1275607fca9aSDavid Ahern dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 1276e314dbdcSPavel Emelyanov } 1277e314dbdcSPavel Emelyanov 1278e314dbdcSPavel Emelyanov /* 1279e314dbdcSPavel Emelyanov * netlink interface 1280e314dbdcSPavel Emelyanov */ 1281e314dbdcSPavel Emelyanov 1282a8b8a889SMatthias Schiffer static int veth_validate(struct nlattr *tb[], struct nlattr *data[], 1283a8b8a889SMatthias Schiffer struct netlink_ext_ack *extack) 1284e314dbdcSPavel Emelyanov { 1285e314dbdcSPavel Emelyanov if (tb[IFLA_ADDRESS]) { 1286e314dbdcSPavel Emelyanov if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) 1287e314dbdcSPavel Emelyanov return -EINVAL; 1288e314dbdcSPavel Emelyanov if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) 1289e314dbdcSPavel Emelyanov return -EADDRNOTAVAIL; 1290e314dbdcSPavel Emelyanov } 129138d40815SEric Biederman if (tb[IFLA_MTU]) { 129238d40815SEric Biederman if (!is_valid_veth_mtu(nla_get_u32(tb[IFLA_MTU]))) 129338d40815SEric Biederman return -EINVAL; 129438d40815SEric Biederman } 1295e314dbdcSPavel Emelyanov return 0; 1296e314dbdcSPavel Emelyanov } 1297e314dbdcSPavel Emelyanov 1298e314dbdcSPavel Emelyanov static struct rtnl_link_ops veth_link_ops; 1299e314dbdcSPavel Emelyanov 130081adee47SEric W. Biederman static int veth_newlink(struct net *src_net, struct net_device *dev, 13017a3f4a18SMatthias Schiffer struct nlattr *tb[], struct nlattr *data[], 13027a3f4a18SMatthias Schiffer struct netlink_ext_ack *extack) 1303e314dbdcSPavel Emelyanov { 13047797b93bSToshiaki Makita int err; 1305e314dbdcSPavel Emelyanov struct net_device *peer; 1306e314dbdcSPavel Emelyanov struct veth_priv *priv; 1307e314dbdcSPavel Emelyanov char ifname[IFNAMSIZ]; 1308e314dbdcSPavel Emelyanov struct nlattr *peer_tb[IFLA_MAX + 1], **tbp; 13095517750fSTom Gundersen unsigned char name_assign_type; 13103729d502SPatrick McHardy struct ifinfomsg *ifmp; 131181adee47SEric W. Biederman struct net *net; 1312e314dbdcSPavel Emelyanov 1313e314dbdcSPavel Emelyanov /* 1314e314dbdcSPavel Emelyanov * create and register peer first 1315e314dbdcSPavel Emelyanov */ 1316e314dbdcSPavel Emelyanov if (data != NULL && data[VETH_INFO_PEER] != NULL) { 1317e314dbdcSPavel Emelyanov struct nlattr *nla_peer; 1318e314dbdcSPavel Emelyanov 1319e314dbdcSPavel Emelyanov nla_peer = data[VETH_INFO_PEER]; 13203729d502SPatrick McHardy ifmp = nla_data(nla_peer); 1321f7b12606SJiri Pirko err = rtnl_nla_parse_ifla(peer_tb, 1322e314dbdcSPavel Emelyanov nla_data(nla_peer) + sizeof(struct ifinfomsg), 1323fceb6435SJohannes Berg nla_len(nla_peer) - sizeof(struct ifinfomsg), 1324fceb6435SJohannes Berg NULL); 1325e314dbdcSPavel Emelyanov if (err < 0) 1326e314dbdcSPavel Emelyanov return err; 1327e314dbdcSPavel Emelyanov 1328a8b8a889SMatthias Schiffer err = veth_validate(peer_tb, NULL, extack); 1329e314dbdcSPavel Emelyanov if (err < 0) 1330e314dbdcSPavel Emelyanov return err; 1331e314dbdcSPavel Emelyanov 1332e314dbdcSPavel Emelyanov tbp = peer_tb; 13333729d502SPatrick McHardy } else { 13343729d502SPatrick McHardy ifmp = NULL; 1335e314dbdcSPavel Emelyanov tbp = tb; 13363729d502SPatrick McHardy } 1337e314dbdcSPavel Emelyanov 1338191cdb38SSerhey Popovych if (ifmp && tbp[IFLA_IFNAME]) { 1339e314dbdcSPavel Emelyanov nla_strlcpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 13405517750fSTom Gundersen name_assign_type = NET_NAME_USER; 13415517750fSTom Gundersen } else { 1342e314dbdcSPavel Emelyanov snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d"); 13435517750fSTom Gundersen name_assign_type = NET_NAME_ENUM; 13445517750fSTom Gundersen } 1345e314dbdcSPavel Emelyanov 134681adee47SEric W. Biederman net = rtnl_link_get_net(src_net, tbp); 134781adee47SEric W. Biederman if (IS_ERR(net)) 134881adee47SEric W. Biederman return PTR_ERR(net); 134981adee47SEric W. Biederman 13505517750fSTom Gundersen peer = rtnl_create_link(net, ifname, name_assign_type, 1351d0522f1cSDavid Ahern &veth_link_ops, tbp, extack); 135281adee47SEric W. Biederman if (IS_ERR(peer)) { 135381adee47SEric W. Biederman put_net(net); 1354e314dbdcSPavel Emelyanov return PTR_ERR(peer); 135581adee47SEric W. Biederman } 1356e314dbdcSPavel Emelyanov 1357191cdb38SSerhey Popovych if (!ifmp || !tbp[IFLA_ADDRESS]) 1358f2cedb63SDanny Kukawka eth_hw_addr_random(peer); 1359e314dbdcSPavel Emelyanov 1360e6f8f1a7SPavel Emelyanov if (ifmp && (dev->ifindex != 0)) 1361e6f8f1a7SPavel Emelyanov peer->ifindex = ifmp->ifi_index; 1362e6f8f1a7SPavel Emelyanov 136372d24955SStephen Hemminger peer->gso_max_size = dev->gso_max_size; 136472d24955SStephen Hemminger peer->gso_max_segs = dev->gso_max_segs; 136572d24955SStephen Hemminger 1366e314dbdcSPavel Emelyanov err = register_netdevice(peer); 136781adee47SEric W. Biederman put_net(net); 136881adee47SEric W. Biederman net = NULL; 1369e314dbdcSPavel Emelyanov if (err < 0) 1370e314dbdcSPavel Emelyanov goto err_register_peer; 1371e314dbdcSPavel Emelyanov 1372e314dbdcSPavel Emelyanov netif_carrier_off(peer); 1373e314dbdcSPavel Emelyanov 13743729d502SPatrick McHardy err = rtnl_configure_link(peer, ifmp); 13753729d502SPatrick McHardy if (err < 0) 13763729d502SPatrick McHardy goto err_configure_peer; 13773729d502SPatrick McHardy 1378e314dbdcSPavel Emelyanov /* 1379e314dbdcSPavel Emelyanov * register dev last 1380e314dbdcSPavel Emelyanov * 1381e314dbdcSPavel Emelyanov * note, that since we've registered new device the dev's name 1382e314dbdcSPavel Emelyanov * should be re-allocated 1383e314dbdcSPavel Emelyanov */ 1384e314dbdcSPavel Emelyanov 1385e314dbdcSPavel Emelyanov if (tb[IFLA_ADDRESS] == NULL) 1386f2cedb63SDanny Kukawka eth_hw_addr_random(dev); 1387e314dbdcSPavel Emelyanov 13886c8c4446SJiri Pirko if (tb[IFLA_IFNAME]) 13896c8c4446SJiri Pirko nla_strlcpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 13906c8c4446SJiri Pirko else 13916c8c4446SJiri Pirko snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d"); 13926c8c4446SJiri Pirko 1393e314dbdcSPavel Emelyanov err = register_netdevice(dev); 1394e314dbdcSPavel Emelyanov if (err < 0) 1395e314dbdcSPavel Emelyanov goto err_register_dev; 1396e314dbdcSPavel Emelyanov 1397e314dbdcSPavel Emelyanov netif_carrier_off(dev); 1398e314dbdcSPavel Emelyanov 1399e314dbdcSPavel Emelyanov /* 1400e314dbdcSPavel Emelyanov * tie the deviced together 1401e314dbdcSPavel Emelyanov */ 1402e314dbdcSPavel Emelyanov 1403e314dbdcSPavel Emelyanov priv = netdev_priv(dev); 1404d0e2c55eSEric Dumazet rcu_assign_pointer(priv->peer, peer); 1405e314dbdcSPavel Emelyanov 1406e314dbdcSPavel Emelyanov priv = netdev_priv(peer); 1407d0e2c55eSEric Dumazet rcu_assign_pointer(priv->peer, dev); 1408948d4f21SToshiaki Makita 1409e314dbdcSPavel Emelyanov return 0; 1410e314dbdcSPavel Emelyanov 1411e314dbdcSPavel Emelyanov err_register_dev: 1412e314dbdcSPavel Emelyanov /* nothing to do */ 14133729d502SPatrick McHardy err_configure_peer: 1414e314dbdcSPavel Emelyanov unregister_netdevice(peer); 1415e314dbdcSPavel Emelyanov return err; 1416e314dbdcSPavel Emelyanov 1417e314dbdcSPavel Emelyanov err_register_peer: 1418e314dbdcSPavel Emelyanov free_netdev(peer); 1419e314dbdcSPavel Emelyanov return err; 1420e314dbdcSPavel Emelyanov } 1421e314dbdcSPavel Emelyanov 142223289a37SEric Dumazet static void veth_dellink(struct net_device *dev, struct list_head *head) 1423e314dbdcSPavel Emelyanov { 1424e314dbdcSPavel Emelyanov struct veth_priv *priv; 1425e314dbdcSPavel Emelyanov struct net_device *peer; 1426e314dbdcSPavel Emelyanov 1427e314dbdcSPavel Emelyanov priv = netdev_priv(dev); 1428d0e2c55eSEric Dumazet peer = rtnl_dereference(priv->peer); 1429d0e2c55eSEric Dumazet 1430d0e2c55eSEric Dumazet /* Note : dellink() is called from default_device_exit_batch(), 1431d0e2c55eSEric Dumazet * before a rcu_synchronize() point. The devices are guaranteed 1432d0e2c55eSEric Dumazet * not being freed before one RCU grace period. 1433d0e2c55eSEric Dumazet */ 1434d0e2c55eSEric Dumazet RCU_INIT_POINTER(priv->peer, NULL); 1435f45a5c26SEric Dumazet unregister_netdevice_queue(dev, head); 1436d0e2c55eSEric Dumazet 1437f45a5c26SEric Dumazet if (peer) { 1438d0e2c55eSEric Dumazet priv = netdev_priv(peer); 1439d0e2c55eSEric Dumazet RCU_INIT_POINTER(priv->peer, NULL); 144024540535SEric Dumazet unregister_netdevice_queue(peer, head); 1441e314dbdcSPavel Emelyanov } 1442f45a5c26SEric Dumazet } 1443e314dbdcSPavel Emelyanov 144423711438SThomas Graf static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = { 144523711438SThomas Graf [VETH_INFO_PEER] = { .len = sizeof(struct ifinfomsg) }, 144623711438SThomas Graf }; 1447e314dbdcSPavel Emelyanov 1448e5f4e7b9SNicolas Dichtel static struct net *veth_get_link_net(const struct net_device *dev) 1449e5f4e7b9SNicolas Dichtel { 1450e5f4e7b9SNicolas Dichtel struct veth_priv *priv = netdev_priv(dev); 1451e5f4e7b9SNicolas Dichtel struct net_device *peer = rtnl_dereference(priv->peer); 1452e5f4e7b9SNicolas Dichtel 1453e5f4e7b9SNicolas Dichtel return peer ? dev_net(peer) : dev_net(dev); 1454e5f4e7b9SNicolas Dichtel } 1455e5f4e7b9SNicolas Dichtel 1456e314dbdcSPavel Emelyanov static struct rtnl_link_ops veth_link_ops = { 1457e314dbdcSPavel Emelyanov .kind = DRV_NAME, 1458e314dbdcSPavel Emelyanov .priv_size = sizeof(struct veth_priv), 1459e314dbdcSPavel Emelyanov .setup = veth_setup, 1460e314dbdcSPavel Emelyanov .validate = veth_validate, 1461e314dbdcSPavel Emelyanov .newlink = veth_newlink, 1462e314dbdcSPavel Emelyanov .dellink = veth_dellink, 1463e314dbdcSPavel Emelyanov .policy = veth_policy, 1464e314dbdcSPavel Emelyanov .maxtype = VETH_INFO_MAX, 1465e5f4e7b9SNicolas Dichtel .get_link_net = veth_get_link_net, 1466e314dbdcSPavel Emelyanov }; 1467e314dbdcSPavel Emelyanov 1468e314dbdcSPavel Emelyanov /* 1469e314dbdcSPavel Emelyanov * init/fini 1470e314dbdcSPavel Emelyanov */ 1471e314dbdcSPavel Emelyanov 1472e314dbdcSPavel Emelyanov static __init int veth_init(void) 1473e314dbdcSPavel Emelyanov { 1474e314dbdcSPavel Emelyanov return rtnl_link_register(&veth_link_ops); 1475e314dbdcSPavel Emelyanov } 1476e314dbdcSPavel Emelyanov 1477e314dbdcSPavel Emelyanov static __exit void veth_exit(void) 1478e314dbdcSPavel Emelyanov { 147968365458SPatrick McHardy rtnl_link_unregister(&veth_link_ops); 1480e314dbdcSPavel Emelyanov } 1481e314dbdcSPavel Emelyanov 1482e314dbdcSPavel Emelyanov module_init(veth_init); 1483e314dbdcSPavel Emelyanov module_exit(veth_exit); 1484e314dbdcSPavel Emelyanov 1485e314dbdcSPavel Emelyanov MODULE_DESCRIPTION("Virtual Ethernet Tunnel"); 1486e314dbdcSPavel Emelyanov MODULE_LICENSE("GPL v2"); 1487e314dbdcSPavel Emelyanov MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1488