1e689cf4aSJeff Kirsher /* sunvnet.c: Sun LDOM Virtual Network Driver. 2e689cf4aSJeff Kirsher * 3e689cf4aSJeff Kirsher * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4e689cf4aSJeff Kirsher */ 5e689cf4aSJeff Kirsher 6e689cf4aSJeff Kirsher #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7e689cf4aSJeff Kirsher 8e689cf4aSJeff Kirsher #include <linux/module.h> 9e689cf4aSJeff Kirsher #include <linux/kernel.h> 10e689cf4aSJeff Kirsher #include <linux/types.h> 11e689cf4aSJeff Kirsher #include <linux/slab.h> 12e689cf4aSJeff Kirsher #include <linux/delay.h> 13e689cf4aSJeff Kirsher #include <linux/init.h> 14e689cf4aSJeff Kirsher #include <linux/netdevice.h> 15e689cf4aSJeff Kirsher #include <linux/ethtool.h> 16e689cf4aSJeff Kirsher #include <linux/etherdevice.h> 17e689cf4aSJeff Kirsher #include <linux/mutex.h> 18da38c564SDavid L Stevens #include <linux/highmem.h> 19e4defc77SDavid L Stevens #include <linux/if_vlan.h> 20e689cf4aSJeff Kirsher 21a2b78e9bSDavid L Stevens #if IS_ENABLED(CONFIG_IPV6) 22a2b78e9bSDavid L Stevens #include <linux/icmpv6.h> 23a2b78e9bSDavid L Stevens #endif 24a2b78e9bSDavid L Stevens 256d0ba919SDavid L Stevens #include <net/ip.h> 26a2b78e9bSDavid L Stevens #include <net/icmp.h> 27a2b78e9bSDavid L Stevens #include <net/route.h> 28a2b78e9bSDavid L Stevens 29e689cf4aSJeff Kirsher #include <asm/vio.h> 30e689cf4aSJeff Kirsher #include <asm/ldc.h> 31e689cf4aSJeff Kirsher 32e689cf4aSJeff Kirsher #include "sunvnet.h" 33e689cf4aSJeff Kirsher 34e689cf4aSJeff Kirsher #define DRV_MODULE_NAME "sunvnet" 35e689cf4aSJeff Kirsher #define DRV_MODULE_VERSION "1.0" 36e689cf4aSJeff Kirsher #define DRV_MODULE_RELDATE "June 25, 2007" 37e689cf4aSJeff Kirsher 38f73d12bdSBill Pemberton static char version[] = 39e689cf4aSJeff Kirsher DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 40e689cf4aSJeff Kirsher MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); 41e689cf4aSJeff Kirsher MODULE_DESCRIPTION("Sun LDOM virtual network driver"); 42e689cf4aSJeff Kirsher MODULE_LICENSE("GPL"); 43e689cf4aSJeff Kirsher MODULE_VERSION(DRV_MODULE_VERSION); 44e689cf4aSJeff Kirsher 45d51bffd1SSowmini Varadhan #define VNET_MAX_TXQS 16 46d51bffd1SSowmini Varadhan 47adddc32dSSowmini Varadhan /* Heuristic for the number of times to exponentially backoff and 48adddc32dSSowmini Varadhan * retry sending an LDC trigger when EAGAIN is encountered 49adddc32dSSowmini Varadhan */ 50adddc32dSSowmini Varadhan #define VNET_MAX_RETRIES 10 51adddc32dSSowmini Varadhan 52d1015645SSowmini Varadhan static int __vnet_tx_trigger(struct vnet_port *port, u32 start); 53d1015645SSowmini Varadhan 54e689cf4aSJeff Kirsher /* Ordered from largest major to lowest */ 55e689cf4aSJeff Kirsher static struct vio_version vnet_versions[] = { 566d0ba919SDavid L Stevens { .major = 1, .minor = 8 }, 576d0ba919SDavid L Stevens { .major = 1, .minor = 7 }, 58e4defc77SDavid L Stevens { .major = 1, .minor = 6 }, 59e689cf4aSJeff Kirsher { .major = 1, .minor = 0 }, 60e689cf4aSJeff Kirsher }; 61e689cf4aSJeff Kirsher 62e689cf4aSJeff Kirsher static inline u32 vnet_tx_dring_avail(struct vio_dring_state *dr) 63e689cf4aSJeff Kirsher { 64e689cf4aSJeff Kirsher return vio_dring_avail(dr, VNET_TX_RING_SIZE); 65e689cf4aSJeff Kirsher } 66e689cf4aSJeff Kirsher 67e689cf4aSJeff Kirsher static int vnet_handle_unknown(struct vnet_port *port, void *arg) 68e689cf4aSJeff Kirsher { 69e689cf4aSJeff Kirsher struct vio_msg_tag *pkt = arg; 70e689cf4aSJeff Kirsher 71e689cf4aSJeff Kirsher pr_err("Received unknown msg [%02x:%02x:%04x:%08x]\n", 72e689cf4aSJeff Kirsher pkt->type, pkt->stype, pkt->stype_env, pkt->sid); 73e689cf4aSJeff Kirsher pr_err("Resetting connection\n"); 74e689cf4aSJeff Kirsher 75e689cf4aSJeff Kirsher ldc_disconnect(port->vio.lp); 76e689cf4aSJeff Kirsher 77e689cf4aSJeff Kirsher return -ECONNRESET; 78e689cf4aSJeff Kirsher } 79e689cf4aSJeff Kirsher 80d6732489SDavid L Stevens static int vnet_port_alloc_tx_ring(struct vnet_port *port); 81d6732489SDavid L Stevens 82e689cf4aSJeff Kirsher static int vnet_send_attr(struct vio_driver_state *vio) 83e689cf4aSJeff Kirsher { 84e689cf4aSJeff Kirsher struct vnet_port *port = to_vnet_port(vio); 85e689cf4aSJeff Kirsher struct net_device *dev = port->vp->dev; 86e689cf4aSJeff Kirsher struct vio_net_attr_info pkt; 87e4defc77SDavid L Stevens int framelen = ETH_FRAME_LEN; 88d6732489SDavid L Stevens int i, err; 89d6732489SDavid L Stevens 90d6732489SDavid L Stevens err = vnet_port_alloc_tx_ring(to_vnet_port(vio)); 91d6732489SDavid L Stevens if (err) 92d6732489SDavid L Stevens return err; 93e689cf4aSJeff Kirsher 94e689cf4aSJeff Kirsher memset(&pkt, 0, sizeof(pkt)); 95e689cf4aSJeff Kirsher pkt.tag.type = VIO_TYPE_CTRL; 96e689cf4aSJeff Kirsher pkt.tag.stype = VIO_SUBTYPE_INFO; 97e689cf4aSJeff Kirsher pkt.tag.stype_env = VIO_ATTR_INFO; 98e689cf4aSJeff Kirsher pkt.tag.sid = vio_send_sid(vio); 99e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 2)) 100e689cf4aSJeff Kirsher pkt.xfer_mode = VIO_DRING_MODE; 101e4defc77SDavid L Stevens else 102e4defc77SDavid L Stevens pkt.xfer_mode = VIO_NEW_DRING_MODE; 103e689cf4aSJeff Kirsher pkt.addr_type = VNET_ADDR_ETHERMAC; 104e689cf4aSJeff Kirsher pkt.ack_freq = 0; 105e689cf4aSJeff Kirsher for (i = 0; i < 6; i++) 106e689cf4aSJeff Kirsher pkt.addr |= (u64)dev->dev_addr[i] << ((5 - i) * 8); 107e4defc77SDavid L Stevens if (vio_version_after(vio, 1, 3)) { 108e4defc77SDavid L Stevens if (port->rmtu) { 109e4defc77SDavid L Stevens port->rmtu = min(VNET_MAXPACKET, port->rmtu); 110e4defc77SDavid L Stevens pkt.mtu = port->rmtu; 111e4defc77SDavid L Stevens } else { 112e4defc77SDavid L Stevens port->rmtu = VNET_MAXPACKET; 113e4defc77SDavid L Stevens pkt.mtu = port->rmtu; 114e4defc77SDavid L Stevens } 115e4defc77SDavid L Stevens if (vio_version_after_eq(vio, 1, 6)) 116e4defc77SDavid L Stevens pkt.options = VIO_TX_DRING; 117e4defc77SDavid L Stevens } else if (vio_version_before(vio, 1, 3)) { 118e4defc77SDavid L Stevens pkt.mtu = framelen; 119e4defc77SDavid L Stevens } else { /* v1.3 */ 120e4defc77SDavid L Stevens pkt.mtu = framelen + VLAN_HLEN; 121e4defc77SDavid L Stevens } 122e4defc77SDavid L Stevens 123e4defc77SDavid L Stevens pkt.plnk_updt = PHYSLINK_UPDATE_NONE; 124e4defc77SDavid L Stevens pkt.cflags = 0; 125e689cf4aSJeff Kirsher 126e689cf4aSJeff Kirsher viodbg(HS, "SEND NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 127e4defc77SDavid L Stevens "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 128e4defc77SDavid L Stevens "cflags[0x%04x] lso_max[%u]\n", 129e689cf4aSJeff Kirsher pkt.xfer_mode, pkt.addr_type, 130e689cf4aSJeff Kirsher (unsigned long long)pkt.addr, 131e4defc77SDavid L Stevens pkt.ack_freq, pkt.plnk_updt, pkt.options, 132e4defc77SDavid L Stevens (unsigned long long)pkt.mtu, pkt.cflags, pkt.ipv4_lso_maxlen); 133e4defc77SDavid L Stevens 134e689cf4aSJeff Kirsher 135e689cf4aSJeff Kirsher return vio_ldc_send(vio, &pkt, sizeof(pkt)); 136e689cf4aSJeff Kirsher } 137e689cf4aSJeff Kirsher 138e689cf4aSJeff Kirsher static int handle_attr_info(struct vio_driver_state *vio, 139e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 140e689cf4aSJeff Kirsher { 141e4defc77SDavid L Stevens struct vnet_port *port = to_vnet_port(vio); 142e4defc77SDavid L Stevens u64 localmtu; 143e4defc77SDavid L Stevens u8 xfer_mode; 144e4defc77SDavid L Stevens 145e4defc77SDavid L Stevens viodbg(HS, "GOT NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 146e4defc77SDavid L Stevens "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 147e4defc77SDavid L Stevens " (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 148e689cf4aSJeff Kirsher pkt->xfer_mode, pkt->addr_type, 149e689cf4aSJeff Kirsher (unsigned long long)pkt->addr, 150e4defc77SDavid L Stevens pkt->ack_freq, pkt->plnk_updt, pkt->options, 151e4defc77SDavid L Stevens (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 152e4defc77SDavid L Stevens pkt->ipv4_lso_maxlen); 153e689cf4aSJeff Kirsher 154e689cf4aSJeff Kirsher pkt->tag.sid = vio_send_sid(vio); 155e689cf4aSJeff Kirsher 156e4defc77SDavid L Stevens xfer_mode = pkt->xfer_mode; 157e4defc77SDavid L Stevens /* for version < 1.2, VIO_DRING_MODE = 0x3 and no bitmask */ 158e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 2) && xfer_mode == VIO_DRING_MODE) 159e4defc77SDavid L Stevens xfer_mode = VIO_NEW_DRING_MODE; 160e4defc77SDavid L Stevens 161e4defc77SDavid L Stevens /* MTU negotiation: 162e4defc77SDavid L Stevens * < v1.3 - ETH_FRAME_LEN exactly 163e4defc77SDavid L Stevens * > v1.3 - MIN(pkt.mtu, VNET_MAXPACKET, port->rmtu) and change 164e4defc77SDavid L Stevens * pkt->mtu for ACK 165e4defc77SDavid L Stevens * = v1.3 - ETH_FRAME_LEN + VLAN_HLEN exactly 166e4defc77SDavid L Stevens */ 167e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 3)) { 168e4defc77SDavid L Stevens localmtu = ETH_FRAME_LEN; 169e4defc77SDavid L Stevens } else if (vio_version_after(vio, 1, 3)) { 170e4defc77SDavid L Stevens localmtu = port->rmtu ? port->rmtu : VNET_MAXPACKET; 171e4defc77SDavid L Stevens localmtu = min(pkt->mtu, localmtu); 172e4defc77SDavid L Stevens pkt->mtu = localmtu; 173e4defc77SDavid L Stevens } else { /* v1.3 */ 174e4defc77SDavid L Stevens localmtu = ETH_FRAME_LEN + VLAN_HLEN; 175e4defc77SDavid L Stevens } 176e4defc77SDavid L Stevens port->rmtu = localmtu; 177e4defc77SDavid L Stevens 178e4defc77SDavid L Stevens /* for version >= 1.6, ACK packet mode we support */ 179e4defc77SDavid L Stevens if (vio_version_after_eq(vio, 1, 6)) { 180e4defc77SDavid L Stevens pkt->xfer_mode = VIO_NEW_DRING_MODE; 181e4defc77SDavid L Stevens pkt->options = VIO_TX_DRING; 182e4defc77SDavid L Stevens } 183e4defc77SDavid L Stevens 184e4defc77SDavid L Stevens if (!(xfer_mode | VIO_NEW_DRING_MODE) || 185e689cf4aSJeff Kirsher pkt->addr_type != VNET_ADDR_ETHERMAC || 186e4defc77SDavid L Stevens pkt->mtu != localmtu) { 187e689cf4aSJeff Kirsher viodbg(HS, "SEND NET ATTR NACK\n"); 188e689cf4aSJeff Kirsher 189e689cf4aSJeff Kirsher pkt->tag.stype = VIO_SUBTYPE_NACK; 190e689cf4aSJeff Kirsher 191e689cf4aSJeff Kirsher (void) vio_ldc_send(vio, pkt, sizeof(*pkt)); 192e689cf4aSJeff Kirsher 193e689cf4aSJeff Kirsher return -ECONNRESET; 194e689cf4aSJeff Kirsher } else { 195e4defc77SDavid L Stevens viodbg(HS, "SEND NET ATTR ACK xmode[0x%x] atype[0x%x] " 196e4defc77SDavid L Stevens "addr[%llx] ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] " 197e4defc77SDavid L Stevens "mtu[%llu] (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 198e4defc77SDavid L Stevens pkt->xfer_mode, pkt->addr_type, 199e4defc77SDavid L Stevens (unsigned long long)pkt->addr, 200e4defc77SDavid L Stevens pkt->ack_freq, pkt->plnk_updt, pkt->options, 201e4defc77SDavid L Stevens (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 202e4defc77SDavid L Stevens pkt->ipv4_lso_maxlen); 203e689cf4aSJeff Kirsher 204e689cf4aSJeff Kirsher pkt->tag.stype = VIO_SUBTYPE_ACK; 205e689cf4aSJeff Kirsher 206e689cf4aSJeff Kirsher return vio_ldc_send(vio, pkt, sizeof(*pkt)); 207e689cf4aSJeff Kirsher } 208e689cf4aSJeff Kirsher 209e689cf4aSJeff Kirsher } 210e689cf4aSJeff Kirsher 211e689cf4aSJeff Kirsher static int handle_attr_ack(struct vio_driver_state *vio, 212e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 213e689cf4aSJeff Kirsher { 214e689cf4aSJeff Kirsher viodbg(HS, "GOT NET ATTR ACK\n"); 215e689cf4aSJeff Kirsher 216e689cf4aSJeff Kirsher return 0; 217e689cf4aSJeff Kirsher } 218e689cf4aSJeff Kirsher 219e689cf4aSJeff Kirsher static int handle_attr_nack(struct vio_driver_state *vio, 220e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 221e689cf4aSJeff Kirsher { 222e689cf4aSJeff Kirsher viodbg(HS, "GOT NET ATTR NACK\n"); 223e689cf4aSJeff Kirsher 224e689cf4aSJeff Kirsher return -ECONNRESET; 225e689cf4aSJeff Kirsher } 226e689cf4aSJeff Kirsher 227e689cf4aSJeff Kirsher static int vnet_handle_attr(struct vio_driver_state *vio, void *arg) 228e689cf4aSJeff Kirsher { 229e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt = arg; 230e689cf4aSJeff Kirsher 231e689cf4aSJeff Kirsher switch (pkt->tag.stype) { 232e689cf4aSJeff Kirsher case VIO_SUBTYPE_INFO: 233e689cf4aSJeff Kirsher return handle_attr_info(vio, pkt); 234e689cf4aSJeff Kirsher 235e689cf4aSJeff Kirsher case VIO_SUBTYPE_ACK: 236e689cf4aSJeff Kirsher return handle_attr_ack(vio, pkt); 237e689cf4aSJeff Kirsher 238e689cf4aSJeff Kirsher case VIO_SUBTYPE_NACK: 239e689cf4aSJeff Kirsher return handle_attr_nack(vio, pkt); 240e689cf4aSJeff Kirsher 241e689cf4aSJeff Kirsher default: 242e689cf4aSJeff Kirsher return -ECONNRESET; 243e689cf4aSJeff Kirsher } 244e689cf4aSJeff Kirsher } 245e689cf4aSJeff Kirsher 246e689cf4aSJeff Kirsher static void vnet_handshake_complete(struct vio_driver_state *vio) 247e689cf4aSJeff Kirsher { 248e689cf4aSJeff Kirsher struct vio_dring_state *dr; 249e689cf4aSJeff Kirsher 250e689cf4aSJeff Kirsher dr = &vio->drings[VIO_DRIVER_RX_RING]; 251e689cf4aSJeff Kirsher dr->snd_nxt = dr->rcv_nxt = 1; 252e689cf4aSJeff Kirsher 253e689cf4aSJeff Kirsher dr = &vio->drings[VIO_DRIVER_TX_RING]; 254e689cf4aSJeff Kirsher dr->snd_nxt = dr->rcv_nxt = 1; 255e689cf4aSJeff Kirsher } 256e689cf4aSJeff Kirsher 257e689cf4aSJeff Kirsher /* The hypervisor interface that implements copying to/from imported 258e689cf4aSJeff Kirsher * memory from another domain requires that copies are done to 8-byte 259e689cf4aSJeff Kirsher * aligned buffers, and that the lengths of such copies are also 8-byte 260e689cf4aSJeff Kirsher * multiples. 261e689cf4aSJeff Kirsher * 262e689cf4aSJeff Kirsher * So we align skb->data to an 8-byte multiple and pad-out the data 263e689cf4aSJeff Kirsher * area so we can round the copy length up to the next multiple of 264e689cf4aSJeff Kirsher * 8 for the copy. 265e689cf4aSJeff Kirsher * 266e689cf4aSJeff Kirsher * The transmitter puts the actual start of the packet 6 bytes into 267e689cf4aSJeff Kirsher * the buffer it sends over, so that the IP headers after the ethernet 268e689cf4aSJeff Kirsher * header are aligned properly. These 6 bytes are not in the descriptor 269e689cf4aSJeff Kirsher * length, they are simply implied. This offset is represented using 270e689cf4aSJeff Kirsher * the VNET_PACKET_SKIP macro. 271e689cf4aSJeff Kirsher */ 272e689cf4aSJeff Kirsher static struct sk_buff *alloc_and_align_skb(struct net_device *dev, 273e689cf4aSJeff Kirsher unsigned int len) 274e689cf4aSJeff Kirsher { 275e689cf4aSJeff Kirsher struct sk_buff *skb = netdev_alloc_skb(dev, len+VNET_PACKET_SKIP+8+8); 276e689cf4aSJeff Kirsher unsigned long addr, off; 277e689cf4aSJeff Kirsher 278e689cf4aSJeff Kirsher if (unlikely(!skb)) 279e689cf4aSJeff Kirsher return NULL; 280e689cf4aSJeff Kirsher 281e689cf4aSJeff Kirsher addr = (unsigned long) skb->data; 282e689cf4aSJeff Kirsher off = ((addr + 7UL) & ~7UL) - addr; 283e689cf4aSJeff Kirsher if (off) 284e689cf4aSJeff Kirsher skb_reserve(skb, off); 285e689cf4aSJeff Kirsher 286e689cf4aSJeff Kirsher return skb; 287e689cf4aSJeff Kirsher } 288e689cf4aSJeff Kirsher 2896d0ba919SDavid L Stevens static inline void vnet_fullcsum(struct sk_buff *skb) 2906d0ba919SDavid L Stevens { 2916d0ba919SDavid L Stevens struct iphdr *iph = ip_hdr(skb); 2926d0ba919SDavid L Stevens int offset = skb_transport_offset(skb); 2936d0ba919SDavid L Stevens 2946d0ba919SDavid L Stevens if (skb->protocol != htons(ETH_P_IP)) 2956d0ba919SDavid L Stevens return; 2966d0ba919SDavid L Stevens if (iph->protocol != IPPROTO_TCP && 2976d0ba919SDavid L Stevens iph->protocol != IPPROTO_UDP) 2986d0ba919SDavid L Stevens return; 2996d0ba919SDavid L Stevens skb->ip_summed = CHECKSUM_NONE; 3006d0ba919SDavid L Stevens skb->csum_level = 1; 3016d0ba919SDavid L Stevens skb->csum = 0; 3026d0ba919SDavid L Stevens if (iph->protocol == IPPROTO_TCP) { 3036d0ba919SDavid L Stevens struct tcphdr *ptcp = tcp_hdr(skb); 3046d0ba919SDavid L Stevens 3056d0ba919SDavid L Stevens ptcp->check = 0; 3066d0ba919SDavid L Stevens skb->csum = skb_checksum(skb, offset, skb->len - offset, 0); 3076d0ba919SDavid L Stevens ptcp->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 3086d0ba919SDavid L Stevens skb->len - offset, IPPROTO_TCP, 3096d0ba919SDavid L Stevens skb->csum); 3106d0ba919SDavid L Stevens } else if (iph->protocol == IPPROTO_UDP) { 3116d0ba919SDavid L Stevens struct udphdr *pudp = udp_hdr(skb); 3126d0ba919SDavid L Stevens 3136d0ba919SDavid L Stevens pudp->check = 0; 3146d0ba919SDavid L Stevens skb->csum = skb_checksum(skb, offset, skb->len - offset, 0); 3156d0ba919SDavid L Stevens pudp->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 3166d0ba919SDavid L Stevens skb->len - offset, IPPROTO_UDP, 3176d0ba919SDavid L Stevens skb->csum); 3186d0ba919SDavid L Stevens } 3196d0ba919SDavid L Stevens } 3206d0ba919SDavid L Stevens 3216d0ba919SDavid L Stevens static int vnet_rx_one(struct vnet_port *port, struct vio_net_desc *desc) 322e689cf4aSJeff Kirsher { 323e689cf4aSJeff Kirsher struct net_device *dev = port->vp->dev; 3246d0ba919SDavid L Stevens unsigned int len = desc->size; 325e689cf4aSJeff Kirsher unsigned int copy_len; 326e689cf4aSJeff Kirsher struct sk_buff *skb; 327e689cf4aSJeff Kirsher int err; 328e689cf4aSJeff Kirsher 329e689cf4aSJeff Kirsher err = -EMSGSIZE; 330e4defc77SDavid L Stevens if (unlikely(len < ETH_ZLEN || len > port->rmtu)) { 331e689cf4aSJeff Kirsher dev->stats.rx_length_errors++; 332e689cf4aSJeff Kirsher goto out_dropped; 333e689cf4aSJeff Kirsher } 334e689cf4aSJeff Kirsher 335e689cf4aSJeff Kirsher skb = alloc_and_align_skb(dev, len); 336e689cf4aSJeff Kirsher err = -ENOMEM; 337e689cf4aSJeff Kirsher if (unlikely(!skb)) { 338e689cf4aSJeff Kirsher dev->stats.rx_missed_errors++; 339e689cf4aSJeff Kirsher goto out_dropped; 340e689cf4aSJeff Kirsher } 341e689cf4aSJeff Kirsher 342e689cf4aSJeff Kirsher copy_len = (len + VNET_PACKET_SKIP + 7U) & ~7U; 343e689cf4aSJeff Kirsher skb_put(skb, copy_len); 344e689cf4aSJeff Kirsher err = ldc_copy(port->vio.lp, LDC_COPY_IN, 345e689cf4aSJeff Kirsher skb->data, copy_len, 0, 3466d0ba919SDavid L Stevens desc->cookies, desc->ncookies); 347e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 348e689cf4aSJeff Kirsher dev->stats.rx_frame_errors++; 349e689cf4aSJeff Kirsher goto out_free_skb; 350e689cf4aSJeff Kirsher } 351e689cf4aSJeff Kirsher 352e689cf4aSJeff Kirsher skb_pull(skb, VNET_PACKET_SKIP); 353e689cf4aSJeff Kirsher skb_trim(skb, len); 354e689cf4aSJeff Kirsher skb->protocol = eth_type_trans(skb, dev); 355e689cf4aSJeff Kirsher 3566d0ba919SDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 8)) { 3576d0ba919SDavid L Stevens struct vio_net_dext *dext = vio_net_ext(desc); 3586d0ba919SDavid L Stevens 3596d0ba919SDavid L Stevens if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM) { 3606d0ba919SDavid L Stevens if (skb->protocol == ETH_P_IP) { 3616d0ba919SDavid L Stevens struct iphdr *iph = (struct iphdr *)skb->data; 3626d0ba919SDavid L Stevens 3636d0ba919SDavid L Stevens iph->check = 0; 3646d0ba919SDavid L Stevens ip_send_check(iph); 3656d0ba919SDavid L Stevens } 3666d0ba919SDavid L Stevens } 3676d0ba919SDavid L Stevens if ((dext->flags & VNET_PKT_HCK_FULLCKSUM) && 3686d0ba919SDavid L Stevens skb->ip_summed == CHECKSUM_NONE) 3696d0ba919SDavid L Stevens vnet_fullcsum(skb); 3706d0ba919SDavid L Stevens if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM_OK) { 3716d0ba919SDavid L Stevens skb->ip_summed = CHECKSUM_PARTIAL; 3726d0ba919SDavid L Stevens skb->csum_level = 0; 3736d0ba919SDavid L Stevens if (dext->flags & VNET_PKT_HCK_FULLCKSUM_OK) 3746d0ba919SDavid L Stevens skb->csum_level = 1; 3756d0ba919SDavid L Stevens } 3766d0ba919SDavid L Stevens } 3776d0ba919SDavid L Stevens 3781b6b0a47SDavid L Stevens skb->ip_summed = port->switch_port ? CHECKSUM_NONE : CHECKSUM_PARTIAL; 3791b6b0a47SDavid L Stevens 380e689cf4aSJeff Kirsher dev->stats.rx_packets++; 381e689cf4aSJeff Kirsher dev->stats.rx_bytes += len; 38269088822SSowmini Varadhan napi_gro_receive(&port->napi, skb); 383e689cf4aSJeff Kirsher return 0; 384e689cf4aSJeff Kirsher 385e689cf4aSJeff Kirsher out_free_skb: 386e689cf4aSJeff Kirsher kfree_skb(skb); 387e689cf4aSJeff Kirsher 388e689cf4aSJeff Kirsher out_dropped: 389e689cf4aSJeff Kirsher dev->stats.rx_dropped++; 390e689cf4aSJeff Kirsher return err; 391e689cf4aSJeff Kirsher } 392e689cf4aSJeff Kirsher 393e689cf4aSJeff Kirsher static int vnet_send_ack(struct vnet_port *port, struct vio_dring_state *dr, 394e689cf4aSJeff Kirsher u32 start, u32 end, u8 vio_dring_state) 395e689cf4aSJeff Kirsher { 396e689cf4aSJeff Kirsher struct vio_dring_data hdr = { 397e689cf4aSJeff Kirsher .tag = { 398e689cf4aSJeff Kirsher .type = VIO_TYPE_DATA, 399e689cf4aSJeff Kirsher .stype = VIO_SUBTYPE_ACK, 400e689cf4aSJeff Kirsher .stype_env = VIO_DRING_DATA, 401e689cf4aSJeff Kirsher .sid = vio_send_sid(&port->vio), 402e689cf4aSJeff Kirsher }, 403e689cf4aSJeff Kirsher .dring_ident = dr->ident, 404e689cf4aSJeff Kirsher .start_idx = start, 405e689cf4aSJeff Kirsher .end_idx = end, 406e689cf4aSJeff Kirsher .state = vio_dring_state, 407e689cf4aSJeff Kirsher }; 408e689cf4aSJeff Kirsher int err, delay; 409adddc32dSSowmini Varadhan int retries = 0; 410e689cf4aSJeff Kirsher 411e689cf4aSJeff Kirsher hdr.seq = dr->snd_nxt; 412e689cf4aSJeff Kirsher delay = 1; 413e689cf4aSJeff Kirsher do { 414e689cf4aSJeff Kirsher err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 415e689cf4aSJeff Kirsher if (err > 0) { 416e689cf4aSJeff Kirsher dr->snd_nxt++; 417e689cf4aSJeff Kirsher break; 418e689cf4aSJeff Kirsher } 419e689cf4aSJeff Kirsher udelay(delay); 420e689cf4aSJeff Kirsher if ((delay <<= 1) > 128) 421e689cf4aSJeff Kirsher delay = 128; 422adddc32dSSowmini Varadhan if (retries++ > VNET_MAX_RETRIES) { 423adddc32dSSowmini Varadhan pr_info("ECONNRESET %x:%x:%x:%x:%x:%x\n", 424adddc32dSSowmini Varadhan port->raddr[0], port->raddr[1], 425adddc32dSSowmini Varadhan port->raddr[2], port->raddr[3], 426adddc32dSSowmini Varadhan port->raddr[4], port->raddr[5]); 427d1015645SSowmini Varadhan break; 428adddc32dSSowmini Varadhan } 429e689cf4aSJeff Kirsher } while (err == -EAGAIN); 430e689cf4aSJeff Kirsher 431d1015645SSowmini Varadhan if (err <= 0 && vio_dring_state == VIO_DRING_STOPPED) { 432d1015645SSowmini Varadhan port->stop_rx_idx = end; 433d1015645SSowmini Varadhan port->stop_rx = true; 434d1015645SSowmini Varadhan } else { 435d1015645SSowmini Varadhan port->stop_rx_idx = 0; 436d1015645SSowmini Varadhan port->stop_rx = false; 437d1015645SSowmini Varadhan } 438d1015645SSowmini Varadhan 439e689cf4aSJeff Kirsher return err; 440e689cf4aSJeff Kirsher } 441e689cf4aSJeff Kirsher 442e689cf4aSJeff Kirsher static u32 next_idx(u32 idx, struct vio_dring_state *dr) 443e689cf4aSJeff Kirsher { 444e689cf4aSJeff Kirsher if (++idx == dr->num_entries) 445e689cf4aSJeff Kirsher idx = 0; 446e689cf4aSJeff Kirsher return idx; 447e689cf4aSJeff Kirsher } 448e689cf4aSJeff Kirsher 449e689cf4aSJeff Kirsher static u32 prev_idx(u32 idx, struct vio_dring_state *dr) 450e689cf4aSJeff Kirsher { 451e689cf4aSJeff Kirsher if (idx == 0) 452e689cf4aSJeff Kirsher idx = dr->num_entries - 1; 453e689cf4aSJeff Kirsher else 454e689cf4aSJeff Kirsher idx--; 455e689cf4aSJeff Kirsher 456e689cf4aSJeff Kirsher return idx; 457e689cf4aSJeff Kirsher } 458e689cf4aSJeff Kirsher 459e689cf4aSJeff Kirsher static struct vio_net_desc *get_rx_desc(struct vnet_port *port, 460e689cf4aSJeff Kirsher struct vio_dring_state *dr, 461e689cf4aSJeff Kirsher u32 index) 462e689cf4aSJeff Kirsher { 463e689cf4aSJeff Kirsher struct vio_net_desc *desc = port->vio.desc_buf; 464e689cf4aSJeff Kirsher int err; 465e689cf4aSJeff Kirsher 466e689cf4aSJeff Kirsher err = ldc_get_dring_entry(port->vio.lp, desc, dr->entry_size, 467e689cf4aSJeff Kirsher (index * dr->entry_size), 468e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 469e689cf4aSJeff Kirsher if (err < 0) 470e689cf4aSJeff Kirsher return ERR_PTR(err); 471e689cf4aSJeff Kirsher 472e689cf4aSJeff Kirsher return desc; 473e689cf4aSJeff Kirsher } 474e689cf4aSJeff Kirsher 475e689cf4aSJeff Kirsher static int put_rx_desc(struct vnet_port *port, 476e689cf4aSJeff Kirsher struct vio_dring_state *dr, 477e689cf4aSJeff Kirsher struct vio_net_desc *desc, 478e689cf4aSJeff Kirsher u32 index) 479e689cf4aSJeff Kirsher { 480e689cf4aSJeff Kirsher int err; 481e689cf4aSJeff Kirsher 482e689cf4aSJeff Kirsher err = ldc_put_dring_entry(port->vio.lp, desc, dr->entry_size, 483e689cf4aSJeff Kirsher (index * dr->entry_size), 484e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 485e689cf4aSJeff Kirsher if (err < 0) 486e689cf4aSJeff Kirsher return err; 487e689cf4aSJeff Kirsher 488e689cf4aSJeff Kirsher return 0; 489e689cf4aSJeff Kirsher } 490e689cf4aSJeff Kirsher 491e689cf4aSJeff Kirsher static int vnet_walk_rx_one(struct vnet_port *port, 492e689cf4aSJeff Kirsher struct vio_dring_state *dr, 493e689cf4aSJeff Kirsher u32 index, int *needs_ack) 494e689cf4aSJeff Kirsher { 495e689cf4aSJeff Kirsher struct vio_net_desc *desc = get_rx_desc(port, dr, index); 496e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 497e689cf4aSJeff Kirsher int err; 498e689cf4aSJeff Kirsher 49969088822SSowmini Varadhan BUG_ON(desc == NULL); 500e689cf4aSJeff Kirsher if (IS_ERR(desc)) 501e689cf4aSJeff Kirsher return PTR_ERR(desc); 502e689cf4aSJeff Kirsher 50378dcff7bSDavid L Stevens if (desc->hdr.state != VIO_DESC_READY) 50478dcff7bSDavid L Stevens return 1; 50578dcff7bSDavid L Stevens 50678dcff7bSDavid L Stevens rmb(); 50778dcff7bSDavid L Stevens 508e689cf4aSJeff Kirsher viodbg(DATA, "vio_walk_rx_one desc[%02x:%02x:%08x:%08x:%llx:%llx]\n", 509e689cf4aSJeff Kirsher desc->hdr.state, desc->hdr.ack, 510e689cf4aSJeff Kirsher desc->size, desc->ncookies, 511e689cf4aSJeff Kirsher desc->cookies[0].cookie_addr, 512e689cf4aSJeff Kirsher desc->cookies[0].cookie_size); 513e689cf4aSJeff Kirsher 5146d0ba919SDavid L Stevens err = vnet_rx_one(port, desc); 515e689cf4aSJeff Kirsher if (err == -ECONNRESET) 516e689cf4aSJeff Kirsher return err; 517e689cf4aSJeff Kirsher desc->hdr.state = VIO_DESC_DONE; 518e689cf4aSJeff Kirsher err = put_rx_desc(port, dr, desc, index); 519e689cf4aSJeff Kirsher if (err < 0) 520e689cf4aSJeff Kirsher return err; 521e689cf4aSJeff Kirsher *needs_ack = desc->hdr.ack; 522e689cf4aSJeff Kirsher return 0; 523e689cf4aSJeff Kirsher } 524e689cf4aSJeff Kirsher 525e689cf4aSJeff Kirsher static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr, 52669088822SSowmini Varadhan u32 start, u32 end, int *npkts, int budget) 527e689cf4aSJeff Kirsher { 528e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 529e689cf4aSJeff Kirsher int ack_start = -1, ack_end = -1; 53069088822SSowmini Varadhan bool send_ack = true; 531e689cf4aSJeff Kirsher 532e689cf4aSJeff Kirsher end = (end == (u32) -1) ? prev_idx(start, dr) : next_idx(end, dr); 533e689cf4aSJeff Kirsher 534e689cf4aSJeff Kirsher viodbg(DATA, "vnet_walk_rx start[%08x] end[%08x]\n", start, end); 535e689cf4aSJeff Kirsher 536e689cf4aSJeff Kirsher while (start != end) { 537e689cf4aSJeff Kirsher int ack = 0, err = vnet_walk_rx_one(port, dr, start, &ack); 538e689cf4aSJeff Kirsher if (err == -ECONNRESET) 539e689cf4aSJeff Kirsher return err; 540e689cf4aSJeff Kirsher if (err != 0) 541e689cf4aSJeff Kirsher break; 54269088822SSowmini Varadhan (*npkts)++; 543e689cf4aSJeff Kirsher if (ack_start == -1) 544e689cf4aSJeff Kirsher ack_start = start; 545e689cf4aSJeff Kirsher ack_end = start; 546e689cf4aSJeff Kirsher start = next_idx(start, dr); 547e689cf4aSJeff Kirsher if (ack && start != end) { 548e689cf4aSJeff Kirsher err = vnet_send_ack(port, dr, ack_start, ack_end, 549e689cf4aSJeff Kirsher VIO_DRING_ACTIVE); 550e689cf4aSJeff Kirsher if (err == -ECONNRESET) 551e689cf4aSJeff Kirsher return err; 552e689cf4aSJeff Kirsher ack_start = -1; 553e689cf4aSJeff Kirsher } 55469088822SSowmini Varadhan if ((*npkts) >= budget) { 55569088822SSowmini Varadhan send_ack = false; 55669088822SSowmini Varadhan break; 55769088822SSowmini Varadhan } 558e689cf4aSJeff Kirsher } 559e689cf4aSJeff Kirsher if (unlikely(ack_start == -1)) 560e689cf4aSJeff Kirsher ack_start = ack_end = prev_idx(start, dr); 56169088822SSowmini Varadhan if (send_ack) { 56269088822SSowmini Varadhan port->napi_resume = false; 56369088822SSowmini Varadhan return vnet_send_ack(port, dr, ack_start, ack_end, 56469088822SSowmini Varadhan VIO_DRING_STOPPED); 56569088822SSowmini Varadhan } else { 56669088822SSowmini Varadhan port->napi_resume = true; 56769088822SSowmini Varadhan port->napi_stop_idx = ack_end; 56869088822SSowmini Varadhan return 1; 56969088822SSowmini Varadhan } 570e689cf4aSJeff Kirsher } 571e689cf4aSJeff Kirsher 57269088822SSowmini Varadhan static int vnet_rx(struct vnet_port *port, void *msgbuf, int *npkts, 57369088822SSowmini Varadhan int budget) 574e689cf4aSJeff Kirsher { 575e689cf4aSJeff Kirsher struct vio_dring_data *pkt = msgbuf; 576e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_RX_RING]; 577e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 578e689cf4aSJeff Kirsher 579e689cf4aSJeff Kirsher viodbg(DATA, "vnet_rx stype_env[%04x] seq[%016llx] rcv_nxt[%016llx]\n", 580e689cf4aSJeff Kirsher pkt->tag.stype_env, pkt->seq, dr->rcv_nxt); 581e689cf4aSJeff Kirsher 582e689cf4aSJeff Kirsher if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 583e689cf4aSJeff Kirsher return 0; 584e689cf4aSJeff Kirsher if (unlikely(pkt->seq != dr->rcv_nxt)) { 585e689cf4aSJeff Kirsher pr_err("RX out of sequence seq[0x%llx] rcv_nxt[0x%llx]\n", 586e689cf4aSJeff Kirsher pkt->seq, dr->rcv_nxt); 587e689cf4aSJeff Kirsher return 0; 588e689cf4aSJeff Kirsher } 589e689cf4aSJeff Kirsher 59069088822SSowmini Varadhan if (!port->napi_resume) 591e689cf4aSJeff Kirsher dr->rcv_nxt++; 592e689cf4aSJeff Kirsher 593e689cf4aSJeff Kirsher /* XXX Validate pkt->start_idx and pkt->end_idx XXX */ 594e689cf4aSJeff Kirsher 59569088822SSowmini Varadhan return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx, 59669088822SSowmini Varadhan npkts, budget); 597e689cf4aSJeff Kirsher } 598e689cf4aSJeff Kirsher 599e689cf4aSJeff Kirsher static int idx_is_pending(struct vio_dring_state *dr, u32 end) 600e689cf4aSJeff Kirsher { 601e689cf4aSJeff Kirsher u32 idx = dr->cons; 602e689cf4aSJeff Kirsher int found = 0; 603e689cf4aSJeff Kirsher 604e689cf4aSJeff Kirsher while (idx != dr->prod) { 605e689cf4aSJeff Kirsher if (idx == end) { 606e689cf4aSJeff Kirsher found = 1; 607e689cf4aSJeff Kirsher break; 608e689cf4aSJeff Kirsher } 609e689cf4aSJeff Kirsher idx = next_idx(idx, dr); 610e689cf4aSJeff Kirsher } 611e689cf4aSJeff Kirsher return found; 612e689cf4aSJeff Kirsher } 613e689cf4aSJeff Kirsher 614e689cf4aSJeff Kirsher static int vnet_ack(struct vnet_port *port, void *msgbuf) 615e689cf4aSJeff Kirsher { 616e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 617e689cf4aSJeff Kirsher struct vio_dring_data *pkt = msgbuf; 618e689cf4aSJeff Kirsher struct net_device *dev; 619e689cf4aSJeff Kirsher struct vnet *vp; 620e689cf4aSJeff Kirsher u32 end; 621d1015645SSowmini Varadhan struct vio_net_desc *desc; 622d51bffd1SSowmini Varadhan struct netdev_queue *txq; 623d51bffd1SSowmini Varadhan 624e689cf4aSJeff Kirsher if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 625e689cf4aSJeff Kirsher return 0; 626e689cf4aSJeff Kirsher 627e689cf4aSJeff Kirsher end = pkt->end_idx; 62869088822SSowmini Varadhan vp = port->vp; 62969088822SSowmini Varadhan dev = vp->dev; 630b0cffed5SSowmini Varadhan netif_tx_lock(dev); 631b0cffed5SSowmini Varadhan if (unlikely(!idx_is_pending(dr, end))) { 632b0cffed5SSowmini Varadhan netif_tx_unlock(dev); 633b0cffed5SSowmini Varadhan return 0; 634b0cffed5SSowmini Varadhan } 635b0cffed5SSowmini Varadhan 636d1015645SSowmini Varadhan /* sync for race conditions with vnet_start_xmit() and tell xmit it 637d1015645SSowmini Varadhan * is time to send a trigger. 638d1015645SSowmini Varadhan */ 639e689cf4aSJeff Kirsher dr->cons = next_idx(end, dr); 640d1015645SSowmini Varadhan desc = vio_dring_entry(dr, dr->cons); 641777362d7SSowmini Varadhan if (desc->hdr.state == VIO_DESC_READY && !port->start_cons) { 642d1015645SSowmini Varadhan /* vnet_start_xmit() just populated this dring but missed 643d1015645SSowmini Varadhan * sending the "start" LDC message to the consumer. 644d1015645SSowmini Varadhan * Send a "start" trigger on its behalf. 645d1015645SSowmini Varadhan */ 646d1015645SSowmini Varadhan if (__vnet_tx_trigger(port, dr->cons) > 0) 647d1015645SSowmini Varadhan port->start_cons = false; 648d1015645SSowmini Varadhan else 649d1015645SSowmini Varadhan port->start_cons = true; 650d1015645SSowmini Varadhan } else { 651d1015645SSowmini Varadhan port->start_cons = true; 652d1015645SSowmini Varadhan } 65369088822SSowmini Varadhan netif_tx_unlock(dev); 654d1015645SSowmini Varadhan 655d51bffd1SSowmini Varadhan txq = netdev_get_tx_queue(dev, port->q_index); 656d51bffd1SSowmini Varadhan if (unlikely(netif_tx_queue_stopped(txq) && 657e689cf4aSJeff Kirsher vnet_tx_dring_avail(dr) >= VNET_TX_WAKEUP_THRESH(dr))) 658e689cf4aSJeff Kirsher return 1; 659e689cf4aSJeff Kirsher 660e689cf4aSJeff Kirsher return 0; 661e689cf4aSJeff Kirsher } 662e689cf4aSJeff Kirsher 663e689cf4aSJeff Kirsher static int vnet_nack(struct vnet_port *port, void *msgbuf) 664e689cf4aSJeff Kirsher { 665e689cf4aSJeff Kirsher /* XXX just reset or similar XXX */ 666e689cf4aSJeff Kirsher return 0; 667e689cf4aSJeff Kirsher } 668e689cf4aSJeff Kirsher 669e689cf4aSJeff Kirsher static int handle_mcast(struct vnet_port *port, void *msgbuf) 670e689cf4aSJeff Kirsher { 671e689cf4aSJeff Kirsher struct vio_net_mcast_info *pkt = msgbuf; 672e689cf4aSJeff Kirsher 673e689cf4aSJeff Kirsher if (pkt->tag.stype != VIO_SUBTYPE_ACK) 674e689cf4aSJeff Kirsher pr_err("%s: Got unexpected MCAST reply [%02x:%02x:%04x:%08x]\n", 675e689cf4aSJeff Kirsher port->vp->dev->name, 676e689cf4aSJeff Kirsher pkt->tag.type, 677e689cf4aSJeff Kirsher pkt->tag.stype, 678e689cf4aSJeff Kirsher pkt->tag.stype_env, 679e689cf4aSJeff Kirsher pkt->tag.sid); 680e689cf4aSJeff Kirsher 681e689cf4aSJeff Kirsher return 0; 682e689cf4aSJeff Kirsher } 683e689cf4aSJeff Kirsher 684d51bffd1SSowmini Varadhan /* Got back a STOPPED LDC message on port. If the queue is stopped, 685d51bffd1SSowmini Varadhan * wake it up so that we'll send out another START message at the 686d51bffd1SSowmini Varadhan * next TX. 687d51bffd1SSowmini Varadhan */ 688d51bffd1SSowmini Varadhan static void maybe_tx_wakeup(struct vnet_port *port) 689e689cf4aSJeff Kirsher { 690d51bffd1SSowmini Varadhan struct netdev_queue *txq; 691e689cf4aSJeff Kirsher 692d51bffd1SSowmini Varadhan txq = netdev_get_tx_queue(port->vp->dev, port->q_index); 693d51bffd1SSowmini Varadhan __netif_tx_lock(txq, smp_processor_id()); 694d51bffd1SSowmini Varadhan if (likely(netif_tx_queue_stopped(txq))) { 695e689cf4aSJeff Kirsher struct vio_dring_state *dr; 696e689cf4aSJeff Kirsher 697e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 698d51bffd1SSowmini Varadhan netif_tx_wake_queue(txq); 699e689cf4aSJeff Kirsher } 700d51bffd1SSowmini Varadhan __netif_tx_unlock(txq); 701e689cf4aSJeff Kirsher } 702e689cf4aSJeff Kirsher 70369088822SSowmini Varadhan static inline bool port_is_up(struct vnet_port *vnet) 704e689cf4aSJeff Kirsher { 70569088822SSowmini Varadhan struct vio_driver_state *vio = &vnet->vio; 70669088822SSowmini Varadhan 70769088822SSowmini Varadhan return !!(vio->hs_state & VIO_HS_COMPLETE); 70869088822SSowmini Varadhan } 70969088822SSowmini Varadhan 71069088822SSowmini Varadhan static int vnet_event_napi(struct vnet_port *port, int budget) 71169088822SSowmini Varadhan { 712e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 713e689cf4aSJeff Kirsher int tx_wakeup, err; 71469088822SSowmini Varadhan int npkts = 0; 71569088822SSowmini Varadhan int event = (port->rx_event & LDC_EVENT_RESET); 716e689cf4aSJeff Kirsher 71769088822SSowmini Varadhan ldc_ctrl: 718e689cf4aSJeff Kirsher if (unlikely(event == LDC_EVENT_RESET || 719e689cf4aSJeff Kirsher event == LDC_EVENT_UP)) { 720e689cf4aSJeff Kirsher vio_link_state_change(vio, event); 721e689cf4aSJeff Kirsher 722e4defc77SDavid L Stevens if (event == LDC_EVENT_RESET) { 723e4defc77SDavid L Stevens port->rmtu = 0; 724e689cf4aSJeff Kirsher vio_port_up(vio); 725e4defc77SDavid L Stevens } 72669088822SSowmini Varadhan port->rx_event = 0; 72769088822SSowmini Varadhan return 0; 728e689cf4aSJeff Kirsher } 72969088822SSowmini Varadhan /* We may have multiple LDC events in rx_event. Unroll send_events() */ 73069088822SSowmini Varadhan event = (port->rx_event & LDC_EVENT_UP); 73169088822SSowmini Varadhan port->rx_event &= ~(LDC_EVENT_RESET|LDC_EVENT_UP); 73269088822SSowmini Varadhan if (event == LDC_EVENT_UP) 73369088822SSowmini Varadhan goto ldc_ctrl; 73469088822SSowmini Varadhan event = port->rx_event; 73569088822SSowmini Varadhan if (!(event & LDC_EVENT_DATA_READY)) 73669088822SSowmini Varadhan return 0; 737e689cf4aSJeff Kirsher 73869088822SSowmini Varadhan /* we dont expect any other bits than RESET, UP, DATA_READY */ 73969088822SSowmini Varadhan BUG_ON(event != LDC_EVENT_DATA_READY); 740e689cf4aSJeff Kirsher 741e689cf4aSJeff Kirsher tx_wakeup = err = 0; 742e689cf4aSJeff Kirsher while (1) { 743e689cf4aSJeff Kirsher union { 744e689cf4aSJeff Kirsher struct vio_msg_tag tag; 745e689cf4aSJeff Kirsher u64 raw[8]; 746e689cf4aSJeff Kirsher } msgbuf; 747e689cf4aSJeff Kirsher 74869088822SSowmini Varadhan if (port->napi_resume) { 74969088822SSowmini Varadhan struct vio_dring_data *pkt = 75069088822SSowmini Varadhan (struct vio_dring_data *)&msgbuf; 75169088822SSowmini Varadhan struct vio_dring_state *dr = 75269088822SSowmini Varadhan &port->vio.drings[VIO_DRIVER_RX_RING]; 75369088822SSowmini Varadhan 75469088822SSowmini Varadhan pkt->tag.type = VIO_TYPE_DATA; 75569088822SSowmini Varadhan pkt->tag.stype = VIO_SUBTYPE_INFO; 75669088822SSowmini Varadhan pkt->tag.stype_env = VIO_DRING_DATA; 75769088822SSowmini Varadhan pkt->seq = dr->rcv_nxt; 75869088822SSowmini Varadhan pkt->start_idx = next_idx(port->napi_stop_idx, dr); 75969088822SSowmini Varadhan pkt->end_idx = -1; 76069088822SSowmini Varadhan goto napi_resume; 76169088822SSowmini Varadhan } 762e689cf4aSJeff Kirsher err = ldc_read(vio->lp, &msgbuf, sizeof(msgbuf)); 763e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 764e689cf4aSJeff Kirsher if (err == -ECONNRESET) 765e689cf4aSJeff Kirsher vio_conn_reset(vio); 766e689cf4aSJeff Kirsher break; 767e689cf4aSJeff Kirsher } 768e689cf4aSJeff Kirsher if (err == 0) 769e689cf4aSJeff Kirsher break; 770e689cf4aSJeff Kirsher viodbg(DATA, "TAG [%02x:%02x:%04x:%08x]\n", 771e689cf4aSJeff Kirsher msgbuf.tag.type, 772e689cf4aSJeff Kirsher msgbuf.tag.stype, 773e689cf4aSJeff Kirsher msgbuf.tag.stype_env, 774e689cf4aSJeff Kirsher msgbuf.tag.sid); 775e689cf4aSJeff Kirsher err = vio_validate_sid(vio, &msgbuf.tag); 776e689cf4aSJeff Kirsher if (err < 0) 777e689cf4aSJeff Kirsher break; 77869088822SSowmini Varadhan napi_resume: 779e689cf4aSJeff Kirsher if (likely(msgbuf.tag.type == VIO_TYPE_DATA)) { 780e689cf4aSJeff Kirsher if (msgbuf.tag.stype == VIO_SUBTYPE_INFO) { 78169088822SSowmini Varadhan if (!port_is_up(port)) { 78269088822SSowmini Varadhan /* failures like handshake_failure() 78369088822SSowmini Varadhan * may have cleaned up dring, but 78469088822SSowmini Varadhan * NAPI polling may bring us here. 78569088822SSowmini Varadhan */ 78669088822SSowmini Varadhan err = -ECONNRESET; 78769088822SSowmini Varadhan break; 78869088822SSowmini Varadhan } 78969088822SSowmini Varadhan err = vnet_rx(port, &msgbuf, &npkts, budget); 79069088822SSowmini Varadhan if (npkts >= budget) 79169088822SSowmini Varadhan break; 7928c4ee3e7SSowmini Varadhan if (npkts == 0) 7938c4ee3e7SSowmini Varadhan break; 794e689cf4aSJeff Kirsher } else if (msgbuf.tag.stype == VIO_SUBTYPE_ACK) { 795e689cf4aSJeff Kirsher err = vnet_ack(port, &msgbuf); 796e689cf4aSJeff Kirsher if (err > 0) 797e689cf4aSJeff Kirsher tx_wakeup |= err; 798e689cf4aSJeff Kirsher } else if (msgbuf.tag.stype == VIO_SUBTYPE_NACK) { 799e689cf4aSJeff Kirsher err = vnet_nack(port, &msgbuf); 800e689cf4aSJeff Kirsher } 801e689cf4aSJeff Kirsher } else if (msgbuf.tag.type == VIO_TYPE_CTRL) { 802e689cf4aSJeff Kirsher if (msgbuf.tag.stype_env == VNET_MCAST_INFO) 803e689cf4aSJeff Kirsher err = handle_mcast(port, &msgbuf); 804e689cf4aSJeff Kirsher else 805e689cf4aSJeff Kirsher err = vio_control_pkt_engine(vio, &msgbuf); 806e689cf4aSJeff Kirsher if (err) 807e689cf4aSJeff Kirsher break; 808e689cf4aSJeff Kirsher } else { 809e689cf4aSJeff Kirsher err = vnet_handle_unknown(port, &msgbuf); 810e689cf4aSJeff Kirsher } 811e689cf4aSJeff Kirsher if (err == -ECONNRESET) 812e689cf4aSJeff Kirsher break; 813e689cf4aSJeff Kirsher } 814e689cf4aSJeff Kirsher if (unlikely(tx_wakeup && err != -ECONNRESET)) 815d51bffd1SSowmini Varadhan maybe_tx_wakeup(port); 81669088822SSowmini Varadhan return npkts; 81769088822SSowmini Varadhan } 8181d311ad2SSowmini Varadhan 81969088822SSowmini Varadhan static int vnet_poll(struct napi_struct *napi, int budget) 82069088822SSowmini Varadhan { 82169088822SSowmini Varadhan struct vnet_port *port = container_of(napi, struct vnet_port, napi); 82269088822SSowmini Varadhan struct vio_driver_state *vio = &port->vio; 82369088822SSowmini Varadhan int processed = vnet_event_napi(port, budget); 82469088822SSowmini Varadhan 82569088822SSowmini Varadhan if (processed < budget) { 82669088822SSowmini Varadhan napi_complete(napi); 8277bd68bfdSSowmini Varadhan port->rx_event &= ~LDC_EVENT_DATA_READY; 82869088822SSowmini Varadhan vio_set_intr(vio->vdev->rx_ino, HV_INTR_ENABLED); 82969088822SSowmini Varadhan } 83069088822SSowmini Varadhan return processed; 83169088822SSowmini Varadhan } 83269088822SSowmini Varadhan 83369088822SSowmini Varadhan static void vnet_event(void *arg, int event) 83469088822SSowmini Varadhan { 83569088822SSowmini Varadhan struct vnet_port *port = arg; 83669088822SSowmini Varadhan struct vio_driver_state *vio = &port->vio; 83769088822SSowmini Varadhan 83869088822SSowmini Varadhan port->rx_event |= event; 83969088822SSowmini Varadhan vio_set_intr(vio->vdev->rx_ino, HV_INTR_DISABLED); 84069088822SSowmini Varadhan napi_schedule(&port->napi); 84169088822SSowmini Varadhan 842e689cf4aSJeff Kirsher } 843e689cf4aSJeff Kirsher 844d1015645SSowmini Varadhan static int __vnet_tx_trigger(struct vnet_port *port, u32 start) 845e689cf4aSJeff Kirsher { 846e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 847e689cf4aSJeff Kirsher struct vio_dring_data hdr = { 848e689cf4aSJeff Kirsher .tag = { 849e689cf4aSJeff Kirsher .type = VIO_TYPE_DATA, 850e689cf4aSJeff Kirsher .stype = VIO_SUBTYPE_INFO, 851e689cf4aSJeff Kirsher .stype_env = VIO_DRING_DATA, 852e689cf4aSJeff Kirsher .sid = vio_send_sid(&port->vio), 853e689cf4aSJeff Kirsher }, 854e689cf4aSJeff Kirsher .dring_ident = dr->ident, 855d1015645SSowmini Varadhan .start_idx = start, 856e689cf4aSJeff Kirsher .end_idx = (u32) -1, 857e689cf4aSJeff Kirsher }; 858e689cf4aSJeff Kirsher int err, delay; 859adddc32dSSowmini Varadhan int retries = 0; 860e689cf4aSJeff Kirsher 861d1015645SSowmini Varadhan if (port->stop_rx) { 862d1015645SSowmini Varadhan err = vnet_send_ack(port, 863d1015645SSowmini Varadhan &port->vio.drings[VIO_DRIVER_RX_RING], 864d1015645SSowmini Varadhan port->stop_rx_idx, -1, 865d1015645SSowmini Varadhan VIO_DRING_STOPPED); 866d1015645SSowmini Varadhan if (err <= 0) 867d1015645SSowmini Varadhan return err; 868d1015645SSowmini Varadhan } 869d1015645SSowmini Varadhan 870e689cf4aSJeff Kirsher hdr.seq = dr->snd_nxt; 871e689cf4aSJeff Kirsher delay = 1; 872e689cf4aSJeff Kirsher do { 873e689cf4aSJeff Kirsher err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 874e689cf4aSJeff Kirsher if (err > 0) { 875e689cf4aSJeff Kirsher dr->snd_nxt++; 876e689cf4aSJeff Kirsher break; 877e689cf4aSJeff Kirsher } 878e689cf4aSJeff Kirsher udelay(delay); 879e689cf4aSJeff Kirsher if ((delay <<= 1) > 128) 880e689cf4aSJeff Kirsher delay = 128; 881adddc32dSSowmini Varadhan if (retries++ > VNET_MAX_RETRIES) 882adddc32dSSowmini Varadhan break; 883e689cf4aSJeff Kirsher } while (err == -EAGAIN); 884e689cf4aSJeff Kirsher 885e689cf4aSJeff Kirsher return err; 886e689cf4aSJeff Kirsher } 887e689cf4aSJeff Kirsher 888e689cf4aSJeff Kirsher struct vnet_port *__tx_port_find(struct vnet *vp, struct sk_buff *skb) 889e689cf4aSJeff Kirsher { 890e689cf4aSJeff Kirsher unsigned int hash = vnet_hashfn(skb->data); 891e689cf4aSJeff Kirsher struct hlist_head *hp = &vp->port_hash[hash]; 892e689cf4aSJeff Kirsher struct vnet_port *port; 893e689cf4aSJeff Kirsher 8942a968dd8SSowmini Varadhan hlist_for_each_entry_rcu(port, hp, hash) { 8958266f5fcSDavid L Stevens if (!port_is_up(port)) 8968266f5fcSDavid L Stevens continue; 8972e42e474SJoe Perches if (ether_addr_equal(port->raddr, skb->data)) 898e689cf4aSJeff Kirsher return port; 899e689cf4aSJeff Kirsher } 9002a968dd8SSowmini Varadhan list_for_each_entry_rcu(port, &vp->port_list, list) { 9018266f5fcSDavid L Stevens if (!port->switch_port) 9028266f5fcSDavid L Stevens continue; 9038266f5fcSDavid L Stevens if (!port_is_up(port)) 9048266f5fcSDavid L Stevens continue; 905e689cf4aSJeff Kirsher return port; 906e689cf4aSJeff Kirsher } 9078266f5fcSDavid L Stevens return NULL; 9088266f5fcSDavid L Stevens } 909e689cf4aSJeff Kirsher 9108e845f4cSDavid L Stevens static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port, 9118e845f4cSDavid L Stevens unsigned *pending) 9128e845f4cSDavid L Stevens { 9138e845f4cSDavid L Stevens struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 9148e845f4cSDavid L Stevens struct sk_buff *skb = NULL; 9158e845f4cSDavid L Stevens int i, txi; 9168e845f4cSDavid L Stevens 9178e845f4cSDavid L Stevens *pending = 0; 9188e845f4cSDavid L Stevens 9198e845f4cSDavid L Stevens txi = dr->prod-1; 9208e845f4cSDavid L Stevens if (txi < 0) 9218e845f4cSDavid L Stevens txi = VNET_TX_RING_SIZE-1; 9228e845f4cSDavid L Stevens 9238e845f4cSDavid L Stevens for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 9248e845f4cSDavid L Stevens struct vio_net_desc *d; 9258e845f4cSDavid L Stevens 9268e845f4cSDavid L Stevens d = vio_dring_entry(dr, txi); 9278e845f4cSDavid L Stevens 9288e845f4cSDavid L Stevens if (d->hdr.state == VIO_DESC_DONE) { 9298e845f4cSDavid L Stevens if (port->tx_bufs[txi].skb) { 9308e845f4cSDavid L Stevens BUG_ON(port->tx_bufs[txi].skb->next); 9318e845f4cSDavid L Stevens 9328e845f4cSDavid L Stevens port->tx_bufs[txi].skb->next = skb; 9338e845f4cSDavid L Stevens skb = port->tx_bufs[txi].skb; 9348e845f4cSDavid L Stevens port->tx_bufs[txi].skb = NULL; 9358e845f4cSDavid L Stevens 9368e845f4cSDavid L Stevens ldc_unmap(port->vio.lp, 9378e845f4cSDavid L Stevens port->tx_bufs[txi].cookies, 9388e845f4cSDavid L Stevens port->tx_bufs[txi].ncookies); 9398e845f4cSDavid L Stevens } 9408e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 9418e845f4cSDavid L Stevens } else if (d->hdr.state == VIO_DESC_READY) { 9428e845f4cSDavid L Stevens (*pending)++; 9438e845f4cSDavid L Stevens } else if (d->hdr.state == VIO_DESC_FREE) { 9448e845f4cSDavid L Stevens break; 9458e845f4cSDavid L Stevens } 9468e845f4cSDavid L Stevens --txi; 9478e845f4cSDavid L Stevens if (txi < 0) 9488e845f4cSDavid L Stevens txi = VNET_TX_RING_SIZE-1; 9498e845f4cSDavid L Stevens } 9508e845f4cSDavid L Stevens return skb; 9518e845f4cSDavid L Stevens } 9528e845f4cSDavid L Stevens 9538e845f4cSDavid L Stevens static inline void vnet_free_skbs(struct sk_buff *skb) 9548e845f4cSDavid L Stevens { 9558e845f4cSDavid L Stevens struct sk_buff *next; 9568e845f4cSDavid L Stevens 9578e845f4cSDavid L Stevens while (skb) { 9588e845f4cSDavid L Stevens next = skb->next; 9598e845f4cSDavid L Stevens skb->next = NULL; 9608e845f4cSDavid L Stevens dev_kfree_skb(skb); 9618e845f4cSDavid L Stevens skb = next; 9628e845f4cSDavid L Stevens } 9638e845f4cSDavid L Stevens } 9648e845f4cSDavid L Stevens 9658e845f4cSDavid L Stevens static void vnet_clean_timer_expire(unsigned long port0) 9668e845f4cSDavid L Stevens { 9678e845f4cSDavid L Stevens struct vnet_port *port = (struct vnet_port *)port0; 9688e845f4cSDavid L Stevens struct sk_buff *freeskbs; 9698e845f4cSDavid L Stevens unsigned pending; 9708e845f4cSDavid L Stevens 97113b13dd9SSowmini Varadhan netif_tx_lock(port->vp->dev); 9728e845f4cSDavid L Stevens freeskbs = vnet_clean_tx_ring(port, &pending); 97313b13dd9SSowmini Varadhan netif_tx_unlock(port->vp->dev); 9748e845f4cSDavid L Stevens 9758e845f4cSDavid L Stevens vnet_free_skbs(freeskbs); 9768e845f4cSDavid L Stevens 9778e845f4cSDavid L Stevens if (pending) 9788e845f4cSDavid L Stevens (void)mod_timer(&port->clean_timer, 9798e845f4cSDavid L Stevens jiffies + VNET_CLEAN_TIMEOUT); 9808e845f4cSDavid L Stevens else 9818e845f4cSDavid L Stevens del_timer(&port->clean_timer); 9828e845f4cSDavid L Stevens } 9838e845f4cSDavid L Stevens 984da38c564SDavid L Stevens static inline int vnet_skb_map(struct ldc_channel *lp, struct sk_buff *skb, 985da38c564SDavid L Stevens struct ldc_trans_cookie *cookies, int ncookies, 986da38c564SDavid L Stevens unsigned int map_perm) 987da38c564SDavid L Stevens { 988da38c564SDavid L Stevens int i, nc, err, blen; 989da38c564SDavid L Stevens 990da38c564SDavid L Stevens /* header */ 991da38c564SDavid L Stevens blen = skb_headlen(skb); 992da38c564SDavid L Stevens if (blen < ETH_ZLEN) 993da38c564SDavid L Stevens blen = ETH_ZLEN; 994da38c564SDavid L Stevens blen += VNET_PACKET_SKIP; 995da38c564SDavid L Stevens blen += 8 - (blen & 7); 996da38c564SDavid L Stevens 997da38c564SDavid L Stevens err = ldc_map_single(lp, skb->data-VNET_PACKET_SKIP, blen, cookies, 998da38c564SDavid L Stevens ncookies, map_perm); 999da38c564SDavid L Stevens if (err < 0) 1000da38c564SDavid L Stevens return err; 1001da38c564SDavid L Stevens nc = err; 1002da38c564SDavid L Stevens 1003da38c564SDavid L Stevens for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1004da38c564SDavid L Stevens skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1005da38c564SDavid L Stevens u8 *vaddr; 1006da38c564SDavid L Stevens 1007da38c564SDavid L Stevens if (nc < ncookies) { 1008da38c564SDavid L Stevens vaddr = kmap_atomic(skb_frag_page(f)); 1009da38c564SDavid L Stevens blen = skb_frag_size(f); 1010da38c564SDavid L Stevens blen += 8 - (blen & 7); 1011da38c564SDavid L Stevens err = ldc_map_single(lp, vaddr + f->page_offset, 1012da38c564SDavid L Stevens blen, cookies + nc, ncookies - nc, 1013da38c564SDavid L Stevens map_perm); 1014da38c564SDavid L Stevens kunmap_atomic(vaddr); 1015da38c564SDavid L Stevens } else { 1016da38c564SDavid L Stevens err = -EMSGSIZE; 1017da38c564SDavid L Stevens } 1018da38c564SDavid L Stevens 1019da38c564SDavid L Stevens if (err < 0) { 1020da38c564SDavid L Stevens ldc_unmap(lp, cookies, nc); 1021da38c564SDavid L Stevens return err; 1022da38c564SDavid L Stevens } 1023da38c564SDavid L Stevens nc += err; 1024da38c564SDavid L Stevens } 1025da38c564SDavid L Stevens return nc; 1026da38c564SDavid L Stevens } 1027da38c564SDavid L Stevens 1028da38c564SDavid L Stevens static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, int ncookies) 10298e845f4cSDavid L Stevens { 10308e845f4cSDavid L Stevens struct sk_buff *nskb; 1031da38c564SDavid L Stevens int i, len, pad, docopy; 10328e845f4cSDavid L Stevens 10338e845f4cSDavid L Stevens len = skb->len; 10348e845f4cSDavid L Stevens pad = 0; 10358e845f4cSDavid L Stevens if (len < ETH_ZLEN) { 10368e845f4cSDavid L Stevens pad += ETH_ZLEN - skb->len; 10378e845f4cSDavid L Stevens len += pad; 10388e845f4cSDavid L Stevens } 10398e845f4cSDavid L Stevens len += VNET_PACKET_SKIP; 10408e845f4cSDavid L Stevens pad += 8 - (len & 7); 10418e845f4cSDavid L Stevens 1042da38c564SDavid L Stevens /* make sure we have enough cookies and alignment in every frag */ 1043da38c564SDavid L Stevens docopy = skb_shinfo(skb)->nr_frags >= ncookies; 1044da38c564SDavid L Stevens for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1045da38c564SDavid L Stevens skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1046da38c564SDavid L Stevens 1047da38c564SDavid L Stevens docopy |= f->page_offset & 7; 1048da38c564SDavid L Stevens } 10498e845f4cSDavid L Stevens if (((unsigned long)skb->data & 7) != VNET_PACKET_SKIP || 10508e845f4cSDavid L Stevens skb_tailroom(skb) < pad || 1051da38c564SDavid L Stevens skb_headroom(skb) < VNET_PACKET_SKIP || docopy) { 10521b6b0a47SDavid L Stevens int start = 0, offset; 10531b6b0a47SDavid L Stevens __wsum csum; 10546d0ba919SDavid L Stevens 1055da38c564SDavid L Stevens len = skb->len > ETH_ZLEN ? skb->len : ETH_ZLEN; 1056da38c564SDavid L Stevens nskb = alloc_and_align_skb(skb->dev, len); 1057da38c564SDavid L Stevens if (nskb == NULL) { 1058da38c564SDavid L Stevens dev_kfree_skb(skb); 1059da38c564SDavid L Stevens return NULL; 1060da38c564SDavid L Stevens } 10618e845f4cSDavid L Stevens skb_reserve(nskb, VNET_PACKET_SKIP); 10626d0ba919SDavid L Stevens 10636d0ba919SDavid L Stevens nskb->protocol = skb->protocol; 10646d0ba919SDavid L Stevens offset = skb_mac_header(skb) - skb->data; 10656d0ba919SDavid L Stevens skb_set_mac_header(nskb, offset); 10666d0ba919SDavid L Stevens offset = skb_network_header(skb) - skb->data; 10676d0ba919SDavid L Stevens skb_set_network_header(nskb, offset); 10686d0ba919SDavid L Stevens offset = skb_transport_header(skb) - skb->data; 10696d0ba919SDavid L Stevens skb_set_transport_header(nskb, offset); 10706d0ba919SDavid L Stevens 10711b6b0a47SDavid L Stevens offset = 0; 10726d0ba919SDavid L Stevens nskb->csum_offset = skb->csum_offset; 10736d0ba919SDavid L Stevens nskb->ip_summed = skb->ip_summed; 10746d0ba919SDavid L Stevens 10751b6b0a47SDavid L Stevens if (skb->ip_summed == CHECKSUM_PARTIAL) 10761b6b0a47SDavid L Stevens start = skb_checksum_start_offset(skb); 10771b6b0a47SDavid L Stevens if (start) { 10781b6b0a47SDavid L Stevens struct iphdr *iph = ip_hdr(nskb); 10791b6b0a47SDavid L Stevens int offset = start + nskb->csum_offset; 10801b6b0a47SDavid L Stevens 10811b6b0a47SDavid L Stevens if (skb_copy_bits(skb, 0, nskb->data, start)) { 10821b6b0a47SDavid L Stevens dev_kfree_skb(nskb); 10831b6b0a47SDavid L Stevens dev_kfree_skb(skb); 10841b6b0a47SDavid L Stevens return NULL; 10851b6b0a47SDavid L Stevens } 10861b6b0a47SDavid L Stevens *(__sum16 *)(skb->data + offset) = 0; 10871b6b0a47SDavid L Stevens csum = skb_copy_and_csum_bits(skb, start, 10881b6b0a47SDavid L Stevens nskb->data + start, 10891b6b0a47SDavid L Stevens skb->len - start, 0); 10901b6b0a47SDavid L Stevens if (iph->protocol == IPPROTO_TCP || 10911b6b0a47SDavid L Stevens iph->protocol == IPPROTO_UDP) { 10921b6b0a47SDavid L Stevens csum = csum_tcpudp_magic(iph->saddr, iph->daddr, 10931b6b0a47SDavid L Stevens skb->len - start, 10941b6b0a47SDavid L Stevens iph->protocol, csum); 10951b6b0a47SDavid L Stevens } 10961b6b0a47SDavid L Stevens *(__sum16 *)(nskb->data + offset) = csum; 10971b6b0a47SDavid L Stevens 10981b6b0a47SDavid L Stevens nskb->ip_summed = CHECKSUM_NONE; 10991b6b0a47SDavid L Stevens } else if (skb_copy_bits(skb, 0, nskb->data, skb->len)) { 11008e845f4cSDavid L Stevens dev_kfree_skb(nskb); 11018e845f4cSDavid L Stevens dev_kfree_skb(skb); 11028e845f4cSDavid L Stevens return NULL; 11038e845f4cSDavid L Stevens } 11048e845f4cSDavid L Stevens (void)skb_put(nskb, skb->len); 11059a72dd4dSDavid L Stevens if (skb_is_gso(skb)) { 11069a72dd4dSDavid L Stevens skb_shinfo(nskb)->gso_size = skb_shinfo(skb)->gso_size; 11079a72dd4dSDavid L Stevens skb_shinfo(nskb)->gso_type = skb_shinfo(skb)->gso_type; 11089a72dd4dSDavid L Stevens } 11098e845f4cSDavid L Stevens dev_kfree_skb(skb); 11108e845f4cSDavid L Stevens skb = nskb; 11118e845f4cSDavid L Stevens } 11128e845f4cSDavid L Stevens return skb; 11138e845f4cSDavid L Stevens } 11148e845f4cSDavid L Stevens 1115d51bffd1SSowmini Varadhan static u16 1116d51bffd1SSowmini Varadhan vnet_select_queue(struct net_device *dev, struct sk_buff *skb, 1117d51bffd1SSowmini Varadhan void *accel_priv, select_queue_fallback_t fallback) 1118d51bffd1SSowmini Varadhan { 1119d51bffd1SSowmini Varadhan struct vnet *vp = netdev_priv(dev); 1120d51bffd1SSowmini Varadhan struct vnet_port *port = __tx_port_find(vp, skb); 1121d51bffd1SSowmini Varadhan 1122c647cc3fSDavid L Stevens if (port == NULL) 1123c647cc3fSDavid L Stevens return 0; 1124d51bffd1SSowmini Varadhan return port->q_index; 1125d51bffd1SSowmini Varadhan } 1126d51bffd1SSowmini Varadhan 11279a72dd4dSDavid L Stevens static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev); 11289a72dd4dSDavid L Stevens 11299a72dd4dSDavid L Stevens static int vnet_handle_offloads(struct vnet_port *port, struct sk_buff *skb) 11309a72dd4dSDavid L Stevens { 11319a72dd4dSDavid L Stevens struct net_device *dev = port->vp->dev; 11329a72dd4dSDavid L Stevens struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 11339a72dd4dSDavid L Stevens struct sk_buff *segs; 11349a72dd4dSDavid L Stevens int maclen; 11359a72dd4dSDavid L Stevens int status; 11369a72dd4dSDavid L Stevens 11379a72dd4dSDavid L Stevens if (unlikely(vnet_tx_dring_avail(dr) < skb_shinfo(skb)->gso_segs)) { 11389a72dd4dSDavid L Stevens struct netdev_queue *txq; 11399a72dd4dSDavid L Stevens 11409a72dd4dSDavid L Stevens txq = netdev_get_tx_queue(dev, port->q_index); 11419a72dd4dSDavid L Stevens netif_tx_stop_queue(txq); 11429a72dd4dSDavid L Stevens if (vnet_tx_dring_avail(dr) < skb_shinfo(skb)->gso_segs) 11439a72dd4dSDavid L Stevens return NETDEV_TX_BUSY; 11449a72dd4dSDavid L Stevens netif_tx_wake_queue(txq); 11459a72dd4dSDavid L Stevens } 11469a72dd4dSDavid L Stevens 11479a72dd4dSDavid L Stevens maclen = skb_network_header(skb) - skb_mac_header(skb); 11489a72dd4dSDavid L Stevens skb_pull(skb, maclen); 11499a72dd4dSDavid L Stevens 11509a72dd4dSDavid L Stevens segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO); 11519a72dd4dSDavid L Stevens if (IS_ERR(segs)) { 11529a72dd4dSDavid L Stevens dev->stats.tx_dropped++; 11539a72dd4dSDavid L Stevens return NETDEV_TX_OK; 11549a72dd4dSDavid L Stevens } 11559a72dd4dSDavid L Stevens 11569a72dd4dSDavid L Stevens skb_push(skb, maclen); 11579a72dd4dSDavid L Stevens skb_reset_mac_header(skb); 11589a72dd4dSDavid L Stevens 11599a72dd4dSDavid L Stevens status = 0; 11609a72dd4dSDavid L Stevens while (segs) { 11619a72dd4dSDavid L Stevens struct sk_buff *curr = segs; 11629a72dd4dSDavid L Stevens 11639a72dd4dSDavid L Stevens segs = segs->next; 11649a72dd4dSDavid L Stevens curr->next = NULL; 11659a72dd4dSDavid L Stevens 11669a72dd4dSDavid L Stevens skb_push(curr, maclen); 11679a72dd4dSDavid L Stevens skb_reset_mac_header(curr); 11689a72dd4dSDavid L Stevens memcpy(skb_mac_header(curr), skb_mac_header(skb), 11699a72dd4dSDavid L Stevens maclen); 11709a72dd4dSDavid L Stevens curr->csum_start = skb_transport_header(curr) - curr->head; 11719a72dd4dSDavid L Stevens if (ip_hdr(curr)->protocol == IPPROTO_TCP) 11729a72dd4dSDavid L Stevens curr->csum_offset = offsetof(struct tcphdr, check); 11739a72dd4dSDavid L Stevens else if (ip_hdr(curr)->protocol == IPPROTO_UDP) 11749a72dd4dSDavid L Stevens curr->csum_offset = offsetof(struct udphdr, check); 11759a72dd4dSDavid L Stevens 11769a72dd4dSDavid L Stevens if (!(status & NETDEV_TX_MASK)) 11779a72dd4dSDavid L Stevens status = vnet_start_xmit(curr, dev); 11789a72dd4dSDavid L Stevens if (status & NETDEV_TX_MASK) 11799a72dd4dSDavid L Stevens dev_kfree_skb_any(curr); 11809a72dd4dSDavid L Stevens } 11819a72dd4dSDavid L Stevens 11829a72dd4dSDavid L Stevens if (!(status & NETDEV_TX_MASK)) 11839a72dd4dSDavid L Stevens dev_kfree_skb_any(skb); 11849a72dd4dSDavid L Stevens return status; 11859a72dd4dSDavid L Stevens } 11869a72dd4dSDavid L Stevens 1187e689cf4aSJeff Kirsher static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 1188e689cf4aSJeff Kirsher { 1189e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 11902a968dd8SSowmini Varadhan struct vnet_port *port = NULL; 1191e689cf4aSJeff Kirsher struct vio_dring_state *dr; 1192e689cf4aSJeff Kirsher struct vio_net_desc *d; 1193e689cf4aSJeff Kirsher unsigned int len; 11948e845f4cSDavid L Stevens struct sk_buff *freeskbs = NULL; 11958e845f4cSDavid L Stevens int i, err, txi; 11968e845f4cSDavid L Stevens unsigned pending = 0; 1197d51bffd1SSowmini Varadhan struct netdev_queue *txq; 1198e689cf4aSJeff Kirsher 11992a968dd8SSowmini Varadhan rcu_read_lock(); 120013b13dd9SSowmini Varadhan port = __tx_port_find(vp, skb); 1201df20286aSSowmini Varadhan if (unlikely(!port)) { 1202df20286aSSowmini Varadhan rcu_read_unlock(); 12038e845f4cSDavid L Stevens goto out_dropped; 1204df20286aSSowmini Varadhan } 12058e845f4cSDavid L Stevens 12069a72dd4dSDavid L Stevens if (skb_is_gso(skb)) { 12079a72dd4dSDavid L Stevens err = vnet_handle_offloads(port, skb); 12089a72dd4dSDavid L Stevens rcu_read_unlock(); 12099a72dd4dSDavid L Stevens return err; 12109a72dd4dSDavid L Stevens } 12119a72dd4dSDavid L Stevens 1212a2b78e9bSDavid L Stevens if (skb->len > port->rmtu) { 1213a2b78e9bSDavid L Stevens unsigned long localmtu = port->rmtu - ETH_HLEN; 1214a2b78e9bSDavid L Stevens 1215a2b78e9bSDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 3)) 1216a2b78e9bSDavid L Stevens localmtu -= VLAN_HLEN; 1217a2b78e9bSDavid L Stevens 1218a2b78e9bSDavid L Stevens if (skb->protocol == htons(ETH_P_IP)) { 1219a2b78e9bSDavid L Stevens struct flowi4 fl4; 1220a2b78e9bSDavid L Stevens struct rtable *rt = NULL; 1221a2b78e9bSDavid L Stevens 1222a2b78e9bSDavid L Stevens memset(&fl4, 0, sizeof(fl4)); 1223a2b78e9bSDavid L Stevens fl4.flowi4_oif = dev->ifindex; 1224a2b78e9bSDavid L Stevens fl4.flowi4_tos = RT_TOS(ip_hdr(skb)->tos); 1225a2b78e9bSDavid L Stevens fl4.daddr = ip_hdr(skb)->daddr; 1226a2b78e9bSDavid L Stevens fl4.saddr = ip_hdr(skb)->saddr; 1227a2b78e9bSDavid L Stevens 1228a2b78e9bSDavid L Stevens rt = ip_route_output_key(dev_net(dev), &fl4); 12292a968dd8SSowmini Varadhan rcu_read_unlock(); 1230a2b78e9bSDavid L Stevens if (!IS_ERR(rt)) { 1231a2b78e9bSDavid L Stevens skb_dst_set(skb, &rt->dst); 1232a2b78e9bSDavid L Stevens icmp_send(skb, ICMP_DEST_UNREACH, 1233a2b78e9bSDavid L Stevens ICMP_FRAG_NEEDED, 1234a2b78e9bSDavid L Stevens htonl(localmtu)); 1235a2b78e9bSDavid L Stevens } 1236a2b78e9bSDavid L Stevens } 1237a2b78e9bSDavid L Stevens #if IS_ENABLED(CONFIG_IPV6) 1238a2b78e9bSDavid L Stevens else if (skb->protocol == htons(ETH_P_IPV6)) 1239a2b78e9bSDavid L Stevens icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, localmtu); 1240a2b78e9bSDavid L Stevens #endif 124142db672dSDavid L Stevens goto out_dropped; 1242a2b78e9bSDavid L Stevens } 124342db672dSDavid L Stevens 1244da38c564SDavid L Stevens skb = vnet_skb_shape(skb, 2); 1245da38c564SDavid L Stevens 1246da38c564SDavid L Stevens if (unlikely(!skb)) { 1247da38c564SDavid L Stevens rcu_read_unlock(); 1248da38c564SDavid L Stevens goto out_dropped; 1249da38c564SDavid L Stevens } 1250da38c564SDavid L Stevens 12511b6b0a47SDavid L Stevens if (skb->ip_summed == CHECKSUM_PARTIAL) 12521b6b0a47SDavid L Stevens vnet_fullcsum(skb); 12531b6b0a47SDavid L Stevens 1254e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1255d51bffd1SSowmini Varadhan i = skb_get_queue_mapping(skb); 1256d51bffd1SSowmini Varadhan txq = netdev_get_tx_queue(dev, i); 1257d0aedcd4SDwight Engen if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1258d51bffd1SSowmini Varadhan if (!netif_tx_queue_stopped(txq)) { 1259d51bffd1SSowmini Varadhan netif_tx_stop_queue(txq); 1260e689cf4aSJeff Kirsher 1261e689cf4aSJeff Kirsher /* This is a hard error, log it. */ 1262e689cf4aSJeff Kirsher netdev_err(dev, "BUG! Tx Ring full when queue awake!\n"); 1263e689cf4aSJeff Kirsher dev->stats.tx_errors++; 1264e689cf4aSJeff Kirsher } 12652a968dd8SSowmini Varadhan rcu_read_unlock(); 1266e689cf4aSJeff Kirsher return NETDEV_TX_BUSY; 1267e689cf4aSJeff Kirsher } 1268e689cf4aSJeff Kirsher 1269e689cf4aSJeff Kirsher d = vio_dring_cur(dr); 1270e689cf4aSJeff Kirsher 12718e845f4cSDavid L Stevens txi = dr->prod; 12728e845f4cSDavid L Stevens 12738e845f4cSDavid L Stevens freeskbs = vnet_clean_tx_ring(port, &pending); 12748e845f4cSDavid L Stevens 12758e845f4cSDavid L Stevens BUG_ON(port->tx_bufs[txi].skb); 1276e689cf4aSJeff Kirsher 1277e689cf4aSJeff Kirsher len = skb->len; 12788e845f4cSDavid L Stevens if (len < ETH_ZLEN) 1279e689cf4aSJeff Kirsher len = ETH_ZLEN; 12808e845f4cSDavid L Stevens 1281da38c564SDavid L Stevens err = vnet_skb_map(port->vio.lp, skb, port->tx_bufs[txi].cookies, 2, 12828e845f4cSDavid L Stevens (LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW)); 12838e845f4cSDavid L Stevens if (err < 0) { 12848e845f4cSDavid L Stevens netdev_info(dev, "tx buffer map error %d\n", err); 128513b13dd9SSowmini Varadhan goto out_dropped; 1286e689cf4aSJeff Kirsher } 1287da38c564SDavid L Stevens 1288da38c564SDavid L Stevens port->tx_bufs[txi].skb = skb; 1289da38c564SDavid L Stevens skb = NULL; 12908e845f4cSDavid L Stevens port->tx_bufs[txi].ncookies = err; 1291e689cf4aSJeff Kirsher 12921f6394e3SSowmini Varadhan /* We don't rely on the ACKs to free the skb in vnet_start_xmit(), 12931f6394e3SSowmini Varadhan * thus it is safe to not set VIO_ACK_ENABLE for each transmission: 12941f6394e3SSowmini Varadhan * the protocol itself does not require it as long as the peer 12951f6394e3SSowmini Varadhan * sends a VIO_SUBTYPE_ACK for VIO_DRING_STOPPED. 12961f6394e3SSowmini Varadhan * 12971f6394e3SSowmini Varadhan * An ACK for every packet in the ring is expensive as the 12981f6394e3SSowmini Varadhan * sending of LDC messages is slow and affects performance. 12991f6394e3SSowmini Varadhan */ 13001f6394e3SSowmini Varadhan d->hdr.ack = VIO_ACK_DISABLE; 1301e689cf4aSJeff Kirsher d->size = len; 13028e845f4cSDavid L Stevens d->ncookies = port->tx_bufs[txi].ncookies; 1303e689cf4aSJeff Kirsher for (i = 0; i < d->ncookies; i++) 13048e845f4cSDavid L Stevens d->cookies[i] = port->tx_bufs[txi].cookies[i]; 13056d0ba919SDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 7)) { 13066d0ba919SDavid L Stevens struct vio_net_dext *dext = vio_net_ext(d); 13076d0ba919SDavid L Stevens 13086d0ba919SDavid L Stevens memset(dext, 0, sizeof(*dext)); 13096d0ba919SDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 8) && 13106d0ba919SDavid L Stevens !port->switch_port) { 13116d0ba919SDavid L Stevens dext->flags |= VNET_PKT_HCK_IPV4_HDRCKSUM_OK; 13126d0ba919SDavid L Stevens dext->flags |= VNET_PKT_HCK_FULLCKSUM_OK; 13136d0ba919SDavid L Stevens } 13146d0ba919SDavid L Stevens } 1315e689cf4aSJeff Kirsher 1316e689cf4aSJeff Kirsher /* This has to be a non-SMP write barrier because we are writing 1317e689cf4aSJeff Kirsher * to memory which is shared with the peer LDOM. 1318e689cf4aSJeff Kirsher */ 1319e689cf4aSJeff Kirsher wmb(); 1320e689cf4aSJeff Kirsher 1321e689cf4aSJeff Kirsher d->hdr.state = VIO_DESC_READY; 1322e689cf4aSJeff Kirsher 1323d1015645SSowmini Varadhan /* Exactly one ldc "start" trigger (for dr->cons) needs to be sent 1324d1015645SSowmini Varadhan * to notify the consumer that some descriptors are READY. 1325d1015645SSowmini Varadhan * After that "start" trigger, no additional triggers are needed until 1326d1015645SSowmini Varadhan * a DRING_STOPPED is received from the consumer. The dr->cons field 1327d1015645SSowmini Varadhan * (set up by vnet_ack()) has the value of the next dring index 1328d1015645SSowmini Varadhan * that has not yet been ack-ed. We send a "start" trigger here 1329d1015645SSowmini Varadhan * if, and only if, start_cons is true (reset it afterward). Conversely, 1330d1015645SSowmini Varadhan * vnet_ack() should check if the dring corresponding to cons 1331d1015645SSowmini Varadhan * is marked READY, but start_cons was false. 1332d1015645SSowmini Varadhan * If so, vnet_ack() should send out the missed "start" trigger. 1333d1015645SSowmini Varadhan * 1334d1015645SSowmini Varadhan * Note that the wmb() above makes sure the cookies et al. are 1335d1015645SSowmini Varadhan * not globally visible before the VIO_DESC_READY, and that the 1336d1015645SSowmini Varadhan * stores are ordered correctly by the compiler. The consumer will 1337d1015645SSowmini Varadhan * not proceed until the VIO_DESC_READY is visible assuring that 1338d1015645SSowmini Varadhan * the consumer does not observe anything related to descriptors 1339d1015645SSowmini Varadhan * out of order. The HV trap from the LDC start trigger is the 1340d1015645SSowmini Varadhan * producer to consumer announcement that work is available to the 1341d1015645SSowmini Varadhan * consumer 1342d1015645SSowmini Varadhan */ 1343d1015645SSowmini Varadhan if (!port->start_cons) 1344d1015645SSowmini Varadhan goto ldc_start_done; /* previous trigger suffices */ 1345d1015645SSowmini Varadhan 1346d1015645SSowmini Varadhan err = __vnet_tx_trigger(port, dr->cons); 1347e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 1348e689cf4aSJeff Kirsher netdev_info(dev, "TX trigger error %d\n", err); 1349e689cf4aSJeff Kirsher d->hdr.state = VIO_DESC_FREE; 1350e689cf4aSJeff Kirsher dev->stats.tx_carrier_errors++; 135113b13dd9SSowmini Varadhan goto out_dropped; 1352e689cf4aSJeff Kirsher } 1353e689cf4aSJeff Kirsher 1354d1015645SSowmini Varadhan ldc_start_done: 1355d1015645SSowmini Varadhan port->start_cons = false; 1356d1015645SSowmini Varadhan 1357e689cf4aSJeff Kirsher dev->stats.tx_packets++; 13588e845f4cSDavid L Stevens dev->stats.tx_bytes += port->tx_bufs[txi].skb->len; 1359e689cf4aSJeff Kirsher 1360e689cf4aSJeff Kirsher dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1); 1361d0aedcd4SDwight Engen if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1362d51bffd1SSowmini Varadhan netif_tx_stop_queue(txq); 1363e689cf4aSJeff Kirsher if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr)) 1364d51bffd1SSowmini Varadhan netif_tx_wake_queue(txq); 1365e689cf4aSJeff Kirsher } 1366e689cf4aSJeff Kirsher 13672a968dd8SSowmini Varadhan (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT); 13682a968dd8SSowmini Varadhan rcu_read_unlock(); 1369e689cf4aSJeff Kirsher 13708e845f4cSDavid L Stevens vnet_free_skbs(freeskbs); 13718e845f4cSDavid L Stevens 1372e689cf4aSJeff Kirsher return NETDEV_TX_OK; 1373e689cf4aSJeff Kirsher 1374e689cf4aSJeff Kirsher out_dropped: 13758e845f4cSDavid L Stevens if (pending) 13768e845f4cSDavid L Stevens (void)mod_timer(&port->clean_timer, 13778e845f4cSDavid L Stevens jiffies + VNET_CLEAN_TIMEOUT); 1378a29c9c43SDavid L Stevens else if (port) 13798e845f4cSDavid L Stevens del_timer(&port->clean_timer); 13802a968dd8SSowmini Varadhan if (port) 13812a968dd8SSowmini Varadhan rcu_read_unlock(); 13822a968dd8SSowmini Varadhan if (skb) 13832a968dd8SSowmini Varadhan dev_kfree_skb(skb); 13842a968dd8SSowmini Varadhan vnet_free_skbs(freeskbs); 1385e689cf4aSJeff Kirsher dev->stats.tx_dropped++; 1386e689cf4aSJeff Kirsher return NETDEV_TX_OK; 1387e689cf4aSJeff Kirsher } 1388e689cf4aSJeff Kirsher 1389e689cf4aSJeff Kirsher static void vnet_tx_timeout(struct net_device *dev) 1390e689cf4aSJeff Kirsher { 1391e689cf4aSJeff Kirsher /* XXX Implement me XXX */ 1392e689cf4aSJeff Kirsher } 1393e689cf4aSJeff Kirsher 1394e689cf4aSJeff Kirsher static int vnet_open(struct net_device *dev) 1395e689cf4aSJeff Kirsher { 1396e689cf4aSJeff Kirsher netif_carrier_on(dev); 1397d51bffd1SSowmini Varadhan netif_tx_start_all_queues(dev); 1398e689cf4aSJeff Kirsher 1399e689cf4aSJeff Kirsher return 0; 1400e689cf4aSJeff Kirsher } 1401e689cf4aSJeff Kirsher 1402e689cf4aSJeff Kirsher static int vnet_close(struct net_device *dev) 1403e689cf4aSJeff Kirsher { 1404d51bffd1SSowmini Varadhan netif_tx_stop_all_queues(dev); 1405e689cf4aSJeff Kirsher netif_carrier_off(dev); 1406e689cf4aSJeff Kirsher 1407e689cf4aSJeff Kirsher return 0; 1408e689cf4aSJeff Kirsher } 1409e689cf4aSJeff Kirsher 1410e689cf4aSJeff Kirsher static struct vnet_mcast_entry *__vnet_mc_find(struct vnet *vp, u8 *addr) 1411e689cf4aSJeff Kirsher { 1412e689cf4aSJeff Kirsher struct vnet_mcast_entry *m; 1413e689cf4aSJeff Kirsher 1414e689cf4aSJeff Kirsher for (m = vp->mcast_list; m; m = m->next) { 141500fa4ce9Sdingtianhong if (ether_addr_equal(m->addr, addr)) 1416e689cf4aSJeff Kirsher return m; 1417e689cf4aSJeff Kirsher } 1418e689cf4aSJeff Kirsher return NULL; 1419e689cf4aSJeff Kirsher } 1420e689cf4aSJeff Kirsher 1421e689cf4aSJeff Kirsher static void __update_mc_list(struct vnet *vp, struct net_device *dev) 1422e689cf4aSJeff Kirsher { 1423e689cf4aSJeff Kirsher struct netdev_hw_addr *ha; 1424e689cf4aSJeff Kirsher 1425e689cf4aSJeff Kirsher netdev_for_each_mc_addr(ha, dev) { 1426e689cf4aSJeff Kirsher struct vnet_mcast_entry *m; 1427e689cf4aSJeff Kirsher 1428e689cf4aSJeff Kirsher m = __vnet_mc_find(vp, ha->addr); 1429e689cf4aSJeff Kirsher if (m) { 1430e689cf4aSJeff Kirsher m->hit = 1; 1431e689cf4aSJeff Kirsher continue; 1432e689cf4aSJeff Kirsher } 1433e689cf4aSJeff Kirsher 1434e689cf4aSJeff Kirsher if (!m) { 1435e689cf4aSJeff Kirsher m = kzalloc(sizeof(*m), GFP_ATOMIC); 1436e689cf4aSJeff Kirsher if (!m) 1437e689cf4aSJeff Kirsher continue; 1438e689cf4aSJeff Kirsher memcpy(m->addr, ha->addr, ETH_ALEN); 1439e689cf4aSJeff Kirsher m->hit = 1; 1440e689cf4aSJeff Kirsher 1441e689cf4aSJeff Kirsher m->next = vp->mcast_list; 1442e689cf4aSJeff Kirsher vp->mcast_list = m; 1443e689cf4aSJeff Kirsher } 1444e689cf4aSJeff Kirsher } 1445e689cf4aSJeff Kirsher } 1446e689cf4aSJeff Kirsher 1447e689cf4aSJeff Kirsher static void __send_mc_list(struct vnet *vp, struct vnet_port *port) 1448e689cf4aSJeff Kirsher { 1449e689cf4aSJeff Kirsher struct vio_net_mcast_info info; 1450e689cf4aSJeff Kirsher struct vnet_mcast_entry *m, **pp; 1451e689cf4aSJeff Kirsher int n_addrs; 1452e689cf4aSJeff Kirsher 1453e689cf4aSJeff Kirsher memset(&info, 0, sizeof(info)); 1454e689cf4aSJeff Kirsher 1455e689cf4aSJeff Kirsher info.tag.type = VIO_TYPE_CTRL; 1456e689cf4aSJeff Kirsher info.tag.stype = VIO_SUBTYPE_INFO; 1457e689cf4aSJeff Kirsher info.tag.stype_env = VNET_MCAST_INFO; 1458e689cf4aSJeff Kirsher info.tag.sid = vio_send_sid(&port->vio); 1459e689cf4aSJeff Kirsher info.set = 1; 1460e689cf4aSJeff Kirsher 1461e689cf4aSJeff Kirsher n_addrs = 0; 1462e689cf4aSJeff Kirsher for (m = vp->mcast_list; m; m = m->next) { 1463e689cf4aSJeff Kirsher if (m->sent) 1464e689cf4aSJeff Kirsher continue; 1465e689cf4aSJeff Kirsher m->sent = 1; 1466e689cf4aSJeff Kirsher memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1467e689cf4aSJeff Kirsher m->addr, ETH_ALEN); 1468e689cf4aSJeff Kirsher if (++n_addrs == VNET_NUM_MCAST) { 1469e689cf4aSJeff Kirsher info.count = n_addrs; 1470e689cf4aSJeff Kirsher 1471e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, 1472e689cf4aSJeff Kirsher sizeof(info)); 1473e689cf4aSJeff Kirsher n_addrs = 0; 1474e689cf4aSJeff Kirsher } 1475e689cf4aSJeff Kirsher } 1476e689cf4aSJeff Kirsher if (n_addrs) { 1477e689cf4aSJeff Kirsher info.count = n_addrs; 1478e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1479e689cf4aSJeff Kirsher } 1480e689cf4aSJeff Kirsher 1481e689cf4aSJeff Kirsher info.set = 0; 1482e689cf4aSJeff Kirsher 1483e689cf4aSJeff Kirsher n_addrs = 0; 1484e689cf4aSJeff Kirsher pp = &vp->mcast_list; 1485e689cf4aSJeff Kirsher while ((m = *pp) != NULL) { 1486e689cf4aSJeff Kirsher if (m->hit) { 1487e689cf4aSJeff Kirsher m->hit = 0; 1488e689cf4aSJeff Kirsher pp = &m->next; 1489e689cf4aSJeff Kirsher continue; 1490e689cf4aSJeff Kirsher } 1491e689cf4aSJeff Kirsher 1492e689cf4aSJeff Kirsher memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1493e689cf4aSJeff Kirsher m->addr, ETH_ALEN); 1494e689cf4aSJeff Kirsher if (++n_addrs == VNET_NUM_MCAST) { 1495e689cf4aSJeff Kirsher info.count = n_addrs; 1496e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, 1497e689cf4aSJeff Kirsher sizeof(info)); 1498e689cf4aSJeff Kirsher n_addrs = 0; 1499e689cf4aSJeff Kirsher } 1500e689cf4aSJeff Kirsher 1501e689cf4aSJeff Kirsher *pp = m->next; 1502e689cf4aSJeff Kirsher kfree(m); 1503e689cf4aSJeff Kirsher } 1504e689cf4aSJeff Kirsher if (n_addrs) { 1505e689cf4aSJeff Kirsher info.count = n_addrs; 1506e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1507e689cf4aSJeff Kirsher } 1508e689cf4aSJeff Kirsher } 1509e689cf4aSJeff Kirsher 1510e689cf4aSJeff Kirsher static void vnet_set_rx_mode(struct net_device *dev) 1511e689cf4aSJeff Kirsher { 1512e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1513e689cf4aSJeff Kirsher struct vnet_port *port; 1514e689cf4aSJeff Kirsher 15152a968dd8SSowmini Varadhan rcu_read_lock(); 15162a968dd8SSowmini Varadhan list_for_each_entry_rcu(port, &vp->port_list, list) { 1517e689cf4aSJeff Kirsher 1518e689cf4aSJeff Kirsher if (port->switch_port) { 1519e689cf4aSJeff Kirsher __update_mc_list(vp, dev); 1520e689cf4aSJeff Kirsher __send_mc_list(vp, port); 15212a968dd8SSowmini Varadhan break; 1522e689cf4aSJeff Kirsher } 1523e689cf4aSJeff Kirsher } 15242a968dd8SSowmini Varadhan rcu_read_unlock(); 1525e689cf4aSJeff Kirsher } 1526e689cf4aSJeff Kirsher 1527e689cf4aSJeff Kirsher static int vnet_change_mtu(struct net_device *dev, int new_mtu) 1528e689cf4aSJeff Kirsher { 152942db672dSDavid L Stevens if (new_mtu < 68 || new_mtu > 65535) 1530e689cf4aSJeff Kirsher return -EINVAL; 1531e689cf4aSJeff Kirsher 1532e689cf4aSJeff Kirsher dev->mtu = new_mtu; 1533e689cf4aSJeff Kirsher return 0; 1534e689cf4aSJeff Kirsher } 1535e689cf4aSJeff Kirsher 1536e689cf4aSJeff Kirsher static int vnet_set_mac_addr(struct net_device *dev, void *p) 1537e689cf4aSJeff Kirsher { 1538e689cf4aSJeff Kirsher return -EINVAL; 1539e689cf4aSJeff Kirsher } 1540e689cf4aSJeff Kirsher 1541e689cf4aSJeff Kirsher static void vnet_get_drvinfo(struct net_device *dev, 1542e689cf4aSJeff Kirsher struct ethtool_drvinfo *info) 1543e689cf4aSJeff Kirsher { 15447826d43fSJiri Pirko strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); 15457826d43fSJiri Pirko strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); 1546e689cf4aSJeff Kirsher } 1547e689cf4aSJeff Kirsher 1548e689cf4aSJeff Kirsher static u32 vnet_get_msglevel(struct net_device *dev) 1549e689cf4aSJeff Kirsher { 1550e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1551e689cf4aSJeff Kirsher return vp->msg_enable; 1552e689cf4aSJeff Kirsher } 1553e689cf4aSJeff Kirsher 1554e689cf4aSJeff Kirsher static void vnet_set_msglevel(struct net_device *dev, u32 value) 1555e689cf4aSJeff Kirsher { 1556e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1557e689cf4aSJeff Kirsher vp->msg_enable = value; 1558e689cf4aSJeff Kirsher } 1559e689cf4aSJeff Kirsher 1560e689cf4aSJeff Kirsher static const struct ethtool_ops vnet_ethtool_ops = { 1561e689cf4aSJeff Kirsher .get_drvinfo = vnet_get_drvinfo, 1562e689cf4aSJeff Kirsher .get_msglevel = vnet_get_msglevel, 1563e689cf4aSJeff Kirsher .set_msglevel = vnet_set_msglevel, 1564e689cf4aSJeff Kirsher .get_link = ethtool_op_get_link, 1565e689cf4aSJeff Kirsher }; 1566e689cf4aSJeff Kirsher 1567e689cf4aSJeff Kirsher static void vnet_port_free_tx_bufs(struct vnet_port *port) 1568e689cf4aSJeff Kirsher { 1569e689cf4aSJeff Kirsher struct vio_dring_state *dr; 1570e689cf4aSJeff Kirsher int i; 1571e689cf4aSJeff Kirsher 1572e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1573e689cf4aSJeff Kirsher if (dr->base) { 1574e689cf4aSJeff Kirsher ldc_free_exp_dring(port->vio.lp, dr->base, 1575e689cf4aSJeff Kirsher (dr->entry_size * dr->num_entries), 1576e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 1577e689cf4aSJeff Kirsher dr->base = NULL; 1578e689cf4aSJeff Kirsher dr->entry_size = 0; 1579e689cf4aSJeff Kirsher dr->num_entries = 0; 1580e689cf4aSJeff Kirsher dr->pending = 0; 1581e689cf4aSJeff Kirsher dr->ncookies = 0; 1582e689cf4aSJeff Kirsher } 1583e689cf4aSJeff Kirsher 1584e689cf4aSJeff Kirsher for (i = 0; i < VNET_TX_RING_SIZE; i++) { 15858e845f4cSDavid L Stevens struct vio_net_desc *d; 15868e845f4cSDavid L Stevens void *skb = port->tx_bufs[i].skb; 1587e689cf4aSJeff Kirsher 15888e845f4cSDavid L Stevens if (!skb) 1589e689cf4aSJeff Kirsher continue; 1590e689cf4aSJeff Kirsher 15918e845f4cSDavid L Stevens d = vio_dring_entry(dr, i); 15928e845f4cSDavid L Stevens if (d->hdr.state == VIO_DESC_READY) 15938e845f4cSDavid L Stevens pr_warn("active transmit buffers freed\n"); 15948e845f4cSDavid L Stevens 1595e689cf4aSJeff Kirsher ldc_unmap(port->vio.lp, 1596e689cf4aSJeff Kirsher port->tx_bufs[i].cookies, 1597e689cf4aSJeff Kirsher port->tx_bufs[i].ncookies); 15988e845f4cSDavid L Stevens dev_kfree_skb(skb); 15998e845f4cSDavid L Stevens port->tx_bufs[i].skb = NULL; 16008e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 1601e689cf4aSJeff Kirsher } 1602e689cf4aSJeff Kirsher } 1603e689cf4aSJeff Kirsher 1604d6732489SDavid L Stevens static int vnet_port_alloc_tx_ring(struct vnet_port *port) 1605e689cf4aSJeff Kirsher { 1606e689cf4aSJeff Kirsher struct vio_dring_state *dr; 16076d0ba919SDavid L Stevens unsigned long len, elen; 1608e689cf4aSJeff Kirsher int i, err, ncookies; 1609e689cf4aSJeff Kirsher void *dring; 1610e689cf4aSJeff Kirsher 1611e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1612e689cf4aSJeff Kirsher 16136d0ba919SDavid L Stevens elen = sizeof(struct vio_net_desc) + 16146d0ba919SDavid L Stevens sizeof(struct ldc_trans_cookie) * 2; 16156d0ba919SDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 7)) 16166d0ba919SDavid L Stevens elen += sizeof(struct vio_net_dext); 16176d0ba919SDavid L Stevens len = VNET_TX_RING_SIZE * elen; 1618e689cf4aSJeff Kirsher 1619e689cf4aSJeff Kirsher ncookies = VIO_MAX_RING_COOKIES; 1620e689cf4aSJeff Kirsher dring = ldc_alloc_exp_dring(port->vio.lp, len, 1621e689cf4aSJeff Kirsher dr->cookies, &ncookies, 1622e689cf4aSJeff Kirsher (LDC_MAP_SHADOW | 1623e689cf4aSJeff Kirsher LDC_MAP_DIRECT | 1624e689cf4aSJeff Kirsher LDC_MAP_RW)); 1625e689cf4aSJeff Kirsher if (IS_ERR(dring)) { 1626e689cf4aSJeff Kirsher err = PTR_ERR(dring); 1627e689cf4aSJeff Kirsher goto err_out; 1628e689cf4aSJeff Kirsher } 1629e689cf4aSJeff Kirsher 1630e689cf4aSJeff Kirsher dr->base = dring; 16316d0ba919SDavid L Stevens dr->entry_size = elen; 1632e689cf4aSJeff Kirsher dr->num_entries = VNET_TX_RING_SIZE; 1633e689cf4aSJeff Kirsher dr->prod = dr->cons = 0; 1634d1015645SSowmini Varadhan port->start_cons = true; /* need an initial trigger */ 1635e689cf4aSJeff Kirsher dr->pending = VNET_TX_RING_SIZE; 1636e689cf4aSJeff Kirsher dr->ncookies = ncookies; 1637e689cf4aSJeff Kirsher 16388e845f4cSDavid L Stevens for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 16398e845f4cSDavid L Stevens struct vio_net_desc *d; 16408e845f4cSDavid L Stevens 16418e845f4cSDavid L Stevens d = vio_dring_entry(dr, i); 16428e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 16438e845f4cSDavid L Stevens } 1644e689cf4aSJeff Kirsher return 0; 1645e689cf4aSJeff Kirsher 1646e689cf4aSJeff Kirsher err_out: 1647e689cf4aSJeff Kirsher vnet_port_free_tx_bufs(port); 1648e689cf4aSJeff Kirsher 1649e689cf4aSJeff Kirsher return err; 1650e689cf4aSJeff Kirsher } 1651e689cf4aSJeff Kirsher 165269088822SSowmini Varadhan #ifdef CONFIG_NET_POLL_CONTROLLER 165369088822SSowmini Varadhan static void vnet_poll_controller(struct net_device *dev) 165469088822SSowmini Varadhan { 165569088822SSowmini Varadhan struct vnet *vp = netdev_priv(dev); 165669088822SSowmini Varadhan struct vnet_port *port; 165769088822SSowmini Varadhan unsigned long flags; 165869088822SSowmini Varadhan 165969088822SSowmini Varadhan spin_lock_irqsave(&vp->lock, flags); 166069088822SSowmini Varadhan if (!list_empty(&vp->port_list)) { 166169088822SSowmini Varadhan port = list_entry(vp->port_list.next, struct vnet_port, list); 166269088822SSowmini Varadhan napi_schedule(&port->napi); 166369088822SSowmini Varadhan } 166469088822SSowmini Varadhan spin_unlock_irqrestore(&vp->lock, flags); 166569088822SSowmini Varadhan } 166669088822SSowmini Varadhan #endif 1667e689cf4aSJeff Kirsher static LIST_HEAD(vnet_list); 1668e689cf4aSJeff Kirsher static DEFINE_MUTEX(vnet_list_mutex); 1669e689cf4aSJeff Kirsher 1670e689cf4aSJeff Kirsher static const struct net_device_ops vnet_ops = { 1671e689cf4aSJeff Kirsher .ndo_open = vnet_open, 1672e689cf4aSJeff Kirsher .ndo_stop = vnet_close, 1673afc4b13dSJiri Pirko .ndo_set_rx_mode = vnet_set_rx_mode, 1674e689cf4aSJeff Kirsher .ndo_set_mac_address = vnet_set_mac_addr, 1675e689cf4aSJeff Kirsher .ndo_validate_addr = eth_validate_addr, 1676e689cf4aSJeff Kirsher .ndo_tx_timeout = vnet_tx_timeout, 1677e689cf4aSJeff Kirsher .ndo_change_mtu = vnet_change_mtu, 1678e689cf4aSJeff Kirsher .ndo_start_xmit = vnet_start_xmit, 1679d51bffd1SSowmini Varadhan .ndo_select_queue = vnet_select_queue, 168069088822SSowmini Varadhan #ifdef CONFIG_NET_POLL_CONTROLLER 168169088822SSowmini Varadhan .ndo_poll_controller = vnet_poll_controller, 168269088822SSowmini Varadhan #endif 1683e689cf4aSJeff Kirsher }; 1684e689cf4aSJeff Kirsher 1685f73d12bdSBill Pemberton static struct vnet *vnet_new(const u64 *local_mac) 1686e689cf4aSJeff Kirsher { 1687e689cf4aSJeff Kirsher struct net_device *dev; 1688e689cf4aSJeff Kirsher struct vnet *vp; 1689e689cf4aSJeff Kirsher int err, i; 1690e689cf4aSJeff Kirsher 1691d51bffd1SSowmini Varadhan dev = alloc_etherdev_mqs(sizeof(*vp), VNET_MAX_TXQS, 1); 169241de8d4cSJoe Perches if (!dev) 1693e689cf4aSJeff Kirsher return ERR_PTR(-ENOMEM); 16948e845f4cSDavid L Stevens dev->needed_headroom = VNET_PACKET_SKIP + 8; 16958e845f4cSDavid L Stevens dev->needed_tailroom = 8; 1696e689cf4aSJeff Kirsher 1697e689cf4aSJeff Kirsher for (i = 0; i < ETH_ALEN; i++) 1698e689cf4aSJeff Kirsher dev->dev_addr[i] = (*local_mac >> (5 - i) * 8) & 0xff; 1699e689cf4aSJeff Kirsher 1700e689cf4aSJeff Kirsher vp = netdev_priv(dev); 1701e689cf4aSJeff Kirsher 1702e689cf4aSJeff Kirsher spin_lock_init(&vp->lock); 1703e689cf4aSJeff Kirsher vp->dev = dev; 1704e689cf4aSJeff Kirsher 1705e689cf4aSJeff Kirsher INIT_LIST_HEAD(&vp->port_list); 1706e689cf4aSJeff Kirsher for (i = 0; i < VNET_PORT_HASH_SIZE; i++) 1707e689cf4aSJeff Kirsher INIT_HLIST_HEAD(&vp->port_hash[i]); 1708e689cf4aSJeff Kirsher INIT_LIST_HEAD(&vp->list); 1709e689cf4aSJeff Kirsher vp->local_mac = *local_mac; 1710e689cf4aSJeff Kirsher 1711e689cf4aSJeff Kirsher dev->netdev_ops = &vnet_ops; 1712e689cf4aSJeff Kirsher dev->ethtool_ops = &vnet_ethtool_ops; 1713e689cf4aSJeff Kirsher dev->watchdog_timeo = VNET_TX_TIMEOUT; 1714e689cf4aSJeff Kirsher 17159a72dd4dSDavid L Stevens dev->hw_features = NETIF_F_GSO | NETIF_F_GSO_SOFTWARE | 17169a72dd4dSDavid L Stevens NETIF_F_HW_CSUM | NETIF_F_SG; 1717da38c564SDavid L Stevens dev->features = dev->hw_features; 1718da38c564SDavid L Stevens 1719e689cf4aSJeff Kirsher err = register_netdev(dev); 1720e689cf4aSJeff Kirsher if (err) { 1721e689cf4aSJeff Kirsher pr_err("Cannot register net device, aborting\n"); 1722e689cf4aSJeff Kirsher goto err_out_free_dev; 1723e689cf4aSJeff Kirsher } 1724e689cf4aSJeff Kirsher 1725e689cf4aSJeff Kirsher netdev_info(dev, "Sun LDOM vnet %pM\n", dev->dev_addr); 1726e689cf4aSJeff Kirsher 1727e689cf4aSJeff Kirsher list_add(&vp->list, &vnet_list); 1728e689cf4aSJeff Kirsher 1729e689cf4aSJeff Kirsher return vp; 1730e689cf4aSJeff Kirsher 1731e689cf4aSJeff Kirsher err_out_free_dev: 1732e689cf4aSJeff Kirsher free_netdev(dev); 1733e689cf4aSJeff Kirsher 1734e689cf4aSJeff Kirsher return ERR_PTR(err); 1735e689cf4aSJeff Kirsher } 1736e689cf4aSJeff Kirsher 1737f73d12bdSBill Pemberton static struct vnet *vnet_find_or_create(const u64 *local_mac) 1738e689cf4aSJeff Kirsher { 1739e689cf4aSJeff Kirsher struct vnet *iter, *vp; 1740e689cf4aSJeff Kirsher 1741e689cf4aSJeff Kirsher mutex_lock(&vnet_list_mutex); 1742e689cf4aSJeff Kirsher vp = NULL; 1743e689cf4aSJeff Kirsher list_for_each_entry(iter, &vnet_list, list) { 1744e689cf4aSJeff Kirsher if (iter->local_mac == *local_mac) { 1745e689cf4aSJeff Kirsher vp = iter; 1746e689cf4aSJeff Kirsher break; 1747e689cf4aSJeff Kirsher } 1748e689cf4aSJeff Kirsher } 1749e689cf4aSJeff Kirsher if (!vp) 1750e689cf4aSJeff Kirsher vp = vnet_new(local_mac); 1751e689cf4aSJeff Kirsher mutex_unlock(&vnet_list_mutex); 1752e689cf4aSJeff Kirsher 1753e689cf4aSJeff Kirsher return vp; 1754e689cf4aSJeff Kirsher } 1755e689cf4aSJeff Kirsher 1756a4b70a07SSowmini Varadhan static void vnet_cleanup(void) 1757a4b70a07SSowmini Varadhan { 1758a4b70a07SSowmini Varadhan struct vnet *vp; 1759a4b70a07SSowmini Varadhan struct net_device *dev; 1760a4b70a07SSowmini Varadhan 1761a4b70a07SSowmini Varadhan mutex_lock(&vnet_list_mutex); 1762a4b70a07SSowmini Varadhan while (!list_empty(&vnet_list)) { 1763a4b70a07SSowmini Varadhan vp = list_first_entry(&vnet_list, struct vnet, list); 1764a4b70a07SSowmini Varadhan list_del(&vp->list); 1765a4b70a07SSowmini Varadhan dev = vp->dev; 1766a4b70a07SSowmini Varadhan /* vio_unregister_driver() should have cleaned up port_list */ 1767a4b70a07SSowmini Varadhan BUG_ON(!list_empty(&vp->port_list)); 1768a4b70a07SSowmini Varadhan unregister_netdev(dev); 1769a4b70a07SSowmini Varadhan free_netdev(dev); 1770a4b70a07SSowmini Varadhan } 1771a4b70a07SSowmini Varadhan mutex_unlock(&vnet_list_mutex); 1772a4b70a07SSowmini Varadhan } 1773a4b70a07SSowmini Varadhan 1774e689cf4aSJeff Kirsher static const char *local_mac_prop = "local-mac-address"; 1775e689cf4aSJeff Kirsher 1776f73d12bdSBill Pemberton static struct vnet *vnet_find_parent(struct mdesc_handle *hp, 1777e689cf4aSJeff Kirsher u64 port_node) 1778e689cf4aSJeff Kirsher { 1779e689cf4aSJeff Kirsher const u64 *local_mac = NULL; 1780e689cf4aSJeff Kirsher u64 a; 1781e689cf4aSJeff Kirsher 1782e689cf4aSJeff Kirsher mdesc_for_each_arc(a, hp, port_node, MDESC_ARC_TYPE_BACK) { 1783e689cf4aSJeff Kirsher u64 target = mdesc_arc_target(hp, a); 1784e689cf4aSJeff Kirsher const char *name; 1785e689cf4aSJeff Kirsher 1786e689cf4aSJeff Kirsher name = mdesc_get_property(hp, target, "name", NULL); 1787e689cf4aSJeff Kirsher if (!name || strcmp(name, "network")) 1788e689cf4aSJeff Kirsher continue; 1789e689cf4aSJeff Kirsher 1790e689cf4aSJeff Kirsher local_mac = mdesc_get_property(hp, target, 1791e689cf4aSJeff Kirsher local_mac_prop, NULL); 1792e689cf4aSJeff Kirsher if (local_mac) 1793e689cf4aSJeff Kirsher break; 1794e689cf4aSJeff Kirsher } 1795e689cf4aSJeff Kirsher if (!local_mac) 1796e689cf4aSJeff Kirsher return ERR_PTR(-ENODEV); 1797e689cf4aSJeff Kirsher 1798e689cf4aSJeff Kirsher return vnet_find_or_create(local_mac); 1799e689cf4aSJeff Kirsher } 1800e689cf4aSJeff Kirsher 1801e689cf4aSJeff Kirsher static struct ldc_channel_config vnet_ldc_cfg = { 1802e689cf4aSJeff Kirsher .event = vnet_event, 1803e689cf4aSJeff Kirsher .mtu = 64, 1804e689cf4aSJeff Kirsher .mode = LDC_MODE_UNRELIABLE, 1805e689cf4aSJeff Kirsher }; 1806e689cf4aSJeff Kirsher 1807e689cf4aSJeff Kirsher static struct vio_driver_ops vnet_vio_ops = { 1808e689cf4aSJeff Kirsher .send_attr = vnet_send_attr, 1809e689cf4aSJeff Kirsher .handle_attr = vnet_handle_attr, 1810e689cf4aSJeff Kirsher .handshake_complete = vnet_handshake_complete, 1811e689cf4aSJeff Kirsher }; 1812e689cf4aSJeff Kirsher 1813f73d12bdSBill Pemberton static void print_version(void) 1814e689cf4aSJeff Kirsher { 1815e689cf4aSJeff Kirsher printk_once(KERN_INFO "%s", version); 1816e689cf4aSJeff Kirsher } 1817e689cf4aSJeff Kirsher 1818e689cf4aSJeff Kirsher const char *remote_macaddr_prop = "remote-mac-address"; 1819e689cf4aSJeff Kirsher 1820d51bffd1SSowmini Varadhan static void 1821d51bffd1SSowmini Varadhan vnet_port_add_txq(struct vnet_port *port) 1822d51bffd1SSowmini Varadhan { 1823d51bffd1SSowmini Varadhan struct vnet *vp = port->vp; 1824d51bffd1SSowmini Varadhan int n; 1825d51bffd1SSowmini Varadhan 1826d51bffd1SSowmini Varadhan n = vp->nports++; 1827d51bffd1SSowmini Varadhan n = n & (VNET_MAX_TXQS - 1); 1828d51bffd1SSowmini Varadhan port->q_index = n; 1829d51bffd1SSowmini Varadhan netif_tx_wake_queue(netdev_get_tx_queue(vp->dev, port->q_index)); 1830d51bffd1SSowmini Varadhan } 1831d51bffd1SSowmini Varadhan 1832d51bffd1SSowmini Varadhan static void 1833d51bffd1SSowmini Varadhan vnet_port_rm_txq(struct vnet_port *port) 1834d51bffd1SSowmini Varadhan { 1835d51bffd1SSowmini Varadhan port->vp->nports--; 1836d51bffd1SSowmini Varadhan netif_tx_stop_queue(netdev_get_tx_queue(port->vp->dev, port->q_index)); 1837d51bffd1SSowmini Varadhan } 1838d51bffd1SSowmini Varadhan 18391dd06ae8SGreg Kroah-Hartman static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) 1840e689cf4aSJeff Kirsher { 1841e689cf4aSJeff Kirsher struct mdesc_handle *hp; 1842e689cf4aSJeff Kirsher struct vnet_port *port; 1843e689cf4aSJeff Kirsher unsigned long flags; 1844e689cf4aSJeff Kirsher struct vnet *vp; 1845e689cf4aSJeff Kirsher const u64 *rmac; 1846e689cf4aSJeff Kirsher int len, i, err, switch_port; 1847e689cf4aSJeff Kirsher 1848e689cf4aSJeff Kirsher print_version(); 1849e689cf4aSJeff Kirsher 1850e689cf4aSJeff Kirsher hp = mdesc_grab(); 1851e689cf4aSJeff Kirsher 1852e689cf4aSJeff Kirsher vp = vnet_find_parent(hp, vdev->mp); 1853e689cf4aSJeff Kirsher if (IS_ERR(vp)) { 1854e689cf4aSJeff Kirsher pr_err("Cannot find port parent vnet\n"); 1855e689cf4aSJeff Kirsher err = PTR_ERR(vp); 1856e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1857e689cf4aSJeff Kirsher } 1858e689cf4aSJeff Kirsher 1859e689cf4aSJeff Kirsher rmac = mdesc_get_property(hp, vdev->mp, remote_macaddr_prop, &len); 1860e689cf4aSJeff Kirsher err = -ENODEV; 1861e689cf4aSJeff Kirsher if (!rmac) { 1862e689cf4aSJeff Kirsher pr_err("Port lacks %s property\n", remote_macaddr_prop); 1863e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1864e689cf4aSJeff Kirsher } 1865e689cf4aSJeff Kirsher 1866e689cf4aSJeff Kirsher port = kzalloc(sizeof(*port), GFP_KERNEL); 1867e689cf4aSJeff Kirsher err = -ENOMEM; 1868e404decbSJoe Perches if (!port) 1869e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1870e689cf4aSJeff Kirsher 1871e689cf4aSJeff Kirsher for (i = 0; i < ETH_ALEN; i++) 1872e689cf4aSJeff Kirsher port->raddr[i] = (*rmac >> (5 - i) * 8) & 0xff; 1873e689cf4aSJeff Kirsher 1874e689cf4aSJeff Kirsher port->vp = vp; 1875e689cf4aSJeff Kirsher 1876e689cf4aSJeff Kirsher err = vio_driver_init(&port->vio, vdev, VDEV_NETWORK, 1877e689cf4aSJeff Kirsher vnet_versions, ARRAY_SIZE(vnet_versions), 1878e689cf4aSJeff Kirsher &vnet_vio_ops, vp->dev->name); 1879e689cf4aSJeff Kirsher if (err) 1880e689cf4aSJeff Kirsher goto err_out_free_port; 1881e689cf4aSJeff Kirsher 1882e689cf4aSJeff Kirsher err = vio_ldc_alloc(&port->vio, &vnet_ldc_cfg, port); 1883e689cf4aSJeff Kirsher if (err) 1884e689cf4aSJeff Kirsher goto err_out_free_port; 1885e689cf4aSJeff Kirsher 188669088822SSowmini Varadhan netif_napi_add(port->vp->dev, &port->napi, vnet_poll, NAPI_POLL_WEIGHT); 188769088822SSowmini Varadhan 1888e689cf4aSJeff Kirsher INIT_HLIST_NODE(&port->hash); 1889e689cf4aSJeff Kirsher INIT_LIST_HEAD(&port->list); 1890e689cf4aSJeff Kirsher 1891e689cf4aSJeff Kirsher switch_port = 0; 1892e689cf4aSJeff Kirsher if (mdesc_get_property(hp, vdev->mp, "switch-port", NULL) != NULL) 1893e689cf4aSJeff Kirsher switch_port = 1; 1894e689cf4aSJeff Kirsher port->switch_port = switch_port; 1895e689cf4aSJeff Kirsher 1896e689cf4aSJeff Kirsher spin_lock_irqsave(&vp->lock, flags); 1897e689cf4aSJeff Kirsher if (switch_port) 18982a968dd8SSowmini Varadhan list_add_rcu(&port->list, &vp->port_list); 1899e689cf4aSJeff Kirsher else 19002a968dd8SSowmini Varadhan list_add_tail_rcu(&port->list, &vp->port_list); 19012a968dd8SSowmini Varadhan hlist_add_head_rcu(&port->hash, 19022a968dd8SSowmini Varadhan &vp->port_hash[vnet_hashfn(port->raddr)]); 1903d51bffd1SSowmini Varadhan vnet_port_add_txq(port); 1904e689cf4aSJeff Kirsher spin_unlock_irqrestore(&vp->lock, flags); 1905e689cf4aSJeff Kirsher 1906e689cf4aSJeff Kirsher dev_set_drvdata(&vdev->dev, port); 1907e689cf4aSJeff Kirsher 1908e689cf4aSJeff Kirsher pr_info("%s: PORT ( remote-mac %pM%s )\n", 1909e689cf4aSJeff Kirsher vp->dev->name, port->raddr, switch_port ? " switch-port" : ""); 1910e689cf4aSJeff Kirsher 19118e845f4cSDavid L Stevens setup_timer(&port->clean_timer, vnet_clean_timer_expire, 19128e845f4cSDavid L Stevens (unsigned long)port); 19138e845f4cSDavid L Stevens 191469088822SSowmini Varadhan napi_enable(&port->napi); 1915e689cf4aSJeff Kirsher vio_port_up(&port->vio); 1916e689cf4aSJeff Kirsher 1917e689cf4aSJeff Kirsher mdesc_release(hp); 1918e689cf4aSJeff Kirsher 1919e689cf4aSJeff Kirsher return 0; 1920e689cf4aSJeff Kirsher 1921e689cf4aSJeff Kirsher err_out_free_port: 1922e689cf4aSJeff Kirsher kfree(port); 1923e689cf4aSJeff Kirsher 1924e689cf4aSJeff Kirsher err_out_put_mdesc: 1925e689cf4aSJeff Kirsher mdesc_release(hp); 1926e689cf4aSJeff Kirsher return err; 1927e689cf4aSJeff Kirsher } 1928e689cf4aSJeff Kirsher 1929e689cf4aSJeff Kirsher static int vnet_port_remove(struct vio_dev *vdev) 1930e689cf4aSJeff Kirsher { 1931e689cf4aSJeff Kirsher struct vnet_port *port = dev_get_drvdata(&vdev->dev); 1932e689cf4aSJeff Kirsher 1933e689cf4aSJeff Kirsher if (port) { 1934e689cf4aSJeff Kirsher 1935e689cf4aSJeff Kirsher del_timer_sync(&port->vio.timer); 1936e689cf4aSJeff Kirsher 193769088822SSowmini Varadhan napi_disable(&port->napi); 1938e689cf4aSJeff Kirsher 19392a968dd8SSowmini Varadhan list_del_rcu(&port->list); 19402a968dd8SSowmini Varadhan hlist_del_rcu(&port->hash); 19412a968dd8SSowmini Varadhan 19422a968dd8SSowmini Varadhan synchronize_rcu(); 19432a968dd8SSowmini Varadhan del_timer_sync(&port->clean_timer); 1944d51bffd1SSowmini Varadhan vnet_port_rm_txq(port); 194569088822SSowmini Varadhan netif_napi_del(&port->napi); 1946e689cf4aSJeff Kirsher vnet_port_free_tx_bufs(port); 1947e689cf4aSJeff Kirsher vio_ldc_free(&port->vio); 1948e689cf4aSJeff Kirsher 1949e689cf4aSJeff Kirsher dev_set_drvdata(&vdev->dev, NULL); 1950e689cf4aSJeff Kirsher 1951e689cf4aSJeff Kirsher kfree(port); 1952aabb9875SDave Kleikamp 1953e689cf4aSJeff Kirsher } 1954e689cf4aSJeff Kirsher return 0; 1955e689cf4aSJeff Kirsher } 1956e689cf4aSJeff Kirsher 1957e689cf4aSJeff Kirsher static const struct vio_device_id vnet_port_match[] = { 1958e689cf4aSJeff Kirsher { 1959e689cf4aSJeff Kirsher .type = "vnet-port", 1960e689cf4aSJeff Kirsher }, 1961e689cf4aSJeff Kirsher {}, 1962e689cf4aSJeff Kirsher }; 1963e689cf4aSJeff Kirsher MODULE_DEVICE_TABLE(vio, vnet_port_match); 1964e689cf4aSJeff Kirsher 1965e689cf4aSJeff Kirsher static struct vio_driver vnet_port_driver = { 1966e689cf4aSJeff Kirsher .id_table = vnet_port_match, 1967e689cf4aSJeff Kirsher .probe = vnet_port_probe, 1968e689cf4aSJeff Kirsher .remove = vnet_port_remove, 1969e689cf4aSJeff Kirsher .name = "vnet_port", 1970e689cf4aSJeff Kirsher }; 1971e689cf4aSJeff Kirsher 1972e689cf4aSJeff Kirsher static int __init vnet_init(void) 1973e689cf4aSJeff Kirsher { 1974e689cf4aSJeff Kirsher return vio_register_driver(&vnet_port_driver); 1975e689cf4aSJeff Kirsher } 1976e689cf4aSJeff Kirsher 1977e689cf4aSJeff Kirsher static void __exit vnet_exit(void) 1978e689cf4aSJeff Kirsher { 1979e689cf4aSJeff Kirsher vio_unregister_driver(&vnet_port_driver); 1980a4b70a07SSowmini Varadhan vnet_cleanup(); 1981e689cf4aSJeff Kirsher } 1982e689cf4aSJeff Kirsher 1983e689cf4aSJeff Kirsher module_init(vnet_init); 1984e689cf4aSJeff Kirsher module_exit(vnet_exit); 1985