1e689cf4aSJeff Kirsher /* sunvnet.c: Sun LDOM Virtual Network Driver. 2e689cf4aSJeff Kirsher * 3e689cf4aSJeff Kirsher * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4e689cf4aSJeff Kirsher */ 5e689cf4aSJeff Kirsher 6e689cf4aSJeff Kirsher #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7e689cf4aSJeff Kirsher 8e689cf4aSJeff Kirsher #include <linux/module.h> 9e689cf4aSJeff Kirsher #include <linux/kernel.h> 10e689cf4aSJeff Kirsher #include <linux/types.h> 11e689cf4aSJeff Kirsher #include <linux/slab.h> 12e689cf4aSJeff Kirsher #include <linux/delay.h> 13e689cf4aSJeff Kirsher #include <linux/init.h> 14e689cf4aSJeff Kirsher #include <linux/netdevice.h> 15e689cf4aSJeff Kirsher #include <linux/ethtool.h> 16e689cf4aSJeff Kirsher #include <linux/etherdevice.h> 17e689cf4aSJeff Kirsher #include <linux/mutex.h> 18e4defc77SDavid L Stevens #include <linux/if_vlan.h> 19e689cf4aSJeff Kirsher 20a2b78e9bSDavid L Stevens #if IS_ENABLED(CONFIG_IPV6) 21a2b78e9bSDavid L Stevens #include <linux/icmpv6.h> 22a2b78e9bSDavid L Stevens #endif 23a2b78e9bSDavid L Stevens 24a2b78e9bSDavid L Stevens #include <net/icmp.h> 25a2b78e9bSDavid L Stevens #include <net/route.h> 26a2b78e9bSDavid L Stevens 27e689cf4aSJeff Kirsher #include <asm/vio.h> 28e689cf4aSJeff Kirsher #include <asm/ldc.h> 29e689cf4aSJeff Kirsher 30e689cf4aSJeff Kirsher #include "sunvnet.h" 31e689cf4aSJeff Kirsher 32e689cf4aSJeff Kirsher #define DRV_MODULE_NAME "sunvnet" 33e689cf4aSJeff Kirsher #define DRV_MODULE_VERSION "1.0" 34e689cf4aSJeff Kirsher #define DRV_MODULE_RELDATE "June 25, 2007" 35e689cf4aSJeff Kirsher 36f73d12bdSBill Pemberton static char version[] = 37e689cf4aSJeff Kirsher DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 38e689cf4aSJeff Kirsher MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); 39e689cf4aSJeff Kirsher MODULE_DESCRIPTION("Sun LDOM virtual network driver"); 40e689cf4aSJeff Kirsher MODULE_LICENSE("GPL"); 41e689cf4aSJeff Kirsher MODULE_VERSION(DRV_MODULE_VERSION); 42e689cf4aSJeff Kirsher 43adddc32dSSowmini Varadhan /* Heuristic for the number of times to exponentially backoff and 44adddc32dSSowmini Varadhan * retry sending an LDC trigger when EAGAIN is encountered 45adddc32dSSowmini Varadhan */ 46adddc32dSSowmini Varadhan #define VNET_MAX_RETRIES 10 47adddc32dSSowmini Varadhan 48d1015645SSowmini Varadhan static int __vnet_tx_trigger(struct vnet_port *port, u32 start); 49d1015645SSowmini Varadhan 50e689cf4aSJeff Kirsher /* Ordered from largest major to lowest */ 51e689cf4aSJeff Kirsher static struct vio_version vnet_versions[] = { 52e4defc77SDavid L Stevens { .major = 1, .minor = 6 }, 53e689cf4aSJeff Kirsher { .major = 1, .minor = 0 }, 54e689cf4aSJeff Kirsher }; 55e689cf4aSJeff Kirsher 56e689cf4aSJeff Kirsher static inline u32 vnet_tx_dring_avail(struct vio_dring_state *dr) 57e689cf4aSJeff Kirsher { 58e689cf4aSJeff Kirsher return vio_dring_avail(dr, VNET_TX_RING_SIZE); 59e689cf4aSJeff Kirsher } 60e689cf4aSJeff Kirsher 61e689cf4aSJeff Kirsher static int vnet_handle_unknown(struct vnet_port *port, void *arg) 62e689cf4aSJeff Kirsher { 63e689cf4aSJeff Kirsher struct vio_msg_tag *pkt = arg; 64e689cf4aSJeff Kirsher 65e689cf4aSJeff Kirsher pr_err("Received unknown msg [%02x:%02x:%04x:%08x]\n", 66e689cf4aSJeff Kirsher pkt->type, pkt->stype, pkt->stype_env, pkt->sid); 67e689cf4aSJeff Kirsher pr_err("Resetting connection\n"); 68e689cf4aSJeff Kirsher 69e689cf4aSJeff Kirsher ldc_disconnect(port->vio.lp); 70e689cf4aSJeff Kirsher 71e689cf4aSJeff Kirsher return -ECONNRESET; 72e689cf4aSJeff Kirsher } 73e689cf4aSJeff Kirsher 74e689cf4aSJeff Kirsher static int vnet_send_attr(struct vio_driver_state *vio) 75e689cf4aSJeff Kirsher { 76e689cf4aSJeff Kirsher struct vnet_port *port = to_vnet_port(vio); 77e689cf4aSJeff Kirsher struct net_device *dev = port->vp->dev; 78e689cf4aSJeff Kirsher struct vio_net_attr_info pkt; 79e4defc77SDavid L Stevens int framelen = ETH_FRAME_LEN; 80e689cf4aSJeff Kirsher int i; 81e689cf4aSJeff Kirsher 82e689cf4aSJeff Kirsher memset(&pkt, 0, sizeof(pkt)); 83e689cf4aSJeff Kirsher pkt.tag.type = VIO_TYPE_CTRL; 84e689cf4aSJeff Kirsher pkt.tag.stype = VIO_SUBTYPE_INFO; 85e689cf4aSJeff Kirsher pkt.tag.stype_env = VIO_ATTR_INFO; 86e689cf4aSJeff Kirsher pkt.tag.sid = vio_send_sid(vio); 87e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 2)) 88e689cf4aSJeff Kirsher pkt.xfer_mode = VIO_DRING_MODE; 89e4defc77SDavid L Stevens else 90e4defc77SDavid L Stevens pkt.xfer_mode = VIO_NEW_DRING_MODE; 91e689cf4aSJeff Kirsher pkt.addr_type = VNET_ADDR_ETHERMAC; 92e689cf4aSJeff Kirsher pkt.ack_freq = 0; 93e689cf4aSJeff Kirsher for (i = 0; i < 6; i++) 94e689cf4aSJeff Kirsher pkt.addr |= (u64)dev->dev_addr[i] << ((5 - i) * 8); 95e4defc77SDavid L Stevens if (vio_version_after(vio, 1, 3)) { 96e4defc77SDavid L Stevens if (port->rmtu) { 97e4defc77SDavid L Stevens port->rmtu = min(VNET_MAXPACKET, port->rmtu); 98e4defc77SDavid L Stevens pkt.mtu = port->rmtu; 99e4defc77SDavid L Stevens } else { 100e4defc77SDavid L Stevens port->rmtu = VNET_MAXPACKET; 101e4defc77SDavid L Stevens pkt.mtu = port->rmtu; 102e4defc77SDavid L Stevens } 103e4defc77SDavid L Stevens if (vio_version_after_eq(vio, 1, 6)) 104e4defc77SDavid L Stevens pkt.options = VIO_TX_DRING; 105e4defc77SDavid L Stevens } else if (vio_version_before(vio, 1, 3)) { 106e4defc77SDavid L Stevens pkt.mtu = framelen; 107e4defc77SDavid L Stevens } else { /* v1.3 */ 108e4defc77SDavid L Stevens pkt.mtu = framelen + VLAN_HLEN; 109e4defc77SDavid L Stevens } 110e4defc77SDavid L Stevens 111e4defc77SDavid L Stevens pkt.plnk_updt = PHYSLINK_UPDATE_NONE; 112e4defc77SDavid L Stevens pkt.cflags = 0; 113e689cf4aSJeff Kirsher 114e689cf4aSJeff Kirsher viodbg(HS, "SEND NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 115e4defc77SDavid L Stevens "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 116e4defc77SDavid L Stevens "cflags[0x%04x] lso_max[%u]\n", 117e689cf4aSJeff Kirsher pkt.xfer_mode, pkt.addr_type, 118e689cf4aSJeff Kirsher (unsigned long long)pkt.addr, 119e4defc77SDavid L Stevens pkt.ack_freq, pkt.plnk_updt, pkt.options, 120e4defc77SDavid L Stevens (unsigned long long)pkt.mtu, pkt.cflags, pkt.ipv4_lso_maxlen); 121e4defc77SDavid L Stevens 122e689cf4aSJeff Kirsher 123e689cf4aSJeff Kirsher return vio_ldc_send(vio, &pkt, sizeof(pkt)); 124e689cf4aSJeff Kirsher } 125e689cf4aSJeff Kirsher 126e689cf4aSJeff Kirsher static int handle_attr_info(struct vio_driver_state *vio, 127e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 128e689cf4aSJeff Kirsher { 129e4defc77SDavid L Stevens struct vnet_port *port = to_vnet_port(vio); 130e4defc77SDavid L Stevens u64 localmtu; 131e4defc77SDavid L Stevens u8 xfer_mode; 132e4defc77SDavid L Stevens 133e4defc77SDavid L Stevens viodbg(HS, "GOT NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 134e4defc77SDavid L Stevens "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 135e4defc77SDavid L Stevens " (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 136e689cf4aSJeff Kirsher pkt->xfer_mode, pkt->addr_type, 137e689cf4aSJeff Kirsher (unsigned long long)pkt->addr, 138e4defc77SDavid L Stevens pkt->ack_freq, pkt->plnk_updt, pkt->options, 139e4defc77SDavid L Stevens (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 140e4defc77SDavid L Stevens pkt->ipv4_lso_maxlen); 141e689cf4aSJeff Kirsher 142e689cf4aSJeff Kirsher pkt->tag.sid = vio_send_sid(vio); 143e689cf4aSJeff Kirsher 144e4defc77SDavid L Stevens xfer_mode = pkt->xfer_mode; 145e4defc77SDavid L Stevens /* for version < 1.2, VIO_DRING_MODE = 0x3 and no bitmask */ 146e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 2) && xfer_mode == VIO_DRING_MODE) 147e4defc77SDavid L Stevens xfer_mode = VIO_NEW_DRING_MODE; 148e4defc77SDavid L Stevens 149e4defc77SDavid L Stevens /* MTU negotiation: 150e4defc77SDavid L Stevens * < v1.3 - ETH_FRAME_LEN exactly 151e4defc77SDavid L Stevens * > v1.3 - MIN(pkt.mtu, VNET_MAXPACKET, port->rmtu) and change 152e4defc77SDavid L Stevens * pkt->mtu for ACK 153e4defc77SDavid L Stevens * = v1.3 - ETH_FRAME_LEN + VLAN_HLEN exactly 154e4defc77SDavid L Stevens */ 155e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 3)) { 156e4defc77SDavid L Stevens localmtu = ETH_FRAME_LEN; 157e4defc77SDavid L Stevens } else if (vio_version_after(vio, 1, 3)) { 158e4defc77SDavid L Stevens localmtu = port->rmtu ? port->rmtu : VNET_MAXPACKET; 159e4defc77SDavid L Stevens localmtu = min(pkt->mtu, localmtu); 160e4defc77SDavid L Stevens pkt->mtu = localmtu; 161e4defc77SDavid L Stevens } else { /* v1.3 */ 162e4defc77SDavid L Stevens localmtu = ETH_FRAME_LEN + VLAN_HLEN; 163e4defc77SDavid L Stevens } 164e4defc77SDavid L Stevens port->rmtu = localmtu; 165e4defc77SDavid L Stevens 166e4defc77SDavid L Stevens /* for version >= 1.6, ACK packet mode we support */ 167e4defc77SDavid L Stevens if (vio_version_after_eq(vio, 1, 6)) { 168e4defc77SDavid L Stevens pkt->xfer_mode = VIO_NEW_DRING_MODE; 169e4defc77SDavid L Stevens pkt->options = VIO_TX_DRING; 170e4defc77SDavid L Stevens } 171e4defc77SDavid L Stevens 172e4defc77SDavid L Stevens if (!(xfer_mode | VIO_NEW_DRING_MODE) || 173e689cf4aSJeff Kirsher pkt->addr_type != VNET_ADDR_ETHERMAC || 174e4defc77SDavid L Stevens pkt->mtu != localmtu) { 175e689cf4aSJeff Kirsher viodbg(HS, "SEND NET ATTR NACK\n"); 176e689cf4aSJeff Kirsher 177e689cf4aSJeff Kirsher pkt->tag.stype = VIO_SUBTYPE_NACK; 178e689cf4aSJeff Kirsher 179e689cf4aSJeff Kirsher (void) vio_ldc_send(vio, pkt, sizeof(*pkt)); 180e689cf4aSJeff Kirsher 181e689cf4aSJeff Kirsher return -ECONNRESET; 182e689cf4aSJeff Kirsher } else { 183e4defc77SDavid L Stevens viodbg(HS, "SEND NET ATTR ACK xmode[0x%x] atype[0x%x] " 184e4defc77SDavid L Stevens "addr[%llx] ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] " 185e4defc77SDavid L Stevens "mtu[%llu] (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 186e4defc77SDavid L Stevens pkt->xfer_mode, pkt->addr_type, 187e4defc77SDavid L Stevens (unsigned long long)pkt->addr, 188e4defc77SDavid L Stevens pkt->ack_freq, pkt->plnk_updt, pkt->options, 189e4defc77SDavid L Stevens (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 190e4defc77SDavid L Stevens pkt->ipv4_lso_maxlen); 191e689cf4aSJeff Kirsher 192e689cf4aSJeff Kirsher pkt->tag.stype = VIO_SUBTYPE_ACK; 193e689cf4aSJeff Kirsher 194e689cf4aSJeff Kirsher return vio_ldc_send(vio, pkt, sizeof(*pkt)); 195e689cf4aSJeff Kirsher } 196e689cf4aSJeff Kirsher 197e689cf4aSJeff Kirsher } 198e689cf4aSJeff Kirsher 199e689cf4aSJeff Kirsher static int handle_attr_ack(struct vio_driver_state *vio, 200e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 201e689cf4aSJeff Kirsher { 202e689cf4aSJeff Kirsher viodbg(HS, "GOT NET ATTR ACK\n"); 203e689cf4aSJeff Kirsher 204e689cf4aSJeff Kirsher return 0; 205e689cf4aSJeff Kirsher } 206e689cf4aSJeff Kirsher 207e689cf4aSJeff Kirsher static int handle_attr_nack(struct vio_driver_state *vio, 208e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 209e689cf4aSJeff Kirsher { 210e689cf4aSJeff Kirsher viodbg(HS, "GOT NET ATTR NACK\n"); 211e689cf4aSJeff Kirsher 212e689cf4aSJeff Kirsher return -ECONNRESET; 213e689cf4aSJeff Kirsher } 214e689cf4aSJeff Kirsher 215e689cf4aSJeff Kirsher static int vnet_handle_attr(struct vio_driver_state *vio, void *arg) 216e689cf4aSJeff Kirsher { 217e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt = arg; 218e689cf4aSJeff Kirsher 219e689cf4aSJeff Kirsher switch (pkt->tag.stype) { 220e689cf4aSJeff Kirsher case VIO_SUBTYPE_INFO: 221e689cf4aSJeff Kirsher return handle_attr_info(vio, pkt); 222e689cf4aSJeff Kirsher 223e689cf4aSJeff Kirsher case VIO_SUBTYPE_ACK: 224e689cf4aSJeff Kirsher return handle_attr_ack(vio, pkt); 225e689cf4aSJeff Kirsher 226e689cf4aSJeff Kirsher case VIO_SUBTYPE_NACK: 227e689cf4aSJeff Kirsher return handle_attr_nack(vio, pkt); 228e689cf4aSJeff Kirsher 229e689cf4aSJeff Kirsher default: 230e689cf4aSJeff Kirsher return -ECONNRESET; 231e689cf4aSJeff Kirsher } 232e689cf4aSJeff Kirsher } 233e689cf4aSJeff Kirsher 234e689cf4aSJeff Kirsher static void vnet_handshake_complete(struct vio_driver_state *vio) 235e689cf4aSJeff Kirsher { 236e689cf4aSJeff Kirsher struct vio_dring_state *dr; 237e689cf4aSJeff Kirsher 238e689cf4aSJeff Kirsher dr = &vio->drings[VIO_DRIVER_RX_RING]; 239e689cf4aSJeff Kirsher dr->snd_nxt = dr->rcv_nxt = 1; 240e689cf4aSJeff Kirsher 241e689cf4aSJeff Kirsher dr = &vio->drings[VIO_DRIVER_TX_RING]; 242e689cf4aSJeff Kirsher dr->snd_nxt = dr->rcv_nxt = 1; 243e689cf4aSJeff Kirsher } 244e689cf4aSJeff Kirsher 245e689cf4aSJeff Kirsher /* The hypervisor interface that implements copying to/from imported 246e689cf4aSJeff Kirsher * memory from another domain requires that copies are done to 8-byte 247e689cf4aSJeff Kirsher * aligned buffers, and that the lengths of such copies are also 8-byte 248e689cf4aSJeff Kirsher * multiples. 249e689cf4aSJeff Kirsher * 250e689cf4aSJeff Kirsher * So we align skb->data to an 8-byte multiple and pad-out the data 251e689cf4aSJeff Kirsher * area so we can round the copy length up to the next multiple of 252e689cf4aSJeff Kirsher * 8 for the copy. 253e689cf4aSJeff Kirsher * 254e689cf4aSJeff Kirsher * The transmitter puts the actual start of the packet 6 bytes into 255e689cf4aSJeff Kirsher * the buffer it sends over, so that the IP headers after the ethernet 256e689cf4aSJeff Kirsher * header are aligned properly. These 6 bytes are not in the descriptor 257e689cf4aSJeff Kirsher * length, they are simply implied. This offset is represented using 258e689cf4aSJeff Kirsher * the VNET_PACKET_SKIP macro. 259e689cf4aSJeff Kirsher */ 260e689cf4aSJeff Kirsher static struct sk_buff *alloc_and_align_skb(struct net_device *dev, 261e689cf4aSJeff Kirsher unsigned int len) 262e689cf4aSJeff Kirsher { 263e689cf4aSJeff Kirsher struct sk_buff *skb = netdev_alloc_skb(dev, len+VNET_PACKET_SKIP+8+8); 264e689cf4aSJeff Kirsher unsigned long addr, off; 265e689cf4aSJeff Kirsher 266e689cf4aSJeff Kirsher if (unlikely(!skb)) 267e689cf4aSJeff Kirsher return NULL; 268e689cf4aSJeff Kirsher 269e689cf4aSJeff Kirsher addr = (unsigned long) skb->data; 270e689cf4aSJeff Kirsher off = ((addr + 7UL) & ~7UL) - addr; 271e689cf4aSJeff Kirsher if (off) 272e689cf4aSJeff Kirsher skb_reserve(skb, off); 273e689cf4aSJeff Kirsher 274e689cf4aSJeff Kirsher return skb; 275e689cf4aSJeff Kirsher } 276e689cf4aSJeff Kirsher 277e689cf4aSJeff Kirsher static int vnet_rx_one(struct vnet_port *port, unsigned int len, 278e689cf4aSJeff Kirsher struct ldc_trans_cookie *cookies, int ncookies) 279e689cf4aSJeff Kirsher { 280e689cf4aSJeff Kirsher struct net_device *dev = port->vp->dev; 281e689cf4aSJeff Kirsher unsigned int copy_len; 282e689cf4aSJeff Kirsher struct sk_buff *skb; 283e689cf4aSJeff Kirsher int err; 284e689cf4aSJeff Kirsher 285e689cf4aSJeff Kirsher err = -EMSGSIZE; 286e4defc77SDavid L Stevens if (unlikely(len < ETH_ZLEN || len > port->rmtu)) { 287e689cf4aSJeff Kirsher dev->stats.rx_length_errors++; 288e689cf4aSJeff Kirsher goto out_dropped; 289e689cf4aSJeff Kirsher } 290e689cf4aSJeff Kirsher 291e689cf4aSJeff Kirsher skb = alloc_and_align_skb(dev, len); 292e689cf4aSJeff Kirsher err = -ENOMEM; 293e689cf4aSJeff Kirsher if (unlikely(!skb)) { 294e689cf4aSJeff Kirsher dev->stats.rx_missed_errors++; 295e689cf4aSJeff Kirsher goto out_dropped; 296e689cf4aSJeff Kirsher } 297e689cf4aSJeff Kirsher 298e689cf4aSJeff Kirsher copy_len = (len + VNET_PACKET_SKIP + 7U) & ~7U; 299e689cf4aSJeff Kirsher skb_put(skb, copy_len); 300e689cf4aSJeff Kirsher err = ldc_copy(port->vio.lp, LDC_COPY_IN, 301e689cf4aSJeff Kirsher skb->data, copy_len, 0, 302e689cf4aSJeff Kirsher cookies, ncookies); 303e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 304e689cf4aSJeff Kirsher dev->stats.rx_frame_errors++; 305e689cf4aSJeff Kirsher goto out_free_skb; 306e689cf4aSJeff Kirsher } 307e689cf4aSJeff Kirsher 308e689cf4aSJeff Kirsher skb_pull(skb, VNET_PACKET_SKIP); 309e689cf4aSJeff Kirsher skb_trim(skb, len); 310e689cf4aSJeff Kirsher skb->protocol = eth_type_trans(skb, dev); 311e689cf4aSJeff Kirsher 312e689cf4aSJeff Kirsher dev->stats.rx_packets++; 313e689cf4aSJeff Kirsher dev->stats.rx_bytes += len; 31469088822SSowmini Varadhan napi_gro_receive(&port->napi, skb); 315e689cf4aSJeff Kirsher return 0; 316e689cf4aSJeff Kirsher 317e689cf4aSJeff Kirsher out_free_skb: 318e689cf4aSJeff Kirsher kfree_skb(skb); 319e689cf4aSJeff Kirsher 320e689cf4aSJeff Kirsher out_dropped: 321e689cf4aSJeff Kirsher dev->stats.rx_dropped++; 322e689cf4aSJeff Kirsher return err; 323e689cf4aSJeff Kirsher } 324e689cf4aSJeff Kirsher 325e689cf4aSJeff Kirsher static int vnet_send_ack(struct vnet_port *port, struct vio_dring_state *dr, 326e689cf4aSJeff Kirsher u32 start, u32 end, u8 vio_dring_state) 327e689cf4aSJeff Kirsher { 328e689cf4aSJeff Kirsher struct vio_dring_data hdr = { 329e689cf4aSJeff Kirsher .tag = { 330e689cf4aSJeff Kirsher .type = VIO_TYPE_DATA, 331e689cf4aSJeff Kirsher .stype = VIO_SUBTYPE_ACK, 332e689cf4aSJeff Kirsher .stype_env = VIO_DRING_DATA, 333e689cf4aSJeff Kirsher .sid = vio_send_sid(&port->vio), 334e689cf4aSJeff Kirsher }, 335e689cf4aSJeff Kirsher .dring_ident = dr->ident, 336e689cf4aSJeff Kirsher .start_idx = start, 337e689cf4aSJeff Kirsher .end_idx = end, 338e689cf4aSJeff Kirsher .state = vio_dring_state, 339e689cf4aSJeff Kirsher }; 340e689cf4aSJeff Kirsher int err, delay; 341adddc32dSSowmini Varadhan int retries = 0; 342e689cf4aSJeff Kirsher 343e689cf4aSJeff Kirsher hdr.seq = dr->snd_nxt; 344e689cf4aSJeff Kirsher delay = 1; 345e689cf4aSJeff Kirsher do { 346e689cf4aSJeff Kirsher err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 347e689cf4aSJeff Kirsher if (err > 0) { 348e689cf4aSJeff Kirsher dr->snd_nxt++; 349e689cf4aSJeff Kirsher break; 350e689cf4aSJeff Kirsher } 351e689cf4aSJeff Kirsher udelay(delay); 352e689cf4aSJeff Kirsher if ((delay <<= 1) > 128) 353e689cf4aSJeff Kirsher delay = 128; 354adddc32dSSowmini Varadhan if (retries++ > VNET_MAX_RETRIES) { 355adddc32dSSowmini Varadhan pr_info("ECONNRESET %x:%x:%x:%x:%x:%x\n", 356adddc32dSSowmini Varadhan port->raddr[0], port->raddr[1], 357adddc32dSSowmini Varadhan port->raddr[2], port->raddr[3], 358adddc32dSSowmini Varadhan port->raddr[4], port->raddr[5]); 359d1015645SSowmini Varadhan break; 360adddc32dSSowmini Varadhan } 361e689cf4aSJeff Kirsher } while (err == -EAGAIN); 362e689cf4aSJeff Kirsher 363d1015645SSowmini Varadhan if (err <= 0 && vio_dring_state == VIO_DRING_STOPPED) { 364d1015645SSowmini Varadhan port->stop_rx_idx = end; 365d1015645SSowmini Varadhan port->stop_rx = true; 366d1015645SSowmini Varadhan } else { 367d1015645SSowmini Varadhan port->stop_rx_idx = 0; 368d1015645SSowmini Varadhan port->stop_rx = false; 369d1015645SSowmini Varadhan } 370d1015645SSowmini Varadhan 371e689cf4aSJeff Kirsher return err; 372e689cf4aSJeff Kirsher } 373e689cf4aSJeff Kirsher 374e689cf4aSJeff Kirsher static u32 next_idx(u32 idx, struct vio_dring_state *dr) 375e689cf4aSJeff Kirsher { 376e689cf4aSJeff Kirsher if (++idx == dr->num_entries) 377e689cf4aSJeff Kirsher idx = 0; 378e689cf4aSJeff Kirsher return idx; 379e689cf4aSJeff Kirsher } 380e689cf4aSJeff Kirsher 381e689cf4aSJeff Kirsher static u32 prev_idx(u32 idx, struct vio_dring_state *dr) 382e689cf4aSJeff Kirsher { 383e689cf4aSJeff Kirsher if (idx == 0) 384e689cf4aSJeff Kirsher idx = dr->num_entries - 1; 385e689cf4aSJeff Kirsher else 386e689cf4aSJeff Kirsher idx--; 387e689cf4aSJeff Kirsher 388e689cf4aSJeff Kirsher return idx; 389e689cf4aSJeff Kirsher } 390e689cf4aSJeff Kirsher 391e689cf4aSJeff Kirsher static struct vio_net_desc *get_rx_desc(struct vnet_port *port, 392e689cf4aSJeff Kirsher struct vio_dring_state *dr, 393e689cf4aSJeff Kirsher u32 index) 394e689cf4aSJeff Kirsher { 395e689cf4aSJeff Kirsher struct vio_net_desc *desc = port->vio.desc_buf; 396e689cf4aSJeff Kirsher int err; 397e689cf4aSJeff Kirsher 398e689cf4aSJeff Kirsher err = ldc_get_dring_entry(port->vio.lp, desc, dr->entry_size, 399e689cf4aSJeff Kirsher (index * dr->entry_size), 400e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 401e689cf4aSJeff Kirsher if (err < 0) 402e689cf4aSJeff Kirsher return ERR_PTR(err); 403e689cf4aSJeff Kirsher 404e689cf4aSJeff Kirsher return desc; 405e689cf4aSJeff Kirsher } 406e689cf4aSJeff Kirsher 407e689cf4aSJeff Kirsher static int put_rx_desc(struct vnet_port *port, 408e689cf4aSJeff Kirsher struct vio_dring_state *dr, 409e689cf4aSJeff Kirsher struct vio_net_desc *desc, 410e689cf4aSJeff Kirsher u32 index) 411e689cf4aSJeff Kirsher { 412e689cf4aSJeff Kirsher int err; 413e689cf4aSJeff Kirsher 414e689cf4aSJeff Kirsher err = ldc_put_dring_entry(port->vio.lp, desc, dr->entry_size, 415e689cf4aSJeff Kirsher (index * dr->entry_size), 416e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 417e689cf4aSJeff Kirsher if (err < 0) 418e689cf4aSJeff Kirsher return err; 419e689cf4aSJeff Kirsher 420e689cf4aSJeff Kirsher return 0; 421e689cf4aSJeff Kirsher } 422e689cf4aSJeff Kirsher 423e689cf4aSJeff Kirsher static int vnet_walk_rx_one(struct vnet_port *port, 424e689cf4aSJeff Kirsher struct vio_dring_state *dr, 425e689cf4aSJeff Kirsher u32 index, int *needs_ack) 426e689cf4aSJeff Kirsher { 427e689cf4aSJeff Kirsher struct vio_net_desc *desc = get_rx_desc(port, dr, index); 428e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 429e689cf4aSJeff Kirsher int err; 430e689cf4aSJeff Kirsher 43169088822SSowmini Varadhan BUG_ON(desc == NULL); 432e689cf4aSJeff Kirsher if (IS_ERR(desc)) 433e689cf4aSJeff Kirsher return PTR_ERR(desc); 434e689cf4aSJeff Kirsher 43578dcff7bSDavid L Stevens if (desc->hdr.state != VIO_DESC_READY) 43678dcff7bSDavid L Stevens return 1; 43778dcff7bSDavid L Stevens 43878dcff7bSDavid L Stevens rmb(); 43978dcff7bSDavid L Stevens 440e689cf4aSJeff Kirsher viodbg(DATA, "vio_walk_rx_one desc[%02x:%02x:%08x:%08x:%llx:%llx]\n", 441e689cf4aSJeff Kirsher desc->hdr.state, desc->hdr.ack, 442e689cf4aSJeff Kirsher desc->size, desc->ncookies, 443e689cf4aSJeff Kirsher desc->cookies[0].cookie_addr, 444e689cf4aSJeff Kirsher desc->cookies[0].cookie_size); 445e689cf4aSJeff Kirsher 446e689cf4aSJeff Kirsher err = vnet_rx_one(port, desc->size, desc->cookies, desc->ncookies); 447e689cf4aSJeff Kirsher if (err == -ECONNRESET) 448e689cf4aSJeff Kirsher return err; 449e689cf4aSJeff Kirsher desc->hdr.state = VIO_DESC_DONE; 450e689cf4aSJeff Kirsher err = put_rx_desc(port, dr, desc, index); 451e689cf4aSJeff Kirsher if (err < 0) 452e689cf4aSJeff Kirsher return err; 453e689cf4aSJeff Kirsher *needs_ack = desc->hdr.ack; 454e689cf4aSJeff Kirsher return 0; 455e689cf4aSJeff Kirsher } 456e689cf4aSJeff Kirsher 457e689cf4aSJeff Kirsher static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr, 45869088822SSowmini Varadhan u32 start, u32 end, int *npkts, int budget) 459e689cf4aSJeff Kirsher { 460e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 461e689cf4aSJeff Kirsher int ack_start = -1, ack_end = -1; 46269088822SSowmini Varadhan bool send_ack = true; 463e689cf4aSJeff Kirsher 464e689cf4aSJeff Kirsher end = (end == (u32) -1) ? prev_idx(start, dr) : next_idx(end, dr); 465e689cf4aSJeff Kirsher 466e689cf4aSJeff Kirsher viodbg(DATA, "vnet_walk_rx start[%08x] end[%08x]\n", start, end); 467e689cf4aSJeff Kirsher 468e689cf4aSJeff Kirsher while (start != end) { 469e689cf4aSJeff Kirsher int ack = 0, err = vnet_walk_rx_one(port, dr, start, &ack); 470e689cf4aSJeff Kirsher if (err == -ECONNRESET) 471e689cf4aSJeff Kirsher return err; 472e689cf4aSJeff Kirsher if (err != 0) 473e689cf4aSJeff Kirsher break; 47469088822SSowmini Varadhan (*npkts)++; 475e689cf4aSJeff Kirsher if (ack_start == -1) 476e689cf4aSJeff Kirsher ack_start = start; 477e689cf4aSJeff Kirsher ack_end = start; 478e689cf4aSJeff Kirsher start = next_idx(start, dr); 479e689cf4aSJeff Kirsher if (ack && start != end) { 480e689cf4aSJeff Kirsher err = vnet_send_ack(port, dr, ack_start, ack_end, 481e689cf4aSJeff Kirsher VIO_DRING_ACTIVE); 482e689cf4aSJeff Kirsher if (err == -ECONNRESET) 483e689cf4aSJeff Kirsher return err; 484e689cf4aSJeff Kirsher ack_start = -1; 485e689cf4aSJeff Kirsher } 48669088822SSowmini Varadhan if ((*npkts) >= budget) { 48769088822SSowmini Varadhan send_ack = false; 48869088822SSowmini Varadhan break; 48969088822SSowmini Varadhan } 490e689cf4aSJeff Kirsher } 491e689cf4aSJeff Kirsher if (unlikely(ack_start == -1)) 492e689cf4aSJeff Kirsher ack_start = ack_end = prev_idx(start, dr); 49369088822SSowmini Varadhan if (send_ack) { 49469088822SSowmini Varadhan port->napi_resume = false; 49569088822SSowmini Varadhan return vnet_send_ack(port, dr, ack_start, ack_end, 49669088822SSowmini Varadhan VIO_DRING_STOPPED); 49769088822SSowmini Varadhan } else { 49869088822SSowmini Varadhan port->napi_resume = true; 49969088822SSowmini Varadhan port->napi_stop_idx = ack_end; 50069088822SSowmini Varadhan return 1; 50169088822SSowmini Varadhan } 502e689cf4aSJeff Kirsher } 503e689cf4aSJeff Kirsher 50469088822SSowmini Varadhan static int vnet_rx(struct vnet_port *port, void *msgbuf, int *npkts, 50569088822SSowmini Varadhan int budget) 506e689cf4aSJeff Kirsher { 507e689cf4aSJeff Kirsher struct vio_dring_data *pkt = msgbuf; 508e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_RX_RING]; 509e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 510e689cf4aSJeff Kirsher 511e689cf4aSJeff Kirsher viodbg(DATA, "vnet_rx stype_env[%04x] seq[%016llx] rcv_nxt[%016llx]\n", 512e689cf4aSJeff Kirsher pkt->tag.stype_env, pkt->seq, dr->rcv_nxt); 513e689cf4aSJeff Kirsher 514e689cf4aSJeff Kirsher if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 515e689cf4aSJeff Kirsher return 0; 516e689cf4aSJeff Kirsher if (unlikely(pkt->seq != dr->rcv_nxt)) { 517e689cf4aSJeff Kirsher pr_err("RX out of sequence seq[0x%llx] rcv_nxt[0x%llx]\n", 518e689cf4aSJeff Kirsher pkt->seq, dr->rcv_nxt); 519e689cf4aSJeff Kirsher return 0; 520e689cf4aSJeff Kirsher } 521e689cf4aSJeff Kirsher 52269088822SSowmini Varadhan if (!port->napi_resume) 523e689cf4aSJeff Kirsher dr->rcv_nxt++; 524e689cf4aSJeff Kirsher 525e689cf4aSJeff Kirsher /* XXX Validate pkt->start_idx and pkt->end_idx XXX */ 526e689cf4aSJeff Kirsher 52769088822SSowmini Varadhan return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx, 52869088822SSowmini Varadhan npkts, budget); 529e689cf4aSJeff Kirsher } 530e689cf4aSJeff Kirsher 531e689cf4aSJeff Kirsher static int idx_is_pending(struct vio_dring_state *dr, u32 end) 532e689cf4aSJeff Kirsher { 533e689cf4aSJeff Kirsher u32 idx = dr->cons; 534e689cf4aSJeff Kirsher int found = 0; 535e689cf4aSJeff Kirsher 536e689cf4aSJeff Kirsher while (idx != dr->prod) { 537e689cf4aSJeff Kirsher if (idx == end) { 538e689cf4aSJeff Kirsher found = 1; 539e689cf4aSJeff Kirsher break; 540e689cf4aSJeff Kirsher } 541e689cf4aSJeff Kirsher idx = next_idx(idx, dr); 542e689cf4aSJeff Kirsher } 543e689cf4aSJeff Kirsher return found; 544e689cf4aSJeff Kirsher } 545e689cf4aSJeff Kirsher 546e689cf4aSJeff Kirsher static int vnet_ack(struct vnet_port *port, void *msgbuf) 547e689cf4aSJeff Kirsher { 548e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 549e689cf4aSJeff Kirsher struct vio_dring_data *pkt = msgbuf; 550e689cf4aSJeff Kirsher struct net_device *dev; 551e689cf4aSJeff Kirsher struct vnet *vp; 552e689cf4aSJeff Kirsher u32 end; 553d1015645SSowmini Varadhan struct vio_net_desc *desc; 554e689cf4aSJeff Kirsher if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 555e689cf4aSJeff Kirsher return 0; 556e689cf4aSJeff Kirsher 557e689cf4aSJeff Kirsher end = pkt->end_idx; 558e689cf4aSJeff Kirsher if (unlikely(!idx_is_pending(dr, end))) 559e689cf4aSJeff Kirsher return 0; 560e689cf4aSJeff Kirsher 56169088822SSowmini Varadhan vp = port->vp; 56269088822SSowmini Varadhan dev = vp->dev; 563d1015645SSowmini Varadhan /* sync for race conditions with vnet_start_xmit() and tell xmit it 564d1015645SSowmini Varadhan * is time to send a trigger. 565d1015645SSowmini Varadhan */ 56669088822SSowmini Varadhan netif_tx_lock(dev); 567e689cf4aSJeff Kirsher dr->cons = next_idx(end, dr); 568d1015645SSowmini Varadhan desc = vio_dring_entry(dr, dr->cons); 569d1015645SSowmini Varadhan if (desc->hdr.state == VIO_DESC_READY && port->start_cons) { 570d1015645SSowmini Varadhan /* vnet_start_xmit() just populated this dring but missed 571d1015645SSowmini Varadhan * sending the "start" LDC message to the consumer. 572d1015645SSowmini Varadhan * Send a "start" trigger on its behalf. 573d1015645SSowmini Varadhan */ 574d1015645SSowmini Varadhan if (__vnet_tx_trigger(port, dr->cons) > 0) 575d1015645SSowmini Varadhan port->start_cons = false; 576d1015645SSowmini Varadhan else 577d1015645SSowmini Varadhan port->start_cons = true; 578d1015645SSowmini Varadhan } else { 579d1015645SSowmini Varadhan port->start_cons = true; 580d1015645SSowmini Varadhan } 58169088822SSowmini Varadhan netif_tx_unlock(dev); 582d1015645SSowmini Varadhan 583e689cf4aSJeff Kirsher if (unlikely(netif_queue_stopped(dev) && 584e689cf4aSJeff Kirsher vnet_tx_dring_avail(dr) >= VNET_TX_WAKEUP_THRESH(dr))) 585e689cf4aSJeff Kirsher return 1; 586e689cf4aSJeff Kirsher 587e689cf4aSJeff Kirsher return 0; 588e689cf4aSJeff Kirsher } 589e689cf4aSJeff Kirsher 590e689cf4aSJeff Kirsher static int vnet_nack(struct vnet_port *port, void *msgbuf) 591e689cf4aSJeff Kirsher { 592e689cf4aSJeff Kirsher /* XXX just reset or similar XXX */ 593e689cf4aSJeff Kirsher return 0; 594e689cf4aSJeff Kirsher } 595e689cf4aSJeff Kirsher 596e689cf4aSJeff Kirsher static int handle_mcast(struct vnet_port *port, void *msgbuf) 597e689cf4aSJeff Kirsher { 598e689cf4aSJeff Kirsher struct vio_net_mcast_info *pkt = msgbuf; 599e689cf4aSJeff Kirsher 600e689cf4aSJeff Kirsher if (pkt->tag.stype != VIO_SUBTYPE_ACK) 601e689cf4aSJeff Kirsher pr_err("%s: Got unexpected MCAST reply [%02x:%02x:%04x:%08x]\n", 602e689cf4aSJeff Kirsher port->vp->dev->name, 603e689cf4aSJeff Kirsher pkt->tag.type, 604e689cf4aSJeff Kirsher pkt->tag.stype, 605e689cf4aSJeff Kirsher pkt->tag.stype_env, 606e689cf4aSJeff Kirsher pkt->tag.sid); 607e689cf4aSJeff Kirsher 608e689cf4aSJeff Kirsher return 0; 609e689cf4aSJeff Kirsher } 610e689cf4aSJeff Kirsher 61169088822SSowmini Varadhan static void maybe_tx_wakeup(struct vnet *vp) 612e689cf4aSJeff Kirsher { 613e689cf4aSJeff Kirsher struct net_device *dev = vp->dev; 614e689cf4aSJeff Kirsher 615e689cf4aSJeff Kirsher netif_tx_lock(dev); 616e689cf4aSJeff Kirsher if (likely(netif_queue_stopped(dev))) { 617e689cf4aSJeff Kirsher struct vnet_port *port; 618e689cf4aSJeff Kirsher int wake = 1; 619e689cf4aSJeff Kirsher 6202a968dd8SSowmini Varadhan rcu_read_lock(); 6212a968dd8SSowmini Varadhan list_for_each_entry_rcu(port, &vp->port_list, list) { 622e689cf4aSJeff Kirsher struct vio_dring_state *dr; 623e689cf4aSJeff Kirsher 624e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 625e689cf4aSJeff Kirsher if (vnet_tx_dring_avail(dr) < 626e689cf4aSJeff Kirsher VNET_TX_WAKEUP_THRESH(dr)) { 627e689cf4aSJeff Kirsher wake = 0; 628e689cf4aSJeff Kirsher break; 629e689cf4aSJeff Kirsher } 630e689cf4aSJeff Kirsher } 6312a968dd8SSowmini Varadhan rcu_read_unlock(); 632e689cf4aSJeff Kirsher if (wake) 633e689cf4aSJeff Kirsher netif_wake_queue(dev); 634e689cf4aSJeff Kirsher } 635e689cf4aSJeff Kirsher netif_tx_unlock(dev); 636e689cf4aSJeff Kirsher } 637e689cf4aSJeff Kirsher 63869088822SSowmini Varadhan static inline bool port_is_up(struct vnet_port *vnet) 639e689cf4aSJeff Kirsher { 64069088822SSowmini Varadhan struct vio_driver_state *vio = &vnet->vio; 64169088822SSowmini Varadhan 64269088822SSowmini Varadhan return !!(vio->hs_state & VIO_HS_COMPLETE); 64369088822SSowmini Varadhan } 64469088822SSowmini Varadhan 64569088822SSowmini Varadhan static int vnet_event_napi(struct vnet_port *port, int budget) 64669088822SSowmini Varadhan { 647e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 648e689cf4aSJeff Kirsher int tx_wakeup, err; 64969088822SSowmini Varadhan int npkts = 0; 65069088822SSowmini Varadhan int event = (port->rx_event & LDC_EVENT_RESET); 651e689cf4aSJeff Kirsher 65269088822SSowmini Varadhan ldc_ctrl: 653e689cf4aSJeff Kirsher if (unlikely(event == LDC_EVENT_RESET || 654e689cf4aSJeff Kirsher event == LDC_EVENT_UP)) { 655e689cf4aSJeff Kirsher vio_link_state_change(vio, event); 656e689cf4aSJeff Kirsher 657e4defc77SDavid L Stevens if (event == LDC_EVENT_RESET) { 658e4defc77SDavid L Stevens port->rmtu = 0; 659e689cf4aSJeff Kirsher vio_port_up(vio); 660e4defc77SDavid L Stevens } 66169088822SSowmini Varadhan port->rx_event = 0; 66269088822SSowmini Varadhan return 0; 663e689cf4aSJeff Kirsher } 66469088822SSowmini Varadhan /* We may have multiple LDC events in rx_event. Unroll send_events() */ 66569088822SSowmini Varadhan event = (port->rx_event & LDC_EVENT_UP); 66669088822SSowmini Varadhan port->rx_event &= ~(LDC_EVENT_RESET|LDC_EVENT_UP); 66769088822SSowmini Varadhan if (event == LDC_EVENT_UP) 66869088822SSowmini Varadhan goto ldc_ctrl; 66969088822SSowmini Varadhan event = port->rx_event; 67069088822SSowmini Varadhan if (!(event & LDC_EVENT_DATA_READY)) 67169088822SSowmini Varadhan return 0; 672e689cf4aSJeff Kirsher 67369088822SSowmini Varadhan /* we dont expect any other bits than RESET, UP, DATA_READY */ 67469088822SSowmini Varadhan BUG_ON(event != LDC_EVENT_DATA_READY); 675e689cf4aSJeff Kirsher 676e689cf4aSJeff Kirsher tx_wakeup = err = 0; 677e689cf4aSJeff Kirsher while (1) { 678e689cf4aSJeff Kirsher union { 679e689cf4aSJeff Kirsher struct vio_msg_tag tag; 680e689cf4aSJeff Kirsher u64 raw[8]; 681e689cf4aSJeff Kirsher } msgbuf; 682e689cf4aSJeff Kirsher 68369088822SSowmini Varadhan if (port->napi_resume) { 68469088822SSowmini Varadhan struct vio_dring_data *pkt = 68569088822SSowmini Varadhan (struct vio_dring_data *)&msgbuf; 68669088822SSowmini Varadhan struct vio_dring_state *dr = 68769088822SSowmini Varadhan &port->vio.drings[VIO_DRIVER_RX_RING]; 68869088822SSowmini Varadhan 68969088822SSowmini Varadhan pkt->tag.type = VIO_TYPE_DATA; 69069088822SSowmini Varadhan pkt->tag.stype = VIO_SUBTYPE_INFO; 69169088822SSowmini Varadhan pkt->tag.stype_env = VIO_DRING_DATA; 69269088822SSowmini Varadhan pkt->seq = dr->rcv_nxt; 69369088822SSowmini Varadhan pkt->start_idx = next_idx(port->napi_stop_idx, dr); 69469088822SSowmini Varadhan pkt->end_idx = -1; 69569088822SSowmini Varadhan goto napi_resume; 69669088822SSowmini Varadhan } 69769088822SSowmini Varadhan ldc_read: 698e689cf4aSJeff Kirsher err = ldc_read(vio->lp, &msgbuf, sizeof(msgbuf)); 699e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 700e689cf4aSJeff Kirsher if (err == -ECONNRESET) 701e689cf4aSJeff Kirsher vio_conn_reset(vio); 702e689cf4aSJeff Kirsher break; 703e689cf4aSJeff Kirsher } 704e689cf4aSJeff Kirsher if (err == 0) 705e689cf4aSJeff Kirsher break; 706e689cf4aSJeff Kirsher viodbg(DATA, "TAG [%02x:%02x:%04x:%08x]\n", 707e689cf4aSJeff Kirsher msgbuf.tag.type, 708e689cf4aSJeff Kirsher msgbuf.tag.stype, 709e689cf4aSJeff Kirsher msgbuf.tag.stype_env, 710e689cf4aSJeff Kirsher msgbuf.tag.sid); 711e689cf4aSJeff Kirsher err = vio_validate_sid(vio, &msgbuf.tag); 712e689cf4aSJeff Kirsher if (err < 0) 713e689cf4aSJeff Kirsher break; 71469088822SSowmini Varadhan napi_resume: 715e689cf4aSJeff Kirsher if (likely(msgbuf.tag.type == VIO_TYPE_DATA)) { 716e689cf4aSJeff Kirsher if (msgbuf.tag.stype == VIO_SUBTYPE_INFO) { 71769088822SSowmini Varadhan if (!port_is_up(port)) { 71869088822SSowmini Varadhan /* failures like handshake_failure() 71969088822SSowmini Varadhan * may have cleaned up dring, but 72069088822SSowmini Varadhan * NAPI polling may bring us here. 72169088822SSowmini Varadhan */ 72269088822SSowmini Varadhan err = -ECONNRESET; 72369088822SSowmini Varadhan break; 72469088822SSowmini Varadhan } 72569088822SSowmini Varadhan err = vnet_rx(port, &msgbuf, &npkts, budget); 72669088822SSowmini Varadhan if (npkts >= budget) 72769088822SSowmini Varadhan break; 72869088822SSowmini Varadhan if (npkts == 0 && err != -ECONNRESET) 72969088822SSowmini Varadhan goto ldc_read; 730e689cf4aSJeff Kirsher } else if (msgbuf.tag.stype == VIO_SUBTYPE_ACK) { 731e689cf4aSJeff Kirsher err = vnet_ack(port, &msgbuf); 732e689cf4aSJeff Kirsher if (err > 0) 733e689cf4aSJeff Kirsher tx_wakeup |= err; 734e689cf4aSJeff Kirsher } else if (msgbuf.tag.stype == VIO_SUBTYPE_NACK) { 735e689cf4aSJeff Kirsher err = vnet_nack(port, &msgbuf); 736e689cf4aSJeff Kirsher } 737e689cf4aSJeff Kirsher } else if (msgbuf.tag.type == VIO_TYPE_CTRL) { 738e689cf4aSJeff Kirsher if (msgbuf.tag.stype_env == VNET_MCAST_INFO) 739e689cf4aSJeff Kirsher err = handle_mcast(port, &msgbuf); 740e689cf4aSJeff Kirsher else 741e689cf4aSJeff Kirsher err = vio_control_pkt_engine(vio, &msgbuf); 742e689cf4aSJeff Kirsher if (err) 743e689cf4aSJeff Kirsher break; 744e689cf4aSJeff Kirsher } else { 745e689cf4aSJeff Kirsher err = vnet_handle_unknown(port, &msgbuf); 746e689cf4aSJeff Kirsher } 747e689cf4aSJeff Kirsher if (err == -ECONNRESET) 748e689cf4aSJeff Kirsher break; 749e689cf4aSJeff Kirsher } 750e689cf4aSJeff Kirsher if (unlikely(tx_wakeup && err != -ECONNRESET)) 75169088822SSowmini Varadhan maybe_tx_wakeup(port->vp); 75269088822SSowmini Varadhan return npkts; 75369088822SSowmini Varadhan } 7541d311ad2SSowmini Varadhan 75569088822SSowmini Varadhan static int vnet_poll(struct napi_struct *napi, int budget) 75669088822SSowmini Varadhan { 75769088822SSowmini Varadhan struct vnet_port *port = container_of(napi, struct vnet_port, napi); 75869088822SSowmini Varadhan struct vio_driver_state *vio = &port->vio; 75969088822SSowmini Varadhan int processed = vnet_event_napi(port, budget); 76069088822SSowmini Varadhan 76169088822SSowmini Varadhan if (processed < budget) { 76269088822SSowmini Varadhan napi_complete(napi); 7637bd68bfdSSowmini Varadhan port->rx_event &= ~LDC_EVENT_DATA_READY; 76469088822SSowmini Varadhan vio_set_intr(vio->vdev->rx_ino, HV_INTR_ENABLED); 76569088822SSowmini Varadhan } 76669088822SSowmini Varadhan return processed; 76769088822SSowmini Varadhan } 76869088822SSowmini Varadhan 76969088822SSowmini Varadhan static void vnet_event(void *arg, int event) 77069088822SSowmini Varadhan { 77169088822SSowmini Varadhan struct vnet_port *port = arg; 77269088822SSowmini Varadhan struct vio_driver_state *vio = &port->vio; 77369088822SSowmini Varadhan 77469088822SSowmini Varadhan port->rx_event |= event; 77569088822SSowmini Varadhan vio_set_intr(vio->vdev->rx_ino, HV_INTR_DISABLED); 77669088822SSowmini Varadhan napi_schedule(&port->napi); 77769088822SSowmini Varadhan 778e689cf4aSJeff Kirsher } 779e689cf4aSJeff Kirsher 780d1015645SSowmini Varadhan static int __vnet_tx_trigger(struct vnet_port *port, u32 start) 781e689cf4aSJeff Kirsher { 782e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 783e689cf4aSJeff Kirsher struct vio_dring_data hdr = { 784e689cf4aSJeff Kirsher .tag = { 785e689cf4aSJeff Kirsher .type = VIO_TYPE_DATA, 786e689cf4aSJeff Kirsher .stype = VIO_SUBTYPE_INFO, 787e689cf4aSJeff Kirsher .stype_env = VIO_DRING_DATA, 788e689cf4aSJeff Kirsher .sid = vio_send_sid(&port->vio), 789e689cf4aSJeff Kirsher }, 790e689cf4aSJeff Kirsher .dring_ident = dr->ident, 791d1015645SSowmini Varadhan .start_idx = start, 792e689cf4aSJeff Kirsher .end_idx = (u32) -1, 793e689cf4aSJeff Kirsher }; 794e689cf4aSJeff Kirsher int err, delay; 795adddc32dSSowmini Varadhan int retries = 0; 796e689cf4aSJeff Kirsher 797d1015645SSowmini Varadhan if (port->stop_rx) { 798d1015645SSowmini Varadhan err = vnet_send_ack(port, 799d1015645SSowmini Varadhan &port->vio.drings[VIO_DRIVER_RX_RING], 800d1015645SSowmini Varadhan port->stop_rx_idx, -1, 801d1015645SSowmini Varadhan VIO_DRING_STOPPED); 802d1015645SSowmini Varadhan if (err <= 0) 803d1015645SSowmini Varadhan return err; 804d1015645SSowmini Varadhan } 805d1015645SSowmini Varadhan 806e689cf4aSJeff Kirsher hdr.seq = dr->snd_nxt; 807e689cf4aSJeff Kirsher delay = 1; 808e689cf4aSJeff Kirsher do { 809e689cf4aSJeff Kirsher err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 810e689cf4aSJeff Kirsher if (err > 0) { 811e689cf4aSJeff Kirsher dr->snd_nxt++; 812e689cf4aSJeff Kirsher break; 813e689cf4aSJeff Kirsher } 814e689cf4aSJeff Kirsher udelay(delay); 815e689cf4aSJeff Kirsher if ((delay <<= 1) > 128) 816e689cf4aSJeff Kirsher delay = 128; 817adddc32dSSowmini Varadhan if (retries++ > VNET_MAX_RETRIES) 818adddc32dSSowmini Varadhan break; 819e689cf4aSJeff Kirsher } while (err == -EAGAIN); 820e689cf4aSJeff Kirsher 821e689cf4aSJeff Kirsher return err; 822e689cf4aSJeff Kirsher } 823e689cf4aSJeff Kirsher 824e689cf4aSJeff Kirsher struct vnet_port *__tx_port_find(struct vnet *vp, struct sk_buff *skb) 825e689cf4aSJeff Kirsher { 826e689cf4aSJeff Kirsher unsigned int hash = vnet_hashfn(skb->data); 827e689cf4aSJeff Kirsher struct hlist_head *hp = &vp->port_hash[hash]; 828e689cf4aSJeff Kirsher struct vnet_port *port; 829e689cf4aSJeff Kirsher 8302a968dd8SSowmini Varadhan hlist_for_each_entry_rcu(port, hp, hash) { 8318266f5fcSDavid L Stevens if (!port_is_up(port)) 8328266f5fcSDavid L Stevens continue; 8332e42e474SJoe Perches if (ether_addr_equal(port->raddr, skb->data)) 834e689cf4aSJeff Kirsher return port; 835e689cf4aSJeff Kirsher } 8362a968dd8SSowmini Varadhan list_for_each_entry_rcu(port, &vp->port_list, list) { 8378266f5fcSDavid L Stevens if (!port->switch_port) 8388266f5fcSDavid L Stevens continue; 8398266f5fcSDavid L Stevens if (!port_is_up(port)) 8408266f5fcSDavid L Stevens continue; 841e689cf4aSJeff Kirsher return port; 842e689cf4aSJeff Kirsher } 8438266f5fcSDavid L Stevens return NULL; 8448266f5fcSDavid L Stevens } 845e689cf4aSJeff Kirsher 8468e845f4cSDavid L Stevens static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port, 8478e845f4cSDavid L Stevens unsigned *pending) 8488e845f4cSDavid L Stevens { 8498e845f4cSDavid L Stevens struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 8508e845f4cSDavid L Stevens struct sk_buff *skb = NULL; 8518e845f4cSDavid L Stevens int i, txi; 8528e845f4cSDavid L Stevens 8538e845f4cSDavid L Stevens *pending = 0; 8548e845f4cSDavid L Stevens 8558e845f4cSDavid L Stevens txi = dr->prod-1; 8568e845f4cSDavid L Stevens if (txi < 0) 8578e845f4cSDavid L Stevens txi = VNET_TX_RING_SIZE-1; 8588e845f4cSDavid L Stevens 8598e845f4cSDavid L Stevens for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 8608e845f4cSDavid L Stevens struct vio_net_desc *d; 8618e845f4cSDavid L Stevens 8628e845f4cSDavid L Stevens d = vio_dring_entry(dr, txi); 8638e845f4cSDavid L Stevens 8648e845f4cSDavid L Stevens if (d->hdr.state == VIO_DESC_DONE) { 8658e845f4cSDavid L Stevens if (port->tx_bufs[txi].skb) { 8668e845f4cSDavid L Stevens BUG_ON(port->tx_bufs[txi].skb->next); 8678e845f4cSDavid L Stevens 8688e845f4cSDavid L Stevens port->tx_bufs[txi].skb->next = skb; 8698e845f4cSDavid L Stevens skb = port->tx_bufs[txi].skb; 8708e845f4cSDavid L Stevens port->tx_bufs[txi].skb = NULL; 8718e845f4cSDavid L Stevens 8728e845f4cSDavid L Stevens ldc_unmap(port->vio.lp, 8738e845f4cSDavid L Stevens port->tx_bufs[txi].cookies, 8748e845f4cSDavid L Stevens port->tx_bufs[txi].ncookies); 8758e845f4cSDavid L Stevens } 8768e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 8778e845f4cSDavid L Stevens } else if (d->hdr.state == VIO_DESC_READY) { 8788e845f4cSDavid L Stevens (*pending)++; 8798e845f4cSDavid L Stevens } else if (d->hdr.state == VIO_DESC_FREE) { 8808e845f4cSDavid L Stevens break; 8818e845f4cSDavid L Stevens } 8828e845f4cSDavid L Stevens --txi; 8838e845f4cSDavid L Stevens if (txi < 0) 8848e845f4cSDavid L Stevens txi = VNET_TX_RING_SIZE-1; 8858e845f4cSDavid L Stevens } 8868e845f4cSDavid L Stevens return skb; 8878e845f4cSDavid L Stevens } 8888e845f4cSDavid L Stevens 8898e845f4cSDavid L Stevens static inline void vnet_free_skbs(struct sk_buff *skb) 8908e845f4cSDavid L Stevens { 8918e845f4cSDavid L Stevens struct sk_buff *next; 8928e845f4cSDavid L Stevens 8938e845f4cSDavid L Stevens while (skb) { 8948e845f4cSDavid L Stevens next = skb->next; 8958e845f4cSDavid L Stevens skb->next = NULL; 8968e845f4cSDavid L Stevens dev_kfree_skb(skb); 8978e845f4cSDavid L Stevens skb = next; 8988e845f4cSDavid L Stevens } 8998e845f4cSDavid L Stevens } 9008e845f4cSDavid L Stevens 9018e845f4cSDavid L Stevens static void vnet_clean_timer_expire(unsigned long port0) 9028e845f4cSDavid L Stevens { 9038e845f4cSDavid L Stevens struct vnet_port *port = (struct vnet_port *)port0; 9048e845f4cSDavid L Stevens struct sk_buff *freeskbs; 9058e845f4cSDavid L Stevens unsigned pending; 9068e845f4cSDavid L Stevens 90713b13dd9SSowmini Varadhan netif_tx_lock(port->vp->dev); 9088e845f4cSDavid L Stevens freeskbs = vnet_clean_tx_ring(port, &pending); 90913b13dd9SSowmini Varadhan netif_tx_unlock(port->vp->dev); 9108e845f4cSDavid L Stevens 9118e845f4cSDavid L Stevens vnet_free_skbs(freeskbs); 9128e845f4cSDavid L Stevens 9138e845f4cSDavid L Stevens if (pending) 9148e845f4cSDavid L Stevens (void)mod_timer(&port->clean_timer, 9158e845f4cSDavid L Stevens jiffies + VNET_CLEAN_TIMEOUT); 9168e845f4cSDavid L Stevens else 9178e845f4cSDavid L Stevens del_timer(&port->clean_timer); 9188e845f4cSDavid L Stevens } 9198e845f4cSDavid L Stevens 9208e845f4cSDavid L Stevens static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, void **pstart, 9218e845f4cSDavid L Stevens int *plen) 9228e845f4cSDavid L Stevens { 9238e845f4cSDavid L Stevens struct sk_buff *nskb; 9248e845f4cSDavid L Stevens int len, pad; 9258e845f4cSDavid L Stevens 9268e845f4cSDavid L Stevens len = skb->len; 9278e845f4cSDavid L Stevens pad = 0; 9288e845f4cSDavid L Stevens if (len < ETH_ZLEN) { 9298e845f4cSDavid L Stevens pad += ETH_ZLEN - skb->len; 9308e845f4cSDavid L Stevens len += pad; 9318e845f4cSDavid L Stevens } 9328e845f4cSDavid L Stevens len += VNET_PACKET_SKIP; 9338e845f4cSDavid L Stevens pad += 8 - (len & 7); 9348e845f4cSDavid L Stevens len += 8 - (len & 7); 9358e845f4cSDavid L Stevens 9368e845f4cSDavid L Stevens if (((unsigned long)skb->data & 7) != VNET_PACKET_SKIP || 9378e845f4cSDavid L Stevens skb_tailroom(skb) < pad || 9388e845f4cSDavid L Stevens skb_headroom(skb) < VNET_PACKET_SKIP) { 9398e845f4cSDavid L Stevens nskb = alloc_and_align_skb(skb->dev, skb->len); 9408e845f4cSDavid L Stevens skb_reserve(nskb, VNET_PACKET_SKIP); 9418e845f4cSDavid L Stevens if (skb_copy_bits(skb, 0, nskb->data, skb->len)) { 9428e845f4cSDavid L Stevens dev_kfree_skb(nskb); 9438e845f4cSDavid L Stevens dev_kfree_skb(skb); 9448e845f4cSDavid L Stevens return NULL; 9458e845f4cSDavid L Stevens } 9468e845f4cSDavid L Stevens (void)skb_put(nskb, skb->len); 9478e845f4cSDavid L Stevens dev_kfree_skb(skb); 9488e845f4cSDavid L Stevens skb = nskb; 9498e845f4cSDavid L Stevens } 9508e845f4cSDavid L Stevens 9518e845f4cSDavid L Stevens *pstart = skb->data - VNET_PACKET_SKIP; 9528e845f4cSDavid L Stevens *plen = len; 9538e845f4cSDavid L Stevens return skb; 9548e845f4cSDavid L Stevens } 9558e845f4cSDavid L Stevens 956e689cf4aSJeff Kirsher static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 957e689cf4aSJeff Kirsher { 958e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 9592a968dd8SSowmini Varadhan struct vnet_port *port = NULL; 960e689cf4aSJeff Kirsher struct vio_dring_state *dr; 961e689cf4aSJeff Kirsher struct vio_net_desc *d; 962e689cf4aSJeff Kirsher unsigned int len; 9638e845f4cSDavid L Stevens struct sk_buff *freeskbs = NULL; 9648e845f4cSDavid L Stevens int i, err, txi; 9658e845f4cSDavid L Stevens void *start = NULL; 9668e845f4cSDavid L Stevens int nlen = 0; 9678e845f4cSDavid L Stevens unsigned pending = 0; 968e689cf4aSJeff Kirsher 9692a968dd8SSowmini Varadhan skb = vnet_skb_shape(skb, &start, &nlen); 9702a968dd8SSowmini Varadhan if (unlikely(!skb)) 971e689cf4aSJeff Kirsher goto out_dropped; 972e689cf4aSJeff Kirsher 9732a968dd8SSowmini Varadhan rcu_read_lock(); 97413b13dd9SSowmini Varadhan port = __tx_port_find(vp, skb); 9752a968dd8SSowmini Varadhan if (unlikely(!port)) 9768e845f4cSDavid L Stevens goto out_dropped; 9778e845f4cSDavid L Stevens 978a2b78e9bSDavid L Stevens if (skb->len > port->rmtu) { 979a2b78e9bSDavid L Stevens unsigned long localmtu = port->rmtu - ETH_HLEN; 980a2b78e9bSDavid L Stevens 981a2b78e9bSDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 3)) 982a2b78e9bSDavid L Stevens localmtu -= VLAN_HLEN; 983a2b78e9bSDavid L Stevens 984a2b78e9bSDavid L Stevens if (skb->protocol == htons(ETH_P_IP)) { 985a2b78e9bSDavid L Stevens struct flowi4 fl4; 986a2b78e9bSDavid L Stevens struct rtable *rt = NULL; 987a2b78e9bSDavid L Stevens 988a2b78e9bSDavid L Stevens memset(&fl4, 0, sizeof(fl4)); 989a2b78e9bSDavid L Stevens fl4.flowi4_oif = dev->ifindex; 990a2b78e9bSDavid L Stevens fl4.flowi4_tos = RT_TOS(ip_hdr(skb)->tos); 991a2b78e9bSDavid L Stevens fl4.daddr = ip_hdr(skb)->daddr; 992a2b78e9bSDavid L Stevens fl4.saddr = ip_hdr(skb)->saddr; 993a2b78e9bSDavid L Stevens 994a2b78e9bSDavid L Stevens rt = ip_route_output_key(dev_net(dev), &fl4); 9952a968dd8SSowmini Varadhan rcu_read_unlock(); 996a2b78e9bSDavid L Stevens if (!IS_ERR(rt)) { 997a2b78e9bSDavid L Stevens skb_dst_set(skb, &rt->dst); 998a2b78e9bSDavid L Stevens icmp_send(skb, ICMP_DEST_UNREACH, 999a2b78e9bSDavid L Stevens ICMP_FRAG_NEEDED, 1000a2b78e9bSDavid L Stevens htonl(localmtu)); 1001a2b78e9bSDavid L Stevens } 1002a2b78e9bSDavid L Stevens } 1003a2b78e9bSDavid L Stevens #if IS_ENABLED(CONFIG_IPV6) 1004a2b78e9bSDavid L Stevens else if (skb->protocol == htons(ETH_P_IPV6)) 1005a2b78e9bSDavid L Stevens icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, localmtu); 1006a2b78e9bSDavid L Stevens #endif 100742db672dSDavid L Stevens goto out_dropped; 1008a2b78e9bSDavid L Stevens } 100942db672dSDavid L Stevens 1010e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1011d0aedcd4SDwight Engen if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1012e689cf4aSJeff Kirsher if (!netif_queue_stopped(dev)) { 1013e689cf4aSJeff Kirsher netif_stop_queue(dev); 1014e689cf4aSJeff Kirsher 1015e689cf4aSJeff Kirsher /* This is a hard error, log it. */ 1016e689cf4aSJeff Kirsher netdev_err(dev, "BUG! Tx Ring full when queue awake!\n"); 1017e689cf4aSJeff Kirsher dev->stats.tx_errors++; 1018e689cf4aSJeff Kirsher } 10192a968dd8SSowmini Varadhan rcu_read_unlock(); 1020e689cf4aSJeff Kirsher return NETDEV_TX_BUSY; 1021e689cf4aSJeff Kirsher } 1022e689cf4aSJeff Kirsher 1023e689cf4aSJeff Kirsher d = vio_dring_cur(dr); 1024e689cf4aSJeff Kirsher 10258e845f4cSDavid L Stevens txi = dr->prod; 10268e845f4cSDavid L Stevens 10278e845f4cSDavid L Stevens freeskbs = vnet_clean_tx_ring(port, &pending); 10288e845f4cSDavid L Stevens 10298e845f4cSDavid L Stevens BUG_ON(port->tx_bufs[txi].skb); 1030e689cf4aSJeff Kirsher 1031e689cf4aSJeff Kirsher len = skb->len; 10328e845f4cSDavid L Stevens if (len < ETH_ZLEN) 1033e689cf4aSJeff Kirsher len = ETH_ZLEN; 10348e845f4cSDavid L Stevens 10358e845f4cSDavid L Stevens port->tx_bufs[txi].skb = skb; 10368e845f4cSDavid L Stevens skb = NULL; 10378e845f4cSDavid L Stevens 10388e845f4cSDavid L Stevens err = ldc_map_single(port->vio.lp, start, nlen, 103942db672dSDavid L Stevens port->tx_bufs[txi].cookies, VNET_MAXCOOKIES, 10408e845f4cSDavid L Stevens (LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW)); 10418e845f4cSDavid L Stevens if (err < 0) { 10428e845f4cSDavid L Stevens netdev_info(dev, "tx buffer map error %d\n", err); 104313b13dd9SSowmini Varadhan goto out_dropped; 1044e689cf4aSJeff Kirsher } 10458e845f4cSDavid L Stevens port->tx_bufs[txi].ncookies = err; 1046e689cf4aSJeff Kirsher 10471f6394e3SSowmini Varadhan /* We don't rely on the ACKs to free the skb in vnet_start_xmit(), 10481f6394e3SSowmini Varadhan * thus it is safe to not set VIO_ACK_ENABLE for each transmission: 10491f6394e3SSowmini Varadhan * the protocol itself does not require it as long as the peer 10501f6394e3SSowmini Varadhan * sends a VIO_SUBTYPE_ACK for VIO_DRING_STOPPED. 10511f6394e3SSowmini Varadhan * 10521f6394e3SSowmini Varadhan * An ACK for every packet in the ring is expensive as the 10531f6394e3SSowmini Varadhan * sending of LDC messages is slow and affects performance. 10541f6394e3SSowmini Varadhan */ 10551f6394e3SSowmini Varadhan d->hdr.ack = VIO_ACK_DISABLE; 1056e689cf4aSJeff Kirsher d->size = len; 10578e845f4cSDavid L Stevens d->ncookies = port->tx_bufs[txi].ncookies; 1058e689cf4aSJeff Kirsher for (i = 0; i < d->ncookies; i++) 10598e845f4cSDavid L Stevens d->cookies[i] = port->tx_bufs[txi].cookies[i]; 1060e689cf4aSJeff Kirsher 1061e689cf4aSJeff Kirsher /* This has to be a non-SMP write barrier because we are writing 1062e689cf4aSJeff Kirsher * to memory which is shared with the peer LDOM. 1063e689cf4aSJeff Kirsher */ 1064e689cf4aSJeff Kirsher wmb(); 1065e689cf4aSJeff Kirsher 1066e689cf4aSJeff Kirsher d->hdr.state = VIO_DESC_READY; 1067e689cf4aSJeff Kirsher 1068d1015645SSowmini Varadhan /* Exactly one ldc "start" trigger (for dr->cons) needs to be sent 1069d1015645SSowmini Varadhan * to notify the consumer that some descriptors are READY. 1070d1015645SSowmini Varadhan * After that "start" trigger, no additional triggers are needed until 1071d1015645SSowmini Varadhan * a DRING_STOPPED is received from the consumer. The dr->cons field 1072d1015645SSowmini Varadhan * (set up by vnet_ack()) has the value of the next dring index 1073d1015645SSowmini Varadhan * that has not yet been ack-ed. We send a "start" trigger here 1074d1015645SSowmini Varadhan * if, and only if, start_cons is true (reset it afterward). Conversely, 1075d1015645SSowmini Varadhan * vnet_ack() should check if the dring corresponding to cons 1076d1015645SSowmini Varadhan * is marked READY, but start_cons was false. 1077d1015645SSowmini Varadhan * If so, vnet_ack() should send out the missed "start" trigger. 1078d1015645SSowmini Varadhan * 1079d1015645SSowmini Varadhan * Note that the wmb() above makes sure the cookies et al. are 1080d1015645SSowmini Varadhan * not globally visible before the VIO_DESC_READY, and that the 1081d1015645SSowmini Varadhan * stores are ordered correctly by the compiler. The consumer will 1082d1015645SSowmini Varadhan * not proceed until the VIO_DESC_READY is visible assuring that 1083d1015645SSowmini Varadhan * the consumer does not observe anything related to descriptors 1084d1015645SSowmini Varadhan * out of order. The HV trap from the LDC start trigger is the 1085d1015645SSowmini Varadhan * producer to consumer announcement that work is available to the 1086d1015645SSowmini Varadhan * consumer 1087d1015645SSowmini Varadhan */ 1088d1015645SSowmini Varadhan if (!port->start_cons) 1089d1015645SSowmini Varadhan goto ldc_start_done; /* previous trigger suffices */ 1090d1015645SSowmini Varadhan 1091d1015645SSowmini Varadhan err = __vnet_tx_trigger(port, dr->cons); 1092e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 1093e689cf4aSJeff Kirsher netdev_info(dev, "TX trigger error %d\n", err); 1094e689cf4aSJeff Kirsher d->hdr.state = VIO_DESC_FREE; 1095e689cf4aSJeff Kirsher dev->stats.tx_carrier_errors++; 109613b13dd9SSowmini Varadhan goto out_dropped; 1097e689cf4aSJeff Kirsher } 1098e689cf4aSJeff Kirsher 1099d1015645SSowmini Varadhan ldc_start_done: 1100d1015645SSowmini Varadhan port->start_cons = false; 1101d1015645SSowmini Varadhan 1102e689cf4aSJeff Kirsher dev->stats.tx_packets++; 11038e845f4cSDavid L Stevens dev->stats.tx_bytes += port->tx_bufs[txi].skb->len; 1104e689cf4aSJeff Kirsher 1105e689cf4aSJeff Kirsher dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1); 1106d0aedcd4SDwight Engen if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1107e689cf4aSJeff Kirsher netif_stop_queue(dev); 1108e689cf4aSJeff Kirsher if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr)) 1109e689cf4aSJeff Kirsher netif_wake_queue(dev); 1110e689cf4aSJeff Kirsher } 1111e689cf4aSJeff Kirsher 11122a968dd8SSowmini Varadhan (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT); 11132a968dd8SSowmini Varadhan rcu_read_unlock(); 1114e689cf4aSJeff Kirsher 11158e845f4cSDavid L Stevens vnet_free_skbs(freeskbs); 11168e845f4cSDavid L Stevens 1117e689cf4aSJeff Kirsher return NETDEV_TX_OK; 1118e689cf4aSJeff Kirsher 1119e689cf4aSJeff Kirsher out_dropped: 11208e845f4cSDavid L Stevens if (pending) 11218e845f4cSDavid L Stevens (void)mod_timer(&port->clean_timer, 11228e845f4cSDavid L Stevens jiffies + VNET_CLEAN_TIMEOUT); 1123a29c9c43SDavid L Stevens else if (port) 11248e845f4cSDavid L Stevens del_timer(&port->clean_timer); 11252a968dd8SSowmini Varadhan if (port) 11262a968dd8SSowmini Varadhan rcu_read_unlock(); 11272a968dd8SSowmini Varadhan if (skb) 11282a968dd8SSowmini Varadhan dev_kfree_skb(skb); 11292a968dd8SSowmini Varadhan vnet_free_skbs(freeskbs); 1130e689cf4aSJeff Kirsher dev->stats.tx_dropped++; 1131e689cf4aSJeff Kirsher return NETDEV_TX_OK; 1132e689cf4aSJeff Kirsher } 1133e689cf4aSJeff Kirsher 1134e689cf4aSJeff Kirsher static void vnet_tx_timeout(struct net_device *dev) 1135e689cf4aSJeff Kirsher { 1136e689cf4aSJeff Kirsher /* XXX Implement me XXX */ 1137e689cf4aSJeff Kirsher } 1138e689cf4aSJeff Kirsher 1139e689cf4aSJeff Kirsher static int vnet_open(struct net_device *dev) 1140e689cf4aSJeff Kirsher { 1141e689cf4aSJeff Kirsher netif_carrier_on(dev); 1142e689cf4aSJeff Kirsher netif_start_queue(dev); 1143e689cf4aSJeff Kirsher 1144e689cf4aSJeff Kirsher return 0; 1145e689cf4aSJeff Kirsher } 1146e689cf4aSJeff Kirsher 1147e689cf4aSJeff Kirsher static int vnet_close(struct net_device *dev) 1148e689cf4aSJeff Kirsher { 1149e689cf4aSJeff Kirsher netif_stop_queue(dev); 1150e689cf4aSJeff Kirsher netif_carrier_off(dev); 1151e689cf4aSJeff Kirsher 1152e689cf4aSJeff Kirsher return 0; 1153e689cf4aSJeff Kirsher } 1154e689cf4aSJeff Kirsher 1155e689cf4aSJeff Kirsher static struct vnet_mcast_entry *__vnet_mc_find(struct vnet *vp, u8 *addr) 1156e689cf4aSJeff Kirsher { 1157e689cf4aSJeff Kirsher struct vnet_mcast_entry *m; 1158e689cf4aSJeff Kirsher 1159e689cf4aSJeff Kirsher for (m = vp->mcast_list; m; m = m->next) { 116000fa4ce9Sdingtianhong if (ether_addr_equal(m->addr, addr)) 1161e689cf4aSJeff Kirsher return m; 1162e689cf4aSJeff Kirsher } 1163e689cf4aSJeff Kirsher return NULL; 1164e689cf4aSJeff Kirsher } 1165e689cf4aSJeff Kirsher 1166e689cf4aSJeff Kirsher static void __update_mc_list(struct vnet *vp, struct net_device *dev) 1167e689cf4aSJeff Kirsher { 1168e689cf4aSJeff Kirsher struct netdev_hw_addr *ha; 1169e689cf4aSJeff Kirsher 1170e689cf4aSJeff Kirsher netdev_for_each_mc_addr(ha, dev) { 1171e689cf4aSJeff Kirsher struct vnet_mcast_entry *m; 1172e689cf4aSJeff Kirsher 1173e689cf4aSJeff Kirsher m = __vnet_mc_find(vp, ha->addr); 1174e689cf4aSJeff Kirsher if (m) { 1175e689cf4aSJeff Kirsher m->hit = 1; 1176e689cf4aSJeff Kirsher continue; 1177e689cf4aSJeff Kirsher } 1178e689cf4aSJeff Kirsher 1179e689cf4aSJeff Kirsher if (!m) { 1180e689cf4aSJeff Kirsher m = kzalloc(sizeof(*m), GFP_ATOMIC); 1181e689cf4aSJeff Kirsher if (!m) 1182e689cf4aSJeff Kirsher continue; 1183e689cf4aSJeff Kirsher memcpy(m->addr, ha->addr, ETH_ALEN); 1184e689cf4aSJeff Kirsher m->hit = 1; 1185e689cf4aSJeff Kirsher 1186e689cf4aSJeff Kirsher m->next = vp->mcast_list; 1187e689cf4aSJeff Kirsher vp->mcast_list = m; 1188e689cf4aSJeff Kirsher } 1189e689cf4aSJeff Kirsher } 1190e689cf4aSJeff Kirsher } 1191e689cf4aSJeff Kirsher 1192e689cf4aSJeff Kirsher static void __send_mc_list(struct vnet *vp, struct vnet_port *port) 1193e689cf4aSJeff Kirsher { 1194e689cf4aSJeff Kirsher struct vio_net_mcast_info info; 1195e689cf4aSJeff Kirsher struct vnet_mcast_entry *m, **pp; 1196e689cf4aSJeff Kirsher int n_addrs; 1197e689cf4aSJeff Kirsher 1198e689cf4aSJeff Kirsher memset(&info, 0, sizeof(info)); 1199e689cf4aSJeff Kirsher 1200e689cf4aSJeff Kirsher info.tag.type = VIO_TYPE_CTRL; 1201e689cf4aSJeff Kirsher info.tag.stype = VIO_SUBTYPE_INFO; 1202e689cf4aSJeff Kirsher info.tag.stype_env = VNET_MCAST_INFO; 1203e689cf4aSJeff Kirsher info.tag.sid = vio_send_sid(&port->vio); 1204e689cf4aSJeff Kirsher info.set = 1; 1205e689cf4aSJeff Kirsher 1206e689cf4aSJeff Kirsher n_addrs = 0; 1207e689cf4aSJeff Kirsher for (m = vp->mcast_list; m; m = m->next) { 1208e689cf4aSJeff Kirsher if (m->sent) 1209e689cf4aSJeff Kirsher continue; 1210e689cf4aSJeff Kirsher m->sent = 1; 1211e689cf4aSJeff Kirsher memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1212e689cf4aSJeff Kirsher m->addr, ETH_ALEN); 1213e689cf4aSJeff Kirsher if (++n_addrs == VNET_NUM_MCAST) { 1214e689cf4aSJeff Kirsher info.count = n_addrs; 1215e689cf4aSJeff Kirsher 1216e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, 1217e689cf4aSJeff Kirsher sizeof(info)); 1218e689cf4aSJeff Kirsher n_addrs = 0; 1219e689cf4aSJeff Kirsher } 1220e689cf4aSJeff Kirsher } 1221e689cf4aSJeff Kirsher if (n_addrs) { 1222e689cf4aSJeff Kirsher info.count = n_addrs; 1223e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1224e689cf4aSJeff Kirsher } 1225e689cf4aSJeff Kirsher 1226e689cf4aSJeff Kirsher info.set = 0; 1227e689cf4aSJeff Kirsher 1228e689cf4aSJeff Kirsher n_addrs = 0; 1229e689cf4aSJeff Kirsher pp = &vp->mcast_list; 1230e689cf4aSJeff Kirsher while ((m = *pp) != NULL) { 1231e689cf4aSJeff Kirsher if (m->hit) { 1232e689cf4aSJeff Kirsher m->hit = 0; 1233e689cf4aSJeff Kirsher pp = &m->next; 1234e689cf4aSJeff Kirsher continue; 1235e689cf4aSJeff Kirsher } 1236e689cf4aSJeff Kirsher 1237e689cf4aSJeff Kirsher memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1238e689cf4aSJeff Kirsher m->addr, ETH_ALEN); 1239e689cf4aSJeff Kirsher if (++n_addrs == VNET_NUM_MCAST) { 1240e689cf4aSJeff Kirsher info.count = n_addrs; 1241e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, 1242e689cf4aSJeff Kirsher sizeof(info)); 1243e689cf4aSJeff Kirsher n_addrs = 0; 1244e689cf4aSJeff Kirsher } 1245e689cf4aSJeff Kirsher 1246e689cf4aSJeff Kirsher *pp = m->next; 1247e689cf4aSJeff Kirsher kfree(m); 1248e689cf4aSJeff Kirsher } 1249e689cf4aSJeff Kirsher if (n_addrs) { 1250e689cf4aSJeff Kirsher info.count = n_addrs; 1251e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1252e689cf4aSJeff Kirsher } 1253e689cf4aSJeff Kirsher } 1254e689cf4aSJeff Kirsher 1255e689cf4aSJeff Kirsher static void vnet_set_rx_mode(struct net_device *dev) 1256e689cf4aSJeff Kirsher { 1257e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1258e689cf4aSJeff Kirsher struct vnet_port *port; 1259e689cf4aSJeff Kirsher 12602a968dd8SSowmini Varadhan rcu_read_lock(); 12612a968dd8SSowmini Varadhan list_for_each_entry_rcu(port, &vp->port_list, list) { 1262e689cf4aSJeff Kirsher 1263e689cf4aSJeff Kirsher if (port->switch_port) { 1264e689cf4aSJeff Kirsher __update_mc_list(vp, dev); 1265e689cf4aSJeff Kirsher __send_mc_list(vp, port); 12662a968dd8SSowmini Varadhan break; 1267e689cf4aSJeff Kirsher } 1268e689cf4aSJeff Kirsher } 12692a968dd8SSowmini Varadhan rcu_read_unlock(); 1270e689cf4aSJeff Kirsher } 1271e689cf4aSJeff Kirsher 1272e689cf4aSJeff Kirsher static int vnet_change_mtu(struct net_device *dev, int new_mtu) 1273e689cf4aSJeff Kirsher { 127442db672dSDavid L Stevens if (new_mtu < 68 || new_mtu > 65535) 1275e689cf4aSJeff Kirsher return -EINVAL; 1276e689cf4aSJeff Kirsher 1277e689cf4aSJeff Kirsher dev->mtu = new_mtu; 1278e689cf4aSJeff Kirsher return 0; 1279e689cf4aSJeff Kirsher } 1280e689cf4aSJeff Kirsher 1281e689cf4aSJeff Kirsher static int vnet_set_mac_addr(struct net_device *dev, void *p) 1282e689cf4aSJeff Kirsher { 1283e689cf4aSJeff Kirsher return -EINVAL; 1284e689cf4aSJeff Kirsher } 1285e689cf4aSJeff Kirsher 1286e689cf4aSJeff Kirsher static void vnet_get_drvinfo(struct net_device *dev, 1287e689cf4aSJeff Kirsher struct ethtool_drvinfo *info) 1288e689cf4aSJeff Kirsher { 12897826d43fSJiri Pirko strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); 12907826d43fSJiri Pirko strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); 1291e689cf4aSJeff Kirsher } 1292e689cf4aSJeff Kirsher 1293e689cf4aSJeff Kirsher static u32 vnet_get_msglevel(struct net_device *dev) 1294e689cf4aSJeff Kirsher { 1295e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1296e689cf4aSJeff Kirsher return vp->msg_enable; 1297e689cf4aSJeff Kirsher } 1298e689cf4aSJeff Kirsher 1299e689cf4aSJeff Kirsher static void vnet_set_msglevel(struct net_device *dev, u32 value) 1300e689cf4aSJeff Kirsher { 1301e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1302e689cf4aSJeff Kirsher vp->msg_enable = value; 1303e689cf4aSJeff Kirsher } 1304e689cf4aSJeff Kirsher 1305e689cf4aSJeff Kirsher static const struct ethtool_ops vnet_ethtool_ops = { 1306e689cf4aSJeff Kirsher .get_drvinfo = vnet_get_drvinfo, 1307e689cf4aSJeff Kirsher .get_msglevel = vnet_get_msglevel, 1308e689cf4aSJeff Kirsher .set_msglevel = vnet_set_msglevel, 1309e689cf4aSJeff Kirsher .get_link = ethtool_op_get_link, 1310e689cf4aSJeff Kirsher }; 1311e689cf4aSJeff Kirsher 1312e689cf4aSJeff Kirsher static void vnet_port_free_tx_bufs(struct vnet_port *port) 1313e689cf4aSJeff Kirsher { 1314e689cf4aSJeff Kirsher struct vio_dring_state *dr; 1315e689cf4aSJeff Kirsher int i; 1316e689cf4aSJeff Kirsher 1317e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1318e689cf4aSJeff Kirsher if (dr->base) { 1319e689cf4aSJeff Kirsher ldc_free_exp_dring(port->vio.lp, dr->base, 1320e689cf4aSJeff Kirsher (dr->entry_size * dr->num_entries), 1321e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 1322e689cf4aSJeff Kirsher dr->base = NULL; 1323e689cf4aSJeff Kirsher dr->entry_size = 0; 1324e689cf4aSJeff Kirsher dr->num_entries = 0; 1325e689cf4aSJeff Kirsher dr->pending = 0; 1326e689cf4aSJeff Kirsher dr->ncookies = 0; 1327e689cf4aSJeff Kirsher } 1328e689cf4aSJeff Kirsher 1329e689cf4aSJeff Kirsher for (i = 0; i < VNET_TX_RING_SIZE; i++) { 13308e845f4cSDavid L Stevens struct vio_net_desc *d; 13318e845f4cSDavid L Stevens void *skb = port->tx_bufs[i].skb; 1332e689cf4aSJeff Kirsher 13338e845f4cSDavid L Stevens if (!skb) 1334e689cf4aSJeff Kirsher continue; 1335e689cf4aSJeff Kirsher 13368e845f4cSDavid L Stevens d = vio_dring_entry(dr, i); 13378e845f4cSDavid L Stevens if (d->hdr.state == VIO_DESC_READY) 13388e845f4cSDavid L Stevens pr_warn("active transmit buffers freed\n"); 13398e845f4cSDavid L Stevens 1340e689cf4aSJeff Kirsher ldc_unmap(port->vio.lp, 1341e689cf4aSJeff Kirsher port->tx_bufs[i].cookies, 1342e689cf4aSJeff Kirsher port->tx_bufs[i].ncookies); 13438e845f4cSDavid L Stevens dev_kfree_skb(skb); 13448e845f4cSDavid L Stevens port->tx_bufs[i].skb = NULL; 13458e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 1346e689cf4aSJeff Kirsher } 1347e689cf4aSJeff Kirsher } 1348e689cf4aSJeff Kirsher 1349f73d12bdSBill Pemberton static int vnet_port_alloc_tx_bufs(struct vnet_port *port) 1350e689cf4aSJeff Kirsher { 1351e689cf4aSJeff Kirsher struct vio_dring_state *dr; 1352e689cf4aSJeff Kirsher unsigned long len; 1353e689cf4aSJeff Kirsher int i, err, ncookies; 1354e689cf4aSJeff Kirsher void *dring; 1355e689cf4aSJeff Kirsher 1356e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1357e689cf4aSJeff Kirsher 1358e689cf4aSJeff Kirsher len = (VNET_TX_RING_SIZE * 1359e689cf4aSJeff Kirsher (sizeof(struct vio_net_desc) + 1360e689cf4aSJeff Kirsher (sizeof(struct ldc_trans_cookie) * 2))); 1361e689cf4aSJeff Kirsher 1362e689cf4aSJeff Kirsher ncookies = VIO_MAX_RING_COOKIES; 1363e689cf4aSJeff Kirsher dring = ldc_alloc_exp_dring(port->vio.lp, len, 1364e689cf4aSJeff Kirsher dr->cookies, &ncookies, 1365e689cf4aSJeff Kirsher (LDC_MAP_SHADOW | 1366e689cf4aSJeff Kirsher LDC_MAP_DIRECT | 1367e689cf4aSJeff Kirsher LDC_MAP_RW)); 1368e689cf4aSJeff Kirsher if (IS_ERR(dring)) { 1369e689cf4aSJeff Kirsher err = PTR_ERR(dring); 1370e689cf4aSJeff Kirsher goto err_out; 1371e689cf4aSJeff Kirsher } 1372e689cf4aSJeff Kirsher 1373e689cf4aSJeff Kirsher dr->base = dring; 1374e689cf4aSJeff Kirsher dr->entry_size = (sizeof(struct vio_net_desc) + 1375e689cf4aSJeff Kirsher (sizeof(struct ldc_trans_cookie) * 2)); 1376e689cf4aSJeff Kirsher dr->num_entries = VNET_TX_RING_SIZE; 1377e689cf4aSJeff Kirsher dr->prod = dr->cons = 0; 1378d1015645SSowmini Varadhan port->start_cons = true; /* need an initial trigger */ 1379e689cf4aSJeff Kirsher dr->pending = VNET_TX_RING_SIZE; 1380e689cf4aSJeff Kirsher dr->ncookies = ncookies; 1381e689cf4aSJeff Kirsher 13828e845f4cSDavid L Stevens for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 13838e845f4cSDavid L Stevens struct vio_net_desc *d; 13848e845f4cSDavid L Stevens 13858e845f4cSDavid L Stevens d = vio_dring_entry(dr, i); 13868e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 13878e845f4cSDavid L Stevens } 1388e689cf4aSJeff Kirsher return 0; 1389e689cf4aSJeff Kirsher 1390e689cf4aSJeff Kirsher err_out: 1391e689cf4aSJeff Kirsher vnet_port_free_tx_bufs(port); 1392e689cf4aSJeff Kirsher 1393e689cf4aSJeff Kirsher return err; 1394e689cf4aSJeff Kirsher } 1395e689cf4aSJeff Kirsher 139669088822SSowmini Varadhan #ifdef CONFIG_NET_POLL_CONTROLLER 139769088822SSowmini Varadhan static void vnet_poll_controller(struct net_device *dev) 139869088822SSowmini Varadhan { 139969088822SSowmini Varadhan struct vnet *vp = netdev_priv(dev); 140069088822SSowmini Varadhan struct vnet_port *port; 140169088822SSowmini Varadhan unsigned long flags; 140269088822SSowmini Varadhan 140369088822SSowmini Varadhan spin_lock_irqsave(&vp->lock, flags); 140469088822SSowmini Varadhan if (!list_empty(&vp->port_list)) { 140569088822SSowmini Varadhan port = list_entry(vp->port_list.next, struct vnet_port, list); 140669088822SSowmini Varadhan napi_schedule(&port->napi); 140769088822SSowmini Varadhan } 140869088822SSowmini Varadhan spin_unlock_irqrestore(&vp->lock, flags); 140969088822SSowmini Varadhan } 141069088822SSowmini Varadhan #endif 1411e689cf4aSJeff Kirsher static LIST_HEAD(vnet_list); 1412e689cf4aSJeff Kirsher static DEFINE_MUTEX(vnet_list_mutex); 1413e689cf4aSJeff Kirsher 1414e689cf4aSJeff Kirsher static const struct net_device_ops vnet_ops = { 1415e689cf4aSJeff Kirsher .ndo_open = vnet_open, 1416e689cf4aSJeff Kirsher .ndo_stop = vnet_close, 1417afc4b13dSJiri Pirko .ndo_set_rx_mode = vnet_set_rx_mode, 1418e689cf4aSJeff Kirsher .ndo_set_mac_address = vnet_set_mac_addr, 1419e689cf4aSJeff Kirsher .ndo_validate_addr = eth_validate_addr, 1420e689cf4aSJeff Kirsher .ndo_tx_timeout = vnet_tx_timeout, 1421e689cf4aSJeff Kirsher .ndo_change_mtu = vnet_change_mtu, 1422e689cf4aSJeff Kirsher .ndo_start_xmit = vnet_start_xmit, 142369088822SSowmini Varadhan #ifdef CONFIG_NET_POLL_CONTROLLER 142469088822SSowmini Varadhan .ndo_poll_controller = vnet_poll_controller, 142569088822SSowmini Varadhan #endif 1426e689cf4aSJeff Kirsher }; 1427e689cf4aSJeff Kirsher 1428f73d12bdSBill Pemberton static struct vnet *vnet_new(const u64 *local_mac) 1429e689cf4aSJeff Kirsher { 1430e689cf4aSJeff Kirsher struct net_device *dev; 1431e689cf4aSJeff Kirsher struct vnet *vp; 1432e689cf4aSJeff Kirsher int err, i; 1433e689cf4aSJeff Kirsher 1434e689cf4aSJeff Kirsher dev = alloc_etherdev(sizeof(*vp)); 143541de8d4cSJoe Perches if (!dev) 1436e689cf4aSJeff Kirsher return ERR_PTR(-ENOMEM); 14378e845f4cSDavid L Stevens dev->needed_headroom = VNET_PACKET_SKIP + 8; 14388e845f4cSDavid L Stevens dev->needed_tailroom = 8; 1439e689cf4aSJeff Kirsher 1440e689cf4aSJeff Kirsher for (i = 0; i < ETH_ALEN; i++) 1441e689cf4aSJeff Kirsher dev->dev_addr[i] = (*local_mac >> (5 - i) * 8) & 0xff; 1442e689cf4aSJeff Kirsher 1443e689cf4aSJeff Kirsher vp = netdev_priv(dev); 1444e689cf4aSJeff Kirsher 1445e689cf4aSJeff Kirsher spin_lock_init(&vp->lock); 1446e689cf4aSJeff Kirsher vp->dev = dev; 1447e689cf4aSJeff Kirsher 1448e689cf4aSJeff Kirsher INIT_LIST_HEAD(&vp->port_list); 1449e689cf4aSJeff Kirsher for (i = 0; i < VNET_PORT_HASH_SIZE; i++) 1450e689cf4aSJeff Kirsher INIT_HLIST_HEAD(&vp->port_hash[i]); 1451e689cf4aSJeff Kirsher INIT_LIST_HEAD(&vp->list); 1452e689cf4aSJeff Kirsher vp->local_mac = *local_mac; 1453e689cf4aSJeff Kirsher 1454e689cf4aSJeff Kirsher dev->netdev_ops = &vnet_ops; 1455e689cf4aSJeff Kirsher dev->ethtool_ops = &vnet_ethtool_ops; 1456e689cf4aSJeff Kirsher dev->watchdog_timeo = VNET_TX_TIMEOUT; 1457e689cf4aSJeff Kirsher 1458e689cf4aSJeff Kirsher err = register_netdev(dev); 1459e689cf4aSJeff Kirsher if (err) { 1460e689cf4aSJeff Kirsher pr_err("Cannot register net device, aborting\n"); 1461e689cf4aSJeff Kirsher goto err_out_free_dev; 1462e689cf4aSJeff Kirsher } 1463e689cf4aSJeff Kirsher 1464e689cf4aSJeff Kirsher netdev_info(dev, "Sun LDOM vnet %pM\n", dev->dev_addr); 1465e689cf4aSJeff Kirsher 1466e689cf4aSJeff Kirsher list_add(&vp->list, &vnet_list); 1467e689cf4aSJeff Kirsher 1468e689cf4aSJeff Kirsher return vp; 1469e689cf4aSJeff Kirsher 1470e689cf4aSJeff Kirsher err_out_free_dev: 1471e689cf4aSJeff Kirsher free_netdev(dev); 1472e689cf4aSJeff Kirsher 1473e689cf4aSJeff Kirsher return ERR_PTR(err); 1474e689cf4aSJeff Kirsher } 1475e689cf4aSJeff Kirsher 1476f73d12bdSBill Pemberton static struct vnet *vnet_find_or_create(const u64 *local_mac) 1477e689cf4aSJeff Kirsher { 1478e689cf4aSJeff Kirsher struct vnet *iter, *vp; 1479e689cf4aSJeff Kirsher 1480e689cf4aSJeff Kirsher mutex_lock(&vnet_list_mutex); 1481e689cf4aSJeff Kirsher vp = NULL; 1482e689cf4aSJeff Kirsher list_for_each_entry(iter, &vnet_list, list) { 1483e689cf4aSJeff Kirsher if (iter->local_mac == *local_mac) { 1484e689cf4aSJeff Kirsher vp = iter; 1485e689cf4aSJeff Kirsher break; 1486e689cf4aSJeff Kirsher } 1487e689cf4aSJeff Kirsher } 1488e689cf4aSJeff Kirsher if (!vp) 1489e689cf4aSJeff Kirsher vp = vnet_new(local_mac); 1490e689cf4aSJeff Kirsher mutex_unlock(&vnet_list_mutex); 1491e689cf4aSJeff Kirsher 1492e689cf4aSJeff Kirsher return vp; 1493e689cf4aSJeff Kirsher } 1494e689cf4aSJeff Kirsher 1495a4b70a07SSowmini Varadhan static void vnet_cleanup(void) 1496a4b70a07SSowmini Varadhan { 1497a4b70a07SSowmini Varadhan struct vnet *vp; 1498a4b70a07SSowmini Varadhan struct net_device *dev; 1499a4b70a07SSowmini Varadhan 1500a4b70a07SSowmini Varadhan mutex_lock(&vnet_list_mutex); 1501a4b70a07SSowmini Varadhan while (!list_empty(&vnet_list)) { 1502a4b70a07SSowmini Varadhan vp = list_first_entry(&vnet_list, struct vnet, list); 1503a4b70a07SSowmini Varadhan list_del(&vp->list); 1504a4b70a07SSowmini Varadhan dev = vp->dev; 1505a4b70a07SSowmini Varadhan /* vio_unregister_driver() should have cleaned up port_list */ 1506a4b70a07SSowmini Varadhan BUG_ON(!list_empty(&vp->port_list)); 1507a4b70a07SSowmini Varadhan unregister_netdev(dev); 1508a4b70a07SSowmini Varadhan free_netdev(dev); 1509a4b70a07SSowmini Varadhan } 1510a4b70a07SSowmini Varadhan mutex_unlock(&vnet_list_mutex); 1511a4b70a07SSowmini Varadhan } 1512a4b70a07SSowmini Varadhan 1513e689cf4aSJeff Kirsher static const char *local_mac_prop = "local-mac-address"; 1514e689cf4aSJeff Kirsher 1515f73d12bdSBill Pemberton static struct vnet *vnet_find_parent(struct mdesc_handle *hp, 1516e689cf4aSJeff Kirsher u64 port_node) 1517e689cf4aSJeff Kirsher { 1518e689cf4aSJeff Kirsher const u64 *local_mac = NULL; 1519e689cf4aSJeff Kirsher u64 a; 1520e689cf4aSJeff Kirsher 1521e689cf4aSJeff Kirsher mdesc_for_each_arc(a, hp, port_node, MDESC_ARC_TYPE_BACK) { 1522e689cf4aSJeff Kirsher u64 target = mdesc_arc_target(hp, a); 1523e689cf4aSJeff Kirsher const char *name; 1524e689cf4aSJeff Kirsher 1525e689cf4aSJeff Kirsher name = mdesc_get_property(hp, target, "name", NULL); 1526e689cf4aSJeff Kirsher if (!name || strcmp(name, "network")) 1527e689cf4aSJeff Kirsher continue; 1528e689cf4aSJeff Kirsher 1529e689cf4aSJeff Kirsher local_mac = mdesc_get_property(hp, target, 1530e689cf4aSJeff Kirsher local_mac_prop, NULL); 1531e689cf4aSJeff Kirsher if (local_mac) 1532e689cf4aSJeff Kirsher break; 1533e689cf4aSJeff Kirsher } 1534e689cf4aSJeff Kirsher if (!local_mac) 1535e689cf4aSJeff Kirsher return ERR_PTR(-ENODEV); 1536e689cf4aSJeff Kirsher 1537e689cf4aSJeff Kirsher return vnet_find_or_create(local_mac); 1538e689cf4aSJeff Kirsher } 1539e689cf4aSJeff Kirsher 1540e689cf4aSJeff Kirsher static struct ldc_channel_config vnet_ldc_cfg = { 1541e689cf4aSJeff Kirsher .event = vnet_event, 1542e689cf4aSJeff Kirsher .mtu = 64, 1543e689cf4aSJeff Kirsher .mode = LDC_MODE_UNRELIABLE, 1544e689cf4aSJeff Kirsher }; 1545e689cf4aSJeff Kirsher 1546e689cf4aSJeff Kirsher static struct vio_driver_ops vnet_vio_ops = { 1547e689cf4aSJeff Kirsher .send_attr = vnet_send_attr, 1548e689cf4aSJeff Kirsher .handle_attr = vnet_handle_attr, 1549e689cf4aSJeff Kirsher .handshake_complete = vnet_handshake_complete, 1550e689cf4aSJeff Kirsher }; 1551e689cf4aSJeff Kirsher 1552f73d12bdSBill Pemberton static void print_version(void) 1553e689cf4aSJeff Kirsher { 1554e689cf4aSJeff Kirsher printk_once(KERN_INFO "%s", version); 1555e689cf4aSJeff Kirsher } 1556e689cf4aSJeff Kirsher 1557e689cf4aSJeff Kirsher const char *remote_macaddr_prop = "remote-mac-address"; 1558e689cf4aSJeff Kirsher 15591dd06ae8SGreg Kroah-Hartman static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) 1560e689cf4aSJeff Kirsher { 1561e689cf4aSJeff Kirsher struct mdesc_handle *hp; 1562e689cf4aSJeff Kirsher struct vnet_port *port; 1563e689cf4aSJeff Kirsher unsigned long flags; 1564e689cf4aSJeff Kirsher struct vnet *vp; 1565e689cf4aSJeff Kirsher const u64 *rmac; 1566e689cf4aSJeff Kirsher int len, i, err, switch_port; 1567e689cf4aSJeff Kirsher 1568e689cf4aSJeff Kirsher print_version(); 1569e689cf4aSJeff Kirsher 1570e689cf4aSJeff Kirsher hp = mdesc_grab(); 1571e689cf4aSJeff Kirsher 1572e689cf4aSJeff Kirsher vp = vnet_find_parent(hp, vdev->mp); 1573e689cf4aSJeff Kirsher if (IS_ERR(vp)) { 1574e689cf4aSJeff Kirsher pr_err("Cannot find port parent vnet\n"); 1575e689cf4aSJeff Kirsher err = PTR_ERR(vp); 1576e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1577e689cf4aSJeff Kirsher } 1578e689cf4aSJeff Kirsher 1579e689cf4aSJeff Kirsher rmac = mdesc_get_property(hp, vdev->mp, remote_macaddr_prop, &len); 1580e689cf4aSJeff Kirsher err = -ENODEV; 1581e689cf4aSJeff Kirsher if (!rmac) { 1582e689cf4aSJeff Kirsher pr_err("Port lacks %s property\n", remote_macaddr_prop); 1583e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1584e689cf4aSJeff Kirsher } 1585e689cf4aSJeff Kirsher 1586e689cf4aSJeff Kirsher port = kzalloc(sizeof(*port), GFP_KERNEL); 1587e689cf4aSJeff Kirsher err = -ENOMEM; 1588e404decbSJoe Perches if (!port) 1589e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1590e689cf4aSJeff Kirsher 1591e689cf4aSJeff Kirsher for (i = 0; i < ETH_ALEN; i++) 1592e689cf4aSJeff Kirsher port->raddr[i] = (*rmac >> (5 - i) * 8) & 0xff; 1593e689cf4aSJeff Kirsher 1594e689cf4aSJeff Kirsher port->vp = vp; 1595e689cf4aSJeff Kirsher 1596e689cf4aSJeff Kirsher err = vio_driver_init(&port->vio, vdev, VDEV_NETWORK, 1597e689cf4aSJeff Kirsher vnet_versions, ARRAY_SIZE(vnet_versions), 1598e689cf4aSJeff Kirsher &vnet_vio_ops, vp->dev->name); 1599e689cf4aSJeff Kirsher if (err) 1600e689cf4aSJeff Kirsher goto err_out_free_port; 1601e689cf4aSJeff Kirsher 1602e689cf4aSJeff Kirsher err = vio_ldc_alloc(&port->vio, &vnet_ldc_cfg, port); 1603e689cf4aSJeff Kirsher if (err) 1604e689cf4aSJeff Kirsher goto err_out_free_port; 1605e689cf4aSJeff Kirsher 160669088822SSowmini Varadhan netif_napi_add(port->vp->dev, &port->napi, vnet_poll, NAPI_POLL_WEIGHT); 160769088822SSowmini Varadhan 1608e689cf4aSJeff Kirsher err = vnet_port_alloc_tx_bufs(port); 1609e689cf4aSJeff Kirsher if (err) 1610e689cf4aSJeff Kirsher goto err_out_free_ldc; 1611e689cf4aSJeff Kirsher 1612e689cf4aSJeff Kirsher INIT_HLIST_NODE(&port->hash); 1613e689cf4aSJeff Kirsher INIT_LIST_HEAD(&port->list); 1614e689cf4aSJeff Kirsher 1615e689cf4aSJeff Kirsher switch_port = 0; 1616e689cf4aSJeff Kirsher if (mdesc_get_property(hp, vdev->mp, "switch-port", NULL) != NULL) 1617e689cf4aSJeff Kirsher switch_port = 1; 1618e689cf4aSJeff Kirsher port->switch_port = switch_port; 1619e689cf4aSJeff Kirsher 1620e689cf4aSJeff Kirsher spin_lock_irqsave(&vp->lock, flags); 1621e689cf4aSJeff Kirsher if (switch_port) 16222a968dd8SSowmini Varadhan list_add_rcu(&port->list, &vp->port_list); 1623e689cf4aSJeff Kirsher else 16242a968dd8SSowmini Varadhan list_add_tail_rcu(&port->list, &vp->port_list); 16252a968dd8SSowmini Varadhan hlist_add_head_rcu(&port->hash, 16262a968dd8SSowmini Varadhan &vp->port_hash[vnet_hashfn(port->raddr)]); 1627e689cf4aSJeff Kirsher spin_unlock_irqrestore(&vp->lock, flags); 1628e689cf4aSJeff Kirsher 1629e689cf4aSJeff Kirsher dev_set_drvdata(&vdev->dev, port); 1630e689cf4aSJeff Kirsher 1631e689cf4aSJeff Kirsher pr_info("%s: PORT ( remote-mac %pM%s )\n", 1632e689cf4aSJeff Kirsher vp->dev->name, port->raddr, switch_port ? " switch-port" : ""); 1633e689cf4aSJeff Kirsher 16348e845f4cSDavid L Stevens setup_timer(&port->clean_timer, vnet_clean_timer_expire, 16358e845f4cSDavid L Stevens (unsigned long)port); 16368e845f4cSDavid L Stevens 163769088822SSowmini Varadhan napi_enable(&port->napi); 1638e689cf4aSJeff Kirsher vio_port_up(&port->vio); 1639e689cf4aSJeff Kirsher 1640e689cf4aSJeff Kirsher mdesc_release(hp); 1641e689cf4aSJeff Kirsher 1642e689cf4aSJeff Kirsher return 0; 1643e689cf4aSJeff Kirsher 1644e689cf4aSJeff Kirsher err_out_free_ldc: 164569088822SSowmini Varadhan netif_napi_del(&port->napi); 1646e689cf4aSJeff Kirsher vio_ldc_free(&port->vio); 1647e689cf4aSJeff Kirsher 1648e689cf4aSJeff Kirsher err_out_free_port: 1649e689cf4aSJeff Kirsher kfree(port); 1650e689cf4aSJeff Kirsher 1651e689cf4aSJeff Kirsher err_out_put_mdesc: 1652e689cf4aSJeff Kirsher mdesc_release(hp); 1653e689cf4aSJeff Kirsher return err; 1654e689cf4aSJeff Kirsher } 1655e689cf4aSJeff Kirsher 1656e689cf4aSJeff Kirsher static int vnet_port_remove(struct vio_dev *vdev) 1657e689cf4aSJeff Kirsher { 1658e689cf4aSJeff Kirsher struct vnet_port *port = dev_get_drvdata(&vdev->dev); 1659e689cf4aSJeff Kirsher 1660e689cf4aSJeff Kirsher if (port) { 1661e689cf4aSJeff Kirsher 1662e689cf4aSJeff Kirsher del_timer_sync(&port->vio.timer); 1663e689cf4aSJeff Kirsher 166469088822SSowmini Varadhan napi_disable(&port->napi); 1665e689cf4aSJeff Kirsher 16662a968dd8SSowmini Varadhan list_del_rcu(&port->list); 16672a968dd8SSowmini Varadhan hlist_del_rcu(&port->hash); 16682a968dd8SSowmini Varadhan 16692a968dd8SSowmini Varadhan synchronize_rcu(); 16702a968dd8SSowmini Varadhan del_timer_sync(&port->clean_timer); 167169088822SSowmini Varadhan netif_napi_del(&port->napi); 1672e689cf4aSJeff Kirsher vnet_port_free_tx_bufs(port); 1673e689cf4aSJeff Kirsher vio_ldc_free(&port->vio); 1674e689cf4aSJeff Kirsher 1675e689cf4aSJeff Kirsher dev_set_drvdata(&vdev->dev, NULL); 1676e689cf4aSJeff Kirsher 1677e689cf4aSJeff Kirsher kfree(port); 1678aabb9875SDave Kleikamp 1679e689cf4aSJeff Kirsher } 1680e689cf4aSJeff Kirsher return 0; 1681e689cf4aSJeff Kirsher } 1682e689cf4aSJeff Kirsher 1683e689cf4aSJeff Kirsher static const struct vio_device_id vnet_port_match[] = { 1684e689cf4aSJeff Kirsher { 1685e689cf4aSJeff Kirsher .type = "vnet-port", 1686e689cf4aSJeff Kirsher }, 1687e689cf4aSJeff Kirsher {}, 1688e689cf4aSJeff Kirsher }; 1689e689cf4aSJeff Kirsher MODULE_DEVICE_TABLE(vio, vnet_port_match); 1690e689cf4aSJeff Kirsher 1691e689cf4aSJeff Kirsher static struct vio_driver vnet_port_driver = { 1692e689cf4aSJeff Kirsher .id_table = vnet_port_match, 1693e689cf4aSJeff Kirsher .probe = vnet_port_probe, 1694e689cf4aSJeff Kirsher .remove = vnet_port_remove, 1695e689cf4aSJeff Kirsher .name = "vnet_port", 1696e689cf4aSJeff Kirsher }; 1697e689cf4aSJeff Kirsher 1698e689cf4aSJeff Kirsher static int __init vnet_init(void) 1699e689cf4aSJeff Kirsher { 1700e689cf4aSJeff Kirsher return vio_register_driver(&vnet_port_driver); 1701e689cf4aSJeff Kirsher } 1702e689cf4aSJeff Kirsher 1703e689cf4aSJeff Kirsher static void __exit vnet_exit(void) 1704e689cf4aSJeff Kirsher { 1705e689cf4aSJeff Kirsher vio_unregister_driver(&vnet_port_driver); 1706a4b70a07SSowmini Varadhan vnet_cleanup(); 1707e689cf4aSJeff Kirsher } 1708e689cf4aSJeff Kirsher 1709e689cf4aSJeff Kirsher module_init(vnet_init); 1710e689cf4aSJeff Kirsher module_exit(vnet_exit); 1711