1e689cf4aSJeff Kirsher /* sunvnet.c: Sun LDOM Virtual Network Driver. 2e689cf4aSJeff Kirsher * 3e689cf4aSJeff Kirsher * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4e689cf4aSJeff Kirsher */ 5e689cf4aSJeff Kirsher 6e689cf4aSJeff Kirsher #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7e689cf4aSJeff Kirsher 8e689cf4aSJeff Kirsher #include <linux/module.h> 9e689cf4aSJeff Kirsher #include <linux/kernel.h> 10e689cf4aSJeff Kirsher #include <linux/types.h> 11e689cf4aSJeff Kirsher #include <linux/slab.h> 12e689cf4aSJeff Kirsher #include <linux/delay.h> 13e689cf4aSJeff Kirsher #include <linux/init.h> 14e689cf4aSJeff Kirsher #include <linux/netdevice.h> 15e689cf4aSJeff Kirsher #include <linux/ethtool.h> 16e689cf4aSJeff Kirsher #include <linux/etherdevice.h> 17e689cf4aSJeff Kirsher #include <linux/mutex.h> 18e4defc77SDavid L Stevens #include <linux/if_vlan.h> 19e689cf4aSJeff Kirsher 20a2b78e9bSDavid L Stevens #if IS_ENABLED(CONFIG_IPV6) 21a2b78e9bSDavid L Stevens #include <linux/icmpv6.h> 22a2b78e9bSDavid L Stevens #endif 23a2b78e9bSDavid L Stevens 24a2b78e9bSDavid L Stevens #include <net/icmp.h> 25a2b78e9bSDavid L Stevens #include <net/route.h> 26a2b78e9bSDavid L Stevens 27e689cf4aSJeff Kirsher #include <asm/vio.h> 28e689cf4aSJeff Kirsher #include <asm/ldc.h> 29e689cf4aSJeff Kirsher 30e689cf4aSJeff Kirsher #include "sunvnet.h" 31e689cf4aSJeff Kirsher 32e689cf4aSJeff Kirsher #define DRV_MODULE_NAME "sunvnet" 33e689cf4aSJeff Kirsher #define DRV_MODULE_VERSION "1.0" 34e689cf4aSJeff Kirsher #define DRV_MODULE_RELDATE "June 25, 2007" 35e689cf4aSJeff Kirsher 36f73d12bdSBill Pemberton static char version[] = 37e689cf4aSJeff Kirsher DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 38e689cf4aSJeff Kirsher MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); 39e689cf4aSJeff Kirsher MODULE_DESCRIPTION("Sun LDOM virtual network driver"); 40e689cf4aSJeff Kirsher MODULE_LICENSE("GPL"); 41e689cf4aSJeff Kirsher MODULE_VERSION(DRV_MODULE_VERSION); 42e689cf4aSJeff Kirsher 43d51bffd1SSowmini Varadhan #define VNET_MAX_TXQS 16 44d51bffd1SSowmini Varadhan 45adddc32dSSowmini Varadhan /* Heuristic for the number of times to exponentially backoff and 46adddc32dSSowmini Varadhan * retry sending an LDC trigger when EAGAIN is encountered 47adddc32dSSowmini Varadhan */ 48adddc32dSSowmini Varadhan #define VNET_MAX_RETRIES 10 49adddc32dSSowmini Varadhan 50d1015645SSowmini Varadhan static int __vnet_tx_trigger(struct vnet_port *port, u32 start); 51d1015645SSowmini Varadhan 52e689cf4aSJeff Kirsher /* Ordered from largest major to lowest */ 53e689cf4aSJeff Kirsher static struct vio_version vnet_versions[] = { 54e4defc77SDavid L Stevens { .major = 1, .minor = 6 }, 55e689cf4aSJeff Kirsher { .major = 1, .minor = 0 }, 56e689cf4aSJeff Kirsher }; 57e689cf4aSJeff Kirsher 58e689cf4aSJeff Kirsher static inline u32 vnet_tx_dring_avail(struct vio_dring_state *dr) 59e689cf4aSJeff Kirsher { 60e689cf4aSJeff Kirsher return vio_dring_avail(dr, VNET_TX_RING_SIZE); 61e689cf4aSJeff Kirsher } 62e689cf4aSJeff Kirsher 63e689cf4aSJeff Kirsher static int vnet_handle_unknown(struct vnet_port *port, void *arg) 64e689cf4aSJeff Kirsher { 65e689cf4aSJeff Kirsher struct vio_msg_tag *pkt = arg; 66e689cf4aSJeff Kirsher 67e689cf4aSJeff Kirsher pr_err("Received unknown msg [%02x:%02x:%04x:%08x]\n", 68e689cf4aSJeff Kirsher pkt->type, pkt->stype, pkt->stype_env, pkt->sid); 69e689cf4aSJeff Kirsher pr_err("Resetting connection\n"); 70e689cf4aSJeff Kirsher 71e689cf4aSJeff Kirsher ldc_disconnect(port->vio.lp); 72e689cf4aSJeff Kirsher 73e689cf4aSJeff Kirsher return -ECONNRESET; 74e689cf4aSJeff Kirsher } 75e689cf4aSJeff Kirsher 76e689cf4aSJeff Kirsher static int vnet_send_attr(struct vio_driver_state *vio) 77e689cf4aSJeff Kirsher { 78e689cf4aSJeff Kirsher struct vnet_port *port = to_vnet_port(vio); 79e689cf4aSJeff Kirsher struct net_device *dev = port->vp->dev; 80e689cf4aSJeff Kirsher struct vio_net_attr_info pkt; 81e4defc77SDavid L Stevens int framelen = ETH_FRAME_LEN; 82e689cf4aSJeff Kirsher int i; 83e689cf4aSJeff Kirsher 84e689cf4aSJeff Kirsher memset(&pkt, 0, sizeof(pkt)); 85e689cf4aSJeff Kirsher pkt.tag.type = VIO_TYPE_CTRL; 86e689cf4aSJeff Kirsher pkt.tag.stype = VIO_SUBTYPE_INFO; 87e689cf4aSJeff Kirsher pkt.tag.stype_env = VIO_ATTR_INFO; 88e689cf4aSJeff Kirsher pkt.tag.sid = vio_send_sid(vio); 89e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 2)) 90e689cf4aSJeff Kirsher pkt.xfer_mode = VIO_DRING_MODE; 91e4defc77SDavid L Stevens else 92e4defc77SDavid L Stevens pkt.xfer_mode = VIO_NEW_DRING_MODE; 93e689cf4aSJeff Kirsher pkt.addr_type = VNET_ADDR_ETHERMAC; 94e689cf4aSJeff Kirsher pkt.ack_freq = 0; 95e689cf4aSJeff Kirsher for (i = 0; i < 6; i++) 96e689cf4aSJeff Kirsher pkt.addr |= (u64)dev->dev_addr[i] << ((5 - i) * 8); 97e4defc77SDavid L Stevens if (vio_version_after(vio, 1, 3)) { 98e4defc77SDavid L Stevens if (port->rmtu) { 99e4defc77SDavid L Stevens port->rmtu = min(VNET_MAXPACKET, port->rmtu); 100e4defc77SDavid L Stevens pkt.mtu = port->rmtu; 101e4defc77SDavid L Stevens } else { 102e4defc77SDavid L Stevens port->rmtu = VNET_MAXPACKET; 103e4defc77SDavid L Stevens pkt.mtu = port->rmtu; 104e4defc77SDavid L Stevens } 105e4defc77SDavid L Stevens if (vio_version_after_eq(vio, 1, 6)) 106e4defc77SDavid L Stevens pkt.options = VIO_TX_DRING; 107e4defc77SDavid L Stevens } else if (vio_version_before(vio, 1, 3)) { 108e4defc77SDavid L Stevens pkt.mtu = framelen; 109e4defc77SDavid L Stevens } else { /* v1.3 */ 110e4defc77SDavid L Stevens pkt.mtu = framelen + VLAN_HLEN; 111e4defc77SDavid L Stevens } 112e4defc77SDavid L Stevens 113e4defc77SDavid L Stevens pkt.plnk_updt = PHYSLINK_UPDATE_NONE; 114e4defc77SDavid L Stevens pkt.cflags = 0; 115e689cf4aSJeff Kirsher 116e689cf4aSJeff Kirsher viodbg(HS, "SEND NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 117e4defc77SDavid L Stevens "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 118e4defc77SDavid L Stevens "cflags[0x%04x] lso_max[%u]\n", 119e689cf4aSJeff Kirsher pkt.xfer_mode, pkt.addr_type, 120e689cf4aSJeff Kirsher (unsigned long long)pkt.addr, 121e4defc77SDavid L Stevens pkt.ack_freq, pkt.plnk_updt, pkt.options, 122e4defc77SDavid L Stevens (unsigned long long)pkt.mtu, pkt.cflags, pkt.ipv4_lso_maxlen); 123e4defc77SDavid L Stevens 124e689cf4aSJeff Kirsher 125e689cf4aSJeff Kirsher return vio_ldc_send(vio, &pkt, sizeof(pkt)); 126e689cf4aSJeff Kirsher } 127e689cf4aSJeff Kirsher 128e689cf4aSJeff Kirsher static int handle_attr_info(struct vio_driver_state *vio, 129e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 130e689cf4aSJeff Kirsher { 131e4defc77SDavid L Stevens struct vnet_port *port = to_vnet_port(vio); 132e4defc77SDavid L Stevens u64 localmtu; 133e4defc77SDavid L Stevens u8 xfer_mode; 134e4defc77SDavid L Stevens 135e4defc77SDavid L Stevens viodbg(HS, "GOT NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 136e4defc77SDavid L Stevens "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 137e4defc77SDavid L Stevens " (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 138e689cf4aSJeff Kirsher pkt->xfer_mode, pkt->addr_type, 139e689cf4aSJeff Kirsher (unsigned long long)pkt->addr, 140e4defc77SDavid L Stevens pkt->ack_freq, pkt->plnk_updt, pkt->options, 141e4defc77SDavid L Stevens (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 142e4defc77SDavid L Stevens pkt->ipv4_lso_maxlen); 143e689cf4aSJeff Kirsher 144e689cf4aSJeff Kirsher pkt->tag.sid = vio_send_sid(vio); 145e689cf4aSJeff Kirsher 146e4defc77SDavid L Stevens xfer_mode = pkt->xfer_mode; 147e4defc77SDavid L Stevens /* for version < 1.2, VIO_DRING_MODE = 0x3 and no bitmask */ 148e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 2) && xfer_mode == VIO_DRING_MODE) 149e4defc77SDavid L Stevens xfer_mode = VIO_NEW_DRING_MODE; 150e4defc77SDavid L Stevens 151e4defc77SDavid L Stevens /* MTU negotiation: 152e4defc77SDavid L Stevens * < v1.3 - ETH_FRAME_LEN exactly 153e4defc77SDavid L Stevens * > v1.3 - MIN(pkt.mtu, VNET_MAXPACKET, port->rmtu) and change 154e4defc77SDavid L Stevens * pkt->mtu for ACK 155e4defc77SDavid L Stevens * = v1.3 - ETH_FRAME_LEN + VLAN_HLEN exactly 156e4defc77SDavid L Stevens */ 157e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 3)) { 158e4defc77SDavid L Stevens localmtu = ETH_FRAME_LEN; 159e4defc77SDavid L Stevens } else if (vio_version_after(vio, 1, 3)) { 160e4defc77SDavid L Stevens localmtu = port->rmtu ? port->rmtu : VNET_MAXPACKET; 161e4defc77SDavid L Stevens localmtu = min(pkt->mtu, localmtu); 162e4defc77SDavid L Stevens pkt->mtu = localmtu; 163e4defc77SDavid L Stevens } else { /* v1.3 */ 164e4defc77SDavid L Stevens localmtu = ETH_FRAME_LEN + VLAN_HLEN; 165e4defc77SDavid L Stevens } 166e4defc77SDavid L Stevens port->rmtu = localmtu; 167e4defc77SDavid L Stevens 168e4defc77SDavid L Stevens /* for version >= 1.6, ACK packet mode we support */ 169e4defc77SDavid L Stevens if (vio_version_after_eq(vio, 1, 6)) { 170e4defc77SDavid L Stevens pkt->xfer_mode = VIO_NEW_DRING_MODE; 171e4defc77SDavid L Stevens pkt->options = VIO_TX_DRING; 172e4defc77SDavid L Stevens } 173e4defc77SDavid L Stevens 174e4defc77SDavid L Stevens if (!(xfer_mode | VIO_NEW_DRING_MODE) || 175e689cf4aSJeff Kirsher pkt->addr_type != VNET_ADDR_ETHERMAC || 176e4defc77SDavid L Stevens pkt->mtu != localmtu) { 177e689cf4aSJeff Kirsher viodbg(HS, "SEND NET ATTR NACK\n"); 178e689cf4aSJeff Kirsher 179e689cf4aSJeff Kirsher pkt->tag.stype = VIO_SUBTYPE_NACK; 180e689cf4aSJeff Kirsher 181e689cf4aSJeff Kirsher (void) vio_ldc_send(vio, pkt, sizeof(*pkt)); 182e689cf4aSJeff Kirsher 183e689cf4aSJeff Kirsher return -ECONNRESET; 184e689cf4aSJeff Kirsher } else { 185e4defc77SDavid L Stevens viodbg(HS, "SEND NET ATTR ACK xmode[0x%x] atype[0x%x] " 186e4defc77SDavid L Stevens "addr[%llx] ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] " 187e4defc77SDavid L Stevens "mtu[%llu] (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 188e4defc77SDavid L Stevens pkt->xfer_mode, pkt->addr_type, 189e4defc77SDavid L Stevens (unsigned long long)pkt->addr, 190e4defc77SDavid L Stevens pkt->ack_freq, pkt->plnk_updt, pkt->options, 191e4defc77SDavid L Stevens (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 192e4defc77SDavid L Stevens pkt->ipv4_lso_maxlen); 193e689cf4aSJeff Kirsher 194e689cf4aSJeff Kirsher pkt->tag.stype = VIO_SUBTYPE_ACK; 195e689cf4aSJeff Kirsher 196e689cf4aSJeff Kirsher return vio_ldc_send(vio, pkt, sizeof(*pkt)); 197e689cf4aSJeff Kirsher } 198e689cf4aSJeff Kirsher 199e689cf4aSJeff Kirsher } 200e689cf4aSJeff Kirsher 201e689cf4aSJeff Kirsher static int handle_attr_ack(struct vio_driver_state *vio, 202e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 203e689cf4aSJeff Kirsher { 204e689cf4aSJeff Kirsher viodbg(HS, "GOT NET ATTR ACK\n"); 205e689cf4aSJeff Kirsher 206e689cf4aSJeff Kirsher return 0; 207e689cf4aSJeff Kirsher } 208e689cf4aSJeff Kirsher 209e689cf4aSJeff Kirsher static int handle_attr_nack(struct vio_driver_state *vio, 210e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 211e689cf4aSJeff Kirsher { 212e689cf4aSJeff Kirsher viodbg(HS, "GOT NET ATTR NACK\n"); 213e689cf4aSJeff Kirsher 214e689cf4aSJeff Kirsher return -ECONNRESET; 215e689cf4aSJeff Kirsher } 216e689cf4aSJeff Kirsher 217e689cf4aSJeff Kirsher static int vnet_handle_attr(struct vio_driver_state *vio, void *arg) 218e689cf4aSJeff Kirsher { 219e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt = arg; 220e689cf4aSJeff Kirsher 221e689cf4aSJeff Kirsher switch (pkt->tag.stype) { 222e689cf4aSJeff Kirsher case VIO_SUBTYPE_INFO: 223e689cf4aSJeff Kirsher return handle_attr_info(vio, pkt); 224e689cf4aSJeff Kirsher 225e689cf4aSJeff Kirsher case VIO_SUBTYPE_ACK: 226e689cf4aSJeff Kirsher return handle_attr_ack(vio, pkt); 227e689cf4aSJeff Kirsher 228e689cf4aSJeff Kirsher case VIO_SUBTYPE_NACK: 229e689cf4aSJeff Kirsher return handle_attr_nack(vio, pkt); 230e689cf4aSJeff Kirsher 231e689cf4aSJeff Kirsher default: 232e689cf4aSJeff Kirsher return -ECONNRESET; 233e689cf4aSJeff Kirsher } 234e689cf4aSJeff Kirsher } 235e689cf4aSJeff Kirsher 236e689cf4aSJeff Kirsher static void vnet_handshake_complete(struct vio_driver_state *vio) 237e689cf4aSJeff Kirsher { 238e689cf4aSJeff Kirsher struct vio_dring_state *dr; 239e689cf4aSJeff Kirsher 240e689cf4aSJeff Kirsher dr = &vio->drings[VIO_DRIVER_RX_RING]; 241e689cf4aSJeff Kirsher dr->snd_nxt = dr->rcv_nxt = 1; 242e689cf4aSJeff Kirsher 243e689cf4aSJeff Kirsher dr = &vio->drings[VIO_DRIVER_TX_RING]; 244e689cf4aSJeff Kirsher dr->snd_nxt = dr->rcv_nxt = 1; 245e689cf4aSJeff Kirsher } 246e689cf4aSJeff Kirsher 247e689cf4aSJeff Kirsher /* The hypervisor interface that implements copying to/from imported 248e689cf4aSJeff Kirsher * memory from another domain requires that copies are done to 8-byte 249e689cf4aSJeff Kirsher * aligned buffers, and that the lengths of such copies are also 8-byte 250e689cf4aSJeff Kirsher * multiples. 251e689cf4aSJeff Kirsher * 252e689cf4aSJeff Kirsher * So we align skb->data to an 8-byte multiple and pad-out the data 253e689cf4aSJeff Kirsher * area so we can round the copy length up to the next multiple of 254e689cf4aSJeff Kirsher * 8 for the copy. 255e689cf4aSJeff Kirsher * 256e689cf4aSJeff Kirsher * The transmitter puts the actual start of the packet 6 bytes into 257e689cf4aSJeff Kirsher * the buffer it sends over, so that the IP headers after the ethernet 258e689cf4aSJeff Kirsher * header are aligned properly. These 6 bytes are not in the descriptor 259e689cf4aSJeff Kirsher * length, they are simply implied. This offset is represented using 260e689cf4aSJeff Kirsher * the VNET_PACKET_SKIP macro. 261e689cf4aSJeff Kirsher */ 262e689cf4aSJeff Kirsher static struct sk_buff *alloc_and_align_skb(struct net_device *dev, 263e689cf4aSJeff Kirsher unsigned int len) 264e689cf4aSJeff Kirsher { 265e689cf4aSJeff Kirsher struct sk_buff *skb = netdev_alloc_skb(dev, len+VNET_PACKET_SKIP+8+8); 266e689cf4aSJeff Kirsher unsigned long addr, off; 267e689cf4aSJeff Kirsher 268e689cf4aSJeff Kirsher if (unlikely(!skb)) 269e689cf4aSJeff Kirsher return NULL; 270e689cf4aSJeff Kirsher 271e689cf4aSJeff Kirsher addr = (unsigned long) skb->data; 272e689cf4aSJeff Kirsher off = ((addr + 7UL) & ~7UL) - addr; 273e689cf4aSJeff Kirsher if (off) 274e689cf4aSJeff Kirsher skb_reserve(skb, off); 275e689cf4aSJeff Kirsher 276e689cf4aSJeff Kirsher return skb; 277e689cf4aSJeff Kirsher } 278e689cf4aSJeff Kirsher 279e689cf4aSJeff Kirsher static int vnet_rx_one(struct vnet_port *port, unsigned int len, 280e689cf4aSJeff Kirsher struct ldc_trans_cookie *cookies, int ncookies) 281e689cf4aSJeff Kirsher { 282e689cf4aSJeff Kirsher struct net_device *dev = port->vp->dev; 283e689cf4aSJeff Kirsher unsigned int copy_len; 284e689cf4aSJeff Kirsher struct sk_buff *skb; 285e689cf4aSJeff Kirsher int err; 286e689cf4aSJeff Kirsher 287e689cf4aSJeff Kirsher err = -EMSGSIZE; 288e4defc77SDavid L Stevens if (unlikely(len < ETH_ZLEN || len > port->rmtu)) { 289e689cf4aSJeff Kirsher dev->stats.rx_length_errors++; 290e689cf4aSJeff Kirsher goto out_dropped; 291e689cf4aSJeff Kirsher } 292e689cf4aSJeff Kirsher 293e689cf4aSJeff Kirsher skb = alloc_and_align_skb(dev, len); 294e689cf4aSJeff Kirsher err = -ENOMEM; 295e689cf4aSJeff Kirsher if (unlikely(!skb)) { 296e689cf4aSJeff Kirsher dev->stats.rx_missed_errors++; 297e689cf4aSJeff Kirsher goto out_dropped; 298e689cf4aSJeff Kirsher } 299e689cf4aSJeff Kirsher 300e689cf4aSJeff Kirsher copy_len = (len + VNET_PACKET_SKIP + 7U) & ~7U; 301e689cf4aSJeff Kirsher skb_put(skb, copy_len); 302e689cf4aSJeff Kirsher err = ldc_copy(port->vio.lp, LDC_COPY_IN, 303e689cf4aSJeff Kirsher skb->data, copy_len, 0, 304e689cf4aSJeff Kirsher cookies, ncookies); 305e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 306e689cf4aSJeff Kirsher dev->stats.rx_frame_errors++; 307e689cf4aSJeff Kirsher goto out_free_skb; 308e689cf4aSJeff Kirsher } 309e689cf4aSJeff Kirsher 310e689cf4aSJeff Kirsher skb_pull(skb, VNET_PACKET_SKIP); 311e689cf4aSJeff Kirsher skb_trim(skb, len); 312e689cf4aSJeff Kirsher skb->protocol = eth_type_trans(skb, dev); 313e689cf4aSJeff Kirsher 314e689cf4aSJeff Kirsher dev->stats.rx_packets++; 315e689cf4aSJeff Kirsher dev->stats.rx_bytes += len; 31669088822SSowmini Varadhan napi_gro_receive(&port->napi, skb); 317e689cf4aSJeff Kirsher return 0; 318e689cf4aSJeff Kirsher 319e689cf4aSJeff Kirsher out_free_skb: 320e689cf4aSJeff Kirsher kfree_skb(skb); 321e689cf4aSJeff Kirsher 322e689cf4aSJeff Kirsher out_dropped: 323e689cf4aSJeff Kirsher dev->stats.rx_dropped++; 324e689cf4aSJeff Kirsher return err; 325e689cf4aSJeff Kirsher } 326e689cf4aSJeff Kirsher 327e689cf4aSJeff Kirsher static int vnet_send_ack(struct vnet_port *port, struct vio_dring_state *dr, 328e689cf4aSJeff Kirsher u32 start, u32 end, u8 vio_dring_state) 329e689cf4aSJeff Kirsher { 330e689cf4aSJeff Kirsher struct vio_dring_data hdr = { 331e689cf4aSJeff Kirsher .tag = { 332e689cf4aSJeff Kirsher .type = VIO_TYPE_DATA, 333e689cf4aSJeff Kirsher .stype = VIO_SUBTYPE_ACK, 334e689cf4aSJeff Kirsher .stype_env = VIO_DRING_DATA, 335e689cf4aSJeff Kirsher .sid = vio_send_sid(&port->vio), 336e689cf4aSJeff Kirsher }, 337e689cf4aSJeff Kirsher .dring_ident = dr->ident, 338e689cf4aSJeff Kirsher .start_idx = start, 339e689cf4aSJeff Kirsher .end_idx = end, 340e689cf4aSJeff Kirsher .state = vio_dring_state, 341e689cf4aSJeff Kirsher }; 342e689cf4aSJeff Kirsher int err, delay; 343adddc32dSSowmini Varadhan int retries = 0; 344e689cf4aSJeff Kirsher 345e689cf4aSJeff Kirsher hdr.seq = dr->snd_nxt; 346e689cf4aSJeff Kirsher delay = 1; 347e689cf4aSJeff Kirsher do { 348e689cf4aSJeff Kirsher err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 349e689cf4aSJeff Kirsher if (err > 0) { 350e689cf4aSJeff Kirsher dr->snd_nxt++; 351e689cf4aSJeff Kirsher break; 352e689cf4aSJeff Kirsher } 353e689cf4aSJeff Kirsher udelay(delay); 354e689cf4aSJeff Kirsher if ((delay <<= 1) > 128) 355e689cf4aSJeff Kirsher delay = 128; 356adddc32dSSowmini Varadhan if (retries++ > VNET_MAX_RETRIES) { 357adddc32dSSowmini Varadhan pr_info("ECONNRESET %x:%x:%x:%x:%x:%x\n", 358adddc32dSSowmini Varadhan port->raddr[0], port->raddr[1], 359adddc32dSSowmini Varadhan port->raddr[2], port->raddr[3], 360adddc32dSSowmini Varadhan port->raddr[4], port->raddr[5]); 361d1015645SSowmini Varadhan break; 362adddc32dSSowmini Varadhan } 363e689cf4aSJeff Kirsher } while (err == -EAGAIN); 364e689cf4aSJeff Kirsher 365d1015645SSowmini Varadhan if (err <= 0 && vio_dring_state == VIO_DRING_STOPPED) { 366d1015645SSowmini Varadhan port->stop_rx_idx = end; 367d1015645SSowmini Varadhan port->stop_rx = true; 368d1015645SSowmini Varadhan } else { 369d1015645SSowmini Varadhan port->stop_rx_idx = 0; 370d1015645SSowmini Varadhan port->stop_rx = false; 371d1015645SSowmini Varadhan } 372d1015645SSowmini Varadhan 373e689cf4aSJeff Kirsher return err; 374e689cf4aSJeff Kirsher } 375e689cf4aSJeff Kirsher 376e689cf4aSJeff Kirsher static u32 next_idx(u32 idx, struct vio_dring_state *dr) 377e689cf4aSJeff Kirsher { 378e689cf4aSJeff Kirsher if (++idx == dr->num_entries) 379e689cf4aSJeff Kirsher idx = 0; 380e689cf4aSJeff Kirsher return idx; 381e689cf4aSJeff Kirsher } 382e689cf4aSJeff Kirsher 383e689cf4aSJeff Kirsher static u32 prev_idx(u32 idx, struct vio_dring_state *dr) 384e689cf4aSJeff Kirsher { 385e689cf4aSJeff Kirsher if (idx == 0) 386e689cf4aSJeff Kirsher idx = dr->num_entries - 1; 387e689cf4aSJeff Kirsher else 388e689cf4aSJeff Kirsher idx--; 389e689cf4aSJeff Kirsher 390e689cf4aSJeff Kirsher return idx; 391e689cf4aSJeff Kirsher } 392e689cf4aSJeff Kirsher 393e689cf4aSJeff Kirsher static struct vio_net_desc *get_rx_desc(struct vnet_port *port, 394e689cf4aSJeff Kirsher struct vio_dring_state *dr, 395e689cf4aSJeff Kirsher u32 index) 396e689cf4aSJeff Kirsher { 397e689cf4aSJeff Kirsher struct vio_net_desc *desc = port->vio.desc_buf; 398e689cf4aSJeff Kirsher int err; 399e689cf4aSJeff Kirsher 400e689cf4aSJeff Kirsher err = ldc_get_dring_entry(port->vio.lp, desc, dr->entry_size, 401e689cf4aSJeff Kirsher (index * dr->entry_size), 402e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 403e689cf4aSJeff Kirsher if (err < 0) 404e689cf4aSJeff Kirsher return ERR_PTR(err); 405e689cf4aSJeff Kirsher 406e689cf4aSJeff Kirsher return desc; 407e689cf4aSJeff Kirsher } 408e689cf4aSJeff Kirsher 409e689cf4aSJeff Kirsher static int put_rx_desc(struct vnet_port *port, 410e689cf4aSJeff Kirsher struct vio_dring_state *dr, 411e689cf4aSJeff Kirsher struct vio_net_desc *desc, 412e689cf4aSJeff Kirsher u32 index) 413e689cf4aSJeff Kirsher { 414e689cf4aSJeff Kirsher int err; 415e689cf4aSJeff Kirsher 416e689cf4aSJeff Kirsher err = ldc_put_dring_entry(port->vio.lp, desc, dr->entry_size, 417e689cf4aSJeff Kirsher (index * dr->entry_size), 418e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 419e689cf4aSJeff Kirsher if (err < 0) 420e689cf4aSJeff Kirsher return err; 421e689cf4aSJeff Kirsher 422e689cf4aSJeff Kirsher return 0; 423e689cf4aSJeff Kirsher } 424e689cf4aSJeff Kirsher 425e689cf4aSJeff Kirsher static int vnet_walk_rx_one(struct vnet_port *port, 426e689cf4aSJeff Kirsher struct vio_dring_state *dr, 427e689cf4aSJeff Kirsher u32 index, int *needs_ack) 428e689cf4aSJeff Kirsher { 429e689cf4aSJeff Kirsher struct vio_net_desc *desc = get_rx_desc(port, dr, index); 430e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 431e689cf4aSJeff Kirsher int err; 432e689cf4aSJeff Kirsher 43369088822SSowmini Varadhan BUG_ON(desc == NULL); 434e689cf4aSJeff Kirsher if (IS_ERR(desc)) 435e689cf4aSJeff Kirsher return PTR_ERR(desc); 436e689cf4aSJeff Kirsher 43778dcff7bSDavid L Stevens if (desc->hdr.state != VIO_DESC_READY) 43878dcff7bSDavid L Stevens return 1; 43978dcff7bSDavid L Stevens 44078dcff7bSDavid L Stevens rmb(); 44178dcff7bSDavid L Stevens 442e689cf4aSJeff Kirsher viodbg(DATA, "vio_walk_rx_one desc[%02x:%02x:%08x:%08x:%llx:%llx]\n", 443e689cf4aSJeff Kirsher desc->hdr.state, desc->hdr.ack, 444e689cf4aSJeff Kirsher desc->size, desc->ncookies, 445e689cf4aSJeff Kirsher desc->cookies[0].cookie_addr, 446e689cf4aSJeff Kirsher desc->cookies[0].cookie_size); 447e689cf4aSJeff Kirsher 448e689cf4aSJeff Kirsher err = vnet_rx_one(port, desc->size, desc->cookies, desc->ncookies); 449e689cf4aSJeff Kirsher if (err == -ECONNRESET) 450e689cf4aSJeff Kirsher return err; 451e689cf4aSJeff Kirsher desc->hdr.state = VIO_DESC_DONE; 452e689cf4aSJeff Kirsher err = put_rx_desc(port, dr, desc, index); 453e689cf4aSJeff Kirsher if (err < 0) 454e689cf4aSJeff Kirsher return err; 455e689cf4aSJeff Kirsher *needs_ack = desc->hdr.ack; 456e689cf4aSJeff Kirsher return 0; 457e689cf4aSJeff Kirsher } 458e689cf4aSJeff Kirsher 459e689cf4aSJeff Kirsher static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr, 46069088822SSowmini Varadhan u32 start, u32 end, int *npkts, int budget) 461e689cf4aSJeff Kirsher { 462e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 463e689cf4aSJeff Kirsher int ack_start = -1, ack_end = -1; 46469088822SSowmini Varadhan bool send_ack = true; 465e689cf4aSJeff Kirsher 466e689cf4aSJeff Kirsher end = (end == (u32) -1) ? prev_idx(start, dr) : next_idx(end, dr); 467e689cf4aSJeff Kirsher 468e689cf4aSJeff Kirsher viodbg(DATA, "vnet_walk_rx start[%08x] end[%08x]\n", start, end); 469e689cf4aSJeff Kirsher 470e689cf4aSJeff Kirsher while (start != end) { 471e689cf4aSJeff Kirsher int ack = 0, err = vnet_walk_rx_one(port, dr, start, &ack); 472e689cf4aSJeff Kirsher if (err == -ECONNRESET) 473e689cf4aSJeff Kirsher return err; 474e689cf4aSJeff Kirsher if (err != 0) 475e689cf4aSJeff Kirsher break; 47669088822SSowmini Varadhan (*npkts)++; 477e689cf4aSJeff Kirsher if (ack_start == -1) 478e689cf4aSJeff Kirsher ack_start = start; 479e689cf4aSJeff Kirsher ack_end = start; 480e689cf4aSJeff Kirsher start = next_idx(start, dr); 481e689cf4aSJeff Kirsher if (ack && start != end) { 482e689cf4aSJeff Kirsher err = vnet_send_ack(port, dr, ack_start, ack_end, 483e689cf4aSJeff Kirsher VIO_DRING_ACTIVE); 484e689cf4aSJeff Kirsher if (err == -ECONNRESET) 485e689cf4aSJeff Kirsher return err; 486e689cf4aSJeff Kirsher ack_start = -1; 487e689cf4aSJeff Kirsher } 48869088822SSowmini Varadhan if ((*npkts) >= budget) { 48969088822SSowmini Varadhan send_ack = false; 49069088822SSowmini Varadhan break; 49169088822SSowmini Varadhan } 492e689cf4aSJeff Kirsher } 493e689cf4aSJeff Kirsher if (unlikely(ack_start == -1)) 494e689cf4aSJeff Kirsher ack_start = ack_end = prev_idx(start, dr); 49569088822SSowmini Varadhan if (send_ack) { 49669088822SSowmini Varadhan port->napi_resume = false; 49769088822SSowmini Varadhan return vnet_send_ack(port, dr, ack_start, ack_end, 49869088822SSowmini Varadhan VIO_DRING_STOPPED); 49969088822SSowmini Varadhan } else { 50069088822SSowmini Varadhan port->napi_resume = true; 50169088822SSowmini Varadhan port->napi_stop_idx = ack_end; 50269088822SSowmini Varadhan return 1; 50369088822SSowmini Varadhan } 504e689cf4aSJeff Kirsher } 505e689cf4aSJeff Kirsher 50669088822SSowmini Varadhan static int vnet_rx(struct vnet_port *port, void *msgbuf, int *npkts, 50769088822SSowmini Varadhan int budget) 508e689cf4aSJeff Kirsher { 509e689cf4aSJeff Kirsher struct vio_dring_data *pkt = msgbuf; 510e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_RX_RING]; 511e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 512e689cf4aSJeff Kirsher 513e689cf4aSJeff Kirsher viodbg(DATA, "vnet_rx stype_env[%04x] seq[%016llx] rcv_nxt[%016llx]\n", 514e689cf4aSJeff Kirsher pkt->tag.stype_env, pkt->seq, dr->rcv_nxt); 515e689cf4aSJeff Kirsher 516e689cf4aSJeff Kirsher if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 517e689cf4aSJeff Kirsher return 0; 518e689cf4aSJeff Kirsher if (unlikely(pkt->seq != dr->rcv_nxt)) { 519e689cf4aSJeff Kirsher pr_err("RX out of sequence seq[0x%llx] rcv_nxt[0x%llx]\n", 520e689cf4aSJeff Kirsher pkt->seq, dr->rcv_nxt); 521e689cf4aSJeff Kirsher return 0; 522e689cf4aSJeff Kirsher } 523e689cf4aSJeff Kirsher 52469088822SSowmini Varadhan if (!port->napi_resume) 525e689cf4aSJeff Kirsher dr->rcv_nxt++; 526e689cf4aSJeff Kirsher 527e689cf4aSJeff Kirsher /* XXX Validate pkt->start_idx and pkt->end_idx XXX */ 528e689cf4aSJeff Kirsher 52969088822SSowmini Varadhan return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx, 53069088822SSowmini Varadhan npkts, budget); 531e689cf4aSJeff Kirsher } 532e689cf4aSJeff Kirsher 533e689cf4aSJeff Kirsher static int idx_is_pending(struct vio_dring_state *dr, u32 end) 534e689cf4aSJeff Kirsher { 535e689cf4aSJeff Kirsher u32 idx = dr->cons; 536e689cf4aSJeff Kirsher int found = 0; 537e689cf4aSJeff Kirsher 538e689cf4aSJeff Kirsher while (idx != dr->prod) { 539e689cf4aSJeff Kirsher if (idx == end) { 540e689cf4aSJeff Kirsher found = 1; 541e689cf4aSJeff Kirsher break; 542e689cf4aSJeff Kirsher } 543e689cf4aSJeff Kirsher idx = next_idx(idx, dr); 544e689cf4aSJeff Kirsher } 545e689cf4aSJeff Kirsher return found; 546e689cf4aSJeff Kirsher } 547e689cf4aSJeff Kirsher 548e689cf4aSJeff Kirsher static int vnet_ack(struct vnet_port *port, void *msgbuf) 549e689cf4aSJeff Kirsher { 550e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 551e689cf4aSJeff Kirsher struct vio_dring_data *pkt = msgbuf; 552e689cf4aSJeff Kirsher struct net_device *dev; 553e689cf4aSJeff Kirsher struct vnet *vp; 554e689cf4aSJeff Kirsher u32 end; 555d1015645SSowmini Varadhan struct vio_net_desc *desc; 556d51bffd1SSowmini Varadhan struct netdev_queue *txq; 557d51bffd1SSowmini Varadhan 558e689cf4aSJeff Kirsher if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 559e689cf4aSJeff Kirsher return 0; 560e689cf4aSJeff Kirsher 561e689cf4aSJeff Kirsher end = pkt->end_idx; 562e689cf4aSJeff Kirsher if (unlikely(!idx_is_pending(dr, end))) 563e689cf4aSJeff Kirsher return 0; 564e689cf4aSJeff Kirsher 56569088822SSowmini Varadhan vp = port->vp; 56669088822SSowmini Varadhan dev = vp->dev; 567d1015645SSowmini Varadhan /* sync for race conditions with vnet_start_xmit() and tell xmit it 568d1015645SSowmini Varadhan * is time to send a trigger. 569d1015645SSowmini Varadhan */ 57069088822SSowmini Varadhan netif_tx_lock(dev); 571e689cf4aSJeff Kirsher dr->cons = next_idx(end, dr); 572d1015645SSowmini Varadhan desc = vio_dring_entry(dr, dr->cons); 573d1015645SSowmini Varadhan if (desc->hdr.state == VIO_DESC_READY && port->start_cons) { 574d1015645SSowmini Varadhan /* vnet_start_xmit() just populated this dring but missed 575d1015645SSowmini Varadhan * sending the "start" LDC message to the consumer. 576d1015645SSowmini Varadhan * Send a "start" trigger on its behalf. 577d1015645SSowmini Varadhan */ 578d1015645SSowmini Varadhan if (__vnet_tx_trigger(port, dr->cons) > 0) 579d1015645SSowmini Varadhan port->start_cons = false; 580d1015645SSowmini Varadhan else 581d1015645SSowmini Varadhan port->start_cons = true; 582d1015645SSowmini Varadhan } else { 583d1015645SSowmini Varadhan port->start_cons = true; 584d1015645SSowmini Varadhan } 58569088822SSowmini Varadhan netif_tx_unlock(dev); 586d1015645SSowmini Varadhan 587d51bffd1SSowmini Varadhan txq = netdev_get_tx_queue(dev, port->q_index); 588d51bffd1SSowmini Varadhan if (unlikely(netif_tx_queue_stopped(txq) && 589e689cf4aSJeff Kirsher vnet_tx_dring_avail(dr) >= VNET_TX_WAKEUP_THRESH(dr))) 590e689cf4aSJeff Kirsher return 1; 591e689cf4aSJeff Kirsher 592e689cf4aSJeff Kirsher return 0; 593e689cf4aSJeff Kirsher } 594e689cf4aSJeff Kirsher 595e689cf4aSJeff Kirsher static int vnet_nack(struct vnet_port *port, void *msgbuf) 596e689cf4aSJeff Kirsher { 597e689cf4aSJeff Kirsher /* XXX just reset or similar XXX */ 598e689cf4aSJeff Kirsher return 0; 599e689cf4aSJeff Kirsher } 600e689cf4aSJeff Kirsher 601e689cf4aSJeff Kirsher static int handle_mcast(struct vnet_port *port, void *msgbuf) 602e689cf4aSJeff Kirsher { 603e689cf4aSJeff Kirsher struct vio_net_mcast_info *pkt = msgbuf; 604e689cf4aSJeff Kirsher 605e689cf4aSJeff Kirsher if (pkt->tag.stype != VIO_SUBTYPE_ACK) 606e689cf4aSJeff Kirsher pr_err("%s: Got unexpected MCAST reply [%02x:%02x:%04x:%08x]\n", 607e689cf4aSJeff Kirsher port->vp->dev->name, 608e689cf4aSJeff Kirsher pkt->tag.type, 609e689cf4aSJeff Kirsher pkt->tag.stype, 610e689cf4aSJeff Kirsher pkt->tag.stype_env, 611e689cf4aSJeff Kirsher pkt->tag.sid); 612e689cf4aSJeff Kirsher 613e689cf4aSJeff Kirsher return 0; 614e689cf4aSJeff Kirsher } 615e689cf4aSJeff Kirsher 616d51bffd1SSowmini Varadhan /* Got back a STOPPED LDC message on port. If the queue is stopped, 617d51bffd1SSowmini Varadhan * wake it up so that we'll send out another START message at the 618d51bffd1SSowmini Varadhan * next TX. 619d51bffd1SSowmini Varadhan */ 620d51bffd1SSowmini Varadhan static void maybe_tx_wakeup(struct vnet_port *port) 621e689cf4aSJeff Kirsher { 622d51bffd1SSowmini Varadhan struct netdev_queue *txq; 623e689cf4aSJeff Kirsher 624d51bffd1SSowmini Varadhan txq = netdev_get_tx_queue(port->vp->dev, port->q_index); 625d51bffd1SSowmini Varadhan __netif_tx_lock(txq, smp_processor_id()); 626d51bffd1SSowmini Varadhan if (likely(netif_tx_queue_stopped(txq))) { 627e689cf4aSJeff Kirsher struct vio_dring_state *dr; 628e689cf4aSJeff Kirsher 629e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 630d51bffd1SSowmini Varadhan netif_tx_wake_queue(txq); 631e689cf4aSJeff Kirsher } 632d51bffd1SSowmini Varadhan __netif_tx_unlock(txq); 633e689cf4aSJeff Kirsher } 634e689cf4aSJeff Kirsher 63569088822SSowmini Varadhan static inline bool port_is_up(struct vnet_port *vnet) 636e689cf4aSJeff Kirsher { 63769088822SSowmini Varadhan struct vio_driver_state *vio = &vnet->vio; 63869088822SSowmini Varadhan 63969088822SSowmini Varadhan return !!(vio->hs_state & VIO_HS_COMPLETE); 64069088822SSowmini Varadhan } 64169088822SSowmini Varadhan 64269088822SSowmini Varadhan static int vnet_event_napi(struct vnet_port *port, int budget) 64369088822SSowmini Varadhan { 644e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 645e689cf4aSJeff Kirsher int tx_wakeup, err; 64669088822SSowmini Varadhan int npkts = 0; 64769088822SSowmini Varadhan int event = (port->rx_event & LDC_EVENT_RESET); 648e689cf4aSJeff Kirsher 64969088822SSowmini Varadhan ldc_ctrl: 650e689cf4aSJeff Kirsher if (unlikely(event == LDC_EVENT_RESET || 651e689cf4aSJeff Kirsher event == LDC_EVENT_UP)) { 652e689cf4aSJeff Kirsher vio_link_state_change(vio, event); 653e689cf4aSJeff Kirsher 654e4defc77SDavid L Stevens if (event == LDC_EVENT_RESET) { 655e4defc77SDavid L Stevens port->rmtu = 0; 656e689cf4aSJeff Kirsher vio_port_up(vio); 657e4defc77SDavid L Stevens } 65869088822SSowmini Varadhan port->rx_event = 0; 65969088822SSowmini Varadhan return 0; 660e689cf4aSJeff Kirsher } 66169088822SSowmini Varadhan /* We may have multiple LDC events in rx_event. Unroll send_events() */ 66269088822SSowmini Varadhan event = (port->rx_event & LDC_EVENT_UP); 66369088822SSowmini Varadhan port->rx_event &= ~(LDC_EVENT_RESET|LDC_EVENT_UP); 66469088822SSowmini Varadhan if (event == LDC_EVENT_UP) 66569088822SSowmini Varadhan goto ldc_ctrl; 66669088822SSowmini Varadhan event = port->rx_event; 66769088822SSowmini Varadhan if (!(event & LDC_EVENT_DATA_READY)) 66869088822SSowmini Varadhan return 0; 669e689cf4aSJeff Kirsher 67069088822SSowmini Varadhan /* we dont expect any other bits than RESET, UP, DATA_READY */ 67169088822SSowmini Varadhan BUG_ON(event != LDC_EVENT_DATA_READY); 672e689cf4aSJeff Kirsher 673e689cf4aSJeff Kirsher tx_wakeup = err = 0; 674e689cf4aSJeff Kirsher while (1) { 675e689cf4aSJeff Kirsher union { 676e689cf4aSJeff Kirsher struct vio_msg_tag tag; 677e689cf4aSJeff Kirsher u64 raw[8]; 678e689cf4aSJeff Kirsher } msgbuf; 679e689cf4aSJeff Kirsher 68069088822SSowmini Varadhan if (port->napi_resume) { 68169088822SSowmini Varadhan struct vio_dring_data *pkt = 68269088822SSowmini Varadhan (struct vio_dring_data *)&msgbuf; 68369088822SSowmini Varadhan struct vio_dring_state *dr = 68469088822SSowmini Varadhan &port->vio.drings[VIO_DRIVER_RX_RING]; 68569088822SSowmini Varadhan 68669088822SSowmini Varadhan pkt->tag.type = VIO_TYPE_DATA; 68769088822SSowmini Varadhan pkt->tag.stype = VIO_SUBTYPE_INFO; 68869088822SSowmini Varadhan pkt->tag.stype_env = VIO_DRING_DATA; 68969088822SSowmini Varadhan pkt->seq = dr->rcv_nxt; 69069088822SSowmini Varadhan pkt->start_idx = next_idx(port->napi_stop_idx, dr); 69169088822SSowmini Varadhan pkt->end_idx = -1; 69269088822SSowmini Varadhan goto napi_resume; 69369088822SSowmini Varadhan } 694e689cf4aSJeff Kirsher err = ldc_read(vio->lp, &msgbuf, sizeof(msgbuf)); 695e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 696e689cf4aSJeff Kirsher if (err == -ECONNRESET) 697e689cf4aSJeff Kirsher vio_conn_reset(vio); 698e689cf4aSJeff Kirsher break; 699e689cf4aSJeff Kirsher } 700e689cf4aSJeff Kirsher if (err == 0) 701e689cf4aSJeff Kirsher break; 702e689cf4aSJeff Kirsher viodbg(DATA, "TAG [%02x:%02x:%04x:%08x]\n", 703e689cf4aSJeff Kirsher msgbuf.tag.type, 704e689cf4aSJeff Kirsher msgbuf.tag.stype, 705e689cf4aSJeff Kirsher msgbuf.tag.stype_env, 706e689cf4aSJeff Kirsher msgbuf.tag.sid); 707e689cf4aSJeff Kirsher err = vio_validate_sid(vio, &msgbuf.tag); 708e689cf4aSJeff Kirsher if (err < 0) 709e689cf4aSJeff Kirsher break; 71069088822SSowmini Varadhan napi_resume: 711e689cf4aSJeff Kirsher if (likely(msgbuf.tag.type == VIO_TYPE_DATA)) { 712e689cf4aSJeff Kirsher if (msgbuf.tag.stype == VIO_SUBTYPE_INFO) { 71369088822SSowmini Varadhan if (!port_is_up(port)) { 71469088822SSowmini Varadhan /* failures like handshake_failure() 71569088822SSowmini Varadhan * may have cleaned up dring, but 71669088822SSowmini Varadhan * NAPI polling may bring us here. 71769088822SSowmini Varadhan */ 71869088822SSowmini Varadhan err = -ECONNRESET; 71969088822SSowmini Varadhan break; 72069088822SSowmini Varadhan } 72169088822SSowmini Varadhan err = vnet_rx(port, &msgbuf, &npkts, budget); 72269088822SSowmini Varadhan if (npkts >= budget) 72369088822SSowmini Varadhan break; 7248c4ee3e7SSowmini Varadhan if (npkts == 0) 7258c4ee3e7SSowmini Varadhan break; 726e689cf4aSJeff Kirsher } else if (msgbuf.tag.stype == VIO_SUBTYPE_ACK) { 727e689cf4aSJeff Kirsher err = vnet_ack(port, &msgbuf); 728e689cf4aSJeff Kirsher if (err > 0) 729e689cf4aSJeff Kirsher tx_wakeup |= err; 730e689cf4aSJeff Kirsher } else if (msgbuf.tag.stype == VIO_SUBTYPE_NACK) { 731e689cf4aSJeff Kirsher err = vnet_nack(port, &msgbuf); 732e689cf4aSJeff Kirsher } 733e689cf4aSJeff Kirsher } else if (msgbuf.tag.type == VIO_TYPE_CTRL) { 734e689cf4aSJeff Kirsher if (msgbuf.tag.stype_env == VNET_MCAST_INFO) 735e689cf4aSJeff Kirsher err = handle_mcast(port, &msgbuf); 736e689cf4aSJeff Kirsher else 737e689cf4aSJeff Kirsher err = vio_control_pkt_engine(vio, &msgbuf); 738e689cf4aSJeff Kirsher if (err) 739e689cf4aSJeff Kirsher break; 740e689cf4aSJeff Kirsher } else { 741e689cf4aSJeff Kirsher err = vnet_handle_unknown(port, &msgbuf); 742e689cf4aSJeff Kirsher } 743e689cf4aSJeff Kirsher if (err == -ECONNRESET) 744e689cf4aSJeff Kirsher break; 745e689cf4aSJeff Kirsher } 746e689cf4aSJeff Kirsher if (unlikely(tx_wakeup && err != -ECONNRESET)) 747d51bffd1SSowmini Varadhan maybe_tx_wakeup(port); 74869088822SSowmini Varadhan return npkts; 74969088822SSowmini Varadhan } 7501d311ad2SSowmini Varadhan 75169088822SSowmini Varadhan static int vnet_poll(struct napi_struct *napi, int budget) 75269088822SSowmini Varadhan { 75369088822SSowmini Varadhan struct vnet_port *port = container_of(napi, struct vnet_port, napi); 75469088822SSowmini Varadhan struct vio_driver_state *vio = &port->vio; 75569088822SSowmini Varadhan int processed = vnet_event_napi(port, budget); 75669088822SSowmini Varadhan 75769088822SSowmini Varadhan if (processed < budget) { 75869088822SSowmini Varadhan napi_complete(napi); 7597bd68bfdSSowmini Varadhan port->rx_event &= ~LDC_EVENT_DATA_READY; 76069088822SSowmini Varadhan vio_set_intr(vio->vdev->rx_ino, HV_INTR_ENABLED); 76169088822SSowmini Varadhan } 76269088822SSowmini Varadhan return processed; 76369088822SSowmini Varadhan } 76469088822SSowmini Varadhan 76569088822SSowmini Varadhan static void vnet_event(void *arg, int event) 76669088822SSowmini Varadhan { 76769088822SSowmini Varadhan struct vnet_port *port = arg; 76869088822SSowmini Varadhan struct vio_driver_state *vio = &port->vio; 76969088822SSowmini Varadhan 77069088822SSowmini Varadhan port->rx_event |= event; 77169088822SSowmini Varadhan vio_set_intr(vio->vdev->rx_ino, HV_INTR_DISABLED); 77269088822SSowmini Varadhan napi_schedule(&port->napi); 77369088822SSowmini Varadhan 774e689cf4aSJeff Kirsher } 775e689cf4aSJeff Kirsher 776d1015645SSowmini Varadhan static int __vnet_tx_trigger(struct vnet_port *port, u32 start) 777e689cf4aSJeff Kirsher { 778e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 779e689cf4aSJeff Kirsher struct vio_dring_data hdr = { 780e689cf4aSJeff Kirsher .tag = { 781e689cf4aSJeff Kirsher .type = VIO_TYPE_DATA, 782e689cf4aSJeff Kirsher .stype = VIO_SUBTYPE_INFO, 783e689cf4aSJeff Kirsher .stype_env = VIO_DRING_DATA, 784e689cf4aSJeff Kirsher .sid = vio_send_sid(&port->vio), 785e689cf4aSJeff Kirsher }, 786e689cf4aSJeff Kirsher .dring_ident = dr->ident, 787d1015645SSowmini Varadhan .start_idx = start, 788e689cf4aSJeff Kirsher .end_idx = (u32) -1, 789e689cf4aSJeff Kirsher }; 790e689cf4aSJeff Kirsher int err, delay; 791adddc32dSSowmini Varadhan int retries = 0; 792e689cf4aSJeff Kirsher 793d1015645SSowmini Varadhan if (port->stop_rx) { 794d1015645SSowmini Varadhan err = vnet_send_ack(port, 795d1015645SSowmini Varadhan &port->vio.drings[VIO_DRIVER_RX_RING], 796d1015645SSowmini Varadhan port->stop_rx_idx, -1, 797d1015645SSowmini Varadhan VIO_DRING_STOPPED); 798d1015645SSowmini Varadhan if (err <= 0) 799d1015645SSowmini Varadhan return err; 800d1015645SSowmini Varadhan } 801d1015645SSowmini Varadhan 802e689cf4aSJeff Kirsher hdr.seq = dr->snd_nxt; 803e689cf4aSJeff Kirsher delay = 1; 804e689cf4aSJeff Kirsher do { 805e689cf4aSJeff Kirsher err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 806e689cf4aSJeff Kirsher if (err > 0) { 807e689cf4aSJeff Kirsher dr->snd_nxt++; 808e689cf4aSJeff Kirsher break; 809e689cf4aSJeff Kirsher } 810e689cf4aSJeff Kirsher udelay(delay); 811e689cf4aSJeff Kirsher if ((delay <<= 1) > 128) 812e689cf4aSJeff Kirsher delay = 128; 813adddc32dSSowmini Varadhan if (retries++ > VNET_MAX_RETRIES) 814adddc32dSSowmini Varadhan break; 815e689cf4aSJeff Kirsher } while (err == -EAGAIN); 816e689cf4aSJeff Kirsher 817e689cf4aSJeff Kirsher return err; 818e689cf4aSJeff Kirsher } 819e689cf4aSJeff Kirsher 820e689cf4aSJeff Kirsher struct vnet_port *__tx_port_find(struct vnet *vp, struct sk_buff *skb) 821e689cf4aSJeff Kirsher { 822e689cf4aSJeff Kirsher unsigned int hash = vnet_hashfn(skb->data); 823e689cf4aSJeff Kirsher struct hlist_head *hp = &vp->port_hash[hash]; 824e689cf4aSJeff Kirsher struct vnet_port *port; 825e689cf4aSJeff Kirsher 8262a968dd8SSowmini Varadhan hlist_for_each_entry_rcu(port, hp, hash) { 8278266f5fcSDavid L Stevens if (!port_is_up(port)) 8288266f5fcSDavid L Stevens continue; 8292e42e474SJoe Perches if (ether_addr_equal(port->raddr, skb->data)) 830e689cf4aSJeff Kirsher return port; 831e689cf4aSJeff Kirsher } 8322a968dd8SSowmini Varadhan list_for_each_entry_rcu(port, &vp->port_list, list) { 8338266f5fcSDavid L Stevens if (!port->switch_port) 8348266f5fcSDavid L Stevens continue; 8358266f5fcSDavid L Stevens if (!port_is_up(port)) 8368266f5fcSDavid L Stevens continue; 837e689cf4aSJeff Kirsher return port; 838e689cf4aSJeff Kirsher } 8398266f5fcSDavid L Stevens return NULL; 8408266f5fcSDavid L Stevens } 841e689cf4aSJeff Kirsher 8428e845f4cSDavid L Stevens static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port, 8438e845f4cSDavid L Stevens unsigned *pending) 8448e845f4cSDavid L Stevens { 8458e845f4cSDavid L Stevens struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 8468e845f4cSDavid L Stevens struct sk_buff *skb = NULL; 8478e845f4cSDavid L Stevens int i, txi; 8488e845f4cSDavid L Stevens 8498e845f4cSDavid L Stevens *pending = 0; 8508e845f4cSDavid L Stevens 8518e845f4cSDavid L Stevens txi = dr->prod-1; 8528e845f4cSDavid L Stevens if (txi < 0) 8538e845f4cSDavid L Stevens txi = VNET_TX_RING_SIZE-1; 8548e845f4cSDavid L Stevens 8558e845f4cSDavid L Stevens for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 8568e845f4cSDavid L Stevens struct vio_net_desc *d; 8578e845f4cSDavid L Stevens 8588e845f4cSDavid L Stevens d = vio_dring_entry(dr, txi); 8598e845f4cSDavid L Stevens 8608e845f4cSDavid L Stevens if (d->hdr.state == VIO_DESC_DONE) { 8618e845f4cSDavid L Stevens if (port->tx_bufs[txi].skb) { 8628e845f4cSDavid L Stevens BUG_ON(port->tx_bufs[txi].skb->next); 8638e845f4cSDavid L Stevens 8648e845f4cSDavid L Stevens port->tx_bufs[txi].skb->next = skb; 8658e845f4cSDavid L Stevens skb = port->tx_bufs[txi].skb; 8668e845f4cSDavid L Stevens port->tx_bufs[txi].skb = NULL; 8678e845f4cSDavid L Stevens 8688e845f4cSDavid L Stevens ldc_unmap(port->vio.lp, 8698e845f4cSDavid L Stevens port->tx_bufs[txi].cookies, 8708e845f4cSDavid L Stevens port->tx_bufs[txi].ncookies); 8718e845f4cSDavid L Stevens } 8728e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 8738e845f4cSDavid L Stevens } else if (d->hdr.state == VIO_DESC_READY) { 8748e845f4cSDavid L Stevens (*pending)++; 8758e845f4cSDavid L Stevens } else if (d->hdr.state == VIO_DESC_FREE) { 8768e845f4cSDavid L Stevens break; 8778e845f4cSDavid L Stevens } 8788e845f4cSDavid L Stevens --txi; 8798e845f4cSDavid L Stevens if (txi < 0) 8808e845f4cSDavid L Stevens txi = VNET_TX_RING_SIZE-1; 8818e845f4cSDavid L Stevens } 8828e845f4cSDavid L Stevens return skb; 8838e845f4cSDavid L Stevens } 8848e845f4cSDavid L Stevens 8858e845f4cSDavid L Stevens static inline void vnet_free_skbs(struct sk_buff *skb) 8868e845f4cSDavid L Stevens { 8878e845f4cSDavid L Stevens struct sk_buff *next; 8888e845f4cSDavid L Stevens 8898e845f4cSDavid L Stevens while (skb) { 8908e845f4cSDavid L Stevens next = skb->next; 8918e845f4cSDavid L Stevens skb->next = NULL; 8928e845f4cSDavid L Stevens dev_kfree_skb(skb); 8938e845f4cSDavid L Stevens skb = next; 8948e845f4cSDavid L Stevens } 8958e845f4cSDavid L Stevens } 8968e845f4cSDavid L Stevens 8978e845f4cSDavid L Stevens static void vnet_clean_timer_expire(unsigned long port0) 8988e845f4cSDavid L Stevens { 8998e845f4cSDavid L Stevens struct vnet_port *port = (struct vnet_port *)port0; 9008e845f4cSDavid L Stevens struct sk_buff *freeskbs; 9018e845f4cSDavid L Stevens unsigned pending; 9028e845f4cSDavid L Stevens 90313b13dd9SSowmini Varadhan netif_tx_lock(port->vp->dev); 9048e845f4cSDavid L Stevens freeskbs = vnet_clean_tx_ring(port, &pending); 90513b13dd9SSowmini Varadhan netif_tx_unlock(port->vp->dev); 9068e845f4cSDavid L Stevens 9078e845f4cSDavid L Stevens vnet_free_skbs(freeskbs); 9088e845f4cSDavid L Stevens 9098e845f4cSDavid L Stevens if (pending) 9108e845f4cSDavid L Stevens (void)mod_timer(&port->clean_timer, 9118e845f4cSDavid L Stevens jiffies + VNET_CLEAN_TIMEOUT); 9128e845f4cSDavid L Stevens else 9138e845f4cSDavid L Stevens del_timer(&port->clean_timer); 9148e845f4cSDavid L Stevens } 9158e845f4cSDavid L Stevens 9168e845f4cSDavid L Stevens static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, void **pstart, 9178e845f4cSDavid L Stevens int *plen) 9188e845f4cSDavid L Stevens { 9198e845f4cSDavid L Stevens struct sk_buff *nskb; 9208e845f4cSDavid L Stevens int len, pad; 9218e845f4cSDavid L Stevens 9228e845f4cSDavid L Stevens len = skb->len; 9238e845f4cSDavid L Stevens pad = 0; 9248e845f4cSDavid L Stevens if (len < ETH_ZLEN) { 9258e845f4cSDavid L Stevens pad += ETH_ZLEN - skb->len; 9268e845f4cSDavid L Stevens len += pad; 9278e845f4cSDavid L Stevens } 9288e845f4cSDavid L Stevens len += VNET_PACKET_SKIP; 9298e845f4cSDavid L Stevens pad += 8 - (len & 7); 9308e845f4cSDavid L Stevens len += 8 - (len & 7); 9318e845f4cSDavid L Stevens 9328e845f4cSDavid L Stevens if (((unsigned long)skb->data & 7) != VNET_PACKET_SKIP || 9338e845f4cSDavid L Stevens skb_tailroom(skb) < pad || 9348e845f4cSDavid L Stevens skb_headroom(skb) < VNET_PACKET_SKIP) { 9358e845f4cSDavid L Stevens nskb = alloc_and_align_skb(skb->dev, skb->len); 9368e845f4cSDavid L Stevens skb_reserve(nskb, VNET_PACKET_SKIP); 9378e845f4cSDavid L Stevens if (skb_copy_bits(skb, 0, nskb->data, skb->len)) { 9388e845f4cSDavid L Stevens dev_kfree_skb(nskb); 9398e845f4cSDavid L Stevens dev_kfree_skb(skb); 9408e845f4cSDavid L Stevens return NULL; 9418e845f4cSDavid L Stevens } 9428e845f4cSDavid L Stevens (void)skb_put(nskb, skb->len); 9438e845f4cSDavid L Stevens dev_kfree_skb(skb); 9448e845f4cSDavid L Stevens skb = nskb; 9458e845f4cSDavid L Stevens } 9468e845f4cSDavid L Stevens 9478e845f4cSDavid L Stevens *pstart = skb->data - VNET_PACKET_SKIP; 9488e845f4cSDavid L Stevens *plen = len; 9498e845f4cSDavid L Stevens return skb; 9508e845f4cSDavid L Stevens } 9518e845f4cSDavid L Stevens 952d51bffd1SSowmini Varadhan static u16 953d51bffd1SSowmini Varadhan vnet_select_queue(struct net_device *dev, struct sk_buff *skb, 954d51bffd1SSowmini Varadhan void *accel_priv, select_queue_fallback_t fallback) 955d51bffd1SSowmini Varadhan { 956d51bffd1SSowmini Varadhan struct vnet *vp = netdev_priv(dev); 957d51bffd1SSowmini Varadhan struct vnet_port *port = __tx_port_find(vp, skb); 958d51bffd1SSowmini Varadhan 959d51bffd1SSowmini Varadhan return port->q_index; 960d51bffd1SSowmini Varadhan } 961d51bffd1SSowmini Varadhan 962e689cf4aSJeff Kirsher static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 963e689cf4aSJeff Kirsher { 964e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 9652a968dd8SSowmini Varadhan struct vnet_port *port = NULL; 966e689cf4aSJeff Kirsher struct vio_dring_state *dr; 967e689cf4aSJeff Kirsher struct vio_net_desc *d; 968e689cf4aSJeff Kirsher unsigned int len; 9698e845f4cSDavid L Stevens struct sk_buff *freeskbs = NULL; 9708e845f4cSDavid L Stevens int i, err, txi; 9718e845f4cSDavid L Stevens void *start = NULL; 9728e845f4cSDavid L Stevens int nlen = 0; 9738e845f4cSDavid L Stevens unsigned pending = 0; 974d51bffd1SSowmini Varadhan struct netdev_queue *txq; 975e689cf4aSJeff Kirsher 9762a968dd8SSowmini Varadhan skb = vnet_skb_shape(skb, &start, &nlen); 9772a968dd8SSowmini Varadhan if (unlikely(!skb)) 978e689cf4aSJeff Kirsher goto out_dropped; 979e689cf4aSJeff Kirsher 9802a968dd8SSowmini Varadhan rcu_read_lock(); 98113b13dd9SSowmini Varadhan port = __tx_port_find(vp, skb); 9822a968dd8SSowmini Varadhan if (unlikely(!port)) 9838e845f4cSDavid L Stevens goto out_dropped; 9848e845f4cSDavid L Stevens 985a2b78e9bSDavid L Stevens if (skb->len > port->rmtu) { 986a2b78e9bSDavid L Stevens unsigned long localmtu = port->rmtu - ETH_HLEN; 987a2b78e9bSDavid L Stevens 988a2b78e9bSDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 3)) 989a2b78e9bSDavid L Stevens localmtu -= VLAN_HLEN; 990a2b78e9bSDavid L Stevens 991a2b78e9bSDavid L Stevens if (skb->protocol == htons(ETH_P_IP)) { 992a2b78e9bSDavid L Stevens struct flowi4 fl4; 993a2b78e9bSDavid L Stevens struct rtable *rt = NULL; 994a2b78e9bSDavid L Stevens 995a2b78e9bSDavid L Stevens memset(&fl4, 0, sizeof(fl4)); 996a2b78e9bSDavid L Stevens fl4.flowi4_oif = dev->ifindex; 997a2b78e9bSDavid L Stevens fl4.flowi4_tos = RT_TOS(ip_hdr(skb)->tos); 998a2b78e9bSDavid L Stevens fl4.daddr = ip_hdr(skb)->daddr; 999a2b78e9bSDavid L Stevens fl4.saddr = ip_hdr(skb)->saddr; 1000a2b78e9bSDavid L Stevens 1001a2b78e9bSDavid L Stevens rt = ip_route_output_key(dev_net(dev), &fl4); 10022a968dd8SSowmini Varadhan rcu_read_unlock(); 1003a2b78e9bSDavid L Stevens if (!IS_ERR(rt)) { 1004a2b78e9bSDavid L Stevens skb_dst_set(skb, &rt->dst); 1005a2b78e9bSDavid L Stevens icmp_send(skb, ICMP_DEST_UNREACH, 1006a2b78e9bSDavid L Stevens ICMP_FRAG_NEEDED, 1007a2b78e9bSDavid L Stevens htonl(localmtu)); 1008a2b78e9bSDavid L Stevens } 1009a2b78e9bSDavid L Stevens } 1010a2b78e9bSDavid L Stevens #if IS_ENABLED(CONFIG_IPV6) 1011a2b78e9bSDavid L Stevens else if (skb->protocol == htons(ETH_P_IPV6)) 1012a2b78e9bSDavid L Stevens icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, localmtu); 1013a2b78e9bSDavid L Stevens #endif 101442db672dSDavid L Stevens goto out_dropped; 1015a2b78e9bSDavid L Stevens } 101642db672dSDavid L Stevens 1017e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1018d51bffd1SSowmini Varadhan i = skb_get_queue_mapping(skb); 1019d51bffd1SSowmini Varadhan txq = netdev_get_tx_queue(dev, i); 1020d0aedcd4SDwight Engen if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1021d51bffd1SSowmini Varadhan if (!netif_tx_queue_stopped(txq)) { 1022d51bffd1SSowmini Varadhan netif_tx_stop_queue(txq); 1023e689cf4aSJeff Kirsher 1024e689cf4aSJeff Kirsher /* This is a hard error, log it. */ 1025e689cf4aSJeff Kirsher netdev_err(dev, "BUG! Tx Ring full when queue awake!\n"); 1026e689cf4aSJeff Kirsher dev->stats.tx_errors++; 1027e689cf4aSJeff Kirsher } 10282a968dd8SSowmini Varadhan rcu_read_unlock(); 1029e689cf4aSJeff Kirsher return NETDEV_TX_BUSY; 1030e689cf4aSJeff Kirsher } 1031e689cf4aSJeff Kirsher 1032e689cf4aSJeff Kirsher d = vio_dring_cur(dr); 1033e689cf4aSJeff Kirsher 10348e845f4cSDavid L Stevens txi = dr->prod; 10358e845f4cSDavid L Stevens 10368e845f4cSDavid L Stevens freeskbs = vnet_clean_tx_ring(port, &pending); 10378e845f4cSDavid L Stevens 10388e845f4cSDavid L Stevens BUG_ON(port->tx_bufs[txi].skb); 1039e689cf4aSJeff Kirsher 1040e689cf4aSJeff Kirsher len = skb->len; 10418e845f4cSDavid L Stevens if (len < ETH_ZLEN) 1042e689cf4aSJeff Kirsher len = ETH_ZLEN; 10438e845f4cSDavid L Stevens 10448e845f4cSDavid L Stevens port->tx_bufs[txi].skb = skb; 10458e845f4cSDavid L Stevens skb = NULL; 10468e845f4cSDavid L Stevens 10478e845f4cSDavid L Stevens err = ldc_map_single(port->vio.lp, start, nlen, 104842db672dSDavid L Stevens port->tx_bufs[txi].cookies, VNET_MAXCOOKIES, 10498e845f4cSDavid L Stevens (LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW)); 10508e845f4cSDavid L Stevens if (err < 0) { 10518e845f4cSDavid L Stevens netdev_info(dev, "tx buffer map error %d\n", err); 105213b13dd9SSowmini Varadhan goto out_dropped; 1053e689cf4aSJeff Kirsher } 10548e845f4cSDavid L Stevens port->tx_bufs[txi].ncookies = err; 1055e689cf4aSJeff Kirsher 10561f6394e3SSowmini Varadhan /* We don't rely on the ACKs to free the skb in vnet_start_xmit(), 10571f6394e3SSowmini Varadhan * thus it is safe to not set VIO_ACK_ENABLE for each transmission: 10581f6394e3SSowmini Varadhan * the protocol itself does not require it as long as the peer 10591f6394e3SSowmini Varadhan * sends a VIO_SUBTYPE_ACK for VIO_DRING_STOPPED. 10601f6394e3SSowmini Varadhan * 10611f6394e3SSowmini Varadhan * An ACK for every packet in the ring is expensive as the 10621f6394e3SSowmini Varadhan * sending of LDC messages is slow and affects performance. 10631f6394e3SSowmini Varadhan */ 10641f6394e3SSowmini Varadhan d->hdr.ack = VIO_ACK_DISABLE; 1065e689cf4aSJeff Kirsher d->size = len; 10668e845f4cSDavid L Stevens d->ncookies = port->tx_bufs[txi].ncookies; 1067e689cf4aSJeff Kirsher for (i = 0; i < d->ncookies; i++) 10688e845f4cSDavid L Stevens d->cookies[i] = port->tx_bufs[txi].cookies[i]; 1069e689cf4aSJeff Kirsher 1070e689cf4aSJeff Kirsher /* This has to be a non-SMP write barrier because we are writing 1071e689cf4aSJeff Kirsher * to memory which is shared with the peer LDOM. 1072e689cf4aSJeff Kirsher */ 1073e689cf4aSJeff Kirsher wmb(); 1074e689cf4aSJeff Kirsher 1075e689cf4aSJeff Kirsher d->hdr.state = VIO_DESC_READY; 1076e689cf4aSJeff Kirsher 1077d1015645SSowmini Varadhan /* Exactly one ldc "start" trigger (for dr->cons) needs to be sent 1078d1015645SSowmini Varadhan * to notify the consumer that some descriptors are READY. 1079d1015645SSowmini Varadhan * After that "start" trigger, no additional triggers are needed until 1080d1015645SSowmini Varadhan * a DRING_STOPPED is received from the consumer. The dr->cons field 1081d1015645SSowmini Varadhan * (set up by vnet_ack()) has the value of the next dring index 1082d1015645SSowmini Varadhan * that has not yet been ack-ed. We send a "start" trigger here 1083d1015645SSowmini Varadhan * if, and only if, start_cons is true (reset it afterward). Conversely, 1084d1015645SSowmini Varadhan * vnet_ack() should check if the dring corresponding to cons 1085d1015645SSowmini Varadhan * is marked READY, but start_cons was false. 1086d1015645SSowmini Varadhan * If so, vnet_ack() should send out the missed "start" trigger. 1087d1015645SSowmini Varadhan * 1088d1015645SSowmini Varadhan * Note that the wmb() above makes sure the cookies et al. are 1089d1015645SSowmini Varadhan * not globally visible before the VIO_DESC_READY, and that the 1090d1015645SSowmini Varadhan * stores are ordered correctly by the compiler. The consumer will 1091d1015645SSowmini Varadhan * not proceed until the VIO_DESC_READY is visible assuring that 1092d1015645SSowmini Varadhan * the consumer does not observe anything related to descriptors 1093d1015645SSowmini Varadhan * out of order. The HV trap from the LDC start trigger is the 1094d1015645SSowmini Varadhan * producer to consumer announcement that work is available to the 1095d1015645SSowmini Varadhan * consumer 1096d1015645SSowmini Varadhan */ 1097d1015645SSowmini Varadhan if (!port->start_cons) 1098d1015645SSowmini Varadhan goto ldc_start_done; /* previous trigger suffices */ 1099d1015645SSowmini Varadhan 1100d1015645SSowmini Varadhan err = __vnet_tx_trigger(port, dr->cons); 1101e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 1102e689cf4aSJeff Kirsher netdev_info(dev, "TX trigger error %d\n", err); 1103e689cf4aSJeff Kirsher d->hdr.state = VIO_DESC_FREE; 1104e689cf4aSJeff Kirsher dev->stats.tx_carrier_errors++; 110513b13dd9SSowmini Varadhan goto out_dropped; 1106e689cf4aSJeff Kirsher } 1107e689cf4aSJeff Kirsher 1108d1015645SSowmini Varadhan ldc_start_done: 1109d1015645SSowmini Varadhan port->start_cons = false; 1110d1015645SSowmini Varadhan 1111e689cf4aSJeff Kirsher dev->stats.tx_packets++; 11128e845f4cSDavid L Stevens dev->stats.tx_bytes += port->tx_bufs[txi].skb->len; 1113e689cf4aSJeff Kirsher 1114e689cf4aSJeff Kirsher dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1); 1115d0aedcd4SDwight Engen if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1116d51bffd1SSowmini Varadhan netif_tx_stop_queue(txq); 1117e689cf4aSJeff Kirsher if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr)) 1118d51bffd1SSowmini Varadhan netif_tx_wake_queue(txq); 1119e689cf4aSJeff Kirsher } 1120e689cf4aSJeff Kirsher 11212a968dd8SSowmini Varadhan (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT); 11222a968dd8SSowmini Varadhan rcu_read_unlock(); 1123e689cf4aSJeff Kirsher 11248e845f4cSDavid L Stevens vnet_free_skbs(freeskbs); 11258e845f4cSDavid L Stevens 1126e689cf4aSJeff Kirsher return NETDEV_TX_OK; 1127e689cf4aSJeff Kirsher 1128e689cf4aSJeff Kirsher out_dropped: 11298e845f4cSDavid L Stevens if (pending) 11308e845f4cSDavid L Stevens (void)mod_timer(&port->clean_timer, 11318e845f4cSDavid L Stevens jiffies + VNET_CLEAN_TIMEOUT); 1132a29c9c43SDavid L Stevens else if (port) 11338e845f4cSDavid L Stevens del_timer(&port->clean_timer); 11342a968dd8SSowmini Varadhan if (port) 11352a968dd8SSowmini Varadhan rcu_read_unlock(); 11362a968dd8SSowmini Varadhan if (skb) 11372a968dd8SSowmini Varadhan dev_kfree_skb(skb); 11382a968dd8SSowmini Varadhan vnet_free_skbs(freeskbs); 1139e689cf4aSJeff Kirsher dev->stats.tx_dropped++; 1140e689cf4aSJeff Kirsher return NETDEV_TX_OK; 1141e689cf4aSJeff Kirsher } 1142e689cf4aSJeff Kirsher 1143e689cf4aSJeff Kirsher static void vnet_tx_timeout(struct net_device *dev) 1144e689cf4aSJeff Kirsher { 1145e689cf4aSJeff Kirsher /* XXX Implement me XXX */ 1146e689cf4aSJeff Kirsher } 1147e689cf4aSJeff Kirsher 1148e689cf4aSJeff Kirsher static int vnet_open(struct net_device *dev) 1149e689cf4aSJeff Kirsher { 1150e689cf4aSJeff Kirsher netif_carrier_on(dev); 1151d51bffd1SSowmini Varadhan netif_tx_start_all_queues(dev); 1152e689cf4aSJeff Kirsher 1153e689cf4aSJeff Kirsher return 0; 1154e689cf4aSJeff Kirsher } 1155e689cf4aSJeff Kirsher 1156e689cf4aSJeff Kirsher static int vnet_close(struct net_device *dev) 1157e689cf4aSJeff Kirsher { 1158d51bffd1SSowmini Varadhan netif_tx_stop_all_queues(dev); 1159e689cf4aSJeff Kirsher netif_carrier_off(dev); 1160e689cf4aSJeff Kirsher 1161e689cf4aSJeff Kirsher return 0; 1162e689cf4aSJeff Kirsher } 1163e689cf4aSJeff Kirsher 1164e689cf4aSJeff Kirsher static struct vnet_mcast_entry *__vnet_mc_find(struct vnet *vp, u8 *addr) 1165e689cf4aSJeff Kirsher { 1166e689cf4aSJeff Kirsher struct vnet_mcast_entry *m; 1167e689cf4aSJeff Kirsher 1168e689cf4aSJeff Kirsher for (m = vp->mcast_list; m; m = m->next) { 116900fa4ce9Sdingtianhong if (ether_addr_equal(m->addr, addr)) 1170e689cf4aSJeff Kirsher return m; 1171e689cf4aSJeff Kirsher } 1172e689cf4aSJeff Kirsher return NULL; 1173e689cf4aSJeff Kirsher } 1174e689cf4aSJeff Kirsher 1175e689cf4aSJeff Kirsher static void __update_mc_list(struct vnet *vp, struct net_device *dev) 1176e689cf4aSJeff Kirsher { 1177e689cf4aSJeff Kirsher struct netdev_hw_addr *ha; 1178e689cf4aSJeff Kirsher 1179e689cf4aSJeff Kirsher netdev_for_each_mc_addr(ha, dev) { 1180e689cf4aSJeff Kirsher struct vnet_mcast_entry *m; 1181e689cf4aSJeff Kirsher 1182e689cf4aSJeff Kirsher m = __vnet_mc_find(vp, ha->addr); 1183e689cf4aSJeff Kirsher if (m) { 1184e689cf4aSJeff Kirsher m->hit = 1; 1185e689cf4aSJeff Kirsher continue; 1186e689cf4aSJeff Kirsher } 1187e689cf4aSJeff Kirsher 1188e689cf4aSJeff Kirsher if (!m) { 1189e689cf4aSJeff Kirsher m = kzalloc(sizeof(*m), GFP_ATOMIC); 1190e689cf4aSJeff Kirsher if (!m) 1191e689cf4aSJeff Kirsher continue; 1192e689cf4aSJeff Kirsher memcpy(m->addr, ha->addr, ETH_ALEN); 1193e689cf4aSJeff Kirsher m->hit = 1; 1194e689cf4aSJeff Kirsher 1195e689cf4aSJeff Kirsher m->next = vp->mcast_list; 1196e689cf4aSJeff Kirsher vp->mcast_list = m; 1197e689cf4aSJeff Kirsher } 1198e689cf4aSJeff Kirsher } 1199e689cf4aSJeff Kirsher } 1200e689cf4aSJeff Kirsher 1201e689cf4aSJeff Kirsher static void __send_mc_list(struct vnet *vp, struct vnet_port *port) 1202e689cf4aSJeff Kirsher { 1203e689cf4aSJeff Kirsher struct vio_net_mcast_info info; 1204e689cf4aSJeff Kirsher struct vnet_mcast_entry *m, **pp; 1205e689cf4aSJeff Kirsher int n_addrs; 1206e689cf4aSJeff Kirsher 1207e689cf4aSJeff Kirsher memset(&info, 0, sizeof(info)); 1208e689cf4aSJeff Kirsher 1209e689cf4aSJeff Kirsher info.tag.type = VIO_TYPE_CTRL; 1210e689cf4aSJeff Kirsher info.tag.stype = VIO_SUBTYPE_INFO; 1211e689cf4aSJeff Kirsher info.tag.stype_env = VNET_MCAST_INFO; 1212e689cf4aSJeff Kirsher info.tag.sid = vio_send_sid(&port->vio); 1213e689cf4aSJeff Kirsher info.set = 1; 1214e689cf4aSJeff Kirsher 1215e689cf4aSJeff Kirsher n_addrs = 0; 1216e689cf4aSJeff Kirsher for (m = vp->mcast_list; m; m = m->next) { 1217e689cf4aSJeff Kirsher if (m->sent) 1218e689cf4aSJeff Kirsher continue; 1219e689cf4aSJeff Kirsher m->sent = 1; 1220e689cf4aSJeff Kirsher memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1221e689cf4aSJeff Kirsher m->addr, ETH_ALEN); 1222e689cf4aSJeff Kirsher if (++n_addrs == VNET_NUM_MCAST) { 1223e689cf4aSJeff Kirsher info.count = n_addrs; 1224e689cf4aSJeff Kirsher 1225e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, 1226e689cf4aSJeff Kirsher sizeof(info)); 1227e689cf4aSJeff Kirsher n_addrs = 0; 1228e689cf4aSJeff Kirsher } 1229e689cf4aSJeff Kirsher } 1230e689cf4aSJeff Kirsher if (n_addrs) { 1231e689cf4aSJeff Kirsher info.count = n_addrs; 1232e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1233e689cf4aSJeff Kirsher } 1234e689cf4aSJeff Kirsher 1235e689cf4aSJeff Kirsher info.set = 0; 1236e689cf4aSJeff Kirsher 1237e689cf4aSJeff Kirsher n_addrs = 0; 1238e689cf4aSJeff Kirsher pp = &vp->mcast_list; 1239e689cf4aSJeff Kirsher while ((m = *pp) != NULL) { 1240e689cf4aSJeff Kirsher if (m->hit) { 1241e689cf4aSJeff Kirsher m->hit = 0; 1242e689cf4aSJeff Kirsher pp = &m->next; 1243e689cf4aSJeff Kirsher continue; 1244e689cf4aSJeff Kirsher } 1245e689cf4aSJeff Kirsher 1246e689cf4aSJeff Kirsher memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1247e689cf4aSJeff Kirsher m->addr, ETH_ALEN); 1248e689cf4aSJeff Kirsher if (++n_addrs == VNET_NUM_MCAST) { 1249e689cf4aSJeff Kirsher info.count = n_addrs; 1250e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, 1251e689cf4aSJeff Kirsher sizeof(info)); 1252e689cf4aSJeff Kirsher n_addrs = 0; 1253e689cf4aSJeff Kirsher } 1254e689cf4aSJeff Kirsher 1255e689cf4aSJeff Kirsher *pp = m->next; 1256e689cf4aSJeff Kirsher kfree(m); 1257e689cf4aSJeff Kirsher } 1258e689cf4aSJeff Kirsher if (n_addrs) { 1259e689cf4aSJeff Kirsher info.count = n_addrs; 1260e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1261e689cf4aSJeff Kirsher } 1262e689cf4aSJeff Kirsher } 1263e689cf4aSJeff Kirsher 1264e689cf4aSJeff Kirsher static void vnet_set_rx_mode(struct net_device *dev) 1265e689cf4aSJeff Kirsher { 1266e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1267e689cf4aSJeff Kirsher struct vnet_port *port; 1268e689cf4aSJeff Kirsher 12692a968dd8SSowmini Varadhan rcu_read_lock(); 12702a968dd8SSowmini Varadhan list_for_each_entry_rcu(port, &vp->port_list, list) { 1271e689cf4aSJeff Kirsher 1272e689cf4aSJeff Kirsher if (port->switch_port) { 1273e689cf4aSJeff Kirsher __update_mc_list(vp, dev); 1274e689cf4aSJeff Kirsher __send_mc_list(vp, port); 12752a968dd8SSowmini Varadhan break; 1276e689cf4aSJeff Kirsher } 1277e689cf4aSJeff Kirsher } 12782a968dd8SSowmini Varadhan rcu_read_unlock(); 1279e689cf4aSJeff Kirsher } 1280e689cf4aSJeff Kirsher 1281e689cf4aSJeff Kirsher static int vnet_change_mtu(struct net_device *dev, int new_mtu) 1282e689cf4aSJeff Kirsher { 128342db672dSDavid L Stevens if (new_mtu < 68 || new_mtu > 65535) 1284e689cf4aSJeff Kirsher return -EINVAL; 1285e689cf4aSJeff Kirsher 1286e689cf4aSJeff Kirsher dev->mtu = new_mtu; 1287e689cf4aSJeff Kirsher return 0; 1288e689cf4aSJeff Kirsher } 1289e689cf4aSJeff Kirsher 1290e689cf4aSJeff Kirsher static int vnet_set_mac_addr(struct net_device *dev, void *p) 1291e689cf4aSJeff Kirsher { 1292e689cf4aSJeff Kirsher return -EINVAL; 1293e689cf4aSJeff Kirsher } 1294e689cf4aSJeff Kirsher 1295e689cf4aSJeff Kirsher static void vnet_get_drvinfo(struct net_device *dev, 1296e689cf4aSJeff Kirsher struct ethtool_drvinfo *info) 1297e689cf4aSJeff Kirsher { 12987826d43fSJiri Pirko strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); 12997826d43fSJiri Pirko strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); 1300e689cf4aSJeff Kirsher } 1301e689cf4aSJeff Kirsher 1302e689cf4aSJeff Kirsher static u32 vnet_get_msglevel(struct net_device *dev) 1303e689cf4aSJeff Kirsher { 1304e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1305e689cf4aSJeff Kirsher return vp->msg_enable; 1306e689cf4aSJeff Kirsher } 1307e689cf4aSJeff Kirsher 1308e689cf4aSJeff Kirsher static void vnet_set_msglevel(struct net_device *dev, u32 value) 1309e689cf4aSJeff Kirsher { 1310e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1311e689cf4aSJeff Kirsher vp->msg_enable = value; 1312e689cf4aSJeff Kirsher } 1313e689cf4aSJeff Kirsher 1314e689cf4aSJeff Kirsher static const struct ethtool_ops vnet_ethtool_ops = { 1315e689cf4aSJeff Kirsher .get_drvinfo = vnet_get_drvinfo, 1316e689cf4aSJeff Kirsher .get_msglevel = vnet_get_msglevel, 1317e689cf4aSJeff Kirsher .set_msglevel = vnet_set_msglevel, 1318e689cf4aSJeff Kirsher .get_link = ethtool_op_get_link, 1319e689cf4aSJeff Kirsher }; 1320e689cf4aSJeff Kirsher 1321e689cf4aSJeff Kirsher static void vnet_port_free_tx_bufs(struct vnet_port *port) 1322e689cf4aSJeff Kirsher { 1323e689cf4aSJeff Kirsher struct vio_dring_state *dr; 1324e689cf4aSJeff Kirsher int i; 1325e689cf4aSJeff Kirsher 1326e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1327e689cf4aSJeff Kirsher if (dr->base) { 1328e689cf4aSJeff Kirsher ldc_free_exp_dring(port->vio.lp, dr->base, 1329e689cf4aSJeff Kirsher (dr->entry_size * dr->num_entries), 1330e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 1331e689cf4aSJeff Kirsher dr->base = NULL; 1332e689cf4aSJeff Kirsher dr->entry_size = 0; 1333e689cf4aSJeff Kirsher dr->num_entries = 0; 1334e689cf4aSJeff Kirsher dr->pending = 0; 1335e689cf4aSJeff Kirsher dr->ncookies = 0; 1336e689cf4aSJeff Kirsher } 1337e689cf4aSJeff Kirsher 1338e689cf4aSJeff Kirsher for (i = 0; i < VNET_TX_RING_SIZE; i++) { 13398e845f4cSDavid L Stevens struct vio_net_desc *d; 13408e845f4cSDavid L Stevens void *skb = port->tx_bufs[i].skb; 1341e689cf4aSJeff Kirsher 13428e845f4cSDavid L Stevens if (!skb) 1343e689cf4aSJeff Kirsher continue; 1344e689cf4aSJeff Kirsher 13458e845f4cSDavid L Stevens d = vio_dring_entry(dr, i); 13468e845f4cSDavid L Stevens if (d->hdr.state == VIO_DESC_READY) 13478e845f4cSDavid L Stevens pr_warn("active transmit buffers freed\n"); 13488e845f4cSDavid L Stevens 1349e689cf4aSJeff Kirsher ldc_unmap(port->vio.lp, 1350e689cf4aSJeff Kirsher port->tx_bufs[i].cookies, 1351e689cf4aSJeff Kirsher port->tx_bufs[i].ncookies); 13528e845f4cSDavid L Stevens dev_kfree_skb(skb); 13538e845f4cSDavid L Stevens port->tx_bufs[i].skb = NULL; 13548e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 1355e689cf4aSJeff Kirsher } 1356e689cf4aSJeff Kirsher } 1357e689cf4aSJeff Kirsher 1358f73d12bdSBill Pemberton static int vnet_port_alloc_tx_bufs(struct vnet_port *port) 1359e689cf4aSJeff Kirsher { 1360e689cf4aSJeff Kirsher struct vio_dring_state *dr; 1361e689cf4aSJeff Kirsher unsigned long len; 1362e689cf4aSJeff Kirsher int i, err, ncookies; 1363e689cf4aSJeff Kirsher void *dring; 1364e689cf4aSJeff Kirsher 1365e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1366e689cf4aSJeff Kirsher 1367e689cf4aSJeff Kirsher len = (VNET_TX_RING_SIZE * 1368e689cf4aSJeff Kirsher (sizeof(struct vio_net_desc) + 1369e689cf4aSJeff Kirsher (sizeof(struct ldc_trans_cookie) * 2))); 1370e689cf4aSJeff Kirsher 1371e689cf4aSJeff Kirsher ncookies = VIO_MAX_RING_COOKIES; 1372e689cf4aSJeff Kirsher dring = ldc_alloc_exp_dring(port->vio.lp, len, 1373e689cf4aSJeff Kirsher dr->cookies, &ncookies, 1374e689cf4aSJeff Kirsher (LDC_MAP_SHADOW | 1375e689cf4aSJeff Kirsher LDC_MAP_DIRECT | 1376e689cf4aSJeff Kirsher LDC_MAP_RW)); 1377e689cf4aSJeff Kirsher if (IS_ERR(dring)) { 1378e689cf4aSJeff Kirsher err = PTR_ERR(dring); 1379e689cf4aSJeff Kirsher goto err_out; 1380e689cf4aSJeff Kirsher } 1381e689cf4aSJeff Kirsher 1382e689cf4aSJeff Kirsher dr->base = dring; 1383e689cf4aSJeff Kirsher dr->entry_size = (sizeof(struct vio_net_desc) + 1384e689cf4aSJeff Kirsher (sizeof(struct ldc_trans_cookie) * 2)); 1385e689cf4aSJeff Kirsher dr->num_entries = VNET_TX_RING_SIZE; 1386e689cf4aSJeff Kirsher dr->prod = dr->cons = 0; 1387d1015645SSowmini Varadhan port->start_cons = true; /* need an initial trigger */ 1388e689cf4aSJeff Kirsher dr->pending = VNET_TX_RING_SIZE; 1389e689cf4aSJeff Kirsher dr->ncookies = ncookies; 1390e689cf4aSJeff Kirsher 13918e845f4cSDavid L Stevens for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 13928e845f4cSDavid L Stevens struct vio_net_desc *d; 13938e845f4cSDavid L Stevens 13948e845f4cSDavid L Stevens d = vio_dring_entry(dr, i); 13958e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 13968e845f4cSDavid L Stevens } 1397e689cf4aSJeff Kirsher return 0; 1398e689cf4aSJeff Kirsher 1399e689cf4aSJeff Kirsher err_out: 1400e689cf4aSJeff Kirsher vnet_port_free_tx_bufs(port); 1401e689cf4aSJeff Kirsher 1402e689cf4aSJeff Kirsher return err; 1403e689cf4aSJeff Kirsher } 1404e689cf4aSJeff Kirsher 140569088822SSowmini Varadhan #ifdef CONFIG_NET_POLL_CONTROLLER 140669088822SSowmini Varadhan static void vnet_poll_controller(struct net_device *dev) 140769088822SSowmini Varadhan { 140869088822SSowmini Varadhan struct vnet *vp = netdev_priv(dev); 140969088822SSowmini Varadhan struct vnet_port *port; 141069088822SSowmini Varadhan unsigned long flags; 141169088822SSowmini Varadhan 141269088822SSowmini Varadhan spin_lock_irqsave(&vp->lock, flags); 141369088822SSowmini Varadhan if (!list_empty(&vp->port_list)) { 141469088822SSowmini Varadhan port = list_entry(vp->port_list.next, struct vnet_port, list); 141569088822SSowmini Varadhan napi_schedule(&port->napi); 141669088822SSowmini Varadhan } 141769088822SSowmini Varadhan spin_unlock_irqrestore(&vp->lock, flags); 141869088822SSowmini Varadhan } 141969088822SSowmini Varadhan #endif 1420e689cf4aSJeff Kirsher static LIST_HEAD(vnet_list); 1421e689cf4aSJeff Kirsher static DEFINE_MUTEX(vnet_list_mutex); 1422e689cf4aSJeff Kirsher 1423e689cf4aSJeff Kirsher static const struct net_device_ops vnet_ops = { 1424e689cf4aSJeff Kirsher .ndo_open = vnet_open, 1425e689cf4aSJeff Kirsher .ndo_stop = vnet_close, 1426afc4b13dSJiri Pirko .ndo_set_rx_mode = vnet_set_rx_mode, 1427e689cf4aSJeff Kirsher .ndo_set_mac_address = vnet_set_mac_addr, 1428e689cf4aSJeff Kirsher .ndo_validate_addr = eth_validate_addr, 1429e689cf4aSJeff Kirsher .ndo_tx_timeout = vnet_tx_timeout, 1430e689cf4aSJeff Kirsher .ndo_change_mtu = vnet_change_mtu, 1431e689cf4aSJeff Kirsher .ndo_start_xmit = vnet_start_xmit, 1432d51bffd1SSowmini Varadhan .ndo_select_queue = vnet_select_queue, 143369088822SSowmini Varadhan #ifdef CONFIG_NET_POLL_CONTROLLER 143469088822SSowmini Varadhan .ndo_poll_controller = vnet_poll_controller, 143569088822SSowmini Varadhan #endif 1436e689cf4aSJeff Kirsher }; 1437e689cf4aSJeff Kirsher 1438f73d12bdSBill Pemberton static struct vnet *vnet_new(const u64 *local_mac) 1439e689cf4aSJeff Kirsher { 1440e689cf4aSJeff Kirsher struct net_device *dev; 1441e689cf4aSJeff Kirsher struct vnet *vp; 1442e689cf4aSJeff Kirsher int err, i; 1443e689cf4aSJeff Kirsher 1444d51bffd1SSowmini Varadhan dev = alloc_etherdev_mqs(sizeof(*vp), VNET_MAX_TXQS, 1); 144541de8d4cSJoe Perches if (!dev) 1446e689cf4aSJeff Kirsher return ERR_PTR(-ENOMEM); 14478e845f4cSDavid L Stevens dev->needed_headroom = VNET_PACKET_SKIP + 8; 14488e845f4cSDavid L Stevens dev->needed_tailroom = 8; 1449e689cf4aSJeff Kirsher 1450e689cf4aSJeff Kirsher for (i = 0; i < ETH_ALEN; i++) 1451e689cf4aSJeff Kirsher dev->dev_addr[i] = (*local_mac >> (5 - i) * 8) & 0xff; 1452e689cf4aSJeff Kirsher 1453e689cf4aSJeff Kirsher vp = netdev_priv(dev); 1454e689cf4aSJeff Kirsher 1455e689cf4aSJeff Kirsher spin_lock_init(&vp->lock); 1456e689cf4aSJeff Kirsher vp->dev = dev; 1457e689cf4aSJeff Kirsher 1458e689cf4aSJeff Kirsher INIT_LIST_HEAD(&vp->port_list); 1459e689cf4aSJeff Kirsher for (i = 0; i < VNET_PORT_HASH_SIZE; i++) 1460e689cf4aSJeff Kirsher INIT_HLIST_HEAD(&vp->port_hash[i]); 1461e689cf4aSJeff Kirsher INIT_LIST_HEAD(&vp->list); 1462e689cf4aSJeff Kirsher vp->local_mac = *local_mac; 1463e689cf4aSJeff Kirsher 1464e689cf4aSJeff Kirsher dev->netdev_ops = &vnet_ops; 1465e689cf4aSJeff Kirsher dev->ethtool_ops = &vnet_ethtool_ops; 1466e689cf4aSJeff Kirsher dev->watchdog_timeo = VNET_TX_TIMEOUT; 1467e689cf4aSJeff Kirsher 1468e689cf4aSJeff Kirsher err = register_netdev(dev); 1469e689cf4aSJeff Kirsher if (err) { 1470e689cf4aSJeff Kirsher pr_err("Cannot register net device, aborting\n"); 1471e689cf4aSJeff Kirsher goto err_out_free_dev; 1472e689cf4aSJeff Kirsher } 1473e689cf4aSJeff Kirsher 1474e689cf4aSJeff Kirsher netdev_info(dev, "Sun LDOM vnet %pM\n", dev->dev_addr); 1475e689cf4aSJeff Kirsher 1476e689cf4aSJeff Kirsher list_add(&vp->list, &vnet_list); 1477e689cf4aSJeff Kirsher 1478e689cf4aSJeff Kirsher return vp; 1479e689cf4aSJeff Kirsher 1480e689cf4aSJeff Kirsher err_out_free_dev: 1481e689cf4aSJeff Kirsher free_netdev(dev); 1482e689cf4aSJeff Kirsher 1483e689cf4aSJeff Kirsher return ERR_PTR(err); 1484e689cf4aSJeff Kirsher } 1485e689cf4aSJeff Kirsher 1486f73d12bdSBill Pemberton static struct vnet *vnet_find_or_create(const u64 *local_mac) 1487e689cf4aSJeff Kirsher { 1488e689cf4aSJeff Kirsher struct vnet *iter, *vp; 1489e689cf4aSJeff Kirsher 1490e689cf4aSJeff Kirsher mutex_lock(&vnet_list_mutex); 1491e689cf4aSJeff Kirsher vp = NULL; 1492e689cf4aSJeff Kirsher list_for_each_entry(iter, &vnet_list, list) { 1493e689cf4aSJeff Kirsher if (iter->local_mac == *local_mac) { 1494e689cf4aSJeff Kirsher vp = iter; 1495e689cf4aSJeff Kirsher break; 1496e689cf4aSJeff Kirsher } 1497e689cf4aSJeff Kirsher } 1498e689cf4aSJeff Kirsher if (!vp) 1499e689cf4aSJeff Kirsher vp = vnet_new(local_mac); 1500e689cf4aSJeff Kirsher mutex_unlock(&vnet_list_mutex); 1501e689cf4aSJeff Kirsher 1502e689cf4aSJeff Kirsher return vp; 1503e689cf4aSJeff Kirsher } 1504e689cf4aSJeff Kirsher 1505a4b70a07SSowmini Varadhan static void vnet_cleanup(void) 1506a4b70a07SSowmini Varadhan { 1507a4b70a07SSowmini Varadhan struct vnet *vp; 1508a4b70a07SSowmini Varadhan struct net_device *dev; 1509a4b70a07SSowmini Varadhan 1510a4b70a07SSowmini Varadhan mutex_lock(&vnet_list_mutex); 1511a4b70a07SSowmini Varadhan while (!list_empty(&vnet_list)) { 1512a4b70a07SSowmini Varadhan vp = list_first_entry(&vnet_list, struct vnet, list); 1513a4b70a07SSowmini Varadhan list_del(&vp->list); 1514a4b70a07SSowmini Varadhan dev = vp->dev; 1515a4b70a07SSowmini Varadhan /* vio_unregister_driver() should have cleaned up port_list */ 1516a4b70a07SSowmini Varadhan BUG_ON(!list_empty(&vp->port_list)); 1517a4b70a07SSowmini Varadhan unregister_netdev(dev); 1518a4b70a07SSowmini Varadhan free_netdev(dev); 1519a4b70a07SSowmini Varadhan } 1520a4b70a07SSowmini Varadhan mutex_unlock(&vnet_list_mutex); 1521a4b70a07SSowmini Varadhan } 1522a4b70a07SSowmini Varadhan 1523e689cf4aSJeff Kirsher static const char *local_mac_prop = "local-mac-address"; 1524e689cf4aSJeff Kirsher 1525f73d12bdSBill Pemberton static struct vnet *vnet_find_parent(struct mdesc_handle *hp, 1526e689cf4aSJeff Kirsher u64 port_node) 1527e689cf4aSJeff Kirsher { 1528e689cf4aSJeff Kirsher const u64 *local_mac = NULL; 1529e689cf4aSJeff Kirsher u64 a; 1530e689cf4aSJeff Kirsher 1531e689cf4aSJeff Kirsher mdesc_for_each_arc(a, hp, port_node, MDESC_ARC_TYPE_BACK) { 1532e689cf4aSJeff Kirsher u64 target = mdesc_arc_target(hp, a); 1533e689cf4aSJeff Kirsher const char *name; 1534e689cf4aSJeff Kirsher 1535e689cf4aSJeff Kirsher name = mdesc_get_property(hp, target, "name", NULL); 1536e689cf4aSJeff Kirsher if (!name || strcmp(name, "network")) 1537e689cf4aSJeff Kirsher continue; 1538e689cf4aSJeff Kirsher 1539e689cf4aSJeff Kirsher local_mac = mdesc_get_property(hp, target, 1540e689cf4aSJeff Kirsher local_mac_prop, NULL); 1541e689cf4aSJeff Kirsher if (local_mac) 1542e689cf4aSJeff Kirsher break; 1543e689cf4aSJeff Kirsher } 1544e689cf4aSJeff Kirsher if (!local_mac) 1545e689cf4aSJeff Kirsher return ERR_PTR(-ENODEV); 1546e689cf4aSJeff Kirsher 1547e689cf4aSJeff Kirsher return vnet_find_or_create(local_mac); 1548e689cf4aSJeff Kirsher } 1549e689cf4aSJeff Kirsher 1550e689cf4aSJeff Kirsher static struct ldc_channel_config vnet_ldc_cfg = { 1551e689cf4aSJeff Kirsher .event = vnet_event, 1552e689cf4aSJeff Kirsher .mtu = 64, 1553e689cf4aSJeff Kirsher .mode = LDC_MODE_UNRELIABLE, 1554e689cf4aSJeff Kirsher }; 1555e689cf4aSJeff Kirsher 1556e689cf4aSJeff Kirsher static struct vio_driver_ops vnet_vio_ops = { 1557e689cf4aSJeff Kirsher .send_attr = vnet_send_attr, 1558e689cf4aSJeff Kirsher .handle_attr = vnet_handle_attr, 1559e689cf4aSJeff Kirsher .handshake_complete = vnet_handshake_complete, 1560e689cf4aSJeff Kirsher }; 1561e689cf4aSJeff Kirsher 1562f73d12bdSBill Pemberton static void print_version(void) 1563e689cf4aSJeff Kirsher { 1564e689cf4aSJeff Kirsher printk_once(KERN_INFO "%s", version); 1565e689cf4aSJeff Kirsher } 1566e689cf4aSJeff Kirsher 1567e689cf4aSJeff Kirsher const char *remote_macaddr_prop = "remote-mac-address"; 1568e689cf4aSJeff Kirsher 1569d51bffd1SSowmini Varadhan static void 1570d51bffd1SSowmini Varadhan vnet_port_add_txq(struct vnet_port *port) 1571d51bffd1SSowmini Varadhan { 1572d51bffd1SSowmini Varadhan struct vnet *vp = port->vp; 1573d51bffd1SSowmini Varadhan int n; 1574d51bffd1SSowmini Varadhan 1575d51bffd1SSowmini Varadhan n = vp->nports++; 1576d51bffd1SSowmini Varadhan n = n & (VNET_MAX_TXQS - 1); 1577d51bffd1SSowmini Varadhan port->q_index = n; 1578d51bffd1SSowmini Varadhan netif_tx_wake_queue(netdev_get_tx_queue(vp->dev, port->q_index)); 1579d51bffd1SSowmini Varadhan } 1580d51bffd1SSowmini Varadhan 1581d51bffd1SSowmini Varadhan static void 1582d51bffd1SSowmini Varadhan vnet_port_rm_txq(struct vnet_port *port) 1583d51bffd1SSowmini Varadhan { 1584d51bffd1SSowmini Varadhan port->vp->nports--; 1585d51bffd1SSowmini Varadhan netif_tx_stop_queue(netdev_get_tx_queue(port->vp->dev, port->q_index)); 1586d51bffd1SSowmini Varadhan } 1587d51bffd1SSowmini Varadhan 15881dd06ae8SGreg Kroah-Hartman static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) 1589e689cf4aSJeff Kirsher { 1590e689cf4aSJeff Kirsher struct mdesc_handle *hp; 1591e689cf4aSJeff Kirsher struct vnet_port *port; 1592e689cf4aSJeff Kirsher unsigned long flags; 1593e689cf4aSJeff Kirsher struct vnet *vp; 1594e689cf4aSJeff Kirsher const u64 *rmac; 1595e689cf4aSJeff Kirsher int len, i, err, switch_port; 1596e689cf4aSJeff Kirsher 1597e689cf4aSJeff Kirsher print_version(); 1598e689cf4aSJeff Kirsher 1599e689cf4aSJeff Kirsher hp = mdesc_grab(); 1600e689cf4aSJeff Kirsher 1601e689cf4aSJeff Kirsher vp = vnet_find_parent(hp, vdev->mp); 1602e689cf4aSJeff Kirsher if (IS_ERR(vp)) { 1603e689cf4aSJeff Kirsher pr_err("Cannot find port parent vnet\n"); 1604e689cf4aSJeff Kirsher err = PTR_ERR(vp); 1605e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1606e689cf4aSJeff Kirsher } 1607e689cf4aSJeff Kirsher 1608e689cf4aSJeff Kirsher rmac = mdesc_get_property(hp, vdev->mp, remote_macaddr_prop, &len); 1609e689cf4aSJeff Kirsher err = -ENODEV; 1610e689cf4aSJeff Kirsher if (!rmac) { 1611e689cf4aSJeff Kirsher pr_err("Port lacks %s property\n", remote_macaddr_prop); 1612e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1613e689cf4aSJeff Kirsher } 1614e689cf4aSJeff Kirsher 1615e689cf4aSJeff Kirsher port = kzalloc(sizeof(*port), GFP_KERNEL); 1616e689cf4aSJeff Kirsher err = -ENOMEM; 1617e404decbSJoe Perches if (!port) 1618e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1619e689cf4aSJeff Kirsher 1620e689cf4aSJeff Kirsher for (i = 0; i < ETH_ALEN; i++) 1621e689cf4aSJeff Kirsher port->raddr[i] = (*rmac >> (5 - i) * 8) & 0xff; 1622e689cf4aSJeff Kirsher 1623e689cf4aSJeff Kirsher port->vp = vp; 1624e689cf4aSJeff Kirsher 1625e689cf4aSJeff Kirsher err = vio_driver_init(&port->vio, vdev, VDEV_NETWORK, 1626e689cf4aSJeff Kirsher vnet_versions, ARRAY_SIZE(vnet_versions), 1627e689cf4aSJeff Kirsher &vnet_vio_ops, vp->dev->name); 1628e689cf4aSJeff Kirsher if (err) 1629e689cf4aSJeff Kirsher goto err_out_free_port; 1630e689cf4aSJeff Kirsher 1631e689cf4aSJeff Kirsher err = vio_ldc_alloc(&port->vio, &vnet_ldc_cfg, port); 1632e689cf4aSJeff Kirsher if (err) 1633e689cf4aSJeff Kirsher goto err_out_free_port; 1634e689cf4aSJeff Kirsher 163569088822SSowmini Varadhan netif_napi_add(port->vp->dev, &port->napi, vnet_poll, NAPI_POLL_WEIGHT); 163669088822SSowmini Varadhan 1637e689cf4aSJeff Kirsher err = vnet_port_alloc_tx_bufs(port); 1638e689cf4aSJeff Kirsher if (err) 1639e689cf4aSJeff Kirsher goto err_out_free_ldc; 1640e689cf4aSJeff Kirsher 1641e689cf4aSJeff Kirsher INIT_HLIST_NODE(&port->hash); 1642e689cf4aSJeff Kirsher INIT_LIST_HEAD(&port->list); 1643e689cf4aSJeff Kirsher 1644e689cf4aSJeff Kirsher switch_port = 0; 1645e689cf4aSJeff Kirsher if (mdesc_get_property(hp, vdev->mp, "switch-port", NULL) != NULL) 1646e689cf4aSJeff Kirsher switch_port = 1; 1647e689cf4aSJeff Kirsher port->switch_port = switch_port; 1648e689cf4aSJeff Kirsher 1649e689cf4aSJeff Kirsher spin_lock_irqsave(&vp->lock, flags); 1650e689cf4aSJeff Kirsher if (switch_port) 16512a968dd8SSowmini Varadhan list_add_rcu(&port->list, &vp->port_list); 1652e689cf4aSJeff Kirsher else 16532a968dd8SSowmini Varadhan list_add_tail_rcu(&port->list, &vp->port_list); 16542a968dd8SSowmini Varadhan hlist_add_head_rcu(&port->hash, 16552a968dd8SSowmini Varadhan &vp->port_hash[vnet_hashfn(port->raddr)]); 1656d51bffd1SSowmini Varadhan vnet_port_add_txq(port); 1657e689cf4aSJeff Kirsher spin_unlock_irqrestore(&vp->lock, flags); 1658e689cf4aSJeff Kirsher 1659e689cf4aSJeff Kirsher dev_set_drvdata(&vdev->dev, port); 1660e689cf4aSJeff Kirsher 1661e689cf4aSJeff Kirsher pr_info("%s: PORT ( remote-mac %pM%s )\n", 1662e689cf4aSJeff Kirsher vp->dev->name, port->raddr, switch_port ? " switch-port" : ""); 1663e689cf4aSJeff Kirsher 16648e845f4cSDavid L Stevens setup_timer(&port->clean_timer, vnet_clean_timer_expire, 16658e845f4cSDavid L Stevens (unsigned long)port); 16668e845f4cSDavid L Stevens 166769088822SSowmini Varadhan napi_enable(&port->napi); 1668e689cf4aSJeff Kirsher vio_port_up(&port->vio); 1669e689cf4aSJeff Kirsher 1670e689cf4aSJeff Kirsher mdesc_release(hp); 1671e689cf4aSJeff Kirsher 1672e689cf4aSJeff Kirsher return 0; 1673e689cf4aSJeff Kirsher 1674e689cf4aSJeff Kirsher err_out_free_ldc: 167569088822SSowmini Varadhan netif_napi_del(&port->napi); 1676e689cf4aSJeff Kirsher vio_ldc_free(&port->vio); 1677e689cf4aSJeff Kirsher 1678e689cf4aSJeff Kirsher err_out_free_port: 1679e689cf4aSJeff Kirsher kfree(port); 1680e689cf4aSJeff Kirsher 1681e689cf4aSJeff Kirsher err_out_put_mdesc: 1682e689cf4aSJeff Kirsher mdesc_release(hp); 1683e689cf4aSJeff Kirsher return err; 1684e689cf4aSJeff Kirsher } 1685e689cf4aSJeff Kirsher 1686e689cf4aSJeff Kirsher static int vnet_port_remove(struct vio_dev *vdev) 1687e689cf4aSJeff Kirsher { 1688e689cf4aSJeff Kirsher struct vnet_port *port = dev_get_drvdata(&vdev->dev); 1689e689cf4aSJeff Kirsher 1690e689cf4aSJeff Kirsher if (port) { 1691e689cf4aSJeff Kirsher 1692e689cf4aSJeff Kirsher del_timer_sync(&port->vio.timer); 1693e689cf4aSJeff Kirsher 169469088822SSowmini Varadhan napi_disable(&port->napi); 1695e689cf4aSJeff Kirsher 16962a968dd8SSowmini Varadhan list_del_rcu(&port->list); 16972a968dd8SSowmini Varadhan hlist_del_rcu(&port->hash); 16982a968dd8SSowmini Varadhan 16992a968dd8SSowmini Varadhan synchronize_rcu(); 17002a968dd8SSowmini Varadhan del_timer_sync(&port->clean_timer); 1701d51bffd1SSowmini Varadhan vnet_port_rm_txq(port); 170269088822SSowmini Varadhan netif_napi_del(&port->napi); 1703e689cf4aSJeff Kirsher vnet_port_free_tx_bufs(port); 1704e689cf4aSJeff Kirsher vio_ldc_free(&port->vio); 1705e689cf4aSJeff Kirsher 1706e689cf4aSJeff Kirsher dev_set_drvdata(&vdev->dev, NULL); 1707e689cf4aSJeff Kirsher 1708e689cf4aSJeff Kirsher kfree(port); 1709aabb9875SDave Kleikamp 1710e689cf4aSJeff Kirsher } 1711e689cf4aSJeff Kirsher return 0; 1712e689cf4aSJeff Kirsher } 1713e689cf4aSJeff Kirsher 1714e689cf4aSJeff Kirsher static const struct vio_device_id vnet_port_match[] = { 1715e689cf4aSJeff Kirsher { 1716e689cf4aSJeff Kirsher .type = "vnet-port", 1717e689cf4aSJeff Kirsher }, 1718e689cf4aSJeff Kirsher {}, 1719e689cf4aSJeff Kirsher }; 1720e689cf4aSJeff Kirsher MODULE_DEVICE_TABLE(vio, vnet_port_match); 1721e689cf4aSJeff Kirsher 1722e689cf4aSJeff Kirsher static struct vio_driver vnet_port_driver = { 1723e689cf4aSJeff Kirsher .id_table = vnet_port_match, 1724e689cf4aSJeff Kirsher .probe = vnet_port_probe, 1725e689cf4aSJeff Kirsher .remove = vnet_port_remove, 1726e689cf4aSJeff Kirsher .name = "vnet_port", 1727e689cf4aSJeff Kirsher }; 1728e689cf4aSJeff Kirsher 1729e689cf4aSJeff Kirsher static int __init vnet_init(void) 1730e689cf4aSJeff Kirsher { 1731e689cf4aSJeff Kirsher return vio_register_driver(&vnet_port_driver); 1732e689cf4aSJeff Kirsher } 1733e689cf4aSJeff Kirsher 1734e689cf4aSJeff Kirsher static void __exit vnet_exit(void) 1735e689cf4aSJeff Kirsher { 1736e689cf4aSJeff Kirsher vio_unregister_driver(&vnet_port_driver); 1737a4b70a07SSowmini Varadhan vnet_cleanup(); 1738e689cf4aSJeff Kirsher } 1739e689cf4aSJeff Kirsher 1740e689cf4aSJeff Kirsher module_init(vnet_init); 1741e689cf4aSJeff Kirsher module_exit(vnet_exit); 1742