1c861ef83SShannon Nelson // SPDX-License-Identifier: GPL-2.0
231762eaaSAaron Young /* sunvnet.c: Sun LDOM Virtual Network Driver.
331762eaaSAaron Young *
431762eaaSAaron Young * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
5867fa150SShannon Nelson * Copyright (C) 2016-2017 Oracle. All rights reserved.
631762eaaSAaron Young */
731762eaaSAaron Young
831762eaaSAaron Young #include <linux/module.h>
931762eaaSAaron Young #include <linux/kernel.h>
1031762eaaSAaron Young #include <linux/types.h>
1131762eaaSAaron Young #include <linux/slab.h>
1231762eaaSAaron Young #include <linux/delay.h>
1331762eaaSAaron Young #include <linux/init.h>
1431762eaaSAaron Young #include <linux/netdevice.h>
1531762eaaSAaron Young #include <linux/ethtool.h>
1631762eaaSAaron Young #include <linux/etherdevice.h>
1731762eaaSAaron Young #include <linux/mutex.h>
1831762eaaSAaron Young #include <linux/highmem.h>
1931762eaaSAaron Young #include <linux/if_vlan.h>
2031762eaaSAaron Young #define CREATE_TRACE_POINTS
2131762eaaSAaron Young #include <trace/events/sunvnet.h>
2231762eaaSAaron Young
2331762eaaSAaron Young #if IS_ENABLED(CONFIG_IPV6)
2431762eaaSAaron Young #include <linux/icmpv6.h>
2531762eaaSAaron Young #endif
2631762eaaSAaron Young
2731762eaaSAaron Young #include <net/ip.h>
28*d9ffa069SStephen Rothwell #include <net/gso.h>
2931762eaaSAaron Young #include <net/icmp.h>
3031762eaaSAaron Young #include <net/route.h>
3131762eaaSAaron Young
3231762eaaSAaron Young #include <asm/vio.h>
3331762eaaSAaron Young #include <asm/ldc.h>
3431762eaaSAaron Young
3531762eaaSAaron Young #include "sunvnet_common.h"
3631762eaaSAaron Young
3731762eaaSAaron Young /* Heuristic for the number of times to exponentially backoff and
3831762eaaSAaron Young * retry sending an LDC trigger when EAGAIN is encountered
3931762eaaSAaron Young */
4031762eaaSAaron Young #define VNET_MAX_RETRIES 10
4131762eaaSAaron Young
422493b842SShannon Nelson MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
432493b842SShannon Nelson MODULE_DESCRIPTION("Sun LDOM virtual network support library");
442493b842SShannon Nelson MODULE_LICENSE("GPL");
452493b842SShannon Nelson MODULE_VERSION("1.1");
462493b842SShannon Nelson
4731762eaaSAaron Young static int __vnet_tx_trigger(struct vnet_port *port, u32 start);
4831762eaaSAaron Young
vnet_tx_dring_avail(struct vio_dring_state * dr)4931762eaaSAaron Young static inline u32 vnet_tx_dring_avail(struct vio_dring_state *dr)
5031762eaaSAaron Young {
5131762eaaSAaron Young return vio_dring_avail(dr, VNET_TX_RING_SIZE);
5231762eaaSAaron Young }
5331762eaaSAaron Young
vnet_handle_unknown(struct vnet_port * port,void * arg)5431762eaaSAaron Young static int vnet_handle_unknown(struct vnet_port *port, void *arg)
5531762eaaSAaron Young {
5631762eaaSAaron Young struct vio_msg_tag *pkt = arg;
5731762eaaSAaron Young
5831762eaaSAaron Young pr_err("Received unknown msg [%02x:%02x:%04x:%08x]\n",
5931762eaaSAaron Young pkt->type, pkt->stype, pkt->stype_env, pkt->sid);
6031762eaaSAaron Young pr_err("Resetting connection\n");
6131762eaaSAaron Young
6231762eaaSAaron Young ldc_disconnect(port->vio.lp);
6331762eaaSAaron Young
6431762eaaSAaron Young return -ECONNRESET;
6531762eaaSAaron Young }
6631762eaaSAaron Young
6731762eaaSAaron Young static int vnet_port_alloc_tx_ring(struct vnet_port *port);
6831762eaaSAaron Young
sunvnet_send_attr_common(struct vio_driver_state * vio)6931762eaaSAaron Young int sunvnet_send_attr_common(struct vio_driver_state *vio)
7031762eaaSAaron Young {
7131762eaaSAaron Young struct vnet_port *port = to_vnet_port(vio);
7267d0719fSAaron Young struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
7331762eaaSAaron Young struct vio_net_attr_info pkt;
7431762eaaSAaron Young int framelen = ETH_FRAME_LEN;
7531762eaaSAaron Young int i, err;
7631762eaaSAaron Young
7731762eaaSAaron Young err = vnet_port_alloc_tx_ring(to_vnet_port(vio));
7831762eaaSAaron Young if (err)
7931762eaaSAaron Young return err;
8031762eaaSAaron Young
8131762eaaSAaron Young memset(&pkt, 0, sizeof(pkt));
8231762eaaSAaron Young pkt.tag.type = VIO_TYPE_CTRL;
8331762eaaSAaron Young pkt.tag.stype = VIO_SUBTYPE_INFO;
8431762eaaSAaron Young pkt.tag.stype_env = VIO_ATTR_INFO;
8531762eaaSAaron Young pkt.tag.sid = vio_send_sid(vio);
8631762eaaSAaron Young if (vio_version_before(vio, 1, 2))
8731762eaaSAaron Young pkt.xfer_mode = VIO_DRING_MODE;
8831762eaaSAaron Young else
8931762eaaSAaron Young pkt.xfer_mode = VIO_NEW_DRING_MODE;
9031762eaaSAaron Young pkt.addr_type = VNET_ADDR_ETHERMAC;
9131762eaaSAaron Young pkt.ack_freq = 0;
9231762eaaSAaron Young for (i = 0; i < 6; i++)
9331762eaaSAaron Young pkt.addr |= (u64)dev->dev_addr[i] << ((5 - i) * 8);
9431762eaaSAaron Young if (vio_version_after(vio, 1, 3)) {
9531762eaaSAaron Young if (port->rmtu) {
9631762eaaSAaron Young port->rmtu = min(VNET_MAXPACKET, port->rmtu);
9731762eaaSAaron Young pkt.mtu = port->rmtu;
9831762eaaSAaron Young } else {
9931762eaaSAaron Young port->rmtu = VNET_MAXPACKET;
10031762eaaSAaron Young pkt.mtu = port->rmtu;
10131762eaaSAaron Young }
10231762eaaSAaron Young if (vio_version_after_eq(vio, 1, 6))
10331762eaaSAaron Young pkt.options = VIO_TX_DRING;
10431762eaaSAaron Young } else if (vio_version_before(vio, 1, 3)) {
10531762eaaSAaron Young pkt.mtu = framelen;
10631762eaaSAaron Young } else { /* v1.3 */
10731762eaaSAaron Young pkt.mtu = framelen + VLAN_HLEN;
10831762eaaSAaron Young }
10931762eaaSAaron Young
11031762eaaSAaron Young pkt.cflags = 0;
11131762eaaSAaron Young if (vio_version_after_eq(vio, 1, 7) && port->tso) {
11231762eaaSAaron Young pkt.cflags |= VNET_LSO_IPV4_CAPAB;
11331762eaaSAaron Young if (!port->tsolen)
11431762eaaSAaron Young port->tsolen = VNET_MAXTSO;
11531762eaaSAaron Young pkt.ipv4_lso_maxlen = port->tsolen;
11631762eaaSAaron Young }
11731762eaaSAaron Young
11831762eaaSAaron Young pkt.plnk_updt = PHYSLINK_UPDATE_NONE;
11931762eaaSAaron Young
12031762eaaSAaron Young viodbg(HS, "SEND NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] "
12131762eaaSAaron Young "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] "
12231762eaaSAaron Young "cflags[0x%04x] lso_max[%u]\n",
12331762eaaSAaron Young pkt.xfer_mode, pkt.addr_type,
12431762eaaSAaron Young (unsigned long long)pkt.addr,
12531762eaaSAaron Young pkt.ack_freq, pkt.plnk_updt, pkt.options,
12631762eaaSAaron Young (unsigned long long)pkt.mtu, pkt.cflags, pkt.ipv4_lso_maxlen);
12731762eaaSAaron Young
12831762eaaSAaron Young return vio_ldc_send(vio, &pkt, sizeof(pkt));
12931762eaaSAaron Young }
13031762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_send_attr_common);
13131762eaaSAaron Young
handle_attr_info(struct vio_driver_state * vio,struct vio_net_attr_info * pkt)13231762eaaSAaron Young static int handle_attr_info(struct vio_driver_state *vio,
13331762eaaSAaron Young struct vio_net_attr_info *pkt)
13431762eaaSAaron Young {
13531762eaaSAaron Young struct vnet_port *port = to_vnet_port(vio);
13631762eaaSAaron Young u64 localmtu;
13731762eaaSAaron Young u8 xfer_mode;
13831762eaaSAaron Young
13931762eaaSAaron Young viodbg(HS, "GOT NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] "
14031762eaaSAaron Young "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] "
14131762eaaSAaron Young " (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n",
14231762eaaSAaron Young pkt->xfer_mode, pkt->addr_type,
14331762eaaSAaron Young (unsigned long long)pkt->addr,
14431762eaaSAaron Young pkt->ack_freq, pkt->plnk_updt, pkt->options,
14531762eaaSAaron Young (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags,
14631762eaaSAaron Young pkt->ipv4_lso_maxlen);
14731762eaaSAaron Young
14831762eaaSAaron Young pkt->tag.sid = vio_send_sid(vio);
14931762eaaSAaron Young
15031762eaaSAaron Young xfer_mode = pkt->xfer_mode;
15131762eaaSAaron Young /* for version < 1.2, VIO_DRING_MODE = 0x3 and no bitmask */
15231762eaaSAaron Young if (vio_version_before(vio, 1, 2) && xfer_mode == VIO_DRING_MODE)
15331762eaaSAaron Young xfer_mode = VIO_NEW_DRING_MODE;
15431762eaaSAaron Young
15531762eaaSAaron Young /* MTU negotiation:
15631762eaaSAaron Young * < v1.3 - ETH_FRAME_LEN exactly
15731762eaaSAaron Young * > v1.3 - MIN(pkt.mtu, VNET_MAXPACKET, port->rmtu) and change
15831762eaaSAaron Young * pkt->mtu for ACK
15931762eaaSAaron Young * = v1.3 - ETH_FRAME_LEN + VLAN_HLEN exactly
16031762eaaSAaron Young */
16131762eaaSAaron Young if (vio_version_before(vio, 1, 3)) {
16231762eaaSAaron Young localmtu = ETH_FRAME_LEN;
16331762eaaSAaron Young } else if (vio_version_after(vio, 1, 3)) {
16431762eaaSAaron Young localmtu = port->rmtu ? port->rmtu : VNET_MAXPACKET;
16531762eaaSAaron Young localmtu = min(pkt->mtu, localmtu);
16631762eaaSAaron Young pkt->mtu = localmtu;
16731762eaaSAaron Young } else { /* v1.3 */
16831762eaaSAaron Young localmtu = ETH_FRAME_LEN + VLAN_HLEN;
16931762eaaSAaron Young }
17031762eaaSAaron Young port->rmtu = localmtu;
17131762eaaSAaron Young
17231762eaaSAaron Young /* LSO negotiation */
17331762eaaSAaron Young if (vio_version_after_eq(vio, 1, 7))
17431762eaaSAaron Young port->tso &= !!(pkt->cflags & VNET_LSO_IPV4_CAPAB);
17531762eaaSAaron Young else
17631762eaaSAaron Young port->tso = false;
17731762eaaSAaron Young if (port->tso) {
17831762eaaSAaron Young if (!port->tsolen)
17931762eaaSAaron Young port->tsolen = VNET_MAXTSO;
18031762eaaSAaron Young port->tsolen = min(port->tsolen, pkt->ipv4_lso_maxlen);
18131762eaaSAaron Young if (port->tsolen < VNET_MINTSO) {
18231762eaaSAaron Young port->tso = false;
18331762eaaSAaron Young port->tsolen = 0;
18431762eaaSAaron Young pkt->cflags &= ~VNET_LSO_IPV4_CAPAB;
18531762eaaSAaron Young }
18631762eaaSAaron Young pkt->ipv4_lso_maxlen = port->tsolen;
18731762eaaSAaron Young } else {
18831762eaaSAaron Young pkt->cflags &= ~VNET_LSO_IPV4_CAPAB;
18931762eaaSAaron Young pkt->ipv4_lso_maxlen = 0;
190bc221a34SShannon Nelson port->tsolen = 0;
19131762eaaSAaron Young }
19231762eaaSAaron Young
19331762eaaSAaron Young /* for version >= 1.6, ACK packet mode we support */
19431762eaaSAaron Young if (vio_version_after_eq(vio, 1, 6)) {
19531762eaaSAaron Young pkt->xfer_mode = VIO_NEW_DRING_MODE;
19631762eaaSAaron Young pkt->options = VIO_TX_DRING;
19731762eaaSAaron Young }
19831762eaaSAaron Young
19931762eaaSAaron Young if (!(xfer_mode | VIO_NEW_DRING_MODE) ||
20031762eaaSAaron Young pkt->addr_type != VNET_ADDR_ETHERMAC ||
20131762eaaSAaron Young pkt->mtu != localmtu) {
20231762eaaSAaron Young viodbg(HS, "SEND NET ATTR NACK\n");
20331762eaaSAaron Young
20431762eaaSAaron Young pkt->tag.stype = VIO_SUBTYPE_NACK;
20531762eaaSAaron Young
20631762eaaSAaron Young (void)vio_ldc_send(vio, pkt, sizeof(*pkt));
20731762eaaSAaron Young
20831762eaaSAaron Young return -ECONNRESET;
209dc153f85SAaron Young }
210dc153f85SAaron Young
21131762eaaSAaron Young viodbg(HS, "SEND NET ATTR ACK xmode[0x%x] atype[0x%x] "
21231762eaaSAaron Young "addr[%llx] ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] "
21331762eaaSAaron Young "mtu[%llu] (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n",
21431762eaaSAaron Young pkt->xfer_mode, pkt->addr_type,
21531762eaaSAaron Young (unsigned long long)pkt->addr,
21631762eaaSAaron Young pkt->ack_freq, pkt->plnk_updt, pkt->options,
21731762eaaSAaron Young (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags,
21831762eaaSAaron Young pkt->ipv4_lso_maxlen);
21931762eaaSAaron Young
22031762eaaSAaron Young pkt->tag.stype = VIO_SUBTYPE_ACK;
22131762eaaSAaron Young
22231762eaaSAaron Young return vio_ldc_send(vio, pkt, sizeof(*pkt));
22331762eaaSAaron Young }
22431762eaaSAaron Young
handle_attr_ack(struct vio_driver_state * vio,struct vio_net_attr_info * pkt)22531762eaaSAaron Young static int handle_attr_ack(struct vio_driver_state *vio,
22631762eaaSAaron Young struct vio_net_attr_info *pkt)
22731762eaaSAaron Young {
22831762eaaSAaron Young viodbg(HS, "GOT NET ATTR ACK\n");
22931762eaaSAaron Young
23031762eaaSAaron Young return 0;
23131762eaaSAaron Young }
23231762eaaSAaron Young
handle_attr_nack(struct vio_driver_state * vio,struct vio_net_attr_info * pkt)23331762eaaSAaron Young static int handle_attr_nack(struct vio_driver_state *vio,
23431762eaaSAaron Young struct vio_net_attr_info *pkt)
23531762eaaSAaron Young {
23631762eaaSAaron Young viodbg(HS, "GOT NET ATTR NACK\n");
23731762eaaSAaron Young
23831762eaaSAaron Young return -ECONNRESET;
23931762eaaSAaron Young }
24031762eaaSAaron Young
sunvnet_handle_attr_common(struct vio_driver_state * vio,void * arg)24131762eaaSAaron Young int sunvnet_handle_attr_common(struct vio_driver_state *vio, void *arg)
24231762eaaSAaron Young {
24331762eaaSAaron Young struct vio_net_attr_info *pkt = arg;
24431762eaaSAaron Young
24531762eaaSAaron Young switch (pkt->tag.stype) {
24631762eaaSAaron Young case VIO_SUBTYPE_INFO:
24731762eaaSAaron Young return handle_attr_info(vio, pkt);
24831762eaaSAaron Young
24931762eaaSAaron Young case VIO_SUBTYPE_ACK:
25031762eaaSAaron Young return handle_attr_ack(vio, pkt);
25131762eaaSAaron Young
25231762eaaSAaron Young case VIO_SUBTYPE_NACK:
25331762eaaSAaron Young return handle_attr_nack(vio, pkt);
25431762eaaSAaron Young
25531762eaaSAaron Young default:
25631762eaaSAaron Young return -ECONNRESET;
25731762eaaSAaron Young }
25831762eaaSAaron Young }
25931762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_handle_attr_common);
26031762eaaSAaron Young
sunvnet_handshake_complete_common(struct vio_driver_state * vio)26131762eaaSAaron Young void sunvnet_handshake_complete_common(struct vio_driver_state *vio)
26231762eaaSAaron Young {
26331762eaaSAaron Young struct vio_dring_state *dr;
26431762eaaSAaron Young
26531762eaaSAaron Young dr = &vio->drings[VIO_DRIVER_RX_RING];
266dc153f85SAaron Young dr->rcv_nxt = 1;
267dc153f85SAaron Young dr->snd_nxt = 1;
26831762eaaSAaron Young
26931762eaaSAaron Young dr = &vio->drings[VIO_DRIVER_TX_RING];
270dc153f85SAaron Young dr->rcv_nxt = 1;
271dc153f85SAaron Young dr->snd_nxt = 1;
27231762eaaSAaron Young }
27331762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_handshake_complete_common);
27431762eaaSAaron Young
27531762eaaSAaron Young /* The hypervisor interface that implements copying to/from imported
27631762eaaSAaron Young * memory from another domain requires that copies are done to 8-byte
27731762eaaSAaron Young * aligned buffers, and that the lengths of such copies are also 8-byte
27831762eaaSAaron Young * multiples.
27931762eaaSAaron Young *
28031762eaaSAaron Young * So we align skb->data to an 8-byte multiple and pad-out the data
28131762eaaSAaron Young * area so we can round the copy length up to the next multiple of
28231762eaaSAaron Young * 8 for the copy.
28331762eaaSAaron Young *
28431762eaaSAaron Young * The transmitter puts the actual start of the packet 6 bytes into
28531762eaaSAaron Young * the buffer it sends over, so that the IP headers after the ethernet
28631762eaaSAaron Young * header are aligned properly. These 6 bytes are not in the descriptor
28731762eaaSAaron Young * length, they are simply implied. This offset is represented using
28831762eaaSAaron Young * the VNET_PACKET_SKIP macro.
28931762eaaSAaron Young */
alloc_and_align_skb(struct net_device * dev,unsigned int len)29031762eaaSAaron Young static struct sk_buff *alloc_and_align_skb(struct net_device *dev,
29131762eaaSAaron Young unsigned int len)
29231762eaaSAaron Young {
293dc153f85SAaron Young struct sk_buff *skb;
29431762eaaSAaron Young unsigned long addr, off;
29531762eaaSAaron Young
296dc153f85SAaron Young skb = netdev_alloc_skb(dev, len + VNET_PACKET_SKIP + 8 + 8);
29731762eaaSAaron Young if (unlikely(!skb))
29831762eaaSAaron Young return NULL;
29931762eaaSAaron Young
30031762eaaSAaron Young addr = (unsigned long)skb->data;
30131762eaaSAaron Young off = ((addr + 7UL) & ~7UL) - addr;
30231762eaaSAaron Young if (off)
30331762eaaSAaron Young skb_reserve(skb, off);
30431762eaaSAaron Young
30531762eaaSAaron Young return skb;
30631762eaaSAaron Young }
30731762eaaSAaron Young
vnet_fullcsum_ipv4(struct sk_buff * skb)30898524e04SShannon Nelson static inline void vnet_fullcsum_ipv4(struct sk_buff *skb)
30931762eaaSAaron Young {
31031762eaaSAaron Young struct iphdr *iph = ip_hdr(skb);
31131762eaaSAaron Young int offset = skb_transport_offset(skb);
31231762eaaSAaron Young
31331762eaaSAaron Young if (skb->protocol != htons(ETH_P_IP))
31431762eaaSAaron Young return;
31531762eaaSAaron Young if (iph->protocol != IPPROTO_TCP &&
31631762eaaSAaron Young iph->protocol != IPPROTO_UDP)
31731762eaaSAaron Young return;
31831762eaaSAaron Young skb->ip_summed = CHECKSUM_NONE;
31931762eaaSAaron Young skb->csum_level = 1;
32031762eaaSAaron Young skb->csum = 0;
32131762eaaSAaron Young if (iph->protocol == IPPROTO_TCP) {
32231762eaaSAaron Young struct tcphdr *ptcp = tcp_hdr(skb);
32331762eaaSAaron Young
32431762eaaSAaron Young ptcp->check = 0;
32531762eaaSAaron Young skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
32631762eaaSAaron Young ptcp->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
32731762eaaSAaron Young skb->len - offset, IPPROTO_TCP,
32831762eaaSAaron Young skb->csum);
32931762eaaSAaron Young } else if (iph->protocol == IPPROTO_UDP) {
33031762eaaSAaron Young struct udphdr *pudp = udp_hdr(skb);
33131762eaaSAaron Young
33231762eaaSAaron Young pudp->check = 0;
33331762eaaSAaron Young skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
33431762eaaSAaron Young pudp->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
33531762eaaSAaron Young skb->len - offset, IPPROTO_UDP,
33631762eaaSAaron Young skb->csum);
33731762eaaSAaron Young }
33831762eaaSAaron Young }
33931762eaaSAaron Young
34098524e04SShannon Nelson #if IS_ENABLED(CONFIG_IPV6)
vnet_fullcsum_ipv6(struct sk_buff * skb)34198524e04SShannon Nelson static inline void vnet_fullcsum_ipv6(struct sk_buff *skb)
34298524e04SShannon Nelson {
34398524e04SShannon Nelson struct ipv6hdr *ip6h = ipv6_hdr(skb);
34498524e04SShannon Nelson int offset = skb_transport_offset(skb);
34598524e04SShannon Nelson
34698524e04SShannon Nelson if (skb->protocol != htons(ETH_P_IPV6))
34798524e04SShannon Nelson return;
34898524e04SShannon Nelson if (ip6h->nexthdr != IPPROTO_TCP &&
34998524e04SShannon Nelson ip6h->nexthdr != IPPROTO_UDP)
35098524e04SShannon Nelson return;
35198524e04SShannon Nelson skb->ip_summed = CHECKSUM_NONE;
35298524e04SShannon Nelson skb->csum_level = 1;
35398524e04SShannon Nelson skb->csum = 0;
35498524e04SShannon Nelson if (ip6h->nexthdr == IPPROTO_TCP) {
35598524e04SShannon Nelson struct tcphdr *ptcp = tcp_hdr(skb);
35698524e04SShannon Nelson
35798524e04SShannon Nelson ptcp->check = 0;
35898524e04SShannon Nelson skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
35998524e04SShannon Nelson ptcp->check = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
36098524e04SShannon Nelson skb->len - offset, IPPROTO_TCP,
36198524e04SShannon Nelson skb->csum);
36298524e04SShannon Nelson } else if (ip6h->nexthdr == IPPROTO_UDP) {
36398524e04SShannon Nelson struct udphdr *pudp = udp_hdr(skb);
36498524e04SShannon Nelson
36598524e04SShannon Nelson pudp->check = 0;
36698524e04SShannon Nelson skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
36798524e04SShannon Nelson pudp->check = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
36898524e04SShannon Nelson skb->len - offset, IPPROTO_UDP,
36998524e04SShannon Nelson skb->csum);
37098524e04SShannon Nelson }
37198524e04SShannon Nelson }
37298524e04SShannon Nelson #endif
37398524e04SShannon Nelson
vnet_rx_one(struct vnet_port * port,struct vio_net_desc * desc)37431762eaaSAaron Young static int vnet_rx_one(struct vnet_port *port, struct vio_net_desc *desc)
37531762eaaSAaron Young {
37667d0719fSAaron Young struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
37731762eaaSAaron Young unsigned int len = desc->size;
37831762eaaSAaron Young unsigned int copy_len;
37931762eaaSAaron Young struct sk_buff *skb;
38031762eaaSAaron Young int maxlen;
38131762eaaSAaron Young int err;
38231762eaaSAaron Young
38331762eaaSAaron Young err = -EMSGSIZE;
38431762eaaSAaron Young if (port->tso && port->tsolen > port->rmtu)
38531762eaaSAaron Young maxlen = port->tsolen;
38631762eaaSAaron Young else
38731762eaaSAaron Young maxlen = port->rmtu;
38831762eaaSAaron Young if (unlikely(len < ETH_ZLEN || len > maxlen)) {
38931762eaaSAaron Young dev->stats.rx_length_errors++;
39031762eaaSAaron Young goto out_dropped;
39131762eaaSAaron Young }
39231762eaaSAaron Young
39331762eaaSAaron Young skb = alloc_and_align_skb(dev, len);
39431762eaaSAaron Young err = -ENOMEM;
39531762eaaSAaron Young if (unlikely(!skb)) {
39631762eaaSAaron Young dev->stats.rx_missed_errors++;
39731762eaaSAaron Young goto out_dropped;
39831762eaaSAaron Young }
39931762eaaSAaron Young
40031762eaaSAaron Young copy_len = (len + VNET_PACKET_SKIP + 7U) & ~7U;
40131762eaaSAaron Young skb_put(skb, copy_len);
40231762eaaSAaron Young err = ldc_copy(port->vio.lp, LDC_COPY_IN,
40331762eaaSAaron Young skb->data, copy_len, 0,
40431762eaaSAaron Young desc->cookies, desc->ncookies);
40531762eaaSAaron Young if (unlikely(err < 0)) {
40631762eaaSAaron Young dev->stats.rx_frame_errors++;
40731762eaaSAaron Young goto out_free_skb;
40831762eaaSAaron Young }
40931762eaaSAaron Young
41031762eaaSAaron Young skb_pull(skb, VNET_PACKET_SKIP);
41131762eaaSAaron Young skb_trim(skb, len);
41231762eaaSAaron Young skb->protocol = eth_type_trans(skb, dev);
41331762eaaSAaron Young
41431762eaaSAaron Young if (vio_version_after_eq(&port->vio, 1, 8)) {
41531762eaaSAaron Young struct vio_net_dext *dext = vio_net_ext(desc);
41631762eaaSAaron Young
41731762eaaSAaron Young skb_reset_network_header(skb);
41831762eaaSAaron Young
41931762eaaSAaron Young if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM) {
42031762eaaSAaron Young if (skb->protocol == ETH_P_IP) {
42131762eaaSAaron Young struct iphdr *iph = ip_hdr(skb);
42231762eaaSAaron Young
42331762eaaSAaron Young iph->check = 0;
42431762eaaSAaron Young ip_send_check(iph);
42531762eaaSAaron Young }
42631762eaaSAaron Young }
42731762eaaSAaron Young if ((dext->flags & VNET_PKT_HCK_FULLCKSUM) &&
42831762eaaSAaron Young skb->ip_summed == CHECKSUM_NONE) {
42931762eaaSAaron Young if (skb->protocol == htons(ETH_P_IP)) {
43031762eaaSAaron Young struct iphdr *iph = ip_hdr(skb);
43131762eaaSAaron Young int ihl = iph->ihl * 4;
43231762eaaSAaron Young
43331762eaaSAaron Young skb_set_transport_header(skb, ihl);
43498524e04SShannon Nelson vnet_fullcsum_ipv4(skb);
43598524e04SShannon Nelson #if IS_ENABLED(CONFIG_IPV6)
43698524e04SShannon Nelson } else if (skb->protocol == htons(ETH_P_IPV6)) {
43798524e04SShannon Nelson skb_set_transport_header(skb,
43898524e04SShannon Nelson sizeof(struct ipv6hdr));
43998524e04SShannon Nelson vnet_fullcsum_ipv6(skb);
44098524e04SShannon Nelson #endif
44131762eaaSAaron Young }
44231762eaaSAaron Young }
44331762eaaSAaron Young if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM_OK) {
44431762eaaSAaron Young skb->ip_summed = CHECKSUM_PARTIAL;
44531762eaaSAaron Young skb->csum_level = 0;
44631762eaaSAaron Young if (dext->flags & VNET_PKT_HCK_FULLCKSUM_OK)
44731762eaaSAaron Young skb->csum_level = 1;
44831762eaaSAaron Young }
44931762eaaSAaron Young }
45031762eaaSAaron Young
45131762eaaSAaron Young skb->ip_summed = port->switch_port ? CHECKSUM_NONE : CHECKSUM_PARTIAL;
45231762eaaSAaron Young
453b12a96f5SShannon Nelson if (unlikely(is_multicast_ether_addr(eth_hdr(skb)->h_dest)))
454b12a96f5SShannon Nelson dev->stats.multicast++;
45531762eaaSAaron Young dev->stats.rx_packets++;
45631762eaaSAaron Young dev->stats.rx_bytes += len;
4570f512c84SShannon Nelson port->stats.rx_packets++;
4580f512c84SShannon Nelson port->stats.rx_bytes += len;
45931762eaaSAaron Young napi_gro_receive(&port->napi, skb);
46031762eaaSAaron Young return 0;
46131762eaaSAaron Young
46231762eaaSAaron Young out_free_skb:
46331762eaaSAaron Young kfree_skb(skb);
46431762eaaSAaron Young
46531762eaaSAaron Young out_dropped:
46631762eaaSAaron Young dev->stats.rx_dropped++;
46731762eaaSAaron Young return err;
46831762eaaSAaron Young }
46931762eaaSAaron Young
vnet_send_ack(struct vnet_port * port,struct vio_dring_state * dr,u32 start,u32 end,u8 vio_dring_state)47031762eaaSAaron Young static int vnet_send_ack(struct vnet_port *port, struct vio_dring_state *dr,
47131762eaaSAaron Young u32 start, u32 end, u8 vio_dring_state)
47231762eaaSAaron Young {
47331762eaaSAaron Young struct vio_dring_data hdr = {
47431762eaaSAaron Young .tag = {
47531762eaaSAaron Young .type = VIO_TYPE_DATA,
47631762eaaSAaron Young .stype = VIO_SUBTYPE_ACK,
47731762eaaSAaron Young .stype_env = VIO_DRING_DATA,
47831762eaaSAaron Young .sid = vio_send_sid(&port->vio),
47931762eaaSAaron Young },
48031762eaaSAaron Young .dring_ident = dr->ident,
48131762eaaSAaron Young .start_idx = start,
48231762eaaSAaron Young .end_idx = end,
48331762eaaSAaron Young .state = vio_dring_state,
48431762eaaSAaron Young };
48531762eaaSAaron Young int err, delay;
48631762eaaSAaron Young int retries = 0;
48731762eaaSAaron Young
48831762eaaSAaron Young hdr.seq = dr->snd_nxt;
48931762eaaSAaron Young delay = 1;
49031762eaaSAaron Young do {
49131762eaaSAaron Young err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr));
49231762eaaSAaron Young if (err > 0) {
49331762eaaSAaron Young dr->snd_nxt++;
49431762eaaSAaron Young break;
49531762eaaSAaron Young }
49631762eaaSAaron Young udelay(delay);
49731762eaaSAaron Young if ((delay <<= 1) > 128)
49831762eaaSAaron Young delay = 128;
49931762eaaSAaron Young if (retries++ > VNET_MAX_RETRIES) {
50031762eaaSAaron Young pr_info("ECONNRESET %x:%x:%x:%x:%x:%x\n",
50131762eaaSAaron Young port->raddr[0], port->raddr[1],
50231762eaaSAaron Young port->raddr[2], port->raddr[3],
50331762eaaSAaron Young port->raddr[4], port->raddr[5]);
50431762eaaSAaron Young break;
50531762eaaSAaron Young }
50631762eaaSAaron Young } while (err == -EAGAIN);
50731762eaaSAaron Young
50831762eaaSAaron Young if (err <= 0 && vio_dring_state == VIO_DRING_STOPPED) {
50931762eaaSAaron Young port->stop_rx_idx = end;
51031762eaaSAaron Young port->stop_rx = true;
51131762eaaSAaron Young } else {
51231762eaaSAaron Young port->stop_rx_idx = 0;
51331762eaaSAaron Young port->stop_rx = false;
51431762eaaSAaron Young }
51531762eaaSAaron Young
51631762eaaSAaron Young return err;
51731762eaaSAaron Young }
51831762eaaSAaron Young
get_rx_desc(struct vnet_port * port,struct vio_dring_state * dr,u32 index)51931762eaaSAaron Young static struct vio_net_desc *get_rx_desc(struct vnet_port *port,
52031762eaaSAaron Young struct vio_dring_state *dr,
52131762eaaSAaron Young u32 index)
52231762eaaSAaron Young {
52331762eaaSAaron Young struct vio_net_desc *desc = port->vio.desc_buf;
52431762eaaSAaron Young int err;
52531762eaaSAaron Young
52631762eaaSAaron Young err = ldc_get_dring_entry(port->vio.lp, desc, dr->entry_size,
52731762eaaSAaron Young (index * dr->entry_size),
52831762eaaSAaron Young dr->cookies, dr->ncookies);
52931762eaaSAaron Young if (err < 0)
53031762eaaSAaron Young return ERR_PTR(err);
53131762eaaSAaron Young
53231762eaaSAaron Young return desc;
53331762eaaSAaron Young }
53431762eaaSAaron Young
put_rx_desc(struct vnet_port * port,struct vio_dring_state * dr,struct vio_net_desc * desc,u32 index)53531762eaaSAaron Young static int put_rx_desc(struct vnet_port *port,
53631762eaaSAaron Young struct vio_dring_state *dr,
53731762eaaSAaron Young struct vio_net_desc *desc,
53831762eaaSAaron Young u32 index)
53931762eaaSAaron Young {
54031762eaaSAaron Young int err;
54131762eaaSAaron Young
54231762eaaSAaron Young err = ldc_put_dring_entry(port->vio.lp, desc, dr->entry_size,
54331762eaaSAaron Young (index * dr->entry_size),
54431762eaaSAaron Young dr->cookies, dr->ncookies);
54531762eaaSAaron Young if (err < 0)
54631762eaaSAaron Young return err;
54731762eaaSAaron Young
54831762eaaSAaron Young return 0;
54931762eaaSAaron Young }
55031762eaaSAaron Young
vnet_walk_rx_one(struct vnet_port * port,struct vio_dring_state * dr,u32 index,int * needs_ack)55131762eaaSAaron Young static int vnet_walk_rx_one(struct vnet_port *port,
55231762eaaSAaron Young struct vio_dring_state *dr,
55331762eaaSAaron Young u32 index, int *needs_ack)
55431762eaaSAaron Young {
55531762eaaSAaron Young struct vio_net_desc *desc = get_rx_desc(port, dr, index);
55631762eaaSAaron Young struct vio_driver_state *vio = &port->vio;
55731762eaaSAaron Young int err;
55831762eaaSAaron Young
559dc153f85SAaron Young BUG_ON(!desc);
56031762eaaSAaron Young if (IS_ERR(desc))
56131762eaaSAaron Young return PTR_ERR(desc);
56231762eaaSAaron Young
56331762eaaSAaron Young if (desc->hdr.state != VIO_DESC_READY)
56431762eaaSAaron Young return 1;
56531762eaaSAaron Young
56631762eaaSAaron Young dma_rmb();
56731762eaaSAaron Young
56831762eaaSAaron Young viodbg(DATA, "vio_walk_rx_one desc[%02x:%02x:%08x:%08x:%llx:%llx]\n",
56931762eaaSAaron Young desc->hdr.state, desc->hdr.ack,
57031762eaaSAaron Young desc->size, desc->ncookies,
57131762eaaSAaron Young desc->cookies[0].cookie_addr,
57231762eaaSAaron Young desc->cookies[0].cookie_size);
57331762eaaSAaron Young
57431762eaaSAaron Young err = vnet_rx_one(port, desc);
57531762eaaSAaron Young if (err == -ECONNRESET)
57631762eaaSAaron Young return err;
57731762eaaSAaron Young trace_vnet_rx_one(port->vio._local_sid, port->vio._peer_sid,
57831762eaaSAaron Young index, desc->hdr.ack);
57931762eaaSAaron Young desc->hdr.state = VIO_DESC_DONE;
58031762eaaSAaron Young err = put_rx_desc(port, dr, desc, index);
58131762eaaSAaron Young if (err < 0)
58231762eaaSAaron Young return err;
58331762eaaSAaron Young *needs_ack = desc->hdr.ack;
58431762eaaSAaron Young return 0;
58531762eaaSAaron Young }
58631762eaaSAaron Young
vnet_walk_rx(struct vnet_port * port,struct vio_dring_state * dr,u32 start,u32 end,int * npkts,int budget)58731762eaaSAaron Young static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr,
58831762eaaSAaron Young u32 start, u32 end, int *npkts, int budget)
58931762eaaSAaron Young {
59031762eaaSAaron Young struct vio_driver_state *vio = &port->vio;
59131762eaaSAaron Young int ack_start = -1, ack_end = -1;
59231762eaaSAaron Young bool send_ack = true;
59331762eaaSAaron Young
59431762eaaSAaron Young end = (end == (u32)-1) ? vio_dring_prev(dr, start)
59531762eaaSAaron Young : vio_dring_next(dr, end);
59631762eaaSAaron Young
59731762eaaSAaron Young viodbg(DATA, "vnet_walk_rx start[%08x] end[%08x]\n", start, end);
59831762eaaSAaron Young
59931762eaaSAaron Young while (start != end) {
60031762eaaSAaron Young int ack = 0, err = vnet_walk_rx_one(port, dr, start, &ack);
601dc153f85SAaron Young
60231762eaaSAaron Young if (err == -ECONNRESET)
60331762eaaSAaron Young return err;
60431762eaaSAaron Young if (err != 0)
60531762eaaSAaron Young break;
60631762eaaSAaron Young (*npkts)++;
60731762eaaSAaron Young if (ack_start == -1)
60831762eaaSAaron Young ack_start = start;
60931762eaaSAaron Young ack_end = start;
61031762eaaSAaron Young start = vio_dring_next(dr, start);
61131762eaaSAaron Young if (ack && start != end) {
61231762eaaSAaron Young err = vnet_send_ack(port, dr, ack_start, ack_end,
61331762eaaSAaron Young VIO_DRING_ACTIVE);
61431762eaaSAaron Young if (err == -ECONNRESET)
61531762eaaSAaron Young return err;
61631762eaaSAaron Young ack_start = -1;
61731762eaaSAaron Young }
61831762eaaSAaron Young if ((*npkts) >= budget) {
61931762eaaSAaron Young send_ack = false;
62031762eaaSAaron Young break;
62131762eaaSAaron Young }
62231762eaaSAaron Young }
623dc153f85SAaron Young if (unlikely(ack_start == -1)) {
624dc153f85SAaron Young ack_end = vio_dring_prev(dr, start);
625dc153f85SAaron Young ack_start = ack_end;
626dc153f85SAaron Young }
62731762eaaSAaron Young if (send_ack) {
62831762eaaSAaron Young port->napi_resume = false;
62931762eaaSAaron Young trace_vnet_tx_send_stopped_ack(port->vio._local_sid,
63031762eaaSAaron Young port->vio._peer_sid,
63131762eaaSAaron Young ack_end, *npkts);
63231762eaaSAaron Young return vnet_send_ack(port, dr, ack_start, ack_end,
63331762eaaSAaron Young VIO_DRING_STOPPED);
63431762eaaSAaron Young } else {
63531762eaaSAaron Young trace_vnet_tx_defer_stopped_ack(port->vio._local_sid,
63631762eaaSAaron Young port->vio._peer_sid,
63731762eaaSAaron Young ack_end, *npkts);
63831762eaaSAaron Young port->napi_resume = true;
63931762eaaSAaron Young port->napi_stop_idx = ack_end;
64031762eaaSAaron Young return 1;
64131762eaaSAaron Young }
64231762eaaSAaron Young }
64331762eaaSAaron Young
vnet_rx(struct vnet_port * port,void * msgbuf,int * npkts,int budget)64431762eaaSAaron Young static int vnet_rx(struct vnet_port *port, void *msgbuf, int *npkts,
64531762eaaSAaron Young int budget)
64631762eaaSAaron Young {
64731762eaaSAaron Young struct vio_dring_data *pkt = msgbuf;
64831762eaaSAaron Young struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_RX_RING];
64931762eaaSAaron Young struct vio_driver_state *vio = &port->vio;
65031762eaaSAaron Young
65131762eaaSAaron Young viodbg(DATA, "vnet_rx stype_env[%04x] seq[%016llx] rcv_nxt[%016llx]\n",
65231762eaaSAaron Young pkt->tag.stype_env, pkt->seq, dr->rcv_nxt);
65331762eaaSAaron Young
65431762eaaSAaron Young if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA))
65531762eaaSAaron Young return 0;
65631762eaaSAaron Young if (unlikely(pkt->seq != dr->rcv_nxt)) {
65731762eaaSAaron Young pr_err("RX out of sequence seq[0x%llx] rcv_nxt[0x%llx]\n",
65831762eaaSAaron Young pkt->seq, dr->rcv_nxt);
65931762eaaSAaron Young return 0;
66031762eaaSAaron Young }
66131762eaaSAaron Young
66231762eaaSAaron Young if (!port->napi_resume)
66331762eaaSAaron Young dr->rcv_nxt++;
66431762eaaSAaron Young
66531762eaaSAaron Young /* XXX Validate pkt->start_idx and pkt->end_idx XXX */
66631762eaaSAaron Young
66731762eaaSAaron Young return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx,
66831762eaaSAaron Young npkts, budget);
66931762eaaSAaron Young }
67031762eaaSAaron Young
idx_is_pending(struct vio_dring_state * dr,u32 end)67131762eaaSAaron Young static int idx_is_pending(struct vio_dring_state *dr, u32 end)
67231762eaaSAaron Young {
67331762eaaSAaron Young u32 idx = dr->cons;
67431762eaaSAaron Young int found = 0;
67531762eaaSAaron Young
67631762eaaSAaron Young while (idx != dr->prod) {
67731762eaaSAaron Young if (idx == end) {
67831762eaaSAaron Young found = 1;
67931762eaaSAaron Young break;
68031762eaaSAaron Young }
68131762eaaSAaron Young idx = vio_dring_next(dr, idx);
68231762eaaSAaron Young }
68331762eaaSAaron Young return found;
68431762eaaSAaron Young }
68531762eaaSAaron Young
vnet_ack(struct vnet_port * port,void * msgbuf)68631762eaaSAaron Young static int vnet_ack(struct vnet_port *port, void *msgbuf)
68731762eaaSAaron Young {
68831762eaaSAaron Young struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
68931762eaaSAaron Young struct vio_dring_data *pkt = msgbuf;
69031762eaaSAaron Young struct net_device *dev;
69131762eaaSAaron Young u32 end;
69231762eaaSAaron Young struct vio_net_desc *desc;
69331762eaaSAaron Young struct netdev_queue *txq;
69431762eaaSAaron Young
69531762eaaSAaron Young if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA))
69631762eaaSAaron Young return 0;
69731762eaaSAaron Young
69831762eaaSAaron Young end = pkt->end_idx;
69967d0719fSAaron Young dev = VNET_PORT_TO_NET_DEVICE(port);
70031762eaaSAaron Young netif_tx_lock(dev);
70131762eaaSAaron Young if (unlikely(!idx_is_pending(dr, end))) {
70231762eaaSAaron Young netif_tx_unlock(dev);
70331762eaaSAaron Young return 0;
70431762eaaSAaron Young }
70531762eaaSAaron Young
70631762eaaSAaron Young /* sync for race conditions with vnet_start_xmit() and tell xmit it
70731762eaaSAaron Young * is time to send a trigger.
70831762eaaSAaron Young */
70931762eaaSAaron Young trace_vnet_rx_stopped_ack(port->vio._local_sid,
71031762eaaSAaron Young port->vio._peer_sid, end);
71131762eaaSAaron Young dr->cons = vio_dring_next(dr, end);
71231762eaaSAaron Young desc = vio_dring_entry(dr, dr->cons);
71331762eaaSAaron Young if (desc->hdr.state == VIO_DESC_READY && !port->start_cons) {
71431762eaaSAaron Young /* vnet_start_xmit() just populated this dring but missed
71531762eaaSAaron Young * sending the "start" LDC message to the consumer.
71631762eaaSAaron Young * Send a "start" trigger on its behalf.
71731762eaaSAaron Young */
71831762eaaSAaron Young if (__vnet_tx_trigger(port, dr->cons) > 0)
71931762eaaSAaron Young port->start_cons = false;
72031762eaaSAaron Young else
72131762eaaSAaron Young port->start_cons = true;
72231762eaaSAaron Young } else {
72331762eaaSAaron Young port->start_cons = true;
72431762eaaSAaron Young }
72531762eaaSAaron Young netif_tx_unlock(dev);
72631762eaaSAaron Young
72731762eaaSAaron Young txq = netdev_get_tx_queue(dev, port->q_index);
72831762eaaSAaron Young if (unlikely(netif_tx_queue_stopped(txq) &&
72931762eaaSAaron Young vnet_tx_dring_avail(dr) >= VNET_TX_WAKEUP_THRESH(dr)))
73031762eaaSAaron Young return 1;
73131762eaaSAaron Young
73231762eaaSAaron Young return 0;
73331762eaaSAaron Young }
73431762eaaSAaron Young
vnet_nack(struct vnet_port * port,void * msgbuf)73531762eaaSAaron Young static int vnet_nack(struct vnet_port *port, void *msgbuf)
73631762eaaSAaron Young {
73731762eaaSAaron Young /* XXX just reset or similar XXX */
73831762eaaSAaron Young return 0;
73931762eaaSAaron Young }
74031762eaaSAaron Young
handle_mcast(struct vnet_port * port,void * msgbuf)74131762eaaSAaron Young static int handle_mcast(struct vnet_port *port, void *msgbuf)
74231762eaaSAaron Young {
74331762eaaSAaron Young struct vio_net_mcast_info *pkt = msgbuf;
74467d0719fSAaron Young struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
74531762eaaSAaron Young
74631762eaaSAaron Young if (pkt->tag.stype != VIO_SUBTYPE_ACK)
74731762eaaSAaron Young pr_err("%s: Got unexpected MCAST reply [%02x:%02x:%04x:%08x]\n",
74867d0719fSAaron Young dev->name,
74931762eaaSAaron Young pkt->tag.type,
75031762eaaSAaron Young pkt->tag.stype,
75131762eaaSAaron Young pkt->tag.stype_env,
75231762eaaSAaron Young pkt->tag.sid);
75331762eaaSAaron Young
75431762eaaSAaron Young return 0;
75531762eaaSAaron Young }
75631762eaaSAaron Young
7578778b276SAaron Young /* If the queue is stopped, wake it up so that we'll
7588778b276SAaron Young * send out another START message at the next TX.
75931762eaaSAaron Young */
maybe_tx_wakeup(struct vnet_port * port)76031762eaaSAaron Young static void maybe_tx_wakeup(struct vnet_port *port)
76131762eaaSAaron Young {
76231762eaaSAaron Young struct netdev_queue *txq;
76331762eaaSAaron Young
76467d0719fSAaron Young txq = netdev_get_tx_queue(VNET_PORT_TO_NET_DEVICE(port),
76567d0719fSAaron Young port->q_index);
76631762eaaSAaron Young __netif_tx_lock(txq, smp_processor_id());
767d4aa89ccSSowmini Varadhan if (likely(netif_tx_queue_stopped(txq)))
76831762eaaSAaron Young netif_tx_wake_queue(txq);
76931762eaaSAaron Young __netif_tx_unlock(txq);
77031762eaaSAaron Young }
77131762eaaSAaron Young
sunvnet_port_is_up_common(struct vnet_port * vnet)77267d0719fSAaron Young bool sunvnet_port_is_up_common(struct vnet_port *vnet)
77331762eaaSAaron Young {
77431762eaaSAaron Young struct vio_driver_state *vio = &vnet->vio;
77531762eaaSAaron Young
77631762eaaSAaron Young return !!(vio->hs_state & VIO_HS_COMPLETE);
77731762eaaSAaron Young }
77867d0719fSAaron Young EXPORT_SYMBOL_GPL(sunvnet_port_is_up_common);
77931762eaaSAaron Young
vnet_event_napi(struct vnet_port * port,int budget)78031762eaaSAaron Young static int vnet_event_napi(struct vnet_port *port, int budget)
78131762eaaSAaron Young {
7828778b276SAaron Young struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
78331762eaaSAaron Young struct vio_driver_state *vio = &port->vio;
78431762eaaSAaron Young int tx_wakeup, err;
78531762eaaSAaron Young int npkts = 0;
78631762eaaSAaron Young
787bf091f3fSShannon Nelson /* we don't expect any other bits */
788bf091f3fSShannon Nelson BUG_ON(port->rx_event & ~(LDC_EVENT_DATA_READY |
789bf091f3fSShannon Nelson LDC_EVENT_RESET |
790bf091f3fSShannon Nelson LDC_EVENT_UP));
79131762eaaSAaron Young
792bf091f3fSShannon Nelson /* RESET takes precedent over any other event */
793bf091f3fSShannon Nelson if (port->rx_event & LDC_EVENT_RESET) {
794867fa150SShannon Nelson /* a link went down */
795867fa150SShannon Nelson
796867fa150SShannon Nelson if (port->vsw == 1) {
797867fa150SShannon Nelson netif_tx_stop_all_queues(dev);
798867fa150SShannon Nelson netif_carrier_off(dev);
799867fa150SShannon Nelson }
800867fa150SShannon Nelson
801bf091f3fSShannon Nelson vio_link_state_change(vio, LDC_EVENT_RESET);
80231762eaaSAaron Young vnet_port_reset(port);
80331762eaaSAaron Young vio_port_up(vio);
8048778b276SAaron Young
8058778b276SAaron Young /* If the device is running but its tx queue was
8068778b276SAaron Young * stopped (due to flow control), restart it.
8078778b276SAaron Young * This is necessary since vnet_port_reset()
8088778b276SAaron Young * clears the tx drings and thus we may never get
8098778b276SAaron Young * back a VIO_TYPE_DATA ACK packet - which is
8108778b276SAaron Young * the normal mechanism to restart the tx queue.
8118778b276SAaron Young */
8128778b276SAaron Young if (netif_running(dev))
8138778b276SAaron Young maybe_tx_wakeup(port);
814bf091f3fSShannon Nelson
81531762eaaSAaron Young port->rx_event = 0;
8160f512c84SShannon Nelson port->stats.event_reset++;
81731762eaaSAaron Young return 0;
81831762eaaSAaron Young }
81931762eaaSAaron Young
820bf091f3fSShannon Nelson if (port->rx_event & LDC_EVENT_UP) {
821867fa150SShannon Nelson /* a link came up */
822867fa150SShannon Nelson
823867fa150SShannon Nelson if (port->vsw == 1) {
824867fa150SShannon Nelson netif_carrier_on(port->dev);
825867fa150SShannon Nelson netif_tx_start_all_queues(port->dev);
826867fa150SShannon Nelson }
827867fa150SShannon Nelson
828bf091f3fSShannon Nelson vio_link_state_change(vio, LDC_EVENT_UP);
829bf091f3fSShannon Nelson port->rx_event = 0;
8300f512c84SShannon Nelson port->stats.event_up++;
831bf091f3fSShannon Nelson return 0;
832bf091f3fSShannon Nelson }
83331762eaaSAaron Young
834dc153f85SAaron Young err = 0;
835dc153f85SAaron Young tx_wakeup = 0;
83631762eaaSAaron Young while (1) {
83731762eaaSAaron Young union {
83831762eaaSAaron Young struct vio_msg_tag tag;
83931762eaaSAaron Young u64 raw[8];
84031762eaaSAaron Young } msgbuf;
84131762eaaSAaron Young
84231762eaaSAaron Young if (port->napi_resume) {
84331762eaaSAaron Young struct vio_dring_data *pkt =
84431762eaaSAaron Young (struct vio_dring_data *)&msgbuf;
84531762eaaSAaron Young struct vio_dring_state *dr =
84631762eaaSAaron Young &port->vio.drings[VIO_DRIVER_RX_RING];
84731762eaaSAaron Young
84831762eaaSAaron Young pkt->tag.type = VIO_TYPE_DATA;
84931762eaaSAaron Young pkt->tag.stype = VIO_SUBTYPE_INFO;
85031762eaaSAaron Young pkt->tag.stype_env = VIO_DRING_DATA;
85131762eaaSAaron Young pkt->seq = dr->rcv_nxt;
852dc153f85SAaron Young pkt->start_idx = vio_dring_next(dr,
853dc153f85SAaron Young port->napi_stop_idx);
85431762eaaSAaron Young pkt->end_idx = -1;
855bf091f3fSShannon Nelson } else {
85631762eaaSAaron Young err = ldc_read(vio->lp, &msgbuf, sizeof(msgbuf));
85731762eaaSAaron Young if (unlikely(err < 0)) {
85831762eaaSAaron Young if (err == -ECONNRESET)
85931762eaaSAaron Young vio_conn_reset(vio);
86031762eaaSAaron Young break;
86131762eaaSAaron Young }
86231762eaaSAaron Young if (err == 0)
86331762eaaSAaron Young break;
86431762eaaSAaron Young viodbg(DATA, "TAG [%02x:%02x:%04x:%08x]\n",
86531762eaaSAaron Young msgbuf.tag.type,
86631762eaaSAaron Young msgbuf.tag.stype,
86731762eaaSAaron Young msgbuf.tag.stype_env,
86831762eaaSAaron Young msgbuf.tag.sid);
86931762eaaSAaron Young err = vio_validate_sid(vio, &msgbuf.tag);
87031762eaaSAaron Young if (err < 0)
87131762eaaSAaron Young break;
872bf091f3fSShannon Nelson }
873bf091f3fSShannon Nelson
87431762eaaSAaron Young if (likely(msgbuf.tag.type == VIO_TYPE_DATA)) {
87531762eaaSAaron Young if (msgbuf.tag.stype == VIO_SUBTYPE_INFO) {
87667d0719fSAaron Young if (!sunvnet_port_is_up_common(port)) {
87731762eaaSAaron Young /* failures like handshake_failure()
87831762eaaSAaron Young * may have cleaned up dring, but
87931762eaaSAaron Young * NAPI polling may bring us here.
88031762eaaSAaron Young */
88131762eaaSAaron Young err = -ECONNRESET;
88231762eaaSAaron Young break;
88331762eaaSAaron Young }
88431762eaaSAaron Young err = vnet_rx(port, &msgbuf, &npkts, budget);
88531762eaaSAaron Young if (npkts >= budget)
88631762eaaSAaron Young break;
88731762eaaSAaron Young if (npkts == 0)
88831762eaaSAaron Young break;
88931762eaaSAaron Young } else if (msgbuf.tag.stype == VIO_SUBTYPE_ACK) {
89031762eaaSAaron Young err = vnet_ack(port, &msgbuf);
89131762eaaSAaron Young if (err > 0)
89231762eaaSAaron Young tx_wakeup |= err;
89331762eaaSAaron Young } else if (msgbuf.tag.stype == VIO_SUBTYPE_NACK) {
89431762eaaSAaron Young err = vnet_nack(port, &msgbuf);
89531762eaaSAaron Young }
89631762eaaSAaron Young } else if (msgbuf.tag.type == VIO_TYPE_CTRL) {
89731762eaaSAaron Young if (msgbuf.tag.stype_env == VNET_MCAST_INFO)
89831762eaaSAaron Young err = handle_mcast(port, &msgbuf);
89931762eaaSAaron Young else
90031762eaaSAaron Young err = vio_control_pkt_engine(vio, &msgbuf);
90131762eaaSAaron Young if (err)
90231762eaaSAaron Young break;
90331762eaaSAaron Young } else {
90431762eaaSAaron Young err = vnet_handle_unknown(port, &msgbuf);
90531762eaaSAaron Young }
90631762eaaSAaron Young if (err == -ECONNRESET)
90731762eaaSAaron Young break;
90831762eaaSAaron Young }
90931762eaaSAaron Young if (unlikely(tx_wakeup && err != -ECONNRESET))
91031762eaaSAaron Young maybe_tx_wakeup(port);
91131762eaaSAaron Young return npkts;
91231762eaaSAaron Young }
91331762eaaSAaron Young
sunvnet_poll_common(struct napi_struct * napi,int budget)91431762eaaSAaron Young int sunvnet_poll_common(struct napi_struct *napi, int budget)
91531762eaaSAaron Young {
91631762eaaSAaron Young struct vnet_port *port = container_of(napi, struct vnet_port, napi);
91731762eaaSAaron Young struct vio_driver_state *vio = &port->vio;
91831762eaaSAaron Young int processed = vnet_event_napi(port, budget);
91931762eaaSAaron Young
92031762eaaSAaron Young if (processed < budget) {
9216ad20165SEric Dumazet napi_complete_done(napi, processed);
92231762eaaSAaron Young port->rx_event &= ~LDC_EVENT_DATA_READY;
92331762eaaSAaron Young vio_set_intr(vio->vdev->rx_ino, HV_INTR_ENABLED);
92431762eaaSAaron Young }
92531762eaaSAaron Young return processed;
92631762eaaSAaron Young }
92731762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_poll_common);
92831762eaaSAaron Young
sunvnet_event_common(void * arg,int event)92931762eaaSAaron Young void sunvnet_event_common(void *arg, int event)
93031762eaaSAaron Young {
93131762eaaSAaron Young struct vnet_port *port = arg;
93231762eaaSAaron Young struct vio_driver_state *vio = &port->vio;
93331762eaaSAaron Young
93431762eaaSAaron Young port->rx_event |= event;
93531762eaaSAaron Young vio_set_intr(vio->vdev->rx_ino, HV_INTR_DISABLED);
93631762eaaSAaron Young napi_schedule(&port->napi);
93731762eaaSAaron Young }
93831762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_event_common);
93931762eaaSAaron Young
__vnet_tx_trigger(struct vnet_port * port,u32 start)94031762eaaSAaron Young static int __vnet_tx_trigger(struct vnet_port *port, u32 start)
94131762eaaSAaron Young {
94231762eaaSAaron Young struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
94331762eaaSAaron Young struct vio_dring_data hdr = {
94431762eaaSAaron Young .tag = {
94531762eaaSAaron Young .type = VIO_TYPE_DATA,
94631762eaaSAaron Young .stype = VIO_SUBTYPE_INFO,
94731762eaaSAaron Young .stype_env = VIO_DRING_DATA,
94831762eaaSAaron Young .sid = vio_send_sid(&port->vio),
94931762eaaSAaron Young },
95031762eaaSAaron Young .dring_ident = dr->ident,
95131762eaaSAaron Young .start_idx = start,
95231762eaaSAaron Young .end_idx = (u32)-1,
95331762eaaSAaron Young };
95431762eaaSAaron Young int err, delay;
95531762eaaSAaron Young int retries = 0;
95631762eaaSAaron Young
95731762eaaSAaron Young if (port->stop_rx) {
95831762eaaSAaron Young trace_vnet_tx_pending_stopped_ack(port->vio._local_sid,
95931762eaaSAaron Young port->vio._peer_sid,
96031762eaaSAaron Young port->stop_rx_idx, -1);
96131762eaaSAaron Young err = vnet_send_ack(port,
96231762eaaSAaron Young &port->vio.drings[VIO_DRIVER_RX_RING],
96331762eaaSAaron Young port->stop_rx_idx, -1,
96431762eaaSAaron Young VIO_DRING_STOPPED);
96531762eaaSAaron Young if (err <= 0)
96631762eaaSAaron Young return err;
96731762eaaSAaron Young }
96831762eaaSAaron Young
96931762eaaSAaron Young hdr.seq = dr->snd_nxt;
97031762eaaSAaron Young delay = 1;
97131762eaaSAaron Young do {
97231762eaaSAaron Young err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr));
97331762eaaSAaron Young if (err > 0) {
97431762eaaSAaron Young dr->snd_nxt++;
97531762eaaSAaron Young break;
97631762eaaSAaron Young }
97731762eaaSAaron Young udelay(delay);
97831762eaaSAaron Young if ((delay <<= 1) > 128)
97931762eaaSAaron Young delay = 128;
98031762eaaSAaron Young if (retries++ > VNET_MAX_RETRIES)
98131762eaaSAaron Young break;
98231762eaaSAaron Young } while (err == -EAGAIN);
98331762eaaSAaron Young trace_vnet_tx_trigger(port->vio._local_sid,
98431762eaaSAaron Young port->vio._peer_sid, start, err);
98531762eaaSAaron Young
98631762eaaSAaron Young return err;
98731762eaaSAaron Young }
98831762eaaSAaron Young
vnet_clean_tx_ring(struct vnet_port * port,unsigned * pending)98931762eaaSAaron Young static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port,
99031762eaaSAaron Young unsigned *pending)
99131762eaaSAaron Young {
99231762eaaSAaron Young struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
99331762eaaSAaron Young struct sk_buff *skb = NULL;
99431762eaaSAaron Young int i, txi;
99531762eaaSAaron Young
99631762eaaSAaron Young *pending = 0;
99731762eaaSAaron Young
99831762eaaSAaron Young txi = dr->prod;
99931762eaaSAaron Young for (i = 0; i < VNET_TX_RING_SIZE; ++i) {
100031762eaaSAaron Young struct vio_net_desc *d;
100131762eaaSAaron Young
100231762eaaSAaron Young --txi;
100331762eaaSAaron Young if (txi < 0)
100431762eaaSAaron Young txi = VNET_TX_RING_SIZE - 1;
100531762eaaSAaron Young
100631762eaaSAaron Young d = vio_dring_entry(dr, txi);
100731762eaaSAaron Young
100831762eaaSAaron Young if (d->hdr.state == VIO_DESC_READY) {
100931762eaaSAaron Young (*pending)++;
101031762eaaSAaron Young continue;
101131762eaaSAaron Young }
101231762eaaSAaron Young if (port->tx_bufs[txi].skb) {
101331762eaaSAaron Young if (d->hdr.state != VIO_DESC_DONE)
101431762eaaSAaron Young pr_notice("invalid ring buffer state %d\n",
101531762eaaSAaron Young d->hdr.state);
101631762eaaSAaron Young BUG_ON(port->tx_bufs[txi].skb->next);
101731762eaaSAaron Young
101831762eaaSAaron Young port->tx_bufs[txi].skb->next = skb;
101931762eaaSAaron Young skb = port->tx_bufs[txi].skb;
102031762eaaSAaron Young port->tx_bufs[txi].skb = NULL;
102131762eaaSAaron Young
102231762eaaSAaron Young ldc_unmap(port->vio.lp,
102331762eaaSAaron Young port->tx_bufs[txi].cookies,
102431762eaaSAaron Young port->tx_bufs[txi].ncookies);
1025dc153f85SAaron Young } else if (d->hdr.state == VIO_DESC_FREE) {
102631762eaaSAaron Young break;
1027dc153f85SAaron Young }
102831762eaaSAaron Young d->hdr.state = VIO_DESC_FREE;
102931762eaaSAaron Young }
103031762eaaSAaron Young return skb;
103131762eaaSAaron Young }
103231762eaaSAaron Young
vnet_free_skbs(struct sk_buff * skb)103331762eaaSAaron Young static inline void vnet_free_skbs(struct sk_buff *skb)
103431762eaaSAaron Young {
103531762eaaSAaron Young struct sk_buff *next;
103631762eaaSAaron Young
103731762eaaSAaron Young while (skb) {
103831762eaaSAaron Young next = skb->next;
103931762eaaSAaron Young skb->next = NULL;
104031762eaaSAaron Young dev_kfree_skb(skb);
104131762eaaSAaron Young skb = next;
104231762eaaSAaron Young }
104331762eaaSAaron Young }
104431762eaaSAaron Young
sunvnet_clean_timer_expire_common(struct timer_list * t)10450822c5d9SKees Cook void sunvnet_clean_timer_expire_common(struct timer_list *t)
104631762eaaSAaron Young {
10470822c5d9SKees Cook struct vnet_port *port = from_timer(port, t, clean_timer);
104831762eaaSAaron Young struct sk_buff *freeskbs;
104931762eaaSAaron Young unsigned pending;
105031762eaaSAaron Young
105167d0719fSAaron Young netif_tx_lock(VNET_PORT_TO_NET_DEVICE(port));
105231762eaaSAaron Young freeskbs = vnet_clean_tx_ring(port, &pending);
105367d0719fSAaron Young netif_tx_unlock(VNET_PORT_TO_NET_DEVICE(port));
105431762eaaSAaron Young
105531762eaaSAaron Young vnet_free_skbs(freeskbs);
105631762eaaSAaron Young
105731762eaaSAaron Young if (pending)
105831762eaaSAaron Young (void)mod_timer(&port->clean_timer,
105931762eaaSAaron Young jiffies + VNET_CLEAN_TIMEOUT);
106031762eaaSAaron Young else
106131762eaaSAaron Young del_timer(&port->clean_timer);
106231762eaaSAaron Young }
106331762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_clean_timer_expire_common);
106431762eaaSAaron Young
vnet_skb_map(struct ldc_channel * lp,struct sk_buff * skb,struct ldc_trans_cookie * cookies,int ncookies,unsigned int map_perm)106531762eaaSAaron Young static inline int vnet_skb_map(struct ldc_channel *lp, struct sk_buff *skb,
106631762eaaSAaron Young struct ldc_trans_cookie *cookies, int ncookies,
106731762eaaSAaron Young unsigned int map_perm)
106831762eaaSAaron Young {
106931762eaaSAaron Young int i, nc, err, blen;
107031762eaaSAaron Young
107131762eaaSAaron Young /* header */
107231762eaaSAaron Young blen = skb_headlen(skb);
107331762eaaSAaron Young if (blen < ETH_ZLEN)
107431762eaaSAaron Young blen = ETH_ZLEN;
107531762eaaSAaron Young blen += VNET_PACKET_SKIP;
107631762eaaSAaron Young blen += 8 - (blen & 7);
107731762eaaSAaron Young
107831762eaaSAaron Young err = ldc_map_single(lp, skb->data - VNET_PACKET_SKIP, blen, cookies,
107931762eaaSAaron Young ncookies, map_perm);
108031762eaaSAaron Young if (err < 0)
108131762eaaSAaron Young return err;
108231762eaaSAaron Young nc = err;
108331762eaaSAaron Young
108431762eaaSAaron Young for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
108531762eaaSAaron Young skb_frag_t *f = &skb_shinfo(skb)->frags[i];
108631762eaaSAaron Young u8 *vaddr;
108731762eaaSAaron Young
108831762eaaSAaron Young if (nc < ncookies) {
1089350d3513SAnirudh Venkataramanan vaddr = kmap_local_page(skb_frag_page(f));
109031762eaaSAaron Young blen = skb_frag_size(f);
109131762eaaSAaron Young blen += 8 - (blen & 7);
1092b54c9d5bSJonathan Lemon err = ldc_map_single(lp, vaddr + skb_frag_off(f),
109331762eaaSAaron Young blen, cookies + nc, ncookies - nc,
109431762eaaSAaron Young map_perm);
1095350d3513SAnirudh Venkataramanan kunmap_local(vaddr);
109631762eaaSAaron Young } else {
109731762eaaSAaron Young err = -EMSGSIZE;
109831762eaaSAaron Young }
109931762eaaSAaron Young
110031762eaaSAaron Young if (err < 0) {
110131762eaaSAaron Young ldc_unmap(lp, cookies, nc);
110231762eaaSAaron Young return err;
110331762eaaSAaron Young }
110431762eaaSAaron Young nc += err;
110531762eaaSAaron Young }
110631762eaaSAaron Young return nc;
110731762eaaSAaron Young }
110831762eaaSAaron Young
vnet_skb_shape(struct sk_buff * skb,int ncookies)110931762eaaSAaron Young static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, int ncookies)
111031762eaaSAaron Young {
111131762eaaSAaron Young struct sk_buff *nskb;
111231762eaaSAaron Young int i, len, pad, docopy;
111331762eaaSAaron Young
111431762eaaSAaron Young len = skb->len;
111531762eaaSAaron Young pad = 0;
111631762eaaSAaron Young if (len < ETH_ZLEN) {
111731762eaaSAaron Young pad += ETH_ZLEN - skb->len;
111831762eaaSAaron Young len += pad;
111931762eaaSAaron Young }
112031762eaaSAaron Young len += VNET_PACKET_SKIP;
112131762eaaSAaron Young pad += 8 - (len & 7);
112231762eaaSAaron Young
112331762eaaSAaron Young /* make sure we have enough cookies and alignment in every frag */
112431762eaaSAaron Young docopy = skb_shinfo(skb)->nr_frags >= ncookies;
112531762eaaSAaron Young for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
112631762eaaSAaron Young skb_frag_t *f = &skb_shinfo(skb)->frags[i];
112731762eaaSAaron Young
1128b54c9d5bSJonathan Lemon docopy |= skb_frag_off(f) & 7;
112931762eaaSAaron Young }
113031762eaaSAaron Young if (((unsigned long)skb->data & 7) != VNET_PACKET_SKIP ||
113131762eaaSAaron Young skb_tailroom(skb) < pad ||
113231762eaaSAaron Young skb_headroom(skb) < VNET_PACKET_SKIP || docopy) {
113331762eaaSAaron Young int start = 0, offset;
113431762eaaSAaron Young __wsum csum;
113531762eaaSAaron Young
113631762eaaSAaron Young len = skb->len > ETH_ZLEN ? skb->len : ETH_ZLEN;
113731762eaaSAaron Young nskb = alloc_and_align_skb(skb->dev, len);
1138dc153f85SAaron Young if (!nskb) {
113931762eaaSAaron Young dev_kfree_skb(skb);
114031762eaaSAaron Young return NULL;
114131762eaaSAaron Young }
114231762eaaSAaron Young skb_reserve(nskb, VNET_PACKET_SKIP);
114331762eaaSAaron Young
114431762eaaSAaron Young nskb->protocol = skb->protocol;
114531762eaaSAaron Young offset = skb_mac_header(skb) - skb->data;
114631762eaaSAaron Young skb_set_mac_header(nskb, offset);
114731762eaaSAaron Young offset = skb_network_header(skb) - skb->data;
114831762eaaSAaron Young skb_set_network_header(nskb, offset);
114931762eaaSAaron Young offset = skb_transport_header(skb) - skb->data;
115031762eaaSAaron Young skb_set_transport_header(nskb, offset);
115131762eaaSAaron Young
115231762eaaSAaron Young offset = 0;
115331762eaaSAaron Young nskb->csum_offset = skb->csum_offset;
115431762eaaSAaron Young nskb->ip_summed = skb->ip_summed;
115531762eaaSAaron Young
115631762eaaSAaron Young if (skb->ip_summed == CHECKSUM_PARTIAL)
115731762eaaSAaron Young start = skb_checksum_start_offset(skb);
115831762eaaSAaron Young if (start) {
115931762eaaSAaron Young int offset = start + nskb->csum_offset;
116031762eaaSAaron Young
116198524e04SShannon Nelson /* copy the headers, no csum here */
116231762eaaSAaron Young if (skb_copy_bits(skb, 0, nskb->data, start)) {
116331762eaaSAaron Young dev_kfree_skb(nskb);
116431762eaaSAaron Young dev_kfree_skb(skb);
116531762eaaSAaron Young return NULL;
116631762eaaSAaron Young }
116798524e04SShannon Nelson
116898524e04SShannon Nelson /* copy the rest, with csum calculation */
116931762eaaSAaron Young *(__sum16 *)(skb->data + offset) = 0;
117031762eaaSAaron Young csum = skb_copy_and_csum_bits(skb, start,
117131762eaaSAaron Young nskb->data + start,
11728d5930dfSAl Viro skb->len - start);
117398524e04SShannon Nelson
117498524e04SShannon Nelson /* add in the header checksums */
117598524e04SShannon Nelson if (skb->protocol == htons(ETH_P_IP)) {
117698524e04SShannon Nelson struct iphdr *iph = ip_hdr(nskb);
117798524e04SShannon Nelson
117831762eaaSAaron Young if (iph->protocol == IPPROTO_TCP ||
117931762eaaSAaron Young iph->protocol == IPPROTO_UDP) {
118098524e04SShannon Nelson csum = csum_tcpudp_magic(iph->saddr,
118198524e04SShannon Nelson iph->daddr,
118231762eaaSAaron Young skb->len - start,
118398524e04SShannon Nelson iph->protocol,
118498524e04SShannon Nelson csum);
118531762eaaSAaron Young }
118698524e04SShannon Nelson } else if (skb->protocol == htons(ETH_P_IPV6)) {
118798524e04SShannon Nelson struct ipv6hdr *ip6h = ipv6_hdr(nskb);
118898524e04SShannon Nelson
118998524e04SShannon Nelson if (ip6h->nexthdr == IPPROTO_TCP ||
119098524e04SShannon Nelson ip6h->nexthdr == IPPROTO_UDP) {
119198524e04SShannon Nelson csum = csum_ipv6_magic(&ip6h->saddr,
119298524e04SShannon Nelson &ip6h->daddr,
119398524e04SShannon Nelson skb->len - start,
119498524e04SShannon Nelson ip6h->nexthdr,
119598524e04SShannon Nelson csum);
119698524e04SShannon Nelson }
119798524e04SShannon Nelson }
119898524e04SShannon Nelson
119998524e04SShannon Nelson /* save the final result */
120031762eaaSAaron Young *(__sum16 *)(nskb->data + offset) = csum;
120131762eaaSAaron Young
120231762eaaSAaron Young nskb->ip_summed = CHECKSUM_NONE;
120331762eaaSAaron Young } else if (skb_copy_bits(skb, 0, nskb->data, skb->len)) {
120431762eaaSAaron Young dev_kfree_skb(nskb);
120531762eaaSAaron Young dev_kfree_skb(skb);
120631762eaaSAaron Young return NULL;
120731762eaaSAaron Young }
120831762eaaSAaron Young (void)skb_put(nskb, skb->len);
120931762eaaSAaron Young if (skb_is_gso(skb)) {
121031762eaaSAaron Young skb_shinfo(nskb)->gso_size = skb_shinfo(skb)->gso_size;
121131762eaaSAaron Young skb_shinfo(nskb)->gso_type = skb_shinfo(skb)->gso_type;
121231762eaaSAaron Young }
121331762eaaSAaron Young nskb->queue_mapping = skb->queue_mapping;
121431762eaaSAaron Young dev_kfree_skb(skb);
121531762eaaSAaron Young skb = nskb;
121631762eaaSAaron Young }
121731762eaaSAaron Young return skb;
121831762eaaSAaron Young }
121931762eaaSAaron Young
12200e0cc31fSYueHaibing static netdev_tx_t
vnet_handle_offloads(struct vnet_port * port,struct sk_buff * skb,struct vnet_port * (* vnet_tx_port)(struct sk_buff *,struct net_device *))12210e0cc31fSYueHaibing vnet_handle_offloads(struct vnet_port *port, struct sk_buff *skb,
122267d0719fSAaron Young struct vnet_port *(*vnet_tx_port)
122367d0719fSAaron Young (struct sk_buff *, struct net_device *))
122431762eaaSAaron Young {
122567d0719fSAaron Young struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
122631762eaaSAaron Young struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
122790919f14SJason A. Donenfeld struct sk_buff *segs, *curr, *next;
122831762eaaSAaron Young int maclen, datalen;
122931762eaaSAaron Young int status;
123031762eaaSAaron Young int gso_size, gso_type, gso_segs;
123131762eaaSAaron Young int hlen = skb_transport_header(skb) - skb_mac_header(skb);
123231762eaaSAaron Young int proto = IPPROTO_IP;
123331762eaaSAaron Young
123431762eaaSAaron Young if (skb->protocol == htons(ETH_P_IP))
123531762eaaSAaron Young proto = ip_hdr(skb)->protocol;
123631762eaaSAaron Young else if (skb->protocol == htons(ETH_P_IPV6))
123731762eaaSAaron Young proto = ipv6_hdr(skb)->nexthdr;
123831762eaaSAaron Young
1239dc153f85SAaron Young if (proto == IPPROTO_TCP) {
124031762eaaSAaron Young hlen += tcp_hdr(skb)->doff * 4;
1241dc153f85SAaron Young } else if (proto == IPPROTO_UDP) {
124231762eaaSAaron Young hlen += sizeof(struct udphdr);
1243dc153f85SAaron Young } else {
124431762eaaSAaron Young pr_err("vnet_handle_offloads GSO with unknown transport "
124531762eaaSAaron Young "protocol %d tproto %d\n", skb->protocol, proto);
124631762eaaSAaron Young hlen = 128; /* XXX */
124731762eaaSAaron Young }
124831762eaaSAaron Young datalen = port->tsolen - hlen;
124931762eaaSAaron Young
125031762eaaSAaron Young gso_size = skb_shinfo(skb)->gso_size;
125131762eaaSAaron Young gso_type = skb_shinfo(skb)->gso_type;
125231762eaaSAaron Young gso_segs = skb_shinfo(skb)->gso_segs;
125331762eaaSAaron Young
125431762eaaSAaron Young if (port->tso && gso_size < datalen)
125531762eaaSAaron Young gso_segs = DIV_ROUND_UP(skb->len - hlen, datalen);
125631762eaaSAaron Young
125731762eaaSAaron Young if (unlikely(vnet_tx_dring_avail(dr) < gso_segs)) {
125831762eaaSAaron Young struct netdev_queue *txq;
125931762eaaSAaron Young
126031762eaaSAaron Young txq = netdev_get_tx_queue(dev, port->q_index);
126131762eaaSAaron Young netif_tx_stop_queue(txq);
126231762eaaSAaron Young if (vnet_tx_dring_avail(dr) < skb_shinfo(skb)->gso_segs)
126331762eaaSAaron Young return NETDEV_TX_BUSY;
126431762eaaSAaron Young netif_tx_wake_queue(txq);
126531762eaaSAaron Young }
126631762eaaSAaron Young
126731762eaaSAaron Young maclen = skb_network_header(skb) - skb_mac_header(skb);
126831762eaaSAaron Young skb_pull(skb, maclen);
126931762eaaSAaron Young
127031762eaaSAaron Young if (port->tso && gso_size < datalen) {
127131762eaaSAaron Young if (skb_unclone(skb, GFP_ATOMIC))
127231762eaaSAaron Young goto out_dropped;
127331762eaaSAaron Young
127431762eaaSAaron Young /* segment to TSO size */
127531762eaaSAaron Young skb_shinfo(skb)->gso_size = datalen;
127631762eaaSAaron Young skb_shinfo(skb)->gso_segs = gso_segs;
127731762eaaSAaron Young }
127831762eaaSAaron Young segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO);
127931762eaaSAaron Young if (IS_ERR(segs))
128031762eaaSAaron Young goto out_dropped;
128131762eaaSAaron Young
128231762eaaSAaron Young skb_push(skb, maclen);
128331762eaaSAaron Young skb_reset_mac_header(skb);
128431762eaaSAaron Young
128531762eaaSAaron Young status = 0;
128690919f14SJason A. Donenfeld skb_list_walk_safe(segs, curr, next) {
128790919f14SJason A. Donenfeld skb_mark_not_on_list(curr);
128831762eaaSAaron Young if (port->tso && curr->len > dev->mtu) {
128931762eaaSAaron Young skb_shinfo(curr)->gso_size = gso_size;
129031762eaaSAaron Young skb_shinfo(curr)->gso_type = gso_type;
129131762eaaSAaron Young skb_shinfo(curr)->gso_segs =
129231762eaaSAaron Young DIV_ROUND_UP(curr->len - hlen, gso_size);
1293dc153f85SAaron Young } else {
129431762eaaSAaron Young skb_shinfo(curr)->gso_size = 0;
1295dc153f85SAaron Young }
129631762eaaSAaron Young
129731762eaaSAaron Young skb_push(curr, maclen);
129831762eaaSAaron Young skb_reset_mac_header(curr);
129931762eaaSAaron Young memcpy(skb_mac_header(curr), skb_mac_header(skb),
130031762eaaSAaron Young maclen);
130131762eaaSAaron Young curr->csum_start = skb_transport_header(curr) - curr->head;
130231762eaaSAaron Young if (ip_hdr(curr)->protocol == IPPROTO_TCP)
130331762eaaSAaron Young curr->csum_offset = offsetof(struct tcphdr, check);
130431762eaaSAaron Young else if (ip_hdr(curr)->protocol == IPPROTO_UDP)
130531762eaaSAaron Young curr->csum_offset = offsetof(struct udphdr, check);
130631762eaaSAaron Young
130731762eaaSAaron Young if (!(status & NETDEV_TX_MASK))
130867d0719fSAaron Young status = sunvnet_start_xmit_common(curr, dev,
130967d0719fSAaron Young vnet_tx_port);
131031762eaaSAaron Young if (status & NETDEV_TX_MASK)
131131762eaaSAaron Young dev_kfree_skb_any(curr);
131231762eaaSAaron Young }
131331762eaaSAaron Young
131431762eaaSAaron Young if (!(status & NETDEV_TX_MASK))
131531762eaaSAaron Young dev_kfree_skb_any(skb);
131631762eaaSAaron Young return status;
131731762eaaSAaron Young out_dropped:
131831762eaaSAaron Young dev->stats.tx_dropped++;
131931762eaaSAaron Young dev_kfree_skb_any(skb);
132031762eaaSAaron Young return NETDEV_TX_OK;
132131762eaaSAaron Young }
132231762eaaSAaron Young
13230e0cc31fSYueHaibing netdev_tx_t
sunvnet_start_xmit_common(struct sk_buff * skb,struct net_device * dev,struct vnet_port * (* vnet_tx_port)(struct sk_buff *,struct net_device *))13240e0cc31fSYueHaibing sunvnet_start_xmit_common(struct sk_buff *skb, struct net_device *dev,
132567d0719fSAaron Young struct vnet_port *(*vnet_tx_port)
132667d0719fSAaron Young (struct sk_buff *, struct net_device *))
132731762eaaSAaron Young {
132831762eaaSAaron Young struct vnet_port *port = NULL;
132931762eaaSAaron Young struct vio_dring_state *dr;
133031762eaaSAaron Young struct vio_net_desc *d;
133131762eaaSAaron Young unsigned int len;
133231762eaaSAaron Young struct sk_buff *freeskbs = NULL;
133331762eaaSAaron Young int i, err, txi;
133431762eaaSAaron Young unsigned pending = 0;
133531762eaaSAaron Young struct netdev_queue *txq;
133631762eaaSAaron Young
133731762eaaSAaron Young rcu_read_lock();
133867d0719fSAaron Young port = vnet_tx_port(skb, dev);
1339daa86e50SShannon Nelson if (unlikely(!port))
134031762eaaSAaron Young goto out_dropped;
134131762eaaSAaron Young
134231762eaaSAaron Young if (skb_is_gso(skb) && skb->len > port->tsolen) {
134367d0719fSAaron Young err = vnet_handle_offloads(port, skb, vnet_tx_port);
134431762eaaSAaron Young rcu_read_unlock();
134531762eaaSAaron Young return err;
134631762eaaSAaron Young }
134731762eaaSAaron Young
134831762eaaSAaron Young if (!skb_is_gso(skb) && skb->len > port->rmtu) {
134931762eaaSAaron Young unsigned long localmtu = port->rmtu - ETH_HLEN;
135031762eaaSAaron Young
135131762eaaSAaron Young if (vio_version_after_eq(&port->vio, 1, 3))
135231762eaaSAaron Young localmtu -= VLAN_HLEN;
135331762eaaSAaron Young
135467c9a7e1SJason A. Donenfeld if (skb->protocol == htons(ETH_P_IP))
135567c9a7e1SJason A. Donenfeld icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
135631762eaaSAaron Young htonl(localmtu));
135731762eaaSAaron Young #if IS_ENABLED(CONFIG_IPV6)
135831762eaaSAaron Young else if (skb->protocol == htons(ETH_P_IPV6))
135967c9a7e1SJason A. Donenfeld icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, localmtu);
136031762eaaSAaron Young #endif
136131762eaaSAaron Young goto out_dropped;
136231762eaaSAaron Young }
136331762eaaSAaron Young
136431762eaaSAaron Young skb = vnet_skb_shape(skb, 2);
136531762eaaSAaron Young
136631762eaaSAaron Young if (unlikely(!skb))
136731762eaaSAaron Young goto out_dropped;
136831762eaaSAaron Young
136998524e04SShannon Nelson if (skb->ip_summed == CHECKSUM_PARTIAL) {
137098524e04SShannon Nelson if (skb->protocol == htons(ETH_P_IP))
137198524e04SShannon Nelson vnet_fullcsum_ipv4(skb);
137298524e04SShannon Nelson #if IS_ENABLED(CONFIG_IPV6)
137398524e04SShannon Nelson else if (skb->protocol == htons(ETH_P_IPV6))
137498524e04SShannon Nelson vnet_fullcsum_ipv6(skb);
137598524e04SShannon Nelson #endif
137698524e04SShannon Nelson }
137731762eaaSAaron Young
137831762eaaSAaron Young dr = &port->vio.drings[VIO_DRIVER_TX_RING];
137931762eaaSAaron Young i = skb_get_queue_mapping(skb);
138031762eaaSAaron Young txq = netdev_get_tx_queue(dev, i);
138131762eaaSAaron Young if (unlikely(vnet_tx_dring_avail(dr) < 1)) {
138231762eaaSAaron Young if (!netif_tx_queue_stopped(txq)) {
138331762eaaSAaron Young netif_tx_stop_queue(txq);
138431762eaaSAaron Young
138531762eaaSAaron Young /* This is a hard error, log it. */
138631762eaaSAaron Young netdev_err(dev, "BUG! Tx Ring full when queue awake!\n");
138731762eaaSAaron Young dev->stats.tx_errors++;
138831762eaaSAaron Young }
138931762eaaSAaron Young rcu_read_unlock();
139031762eaaSAaron Young return NETDEV_TX_BUSY;
139131762eaaSAaron Young }
139231762eaaSAaron Young
139331762eaaSAaron Young d = vio_dring_cur(dr);
139431762eaaSAaron Young
139531762eaaSAaron Young txi = dr->prod;
139631762eaaSAaron Young
139731762eaaSAaron Young freeskbs = vnet_clean_tx_ring(port, &pending);
139831762eaaSAaron Young
139931762eaaSAaron Young BUG_ON(port->tx_bufs[txi].skb);
140031762eaaSAaron Young
140131762eaaSAaron Young len = skb->len;
140231762eaaSAaron Young if (len < ETH_ZLEN)
140331762eaaSAaron Young len = ETH_ZLEN;
140431762eaaSAaron Young
140531762eaaSAaron Young err = vnet_skb_map(port->vio.lp, skb, port->tx_bufs[txi].cookies, 2,
140631762eaaSAaron Young (LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW));
140731762eaaSAaron Young if (err < 0) {
140831762eaaSAaron Young netdev_info(dev, "tx buffer map error %d\n", err);
140931762eaaSAaron Young goto out_dropped;
141031762eaaSAaron Young }
141131762eaaSAaron Young
141231762eaaSAaron Young port->tx_bufs[txi].skb = skb;
141331762eaaSAaron Young skb = NULL;
141431762eaaSAaron Young port->tx_bufs[txi].ncookies = err;
141531762eaaSAaron Young
141631762eaaSAaron Young /* We don't rely on the ACKs to free the skb in vnet_start_xmit(),
141731762eaaSAaron Young * thus it is safe to not set VIO_ACK_ENABLE for each transmission:
141831762eaaSAaron Young * the protocol itself does not require it as long as the peer
141931762eaaSAaron Young * sends a VIO_SUBTYPE_ACK for VIO_DRING_STOPPED.
142031762eaaSAaron Young *
142131762eaaSAaron Young * An ACK for every packet in the ring is expensive as the
142231762eaaSAaron Young * sending of LDC messages is slow and affects performance.
142331762eaaSAaron Young */
142431762eaaSAaron Young d->hdr.ack = VIO_ACK_DISABLE;
142531762eaaSAaron Young d->size = len;
142631762eaaSAaron Young d->ncookies = port->tx_bufs[txi].ncookies;
142731762eaaSAaron Young for (i = 0; i < d->ncookies; i++)
142831762eaaSAaron Young d->cookies[i] = port->tx_bufs[txi].cookies[i];
142931762eaaSAaron Young if (vio_version_after_eq(&port->vio, 1, 7)) {
143031762eaaSAaron Young struct vio_net_dext *dext = vio_net_ext(d);
143131762eaaSAaron Young
143231762eaaSAaron Young memset(dext, 0, sizeof(*dext));
143331762eaaSAaron Young if (skb_is_gso(port->tx_bufs[txi].skb)) {
143431762eaaSAaron Young dext->ipv4_lso_mss = skb_shinfo(port->tx_bufs[txi].skb)
143531762eaaSAaron Young ->gso_size;
143631762eaaSAaron Young dext->flags |= VNET_PKT_IPV4_LSO;
143731762eaaSAaron Young }
143831762eaaSAaron Young if (vio_version_after_eq(&port->vio, 1, 8) &&
143931762eaaSAaron Young !port->switch_port) {
144031762eaaSAaron Young dext->flags |= VNET_PKT_HCK_IPV4_HDRCKSUM_OK;
144131762eaaSAaron Young dext->flags |= VNET_PKT_HCK_FULLCKSUM_OK;
144231762eaaSAaron Young }
144331762eaaSAaron Young }
144431762eaaSAaron Young
144531762eaaSAaron Young /* This has to be a non-SMP write barrier because we are writing
144631762eaaSAaron Young * to memory which is shared with the peer LDOM.
144731762eaaSAaron Young */
144831762eaaSAaron Young dma_wmb();
144931762eaaSAaron Young
145031762eaaSAaron Young d->hdr.state = VIO_DESC_READY;
145131762eaaSAaron Young
145231762eaaSAaron Young /* Exactly one ldc "start" trigger (for dr->cons) needs to be sent
145331762eaaSAaron Young * to notify the consumer that some descriptors are READY.
145431762eaaSAaron Young * After that "start" trigger, no additional triggers are needed until
145531762eaaSAaron Young * a DRING_STOPPED is received from the consumer. The dr->cons field
145631762eaaSAaron Young * (set up by vnet_ack()) has the value of the next dring index
145731762eaaSAaron Young * that has not yet been ack-ed. We send a "start" trigger here
145831762eaaSAaron Young * if, and only if, start_cons is true (reset it afterward). Conversely,
145931762eaaSAaron Young * vnet_ack() should check if the dring corresponding to cons
146031762eaaSAaron Young * is marked READY, but start_cons was false.
146131762eaaSAaron Young * If so, vnet_ack() should send out the missed "start" trigger.
146231762eaaSAaron Young *
146331762eaaSAaron Young * Note that the dma_wmb() above makes sure the cookies et al. are
146431762eaaSAaron Young * not globally visible before the VIO_DESC_READY, and that the
146531762eaaSAaron Young * stores are ordered correctly by the compiler. The consumer will
146631762eaaSAaron Young * not proceed until the VIO_DESC_READY is visible assuring that
146731762eaaSAaron Young * the consumer does not observe anything related to descriptors
146831762eaaSAaron Young * out of order. The HV trap from the LDC start trigger is the
146931762eaaSAaron Young * producer to consumer announcement that work is available to the
147031762eaaSAaron Young * consumer
147131762eaaSAaron Young */
147231762eaaSAaron Young if (!port->start_cons) { /* previous trigger suffices */
147331762eaaSAaron Young trace_vnet_skip_tx_trigger(port->vio._local_sid,
147431762eaaSAaron Young port->vio._peer_sid, dr->cons);
147531762eaaSAaron Young goto ldc_start_done;
147631762eaaSAaron Young }
147731762eaaSAaron Young
147831762eaaSAaron Young err = __vnet_tx_trigger(port, dr->cons);
147931762eaaSAaron Young if (unlikely(err < 0)) {
148031762eaaSAaron Young netdev_info(dev, "TX trigger error %d\n", err);
148131762eaaSAaron Young d->hdr.state = VIO_DESC_FREE;
148231762eaaSAaron Young skb = port->tx_bufs[txi].skb;
148331762eaaSAaron Young port->tx_bufs[txi].skb = NULL;
148431762eaaSAaron Young dev->stats.tx_carrier_errors++;
148531762eaaSAaron Young goto out_dropped;
148631762eaaSAaron Young }
148731762eaaSAaron Young
148831762eaaSAaron Young ldc_start_done:
148931762eaaSAaron Young port->start_cons = false;
149031762eaaSAaron Young
149131762eaaSAaron Young dev->stats.tx_packets++;
149231762eaaSAaron Young dev->stats.tx_bytes += port->tx_bufs[txi].skb->len;
14930f512c84SShannon Nelson port->stats.tx_packets++;
14940f512c84SShannon Nelson port->stats.tx_bytes += port->tx_bufs[txi].skb->len;
149531762eaaSAaron Young
149631762eaaSAaron Young dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1);
149731762eaaSAaron Young if (unlikely(vnet_tx_dring_avail(dr) < 1)) {
149831762eaaSAaron Young netif_tx_stop_queue(txq);
1499fd263fb6SShannon Nelson smp_rmb();
150031762eaaSAaron Young if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr))
150131762eaaSAaron Young netif_tx_wake_queue(txq);
150231762eaaSAaron Young }
150331762eaaSAaron Young
150431762eaaSAaron Young (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT);
150531762eaaSAaron Young rcu_read_unlock();
150631762eaaSAaron Young
150731762eaaSAaron Young vnet_free_skbs(freeskbs);
150831762eaaSAaron Young
150931762eaaSAaron Young return NETDEV_TX_OK;
151031762eaaSAaron Young
151131762eaaSAaron Young out_dropped:
151231762eaaSAaron Young if (pending)
151331762eaaSAaron Young (void)mod_timer(&port->clean_timer,
151431762eaaSAaron Young jiffies + VNET_CLEAN_TIMEOUT);
151531762eaaSAaron Young else if (port)
151631762eaaSAaron Young del_timer(&port->clean_timer);
151731762eaaSAaron Young rcu_read_unlock();
151831762eaaSAaron Young dev_kfree_skb(skb);
151931762eaaSAaron Young vnet_free_skbs(freeskbs);
152031762eaaSAaron Young dev->stats.tx_dropped++;
152131762eaaSAaron Young return NETDEV_TX_OK;
152231762eaaSAaron Young }
152331762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_start_xmit_common);
152431762eaaSAaron Young
sunvnet_tx_timeout_common(struct net_device * dev,unsigned int txqueue)15250290bd29SMichael S. Tsirkin void sunvnet_tx_timeout_common(struct net_device *dev, unsigned int txqueue)
152631762eaaSAaron Young {
152731762eaaSAaron Young /* XXX Implement me XXX */
152831762eaaSAaron Young }
152931762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_tx_timeout_common);
153031762eaaSAaron Young
sunvnet_open_common(struct net_device * dev)153131762eaaSAaron Young int sunvnet_open_common(struct net_device *dev)
153231762eaaSAaron Young {
153331762eaaSAaron Young netif_carrier_on(dev);
153431762eaaSAaron Young netif_tx_start_all_queues(dev);
153531762eaaSAaron Young
153631762eaaSAaron Young return 0;
153731762eaaSAaron Young }
153831762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_open_common);
153931762eaaSAaron Young
sunvnet_close_common(struct net_device * dev)154031762eaaSAaron Young int sunvnet_close_common(struct net_device *dev)
154131762eaaSAaron Young {
154231762eaaSAaron Young netif_tx_stop_all_queues(dev);
154331762eaaSAaron Young netif_carrier_off(dev);
154431762eaaSAaron Young
154531762eaaSAaron Young return 0;
154631762eaaSAaron Young }
154731762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_close_common);
154831762eaaSAaron Young
__vnet_mc_find(struct vnet * vp,u8 * addr)154931762eaaSAaron Young static struct vnet_mcast_entry *__vnet_mc_find(struct vnet *vp, u8 *addr)
155031762eaaSAaron Young {
155131762eaaSAaron Young struct vnet_mcast_entry *m;
155231762eaaSAaron Young
155331762eaaSAaron Young for (m = vp->mcast_list; m; m = m->next) {
155431762eaaSAaron Young if (ether_addr_equal(m->addr, addr))
155531762eaaSAaron Young return m;
155631762eaaSAaron Young }
155731762eaaSAaron Young return NULL;
155831762eaaSAaron Young }
155931762eaaSAaron Young
__update_mc_list(struct vnet * vp,struct net_device * dev)156031762eaaSAaron Young static void __update_mc_list(struct vnet *vp, struct net_device *dev)
156131762eaaSAaron Young {
156231762eaaSAaron Young struct netdev_hw_addr *ha;
156331762eaaSAaron Young
156431762eaaSAaron Young netdev_for_each_mc_addr(ha, dev) {
156531762eaaSAaron Young struct vnet_mcast_entry *m;
156631762eaaSAaron Young
156731762eaaSAaron Young m = __vnet_mc_find(vp, ha->addr);
156831762eaaSAaron Young if (m) {
156931762eaaSAaron Young m->hit = 1;
157031762eaaSAaron Young continue;
157131762eaaSAaron Young }
157231762eaaSAaron Young
157331762eaaSAaron Young if (!m) {
157431762eaaSAaron Young m = kzalloc(sizeof(*m), GFP_ATOMIC);
157531762eaaSAaron Young if (!m)
157631762eaaSAaron Young continue;
157731762eaaSAaron Young memcpy(m->addr, ha->addr, ETH_ALEN);
157831762eaaSAaron Young m->hit = 1;
157931762eaaSAaron Young
158031762eaaSAaron Young m->next = vp->mcast_list;
158131762eaaSAaron Young vp->mcast_list = m;
158231762eaaSAaron Young }
158331762eaaSAaron Young }
158431762eaaSAaron Young }
158531762eaaSAaron Young
__send_mc_list(struct vnet * vp,struct vnet_port * port)158631762eaaSAaron Young static void __send_mc_list(struct vnet *vp, struct vnet_port *port)
158731762eaaSAaron Young {
158831762eaaSAaron Young struct vio_net_mcast_info info;
158931762eaaSAaron Young struct vnet_mcast_entry *m, **pp;
159031762eaaSAaron Young int n_addrs;
159131762eaaSAaron Young
159231762eaaSAaron Young memset(&info, 0, sizeof(info));
159331762eaaSAaron Young
159431762eaaSAaron Young info.tag.type = VIO_TYPE_CTRL;
159531762eaaSAaron Young info.tag.stype = VIO_SUBTYPE_INFO;
159631762eaaSAaron Young info.tag.stype_env = VNET_MCAST_INFO;
159731762eaaSAaron Young info.tag.sid = vio_send_sid(&port->vio);
159831762eaaSAaron Young info.set = 1;
159931762eaaSAaron Young
160031762eaaSAaron Young n_addrs = 0;
160131762eaaSAaron Young for (m = vp->mcast_list; m; m = m->next) {
160231762eaaSAaron Young if (m->sent)
160331762eaaSAaron Young continue;
160431762eaaSAaron Young m->sent = 1;
160531762eaaSAaron Young memcpy(&info.mcast_addr[n_addrs * ETH_ALEN],
160631762eaaSAaron Young m->addr, ETH_ALEN);
160731762eaaSAaron Young if (++n_addrs == VNET_NUM_MCAST) {
160831762eaaSAaron Young info.count = n_addrs;
160931762eaaSAaron Young
161031762eaaSAaron Young (void)vio_ldc_send(&port->vio, &info,
161131762eaaSAaron Young sizeof(info));
161231762eaaSAaron Young n_addrs = 0;
161331762eaaSAaron Young }
161431762eaaSAaron Young }
161531762eaaSAaron Young if (n_addrs) {
161631762eaaSAaron Young info.count = n_addrs;
161731762eaaSAaron Young (void)vio_ldc_send(&port->vio, &info, sizeof(info));
161831762eaaSAaron Young }
161931762eaaSAaron Young
162031762eaaSAaron Young info.set = 0;
162131762eaaSAaron Young
162231762eaaSAaron Young n_addrs = 0;
162331762eaaSAaron Young pp = &vp->mcast_list;
162431762eaaSAaron Young while ((m = *pp) != NULL) {
162531762eaaSAaron Young if (m->hit) {
162631762eaaSAaron Young m->hit = 0;
162731762eaaSAaron Young pp = &m->next;
162831762eaaSAaron Young continue;
162931762eaaSAaron Young }
163031762eaaSAaron Young
163131762eaaSAaron Young memcpy(&info.mcast_addr[n_addrs * ETH_ALEN],
163231762eaaSAaron Young m->addr, ETH_ALEN);
163331762eaaSAaron Young if (++n_addrs == VNET_NUM_MCAST) {
163431762eaaSAaron Young info.count = n_addrs;
163531762eaaSAaron Young (void)vio_ldc_send(&port->vio, &info,
163631762eaaSAaron Young sizeof(info));
163731762eaaSAaron Young n_addrs = 0;
163831762eaaSAaron Young }
163931762eaaSAaron Young
164031762eaaSAaron Young *pp = m->next;
164131762eaaSAaron Young kfree(m);
164231762eaaSAaron Young }
164331762eaaSAaron Young if (n_addrs) {
164431762eaaSAaron Young info.count = n_addrs;
164531762eaaSAaron Young (void)vio_ldc_send(&port->vio, &info, sizeof(info));
164631762eaaSAaron Young }
164731762eaaSAaron Young }
164831762eaaSAaron Young
sunvnet_set_rx_mode_common(struct net_device * dev,struct vnet * vp)164967d0719fSAaron Young void sunvnet_set_rx_mode_common(struct net_device *dev, struct vnet *vp)
165031762eaaSAaron Young {
165131762eaaSAaron Young struct vnet_port *port;
165231762eaaSAaron Young
165331762eaaSAaron Young rcu_read_lock();
165431762eaaSAaron Young list_for_each_entry_rcu(port, &vp->port_list, list) {
165531762eaaSAaron Young if (port->switch_port) {
165631762eaaSAaron Young __update_mc_list(vp, dev);
165731762eaaSAaron Young __send_mc_list(vp, port);
165831762eaaSAaron Young break;
165931762eaaSAaron Young }
166031762eaaSAaron Young }
166131762eaaSAaron Young rcu_read_unlock();
166231762eaaSAaron Young }
166331762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_set_rx_mode_common);
166431762eaaSAaron Young
sunvnet_set_mac_addr_common(struct net_device * dev,void * p)166531762eaaSAaron Young int sunvnet_set_mac_addr_common(struct net_device *dev, void *p)
166631762eaaSAaron Young {
166731762eaaSAaron Young return -EINVAL;
166831762eaaSAaron Young }
166931762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_set_mac_addr_common);
167031762eaaSAaron Young
sunvnet_port_free_tx_bufs_common(struct vnet_port * port)167131762eaaSAaron Young void sunvnet_port_free_tx_bufs_common(struct vnet_port *port)
167231762eaaSAaron Young {
167331762eaaSAaron Young struct vio_dring_state *dr;
167431762eaaSAaron Young int i;
167531762eaaSAaron Young
167631762eaaSAaron Young dr = &port->vio.drings[VIO_DRIVER_TX_RING];
167731762eaaSAaron Young
1678dc153f85SAaron Young if (!dr->base)
167931762eaaSAaron Young return;
168031762eaaSAaron Young
168131762eaaSAaron Young for (i = 0; i < VNET_TX_RING_SIZE; i++) {
168231762eaaSAaron Young struct vio_net_desc *d;
168331762eaaSAaron Young void *skb = port->tx_bufs[i].skb;
168431762eaaSAaron Young
168531762eaaSAaron Young if (!skb)
168631762eaaSAaron Young continue;
168731762eaaSAaron Young
168831762eaaSAaron Young d = vio_dring_entry(dr, i);
168931762eaaSAaron Young
169031762eaaSAaron Young ldc_unmap(port->vio.lp,
169131762eaaSAaron Young port->tx_bufs[i].cookies,
169231762eaaSAaron Young port->tx_bufs[i].ncookies);
169331762eaaSAaron Young dev_kfree_skb(skb);
169431762eaaSAaron Young port->tx_bufs[i].skb = NULL;
169531762eaaSAaron Young d->hdr.state = VIO_DESC_FREE;
169631762eaaSAaron Young }
169731762eaaSAaron Young ldc_free_exp_dring(port->vio.lp, dr->base,
169831762eaaSAaron Young (dr->entry_size * dr->num_entries),
169931762eaaSAaron Young dr->cookies, dr->ncookies);
170031762eaaSAaron Young dr->base = NULL;
170131762eaaSAaron Young dr->entry_size = 0;
170231762eaaSAaron Young dr->num_entries = 0;
170331762eaaSAaron Young dr->pending = 0;
170431762eaaSAaron Young dr->ncookies = 0;
170531762eaaSAaron Young }
170631762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_port_free_tx_bufs_common);
170731762eaaSAaron Young
vnet_port_reset(struct vnet_port * port)1708867fa150SShannon Nelson void vnet_port_reset(struct vnet_port *port)
170931762eaaSAaron Young {
171031762eaaSAaron Young del_timer(&port->clean_timer);
171131762eaaSAaron Young sunvnet_port_free_tx_bufs_common(port);
171231762eaaSAaron Young port->rmtu = 0;
1713bc221a34SShannon Nelson port->tso = (port->vsw == 0); /* no tso in vsw, misbehaves in bridge */
171431762eaaSAaron Young port->tsolen = 0;
171531762eaaSAaron Young }
1716867fa150SShannon Nelson EXPORT_SYMBOL_GPL(vnet_port_reset);
171731762eaaSAaron Young
vnet_port_alloc_tx_ring(struct vnet_port * port)171831762eaaSAaron Young static int vnet_port_alloc_tx_ring(struct vnet_port *port)
171931762eaaSAaron Young {
172031762eaaSAaron Young struct vio_dring_state *dr;
172131762eaaSAaron Young unsigned long len, elen;
172231762eaaSAaron Young int i, err, ncookies;
172331762eaaSAaron Young void *dring;
172431762eaaSAaron Young
172531762eaaSAaron Young dr = &port->vio.drings[VIO_DRIVER_TX_RING];
172631762eaaSAaron Young
172731762eaaSAaron Young elen = sizeof(struct vio_net_desc) +
172831762eaaSAaron Young sizeof(struct ldc_trans_cookie) * 2;
172931762eaaSAaron Young if (vio_version_after_eq(&port->vio, 1, 7))
173031762eaaSAaron Young elen += sizeof(struct vio_net_dext);
173131762eaaSAaron Young len = VNET_TX_RING_SIZE * elen;
173231762eaaSAaron Young
173331762eaaSAaron Young ncookies = VIO_MAX_RING_COOKIES;
173431762eaaSAaron Young dring = ldc_alloc_exp_dring(port->vio.lp, len,
173531762eaaSAaron Young dr->cookies, &ncookies,
173631762eaaSAaron Young (LDC_MAP_SHADOW |
173731762eaaSAaron Young LDC_MAP_DIRECT |
173831762eaaSAaron Young LDC_MAP_RW));
173931762eaaSAaron Young if (IS_ERR(dring)) {
174031762eaaSAaron Young err = PTR_ERR(dring);
174131762eaaSAaron Young goto err_out;
174231762eaaSAaron Young }
174331762eaaSAaron Young
174431762eaaSAaron Young dr->base = dring;
174531762eaaSAaron Young dr->entry_size = elen;
174631762eaaSAaron Young dr->num_entries = VNET_TX_RING_SIZE;
1747dc153f85SAaron Young dr->prod = 0;
1748dc153f85SAaron Young dr->cons = 0;
174931762eaaSAaron Young port->start_cons = true; /* need an initial trigger */
175031762eaaSAaron Young dr->pending = VNET_TX_RING_SIZE;
175131762eaaSAaron Young dr->ncookies = ncookies;
175231762eaaSAaron Young
175331762eaaSAaron Young for (i = 0; i < VNET_TX_RING_SIZE; ++i) {
175431762eaaSAaron Young struct vio_net_desc *d;
175531762eaaSAaron Young
175631762eaaSAaron Young d = vio_dring_entry(dr, i);
175731762eaaSAaron Young d->hdr.state = VIO_DESC_FREE;
175831762eaaSAaron Young }
175931762eaaSAaron Young return 0;
176031762eaaSAaron Young
176131762eaaSAaron Young err_out:
176231762eaaSAaron Young sunvnet_port_free_tx_bufs_common(port);
176331762eaaSAaron Young
176431762eaaSAaron Young return err;
176531762eaaSAaron Young }
176631762eaaSAaron Young
176731762eaaSAaron Young #ifdef CONFIG_NET_POLL_CONTROLLER
sunvnet_poll_controller_common(struct net_device * dev,struct vnet * vp)176867d0719fSAaron Young void sunvnet_poll_controller_common(struct net_device *dev, struct vnet *vp)
176931762eaaSAaron Young {
177031762eaaSAaron Young struct vnet_port *port;
177131762eaaSAaron Young unsigned long flags;
177231762eaaSAaron Young
177331762eaaSAaron Young spin_lock_irqsave(&vp->lock, flags);
177431762eaaSAaron Young if (!list_empty(&vp->port_list)) {
177531762eaaSAaron Young port = list_entry(vp->port_list.next, struct vnet_port, list);
177631762eaaSAaron Young napi_schedule(&port->napi);
177731762eaaSAaron Young }
177831762eaaSAaron Young spin_unlock_irqrestore(&vp->lock, flags);
177931762eaaSAaron Young }
178031762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_poll_controller_common);
178131762eaaSAaron Young #endif
178231762eaaSAaron Young
sunvnet_port_add_txq_common(struct vnet_port * port)178331762eaaSAaron Young void sunvnet_port_add_txq_common(struct vnet_port *port)
178431762eaaSAaron Young {
178531762eaaSAaron Young struct vnet *vp = port->vp;
1786e1f1e5f7SShannon Nelson int smallest = 0;
1787e1f1e5f7SShannon Nelson int i;
178831762eaaSAaron Young
1789e1f1e5f7SShannon Nelson /* find the first least-used q
1790e1f1e5f7SShannon Nelson * When there are more ldoms than q's, we start to
1791e1f1e5f7SShannon Nelson * double up on ports per queue.
1792e1f1e5f7SShannon Nelson */
1793e1f1e5f7SShannon Nelson for (i = 0; i < VNET_MAX_TXQS; i++) {
1794e1f1e5f7SShannon Nelson if (vp->q_used[i] == 0) {
1795e1f1e5f7SShannon Nelson smallest = i;
1796e1f1e5f7SShannon Nelson break;
1797e1f1e5f7SShannon Nelson }
1798e1f1e5f7SShannon Nelson if (vp->q_used[i] < vp->q_used[smallest])
1799e1f1e5f7SShannon Nelson smallest = i;
1800e1f1e5f7SShannon Nelson }
1801e1f1e5f7SShannon Nelson
1802e1f1e5f7SShannon Nelson vp->nports++;
1803e1f1e5f7SShannon Nelson vp->q_used[smallest]++;
1804e1f1e5f7SShannon Nelson port->q_index = smallest;
180531762eaaSAaron Young }
180631762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_port_add_txq_common);
180731762eaaSAaron Young
sunvnet_port_rm_txq_common(struct vnet_port * port)180831762eaaSAaron Young void sunvnet_port_rm_txq_common(struct vnet_port *port)
180931762eaaSAaron Young {
181031762eaaSAaron Young port->vp->nports--;
1811e1f1e5f7SShannon Nelson port->vp->q_used[port->q_index]--;
1812e1f1e5f7SShannon Nelson port->q_index = 0;
181331762eaaSAaron Young }
181431762eaaSAaron Young EXPORT_SYMBOL_GPL(sunvnet_port_rm_txq_common);
1815