xref: /openbmc/linux/drivers/net/ethernet/sun/sunvnet.c (revision b34e08d5)
1 /* sunvnet.c: Sun LDOM Virtual Network Driver.
2  *
3  * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
4  */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <linux/module.h>
9 #include <linux/kernel.h>
10 #include <linux/types.h>
11 #include <linux/slab.h>
12 #include <linux/delay.h>
13 #include <linux/init.h>
14 #include <linux/netdevice.h>
15 #include <linux/ethtool.h>
16 #include <linux/etherdevice.h>
17 #include <linux/mutex.h>
18 
19 #include <asm/vio.h>
20 #include <asm/ldc.h>
21 
22 #include "sunvnet.h"
23 
24 #define DRV_MODULE_NAME		"sunvnet"
25 #define DRV_MODULE_VERSION	"1.0"
26 #define DRV_MODULE_RELDATE	"June 25, 2007"
27 
28 static char version[] =
29 	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
30 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
31 MODULE_DESCRIPTION("Sun LDOM virtual network driver");
32 MODULE_LICENSE("GPL");
33 MODULE_VERSION(DRV_MODULE_VERSION);
34 
35 /* Ordered from largest major to lowest */
36 static struct vio_version vnet_versions[] = {
37 	{ .major = 1, .minor = 0 },
38 };
39 
40 static inline u32 vnet_tx_dring_avail(struct vio_dring_state *dr)
41 {
42 	return vio_dring_avail(dr, VNET_TX_RING_SIZE);
43 }
44 
45 static int vnet_handle_unknown(struct vnet_port *port, void *arg)
46 {
47 	struct vio_msg_tag *pkt = arg;
48 
49 	pr_err("Received unknown msg [%02x:%02x:%04x:%08x]\n",
50 	       pkt->type, pkt->stype, pkt->stype_env, pkt->sid);
51 	pr_err("Resetting connection\n");
52 
53 	ldc_disconnect(port->vio.lp);
54 
55 	return -ECONNRESET;
56 }
57 
58 static int vnet_send_attr(struct vio_driver_state *vio)
59 {
60 	struct vnet_port *port = to_vnet_port(vio);
61 	struct net_device *dev = port->vp->dev;
62 	struct vio_net_attr_info pkt;
63 	int i;
64 
65 	memset(&pkt, 0, sizeof(pkt));
66 	pkt.tag.type = VIO_TYPE_CTRL;
67 	pkt.tag.stype = VIO_SUBTYPE_INFO;
68 	pkt.tag.stype_env = VIO_ATTR_INFO;
69 	pkt.tag.sid = vio_send_sid(vio);
70 	pkt.xfer_mode = VIO_DRING_MODE;
71 	pkt.addr_type = VNET_ADDR_ETHERMAC;
72 	pkt.ack_freq = 0;
73 	for (i = 0; i < 6; i++)
74 		pkt.addr |= (u64)dev->dev_addr[i] << ((5 - i) * 8);
75 	pkt.mtu = ETH_FRAME_LEN;
76 
77 	viodbg(HS, "SEND NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] "
78 	       "ackfreq[%u] mtu[%llu]\n",
79 	       pkt.xfer_mode, pkt.addr_type,
80 	       (unsigned long long) pkt.addr,
81 	       pkt.ack_freq,
82 	       (unsigned long long) pkt.mtu);
83 
84 	return vio_ldc_send(vio, &pkt, sizeof(pkt));
85 }
86 
87 static int handle_attr_info(struct vio_driver_state *vio,
88 			    struct vio_net_attr_info *pkt)
89 {
90 	viodbg(HS, "GOT NET ATTR INFO xmode[0x%x] atype[0x%x] addr[%llx] "
91 	       "ackfreq[%u] mtu[%llu]\n",
92 	       pkt->xfer_mode, pkt->addr_type,
93 	       (unsigned long long) pkt->addr,
94 	       pkt->ack_freq,
95 	       (unsigned long long) pkt->mtu);
96 
97 	pkt->tag.sid = vio_send_sid(vio);
98 
99 	if (pkt->xfer_mode != VIO_DRING_MODE ||
100 	    pkt->addr_type != VNET_ADDR_ETHERMAC ||
101 	    pkt->mtu != ETH_FRAME_LEN) {
102 		viodbg(HS, "SEND NET ATTR NACK\n");
103 
104 		pkt->tag.stype = VIO_SUBTYPE_NACK;
105 
106 		(void) vio_ldc_send(vio, pkt, sizeof(*pkt));
107 
108 		return -ECONNRESET;
109 	} else {
110 		viodbg(HS, "SEND NET ATTR ACK\n");
111 
112 		pkt->tag.stype = VIO_SUBTYPE_ACK;
113 
114 		return vio_ldc_send(vio, pkt, sizeof(*pkt));
115 	}
116 
117 }
118 
119 static int handle_attr_ack(struct vio_driver_state *vio,
120 			   struct vio_net_attr_info *pkt)
121 {
122 	viodbg(HS, "GOT NET ATTR ACK\n");
123 
124 	return 0;
125 }
126 
127 static int handle_attr_nack(struct vio_driver_state *vio,
128 			    struct vio_net_attr_info *pkt)
129 {
130 	viodbg(HS, "GOT NET ATTR NACK\n");
131 
132 	return -ECONNRESET;
133 }
134 
135 static int vnet_handle_attr(struct vio_driver_state *vio, void *arg)
136 {
137 	struct vio_net_attr_info *pkt = arg;
138 
139 	switch (pkt->tag.stype) {
140 	case VIO_SUBTYPE_INFO:
141 		return handle_attr_info(vio, pkt);
142 
143 	case VIO_SUBTYPE_ACK:
144 		return handle_attr_ack(vio, pkt);
145 
146 	case VIO_SUBTYPE_NACK:
147 		return handle_attr_nack(vio, pkt);
148 
149 	default:
150 		return -ECONNRESET;
151 	}
152 }
153 
154 static void vnet_handshake_complete(struct vio_driver_state *vio)
155 {
156 	struct vio_dring_state *dr;
157 
158 	dr = &vio->drings[VIO_DRIVER_RX_RING];
159 	dr->snd_nxt = dr->rcv_nxt = 1;
160 
161 	dr = &vio->drings[VIO_DRIVER_TX_RING];
162 	dr->snd_nxt = dr->rcv_nxt = 1;
163 }
164 
165 /* The hypervisor interface that implements copying to/from imported
166  * memory from another domain requires that copies are done to 8-byte
167  * aligned buffers, and that the lengths of such copies are also 8-byte
168  * multiples.
169  *
170  * So we align skb->data to an 8-byte multiple and pad-out the data
171  * area so we can round the copy length up to the next multiple of
172  * 8 for the copy.
173  *
174  * The transmitter puts the actual start of the packet 6 bytes into
175  * the buffer it sends over, so that the IP headers after the ethernet
176  * header are aligned properly.  These 6 bytes are not in the descriptor
177  * length, they are simply implied.  This offset is represented using
178  * the VNET_PACKET_SKIP macro.
179  */
180 static struct sk_buff *alloc_and_align_skb(struct net_device *dev,
181 					   unsigned int len)
182 {
183 	struct sk_buff *skb = netdev_alloc_skb(dev, len+VNET_PACKET_SKIP+8+8);
184 	unsigned long addr, off;
185 
186 	if (unlikely(!skb))
187 		return NULL;
188 
189 	addr = (unsigned long) skb->data;
190 	off = ((addr + 7UL) & ~7UL) - addr;
191 	if (off)
192 		skb_reserve(skb, off);
193 
194 	return skb;
195 }
196 
197 static int vnet_rx_one(struct vnet_port *port, unsigned int len,
198 		       struct ldc_trans_cookie *cookies, int ncookies)
199 {
200 	struct net_device *dev = port->vp->dev;
201 	unsigned int copy_len;
202 	struct sk_buff *skb;
203 	int err;
204 
205 	err = -EMSGSIZE;
206 	if (unlikely(len < ETH_ZLEN || len > ETH_FRAME_LEN)) {
207 		dev->stats.rx_length_errors++;
208 		goto out_dropped;
209 	}
210 
211 	skb = alloc_and_align_skb(dev, len);
212 	err = -ENOMEM;
213 	if (unlikely(!skb)) {
214 		dev->stats.rx_missed_errors++;
215 		goto out_dropped;
216 	}
217 
218 	copy_len = (len + VNET_PACKET_SKIP + 7U) & ~7U;
219 	skb_put(skb, copy_len);
220 	err = ldc_copy(port->vio.lp, LDC_COPY_IN,
221 		       skb->data, copy_len, 0,
222 		       cookies, ncookies);
223 	if (unlikely(err < 0)) {
224 		dev->stats.rx_frame_errors++;
225 		goto out_free_skb;
226 	}
227 
228 	skb_pull(skb, VNET_PACKET_SKIP);
229 	skb_trim(skb, len);
230 	skb->protocol = eth_type_trans(skb, dev);
231 
232 	dev->stats.rx_packets++;
233 	dev->stats.rx_bytes += len;
234 
235 	netif_rx(skb);
236 
237 	return 0;
238 
239 out_free_skb:
240 	kfree_skb(skb);
241 
242 out_dropped:
243 	dev->stats.rx_dropped++;
244 	return err;
245 }
246 
247 static int vnet_send_ack(struct vnet_port *port, struct vio_dring_state *dr,
248 			 u32 start, u32 end, u8 vio_dring_state)
249 {
250 	struct vio_dring_data hdr = {
251 		.tag = {
252 			.type		= VIO_TYPE_DATA,
253 			.stype		= VIO_SUBTYPE_ACK,
254 			.stype_env	= VIO_DRING_DATA,
255 			.sid		= vio_send_sid(&port->vio),
256 		},
257 		.dring_ident		= dr->ident,
258 		.start_idx		= start,
259 		.end_idx		= end,
260 		.state			= vio_dring_state,
261 	};
262 	int err, delay;
263 
264 	hdr.seq = dr->snd_nxt;
265 	delay = 1;
266 	do {
267 		err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr));
268 		if (err > 0) {
269 			dr->snd_nxt++;
270 			break;
271 		}
272 		udelay(delay);
273 		if ((delay <<= 1) > 128)
274 			delay = 128;
275 	} while (err == -EAGAIN);
276 
277 	return err;
278 }
279 
280 static u32 next_idx(u32 idx, struct vio_dring_state *dr)
281 {
282 	if (++idx == dr->num_entries)
283 		idx = 0;
284 	return idx;
285 }
286 
287 static u32 prev_idx(u32 idx, struct vio_dring_state *dr)
288 {
289 	if (idx == 0)
290 		idx = dr->num_entries - 1;
291 	else
292 		idx--;
293 
294 	return idx;
295 }
296 
297 static struct vio_net_desc *get_rx_desc(struct vnet_port *port,
298 					struct vio_dring_state *dr,
299 					u32 index)
300 {
301 	struct vio_net_desc *desc = port->vio.desc_buf;
302 	int err;
303 
304 	err = ldc_get_dring_entry(port->vio.lp, desc, dr->entry_size,
305 				  (index * dr->entry_size),
306 				  dr->cookies, dr->ncookies);
307 	if (err < 0)
308 		return ERR_PTR(err);
309 
310 	return desc;
311 }
312 
313 static int put_rx_desc(struct vnet_port *port,
314 		       struct vio_dring_state *dr,
315 		       struct vio_net_desc *desc,
316 		       u32 index)
317 {
318 	int err;
319 
320 	err = ldc_put_dring_entry(port->vio.lp, desc, dr->entry_size,
321 				  (index * dr->entry_size),
322 				  dr->cookies, dr->ncookies);
323 	if (err < 0)
324 		return err;
325 
326 	return 0;
327 }
328 
329 static int vnet_walk_rx_one(struct vnet_port *port,
330 			    struct vio_dring_state *dr,
331 			    u32 index, int *needs_ack)
332 {
333 	struct vio_net_desc *desc = get_rx_desc(port, dr, index);
334 	struct vio_driver_state *vio = &port->vio;
335 	int err;
336 
337 	if (IS_ERR(desc))
338 		return PTR_ERR(desc);
339 
340 	viodbg(DATA, "vio_walk_rx_one desc[%02x:%02x:%08x:%08x:%llx:%llx]\n",
341 	       desc->hdr.state, desc->hdr.ack,
342 	       desc->size, desc->ncookies,
343 	       desc->cookies[0].cookie_addr,
344 	       desc->cookies[0].cookie_size);
345 
346 	if (desc->hdr.state != VIO_DESC_READY)
347 		return 1;
348 	err = vnet_rx_one(port, desc->size, desc->cookies, desc->ncookies);
349 	if (err == -ECONNRESET)
350 		return err;
351 	desc->hdr.state = VIO_DESC_DONE;
352 	err = put_rx_desc(port, dr, desc, index);
353 	if (err < 0)
354 		return err;
355 	*needs_ack = desc->hdr.ack;
356 	return 0;
357 }
358 
359 static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr,
360 			u32 start, u32 end)
361 {
362 	struct vio_driver_state *vio = &port->vio;
363 	int ack_start = -1, ack_end = -1;
364 
365 	end = (end == (u32) -1) ? prev_idx(start, dr) : next_idx(end, dr);
366 
367 	viodbg(DATA, "vnet_walk_rx start[%08x] end[%08x]\n", start, end);
368 
369 	while (start != end) {
370 		int ack = 0, err = vnet_walk_rx_one(port, dr, start, &ack);
371 		if (err == -ECONNRESET)
372 			return err;
373 		if (err != 0)
374 			break;
375 		if (ack_start == -1)
376 			ack_start = start;
377 		ack_end = start;
378 		start = next_idx(start, dr);
379 		if (ack && start != end) {
380 			err = vnet_send_ack(port, dr, ack_start, ack_end,
381 					    VIO_DRING_ACTIVE);
382 			if (err == -ECONNRESET)
383 				return err;
384 			ack_start = -1;
385 		}
386 	}
387 	if (unlikely(ack_start == -1))
388 		ack_start = ack_end = prev_idx(start, dr);
389 	return vnet_send_ack(port, dr, ack_start, ack_end, VIO_DRING_STOPPED);
390 }
391 
392 static int vnet_rx(struct vnet_port *port, void *msgbuf)
393 {
394 	struct vio_dring_data *pkt = msgbuf;
395 	struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_RX_RING];
396 	struct vio_driver_state *vio = &port->vio;
397 
398 	viodbg(DATA, "vnet_rx stype_env[%04x] seq[%016llx] rcv_nxt[%016llx]\n",
399 	       pkt->tag.stype_env, pkt->seq, dr->rcv_nxt);
400 
401 	if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA))
402 		return 0;
403 	if (unlikely(pkt->seq != dr->rcv_nxt)) {
404 		pr_err("RX out of sequence seq[0x%llx] rcv_nxt[0x%llx]\n",
405 		       pkt->seq, dr->rcv_nxt);
406 		return 0;
407 	}
408 
409 	dr->rcv_nxt++;
410 
411 	/* XXX Validate pkt->start_idx and pkt->end_idx XXX */
412 
413 	return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx);
414 }
415 
416 static int idx_is_pending(struct vio_dring_state *dr, u32 end)
417 {
418 	u32 idx = dr->cons;
419 	int found = 0;
420 
421 	while (idx != dr->prod) {
422 		if (idx == end) {
423 			found = 1;
424 			break;
425 		}
426 		idx = next_idx(idx, dr);
427 	}
428 	return found;
429 }
430 
431 static int vnet_ack(struct vnet_port *port, void *msgbuf)
432 {
433 	struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
434 	struct vio_dring_data *pkt = msgbuf;
435 	struct net_device *dev;
436 	struct vnet *vp;
437 	u32 end;
438 
439 	if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA))
440 		return 0;
441 
442 	end = pkt->end_idx;
443 	if (unlikely(!idx_is_pending(dr, end)))
444 		return 0;
445 
446 	dr->cons = next_idx(end, dr);
447 
448 	vp = port->vp;
449 	dev = vp->dev;
450 	if (unlikely(netif_queue_stopped(dev) &&
451 		     vnet_tx_dring_avail(dr) >= VNET_TX_WAKEUP_THRESH(dr)))
452 		return 1;
453 
454 	return 0;
455 }
456 
457 static int vnet_nack(struct vnet_port *port, void *msgbuf)
458 {
459 	/* XXX just reset or similar XXX */
460 	return 0;
461 }
462 
463 static int handle_mcast(struct vnet_port *port, void *msgbuf)
464 {
465 	struct vio_net_mcast_info *pkt = msgbuf;
466 
467 	if (pkt->tag.stype != VIO_SUBTYPE_ACK)
468 		pr_err("%s: Got unexpected MCAST reply [%02x:%02x:%04x:%08x]\n",
469 		       port->vp->dev->name,
470 		       pkt->tag.type,
471 		       pkt->tag.stype,
472 		       pkt->tag.stype_env,
473 		       pkt->tag.sid);
474 
475 	return 0;
476 }
477 
478 static void maybe_tx_wakeup(struct vnet *vp)
479 {
480 	struct net_device *dev = vp->dev;
481 
482 	netif_tx_lock(dev);
483 	if (likely(netif_queue_stopped(dev))) {
484 		struct vnet_port *port;
485 		int wake = 1;
486 
487 		list_for_each_entry(port, &vp->port_list, list) {
488 			struct vio_dring_state *dr;
489 
490 			dr = &port->vio.drings[VIO_DRIVER_TX_RING];
491 			if (vnet_tx_dring_avail(dr) <
492 			    VNET_TX_WAKEUP_THRESH(dr)) {
493 				wake = 0;
494 				break;
495 			}
496 		}
497 		if (wake)
498 			netif_wake_queue(dev);
499 	}
500 	netif_tx_unlock(dev);
501 }
502 
503 static void vnet_event(void *arg, int event)
504 {
505 	struct vnet_port *port = arg;
506 	struct vio_driver_state *vio = &port->vio;
507 	unsigned long flags;
508 	int tx_wakeup, err;
509 
510 	spin_lock_irqsave(&vio->lock, flags);
511 
512 	if (unlikely(event == LDC_EVENT_RESET ||
513 		     event == LDC_EVENT_UP)) {
514 		vio_link_state_change(vio, event);
515 		spin_unlock_irqrestore(&vio->lock, flags);
516 
517 		if (event == LDC_EVENT_RESET)
518 			vio_port_up(vio);
519 		return;
520 	}
521 
522 	if (unlikely(event != LDC_EVENT_DATA_READY)) {
523 		pr_warning("Unexpected LDC event %d\n", event);
524 		spin_unlock_irqrestore(&vio->lock, flags);
525 		return;
526 	}
527 
528 	tx_wakeup = err = 0;
529 	while (1) {
530 		union {
531 			struct vio_msg_tag tag;
532 			u64 raw[8];
533 		} msgbuf;
534 
535 		err = ldc_read(vio->lp, &msgbuf, sizeof(msgbuf));
536 		if (unlikely(err < 0)) {
537 			if (err == -ECONNRESET)
538 				vio_conn_reset(vio);
539 			break;
540 		}
541 		if (err == 0)
542 			break;
543 		viodbg(DATA, "TAG [%02x:%02x:%04x:%08x]\n",
544 		       msgbuf.tag.type,
545 		       msgbuf.tag.stype,
546 		       msgbuf.tag.stype_env,
547 		       msgbuf.tag.sid);
548 		err = vio_validate_sid(vio, &msgbuf.tag);
549 		if (err < 0)
550 			break;
551 
552 		if (likely(msgbuf.tag.type == VIO_TYPE_DATA)) {
553 			if (msgbuf.tag.stype == VIO_SUBTYPE_INFO) {
554 				err = vnet_rx(port, &msgbuf);
555 			} else if (msgbuf.tag.stype == VIO_SUBTYPE_ACK) {
556 				err = vnet_ack(port, &msgbuf);
557 				if (err > 0)
558 					tx_wakeup |= err;
559 			} else if (msgbuf.tag.stype == VIO_SUBTYPE_NACK) {
560 				err = vnet_nack(port, &msgbuf);
561 			}
562 		} else if (msgbuf.tag.type == VIO_TYPE_CTRL) {
563 			if (msgbuf.tag.stype_env == VNET_MCAST_INFO)
564 				err = handle_mcast(port, &msgbuf);
565 			else
566 				err = vio_control_pkt_engine(vio, &msgbuf);
567 			if (err)
568 				break;
569 		} else {
570 			err = vnet_handle_unknown(port, &msgbuf);
571 		}
572 		if (err == -ECONNRESET)
573 			break;
574 	}
575 	spin_unlock(&vio->lock);
576 	if (unlikely(tx_wakeup && err != -ECONNRESET))
577 		maybe_tx_wakeup(port->vp);
578 	local_irq_restore(flags);
579 }
580 
581 static int __vnet_tx_trigger(struct vnet_port *port)
582 {
583 	struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
584 	struct vio_dring_data hdr = {
585 		.tag = {
586 			.type		= VIO_TYPE_DATA,
587 			.stype		= VIO_SUBTYPE_INFO,
588 			.stype_env	= VIO_DRING_DATA,
589 			.sid		= vio_send_sid(&port->vio),
590 		},
591 		.dring_ident		= dr->ident,
592 		.start_idx		= dr->prod,
593 		.end_idx		= (u32) -1,
594 	};
595 	int err, delay;
596 
597 	hdr.seq = dr->snd_nxt;
598 	delay = 1;
599 	do {
600 		err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr));
601 		if (err > 0) {
602 			dr->snd_nxt++;
603 			break;
604 		}
605 		udelay(delay);
606 		if ((delay <<= 1) > 128)
607 			delay = 128;
608 	} while (err == -EAGAIN);
609 
610 	return err;
611 }
612 
613 struct vnet_port *__tx_port_find(struct vnet *vp, struct sk_buff *skb)
614 {
615 	unsigned int hash = vnet_hashfn(skb->data);
616 	struct hlist_head *hp = &vp->port_hash[hash];
617 	struct vnet_port *port;
618 
619 	hlist_for_each_entry(port, hp, hash) {
620 		if (ether_addr_equal(port->raddr, skb->data))
621 			return port;
622 	}
623 	port = NULL;
624 	if (!list_empty(&vp->port_list))
625 		port = list_entry(vp->port_list.next, struct vnet_port, list);
626 
627 	return port;
628 }
629 
630 struct vnet_port *tx_port_find(struct vnet *vp, struct sk_buff *skb)
631 {
632 	struct vnet_port *ret;
633 	unsigned long flags;
634 
635 	spin_lock_irqsave(&vp->lock, flags);
636 	ret = __tx_port_find(vp, skb);
637 	spin_unlock_irqrestore(&vp->lock, flags);
638 
639 	return ret;
640 }
641 
642 static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev)
643 {
644 	struct vnet *vp = netdev_priv(dev);
645 	struct vnet_port *port = tx_port_find(vp, skb);
646 	struct vio_dring_state *dr;
647 	struct vio_net_desc *d;
648 	unsigned long flags;
649 	unsigned int len;
650 	void *tx_buf;
651 	int i, err;
652 
653 	if (unlikely(!port))
654 		goto out_dropped;
655 
656 	spin_lock_irqsave(&port->vio.lock, flags);
657 
658 	dr = &port->vio.drings[VIO_DRIVER_TX_RING];
659 	if (unlikely(vnet_tx_dring_avail(dr) < 2)) {
660 		if (!netif_queue_stopped(dev)) {
661 			netif_stop_queue(dev);
662 
663 			/* This is a hard error, log it. */
664 			netdev_err(dev, "BUG! Tx Ring full when queue awake!\n");
665 			dev->stats.tx_errors++;
666 		}
667 		spin_unlock_irqrestore(&port->vio.lock, flags);
668 		return NETDEV_TX_BUSY;
669 	}
670 
671 	d = vio_dring_cur(dr);
672 
673 	tx_buf = port->tx_bufs[dr->prod].buf;
674 	skb_copy_from_linear_data(skb, tx_buf + VNET_PACKET_SKIP, skb->len);
675 
676 	len = skb->len;
677 	if (len < ETH_ZLEN) {
678 		len = ETH_ZLEN;
679 		memset(tx_buf+VNET_PACKET_SKIP+skb->len, 0, len - skb->len);
680 	}
681 
682 	d->hdr.ack = VIO_ACK_ENABLE;
683 	d->size = len;
684 	d->ncookies = port->tx_bufs[dr->prod].ncookies;
685 	for (i = 0; i < d->ncookies; i++)
686 		d->cookies[i] = port->tx_bufs[dr->prod].cookies[i];
687 
688 	/* This has to be a non-SMP write barrier because we are writing
689 	 * to memory which is shared with the peer LDOM.
690 	 */
691 	wmb();
692 
693 	d->hdr.state = VIO_DESC_READY;
694 
695 	err = __vnet_tx_trigger(port);
696 	if (unlikely(err < 0)) {
697 		netdev_info(dev, "TX trigger error %d\n", err);
698 		d->hdr.state = VIO_DESC_FREE;
699 		dev->stats.tx_carrier_errors++;
700 		goto out_dropped_unlock;
701 	}
702 
703 	dev->stats.tx_packets++;
704 	dev->stats.tx_bytes += skb->len;
705 
706 	dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1);
707 	if (unlikely(vnet_tx_dring_avail(dr) < 2)) {
708 		netif_stop_queue(dev);
709 		if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr))
710 			netif_wake_queue(dev);
711 	}
712 
713 	spin_unlock_irqrestore(&port->vio.lock, flags);
714 
715 	dev_kfree_skb(skb);
716 
717 	return NETDEV_TX_OK;
718 
719 out_dropped_unlock:
720 	spin_unlock_irqrestore(&port->vio.lock, flags);
721 
722 out_dropped:
723 	dev_kfree_skb(skb);
724 	dev->stats.tx_dropped++;
725 	return NETDEV_TX_OK;
726 }
727 
728 static void vnet_tx_timeout(struct net_device *dev)
729 {
730 	/* XXX Implement me XXX */
731 }
732 
733 static int vnet_open(struct net_device *dev)
734 {
735 	netif_carrier_on(dev);
736 	netif_start_queue(dev);
737 
738 	return 0;
739 }
740 
741 static int vnet_close(struct net_device *dev)
742 {
743 	netif_stop_queue(dev);
744 	netif_carrier_off(dev);
745 
746 	return 0;
747 }
748 
749 static struct vnet_mcast_entry *__vnet_mc_find(struct vnet *vp, u8 *addr)
750 {
751 	struct vnet_mcast_entry *m;
752 
753 	for (m = vp->mcast_list; m; m = m->next) {
754 		if (ether_addr_equal(m->addr, addr))
755 			return m;
756 	}
757 	return NULL;
758 }
759 
760 static void __update_mc_list(struct vnet *vp, struct net_device *dev)
761 {
762 	struct netdev_hw_addr *ha;
763 
764 	netdev_for_each_mc_addr(ha, dev) {
765 		struct vnet_mcast_entry *m;
766 
767 		m = __vnet_mc_find(vp, ha->addr);
768 		if (m) {
769 			m->hit = 1;
770 			continue;
771 		}
772 
773 		if (!m) {
774 			m = kzalloc(sizeof(*m), GFP_ATOMIC);
775 			if (!m)
776 				continue;
777 			memcpy(m->addr, ha->addr, ETH_ALEN);
778 			m->hit = 1;
779 
780 			m->next = vp->mcast_list;
781 			vp->mcast_list = m;
782 		}
783 	}
784 }
785 
786 static void __send_mc_list(struct vnet *vp, struct vnet_port *port)
787 {
788 	struct vio_net_mcast_info info;
789 	struct vnet_mcast_entry *m, **pp;
790 	int n_addrs;
791 
792 	memset(&info, 0, sizeof(info));
793 
794 	info.tag.type = VIO_TYPE_CTRL;
795 	info.tag.stype = VIO_SUBTYPE_INFO;
796 	info.tag.stype_env = VNET_MCAST_INFO;
797 	info.tag.sid = vio_send_sid(&port->vio);
798 	info.set = 1;
799 
800 	n_addrs = 0;
801 	for (m = vp->mcast_list; m; m = m->next) {
802 		if (m->sent)
803 			continue;
804 		m->sent = 1;
805 		memcpy(&info.mcast_addr[n_addrs * ETH_ALEN],
806 		       m->addr, ETH_ALEN);
807 		if (++n_addrs == VNET_NUM_MCAST) {
808 			info.count = n_addrs;
809 
810 			(void) vio_ldc_send(&port->vio, &info,
811 					    sizeof(info));
812 			n_addrs = 0;
813 		}
814 	}
815 	if (n_addrs) {
816 		info.count = n_addrs;
817 		(void) vio_ldc_send(&port->vio, &info, sizeof(info));
818 	}
819 
820 	info.set = 0;
821 
822 	n_addrs = 0;
823 	pp = &vp->mcast_list;
824 	while ((m = *pp) != NULL) {
825 		if (m->hit) {
826 			m->hit = 0;
827 			pp = &m->next;
828 			continue;
829 		}
830 
831 		memcpy(&info.mcast_addr[n_addrs * ETH_ALEN],
832 		       m->addr, ETH_ALEN);
833 		if (++n_addrs == VNET_NUM_MCAST) {
834 			info.count = n_addrs;
835 			(void) vio_ldc_send(&port->vio, &info,
836 					    sizeof(info));
837 			n_addrs = 0;
838 		}
839 
840 		*pp = m->next;
841 		kfree(m);
842 	}
843 	if (n_addrs) {
844 		info.count = n_addrs;
845 		(void) vio_ldc_send(&port->vio, &info, sizeof(info));
846 	}
847 }
848 
849 static void vnet_set_rx_mode(struct net_device *dev)
850 {
851 	struct vnet *vp = netdev_priv(dev);
852 	struct vnet_port *port;
853 	unsigned long flags;
854 
855 	spin_lock_irqsave(&vp->lock, flags);
856 	if (!list_empty(&vp->port_list)) {
857 		port = list_entry(vp->port_list.next, struct vnet_port, list);
858 
859 		if (port->switch_port) {
860 			__update_mc_list(vp, dev);
861 			__send_mc_list(vp, port);
862 		}
863 	}
864 	spin_unlock_irqrestore(&vp->lock, flags);
865 }
866 
867 static int vnet_change_mtu(struct net_device *dev, int new_mtu)
868 {
869 	if (new_mtu != ETH_DATA_LEN)
870 		return -EINVAL;
871 
872 	dev->mtu = new_mtu;
873 	return 0;
874 }
875 
876 static int vnet_set_mac_addr(struct net_device *dev, void *p)
877 {
878 	return -EINVAL;
879 }
880 
881 static void vnet_get_drvinfo(struct net_device *dev,
882 			     struct ethtool_drvinfo *info)
883 {
884 	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
885 	strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
886 }
887 
888 static u32 vnet_get_msglevel(struct net_device *dev)
889 {
890 	struct vnet *vp = netdev_priv(dev);
891 	return vp->msg_enable;
892 }
893 
894 static void vnet_set_msglevel(struct net_device *dev, u32 value)
895 {
896 	struct vnet *vp = netdev_priv(dev);
897 	vp->msg_enable = value;
898 }
899 
900 static const struct ethtool_ops vnet_ethtool_ops = {
901 	.get_drvinfo		= vnet_get_drvinfo,
902 	.get_msglevel		= vnet_get_msglevel,
903 	.set_msglevel		= vnet_set_msglevel,
904 	.get_link		= ethtool_op_get_link,
905 };
906 
907 static void vnet_port_free_tx_bufs(struct vnet_port *port)
908 {
909 	struct vio_dring_state *dr;
910 	int i;
911 
912 	dr = &port->vio.drings[VIO_DRIVER_TX_RING];
913 	if (dr->base) {
914 		ldc_free_exp_dring(port->vio.lp, dr->base,
915 				   (dr->entry_size * dr->num_entries),
916 				   dr->cookies, dr->ncookies);
917 		dr->base = NULL;
918 		dr->entry_size = 0;
919 		dr->num_entries = 0;
920 		dr->pending = 0;
921 		dr->ncookies = 0;
922 	}
923 
924 	for (i = 0; i < VNET_TX_RING_SIZE; i++) {
925 		void *buf = port->tx_bufs[i].buf;
926 
927 		if (!buf)
928 			continue;
929 
930 		ldc_unmap(port->vio.lp,
931 			  port->tx_bufs[i].cookies,
932 			  port->tx_bufs[i].ncookies);
933 
934 		kfree(buf);
935 		port->tx_bufs[i].buf = NULL;
936 	}
937 }
938 
939 static int vnet_port_alloc_tx_bufs(struct vnet_port *port)
940 {
941 	struct vio_dring_state *dr;
942 	unsigned long len;
943 	int i, err, ncookies;
944 	void *dring;
945 
946 	for (i = 0; i < VNET_TX_RING_SIZE; i++) {
947 		void *buf = kzalloc(ETH_FRAME_LEN + 8, GFP_KERNEL);
948 		int map_len = (ETH_FRAME_LEN + 7) & ~7;
949 
950 		err = -ENOMEM;
951 		if (!buf)
952 			goto err_out;
953 
954 		err = -EFAULT;
955 		if ((unsigned long)buf & (8UL - 1)) {
956 			pr_err("TX buffer misaligned\n");
957 			kfree(buf);
958 			goto err_out;
959 		}
960 
961 		err = ldc_map_single(port->vio.lp, buf, map_len,
962 				     port->tx_bufs[i].cookies, 2,
963 				     (LDC_MAP_SHADOW |
964 				      LDC_MAP_DIRECT |
965 				      LDC_MAP_RW));
966 		if (err < 0) {
967 			kfree(buf);
968 			goto err_out;
969 		}
970 		port->tx_bufs[i].buf = buf;
971 		port->tx_bufs[i].ncookies = err;
972 	}
973 
974 	dr = &port->vio.drings[VIO_DRIVER_TX_RING];
975 
976 	len = (VNET_TX_RING_SIZE *
977 	       (sizeof(struct vio_net_desc) +
978 		(sizeof(struct ldc_trans_cookie) * 2)));
979 
980 	ncookies = VIO_MAX_RING_COOKIES;
981 	dring = ldc_alloc_exp_dring(port->vio.lp, len,
982 				    dr->cookies, &ncookies,
983 				    (LDC_MAP_SHADOW |
984 				     LDC_MAP_DIRECT |
985 				     LDC_MAP_RW));
986 	if (IS_ERR(dring)) {
987 		err = PTR_ERR(dring);
988 		goto err_out;
989 	}
990 
991 	dr->base = dring;
992 	dr->entry_size = (sizeof(struct vio_net_desc) +
993 			  (sizeof(struct ldc_trans_cookie) * 2));
994 	dr->num_entries = VNET_TX_RING_SIZE;
995 	dr->prod = dr->cons = 0;
996 	dr->pending = VNET_TX_RING_SIZE;
997 	dr->ncookies = ncookies;
998 
999 	return 0;
1000 
1001 err_out:
1002 	vnet_port_free_tx_bufs(port);
1003 
1004 	return err;
1005 }
1006 
1007 static LIST_HEAD(vnet_list);
1008 static DEFINE_MUTEX(vnet_list_mutex);
1009 
1010 static const struct net_device_ops vnet_ops = {
1011 	.ndo_open		= vnet_open,
1012 	.ndo_stop		= vnet_close,
1013 	.ndo_set_rx_mode	= vnet_set_rx_mode,
1014 	.ndo_set_mac_address	= vnet_set_mac_addr,
1015 	.ndo_validate_addr	= eth_validate_addr,
1016 	.ndo_tx_timeout		= vnet_tx_timeout,
1017 	.ndo_change_mtu		= vnet_change_mtu,
1018 	.ndo_start_xmit		= vnet_start_xmit,
1019 };
1020 
1021 static struct vnet *vnet_new(const u64 *local_mac)
1022 {
1023 	struct net_device *dev;
1024 	struct vnet *vp;
1025 	int err, i;
1026 
1027 	dev = alloc_etherdev(sizeof(*vp));
1028 	if (!dev)
1029 		return ERR_PTR(-ENOMEM);
1030 
1031 	for (i = 0; i < ETH_ALEN; i++)
1032 		dev->dev_addr[i] = (*local_mac >> (5 - i) * 8) & 0xff;
1033 
1034 	vp = netdev_priv(dev);
1035 
1036 	spin_lock_init(&vp->lock);
1037 	vp->dev = dev;
1038 
1039 	INIT_LIST_HEAD(&vp->port_list);
1040 	for (i = 0; i < VNET_PORT_HASH_SIZE; i++)
1041 		INIT_HLIST_HEAD(&vp->port_hash[i]);
1042 	INIT_LIST_HEAD(&vp->list);
1043 	vp->local_mac = *local_mac;
1044 
1045 	dev->netdev_ops = &vnet_ops;
1046 	dev->ethtool_ops = &vnet_ethtool_ops;
1047 	dev->watchdog_timeo = VNET_TX_TIMEOUT;
1048 
1049 	err = register_netdev(dev);
1050 	if (err) {
1051 		pr_err("Cannot register net device, aborting\n");
1052 		goto err_out_free_dev;
1053 	}
1054 
1055 	netdev_info(dev, "Sun LDOM vnet %pM\n", dev->dev_addr);
1056 
1057 	list_add(&vp->list, &vnet_list);
1058 
1059 	return vp;
1060 
1061 err_out_free_dev:
1062 	free_netdev(dev);
1063 
1064 	return ERR_PTR(err);
1065 }
1066 
1067 static struct vnet *vnet_find_or_create(const u64 *local_mac)
1068 {
1069 	struct vnet *iter, *vp;
1070 
1071 	mutex_lock(&vnet_list_mutex);
1072 	vp = NULL;
1073 	list_for_each_entry(iter, &vnet_list, list) {
1074 		if (iter->local_mac == *local_mac) {
1075 			vp = iter;
1076 			break;
1077 		}
1078 	}
1079 	if (!vp)
1080 		vp = vnet_new(local_mac);
1081 	mutex_unlock(&vnet_list_mutex);
1082 
1083 	return vp;
1084 }
1085 
1086 static const char *local_mac_prop = "local-mac-address";
1087 
1088 static struct vnet *vnet_find_parent(struct mdesc_handle *hp,
1089 						u64 port_node)
1090 {
1091 	const u64 *local_mac = NULL;
1092 	u64 a;
1093 
1094 	mdesc_for_each_arc(a, hp, port_node, MDESC_ARC_TYPE_BACK) {
1095 		u64 target = mdesc_arc_target(hp, a);
1096 		const char *name;
1097 
1098 		name = mdesc_get_property(hp, target, "name", NULL);
1099 		if (!name || strcmp(name, "network"))
1100 			continue;
1101 
1102 		local_mac = mdesc_get_property(hp, target,
1103 					       local_mac_prop, NULL);
1104 		if (local_mac)
1105 			break;
1106 	}
1107 	if (!local_mac)
1108 		return ERR_PTR(-ENODEV);
1109 
1110 	return vnet_find_or_create(local_mac);
1111 }
1112 
1113 static struct ldc_channel_config vnet_ldc_cfg = {
1114 	.event		= vnet_event,
1115 	.mtu		= 64,
1116 	.mode		= LDC_MODE_UNRELIABLE,
1117 };
1118 
1119 static struct vio_driver_ops vnet_vio_ops = {
1120 	.send_attr		= vnet_send_attr,
1121 	.handle_attr		= vnet_handle_attr,
1122 	.handshake_complete	= vnet_handshake_complete,
1123 };
1124 
1125 static void print_version(void)
1126 {
1127 	printk_once(KERN_INFO "%s", version);
1128 }
1129 
1130 const char *remote_macaddr_prop = "remote-mac-address";
1131 
1132 static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id)
1133 {
1134 	struct mdesc_handle *hp;
1135 	struct vnet_port *port;
1136 	unsigned long flags;
1137 	struct vnet *vp;
1138 	const u64 *rmac;
1139 	int len, i, err, switch_port;
1140 
1141 	print_version();
1142 
1143 	hp = mdesc_grab();
1144 
1145 	vp = vnet_find_parent(hp, vdev->mp);
1146 	if (IS_ERR(vp)) {
1147 		pr_err("Cannot find port parent vnet\n");
1148 		err = PTR_ERR(vp);
1149 		goto err_out_put_mdesc;
1150 	}
1151 
1152 	rmac = mdesc_get_property(hp, vdev->mp, remote_macaddr_prop, &len);
1153 	err = -ENODEV;
1154 	if (!rmac) {
1155 		pr_err("Port lacks %s property\n", remote_macaddr_prop);
1156 		goto err_out_put_mdesc;
1157 	}
1158 
1159 	port = kzalloc(sizeof(*port), GFP_KERNEL);
1160 	err = -ENOMEM;
1161 	if (!port)
1162 		goto err_out_put_mdesc;
1163 
1164 	for (i = 0; i < ETH_ALEN; i++)
1165 		port->raddr[i] = (*rmac >> (5 - i) * 8) & 0xff;
1166 
1167 	port->vp = vp;
1168 
1169 	err = vio_driver_init(&port->vio, vdev, VDEV_NETWORK,
1170 			      vnet_versions, ARRAY_SIZE(vnet_versions),
1171 			      &vnet_vio_ops, vp->dev->name);
1172 	if (err)
1173 		goto err_out_free_port;
1174 
1175 	err = vio_ldc_alloc(&port->vio, &vnet_ldc_cfg, port);
1176 	if (err)
1177 		goto err_out_free_port;
1178 
1179 	err = vnet_port_alloc_tx_bufs(port);
1180 	if (err)
1181 		goto err_out_free_ldc;
1182 
1183 	INIT_HLIST_NODE(&port->hash);
1184 	INIT_LIST_HEAD(&port->list);
1185 
1186 	switch_port = 0;
1187 	if (mdesc_get_property(hp, vdev->mp, "switch-port", NULL) != NULL)
1188 		switch_port = 1;
1189 	port->switch_port = switch_port;
1190 
1191 	spin_lock_irqsave(&vp->lock, flags);
1192 	if (switch_port)
1193 		list_add(&port->list, &vp->port_list);
1194 	else
1195 		list_add_tail(&port->list, &vp->port_list);
1196 	hlist_add_head(&port->hash, &vp->port_hash[vnet_hashfn(port->raddr)]);
1197 	spin_unlock_irqrestore(&vp->lock, flags);
1198 
1199 	dev_set_drvdata(&vdev->dev, port);
1200 
1201 	pr_info("%s: PORT ( remote-mac %pM%s )\n",
1202 		vp->dev->name, port->raddr, switch_port ? " switch-port" : "");
1203 
1204 	vio_port_up(&port->vio);
1205 
1206 	mdesc_release(hp);
1207 
1208 	return 0;
1209 
1210 err_out_free_ldc:
1211 	vio_ldc_free(&port->vio);
1212 
1213 err_out_free_port:
1214 	kfree(port);
1215 
1216 err_out_put_mdesc:
1217 	mdesc_release(hp);
1218 	return err;
1219 }
1220 
1221 static int vnet_port_remove(struct vio_dev *vdev)
1222 {
1223 	struct vnet_port *port = dev_get_drvdata(&vdev->dev);
1224 
1225 	if (port) {
1226 		struct vnet *vp = port->vp;
1227 		unsigned long flags;
1228 
1229 		del_timer_sync(&port->vio.timer);
1230 
1231 		spin_lock_irqsave(&vp->lock, flags);
1232 		list_del(&port->list);
1233 		hlist_del(&port->hash);
1234 		spin_unlock_irqrestore(&vp->lock, flags);
1235 
1236 		vnet_port_free_tx_bufs(port);
1237 		vio_ldc_free(&port->vio);
1238 
1239 		dev_set_drvdata(&vdev->dev, NULL);
1240 
1241 		kfree(port);
1242 
1243 		unregister_netdev(vp->dev);
1244 	}
1245 	return 0;
1246 }
1247 
1248 static const struct vio_device_id vnet_port_match[] = {
1249 	{
1250 		.type = "vnet-port",
1251 	},
1252 	{},
1253 };
1254 MODULE_DEVICE_TABLE(vio, vnet_port_match);
1255 
1256 static struct vio_driver vnet_port_driver = {
1257 	.id_table	= vnet_port_match,
1258 	.probe		= vnet_port_probe,
1259 	.remove		= vnet_port_remove,
1260 	.name		= "vnet_port",
1261 };
1262 
1263 static int __init vnet_init(void)
1264 {
1265 	return vio_register_driver(&vnet_port_driver);
1266 }
1267 
1268 static void __exit vnet_exit(void)
1269 {
1270 	vio_unregister_driver(&vnet_port_driver);
1271 }
1272 
1273 module_init(vnet_init);
1274 module_exit(vnet_exit);
1275