xref: /openbmc/linux/arch/sparc/kernel/ldc.c (revision ca481398)
1 /* ldc.c: Logical Domain Channel link-layer protocol driver.
2  *
3  * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
4  */
5 
6 #include <linux/kernel.h>
7 #include <linux/export.h>
8 #include <linux/slab.h>
9 #include <linux/spinlock.h>
10 #include <linux/delay.h>
11 #include <linux/errno.h>
12 #include <linux/string.h>
13 #include <linux/scatterlist.h>
14 #include <linux/interrupt.h>
15 #include <linux/list.h>
16 #include <linux/init.h>
17 #include <linux/bitmap.h>
18 #include <linux/iommu-common.h>
19 
20 #include <asm/hypervisor.h>
21 #include <asm/iommu.h>
22 #include <asm/page.h>
23 #include <asm/ldc.h>
24 #include <asm/mdesc.h>
25 
26 #define DRV_MODULE_NAME		"ldc"
27 #define PFX DRV_MODULE_NAME	": "
28 #define DRV_MODULE_VERSION	"1.1"
29 #define DRV_MODULE_RELDATE	"July 22, 2008"
30 
31 #define COOKIE_PGSZ_CODE	0xf000000000000000ULL
32 #define COOKIE_PGSZ_CODE_SHIFT	60ULL
33 
34 
35 static char version[] =
36 	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
37 
38 /* Packet header layout for unreliable and reliable mode frames.
39  * When in RAW mode, packets are simply straight 64-byte payloads
40  * with no headers.
41  */
42 struct ldc_packet {
43 	u8			type;
44 #define LDC_CTRL		0x01
45 #define LDC_DATA		0x02
46 #define LDC_ERR			0x10
47 
48 	u8			stype;
49 #define LDC_INFO		0x01
50 #define LDC_ACK			0x02
51 #define LDC_NACK		0x04
52 
53 	u8			ctrl;
54 #define LDC_VERS		0x01 /* Link Version		*/
55 #define LDC_RTS			0x02 /* Request To Send		*/
56 #define LDC_RTR			0x03 /* Ready To Receive	*/
57 #define LDC_RDX			0x04 /* Ready for Data eXchange	*/
58 #define LDC_CTRL_MSK		0x0f
59 
60 	u8			env;
61 #define LDC_LEN			0x3f
62 #define LDC_FRAG_MASK		0xc0
63 #define LDC_START		0x40
64 #define LDC_STOP		0x80
65 
66 	u32			seqid;
67 
68 	union {
69 		u8		u_data[LDC_PACKET_SIZE - 8];
70 		struct {
71 			u32	pad;
72 			u32	ackid;
73 			u8	r_data[LDC_PACKET_SIZE - 8 - 8];
74 		} r;
75 	} u;
76 };
77 
78 struct ldc_version {
79 	u16 major;
80 	u16 minor;
81 };
82 
83 /* Ordered from largest major to lowest.  */
84 static struct ldc_version ver_arr[] = {
85 	{ .major = 1, .minor = 0 },
86 };
87 
88 #define LDC_DEFAULT_MTU			(4 * LDC_PACKET_SIZE)
89 #define LDC_DEFAULT_NUM_ENTRIES		(PAGE_SIZE / LDC_PACKET_SIZE)
90 
91 struct ldc_channel;
92 
93 struct ldc_mode_ops {
94 	int (*write)(struct ldc_channel *, const void *, unsigned int);
95 	int (*read)(struct ldc_channel *, void *, unsigned int);
96 };
97 
98 static const struct ldc_mode_ops raw_ops;
99 static const struct ldc_mode_ops nonraw_ops;
100 static const struct ldc_mode_ops stream_ops;
101 
102 int ldom_domaining_enabled;
103 
104 struct ldc_iommu {
105 	/* Protects ldc_unmap.  */
106 	spinlock_t			lock;
107 	struct ldc_mtable_entry		*page_table;
108 	struct iommu_map_table		iommu_map_table;
109 };
110 
111 struct ldc_channel {
112 	/* Protects all operations that depend upon channel state.  */
113 	spinlock_t			lock;
114 
115 	unsigned long			id;
116 
117 	u8				*mssbuf;
118 	u32				mssbuf_len;
119 	u32				mssbuf_off;
120 
121 	struct ldc_packet		*tx_base;
122 	unsigned long			tx_head;
123 	unsigned long			tx_tail;
124 	unsigned long			tx_num_entries;
125 	unsigned long			tx_ra;
126 
127 	unsigned long			tx_acked;
128 
129 	struct ldc_packet		*rx_base;
130 	unsigned long			rx_head;
131 	unsigned long			rx_tail;
132 	unsigned long			rx_num_entries;
133 	unsigned long			rx_ra;
134 
135 	u32				rcv_nxt;
136 	u32				snd_nxt;
137 
138 	unsigned long			chan_state;
139 
140 	struct ldc_channel_config	cfg;
141 	void				*event_arg;
142 
143 	const struct ldc_mode_ops	*mops;
144 
145 	struct ldc_iommu		iommu;
146 
147 	struct ldc_version		ver;
148 
149 	u8				hs_state;
150 #define LDC_HS_CLOSED			0x00
151 #define LDC_HS_OPEN			0x01
152 #define LDC_HS_GOTVERS			0x02
153 #define LDC_HS_SENTRTR			0x03
154 #define LDC_HS_GOTRTR			0x04
155 #define LDC_HS_COMPLETE			0x10
156 
157 	u8				flags;
158 #define LDC_FLAG_ALLOCED_QUEUES		0x01
159 #define LDC_FLAG_REGISTERED_QUEUES	0x02
160 #define LDC_FLAG_REGISTERED_IRQS	0x04
161 #define LDC_FLAG_RESET			0x10
162 
163 	u8				mss;
164 	u8				state;
165 
166 #define LDC_IRQ_NAME_MAX		32
167 	char				rx_irq_name[LDC_IRQ_NAME_MAX];
168 	char				tx_irq_name[LDC_IRQ_NAME_MAX];
169 
170 	struct hlist_head		mh_list;
171 
172 	struct hlist_node		list;
173 };
174 
175 #define ldcdbg(TYPE, f, a...) \
176 do {	if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
177 		printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
178 } while (0)
179 
180 #define	LDC_ABORT(lp)	ldc_abort((lp), __func__)
181 
182 static const char *state_to_str(u8 state)
183 {
184 	switch (state) {
185 	case LDC_STATE_INVALID:
186 		return "INVALID";
187 	case LDC_STATE_INIT:
188 		return "INIT";
189 	case LDC_STATE_BOUND:
190 		return "BOUND";
191 	case LDC_STATE_READY:
192 		return "READY";
193 	case LDC_STATE_CONNECTED:
194 		return "CONNECTED";
195 	default:
196 		return "<UNKNOWN>";
197 	}
198 }
199 
200 static unsigned long __advance(unsigned long off, unsigned long num_entries)
201 {
202 	off += LDC_PACKET_SIZE;
203 	if (off == (num_entries * LDC_PACKET_SIZE))
204 		off = 0;
205 
206 	return off;
207 }
208 
209 static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off)
210 {
211 	return __advance(off, lp->rx_num_entries);
212 }
213 
214 static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off)
215 {
216 	return __advance(off, lp->tx_num_entries);
217 }
218 
219 static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp,
220 						  unsigned long *new_tail)
221 {
222 	struct ldc_packet *p;
223 	unsigned long t;
224 
225 	t = tx_advance(lp, lp->tx_tail);
226 	if (t == lp->tx_head)
227 		return NULL;
228 
229 	*new_tail = t;
230 
231 	p = lp->tx_base;
232 	return p + (lp->tx_tail / LDC_PACKET_SIZE);
233 }
234 
235 /* When we are in reliable or stream mode, have to track the next packet
236  * we haven't gotten an ACK for in the TX queue using tx_acked.  We have
237  * to be careful not to stomp over the queue past that point.  During
238  * the handshake, we don't have TX data packets pending in the queue
239  * and that's why handshake_get_tx_packet() need not be mindful of
240  * lp->tx_acked.
241  */
242 static unsigned long head_for_data(struct ldc_channel *lp)
243 {
244 	if (lp->cfg.mode == LDC_MODE_STREAM)
245 		return lp->tx_acked;
246 	return lp->tx_head;
247 }
248 
249 static int tx_has_space_for(struct ldc_channel *lp, unsigned int size)
250 {
251 	unsigned long limit, tail, new_tail, diff;
252 	unsigned int mss;
253 
254 	limit = head_for_data(lp);
255 	tail = lp->tx_tail;
256 	new_tail = tx_advance(lp, tail);
257 	if (new_tail == limit)
258 		return 0;
259 
260 	if (limit > new_tail)
261 		diff = limit - new_tail;
262 	else
263 		diff = (limit +
264 			((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail));
265 	diff /= LDC_PACKET_SIZE;
266 	mss = lp->mss;
267 
268 	if (diff * mss < size)
269 		return 0;
270 
271 	return 1;
272 }
273 
274 static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp,
275 					     unsigned long *new_tail)
276 {
277 	struct ldc_packet *p;
278 	unsigned long h, t;
279 
280 	h = head_for_data(lp);
281 	t = tx_advance(lp, lp->tx_tail);
282 	if (t == h)
283 		return NULL;
284 
285 	*new_tail = t;
286 
287 	p = lp->tx_base;
288 	return p + (lp->tx_tail / LDC_PACKET_SIZE);
289 }
290 
291 static int set_tx_tail(struct ldc_channel *lp, unsigned long tail)
292 {
293 	unsigned long orig_tail = lp->tx_tail;
294 	int limit = 1000;
295 
296 	lp->tx_tail = tail;
297 	while (limit-- > 0) {
298 		unsigned long err;
299 
300 		err = sun4v_ldc_tx_set_qtail(lp->id, tail);
301 		if (!err)
302 			return 0;
303 
304 		if (err != HV_EWOULDBLOCK) {
305 			lp->tx_tail = orig_tail;
306 			return -EINVAL;
307 		}
308 		udelay(1);
309 	}
310 
311 	lp->tx_tail = orig_tail;
312 	return -EBUSY;
313 }
314 
315 /* This just updates the head value in the hypervisor using
316  * a polling loop with a timeout.  The caller takes care of
317  * upating software state representing the head change, if any.
318  */
319 static int __set_rx_head(struct ldc_channel *lp, unsigned long head)
320 {
321 	int limit = 1000;
322 
323 	while (limit-- > 0) {
324 		unsigned long err;
325 
326 		err = sun4v_ldc_rx_set_qhead(lp->id, head);
327 		if (!err)
328 			return 0;
329 
330 		if (err != HV_EWOULDBLOCK)
331 			return -EINVAL;
332 
333 		udelay(1);
334 	}
335 
336 	return -EBUSY;
337 }
338 
339 static int send_tx_packet(struct ldc_channel *lp,
340 			  struct ldc_packet *p,
341 			  unsigned long new_tail)
342 {
343 	BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE)));
344 
345 	return set_tx_tail(lp, new_tail);
346 }
347 
348 static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp,
349 						 u8 stype, u8 ctrl,
350 						 void *data, int dlen,
351 						 unsigned long *new_tail)
352 {
353 	struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail);
354 
355 	if (p) {
356 		memset(p, 0, sizeof(*p));
357 		p->type = LDC_CTRL;
358 		p->stype = stype;
359 		p->ctrl = ctrl;
360 		if (data)
361 			memcpy(p->u.u_data, data, dlen);
362 	}
363 	return p;
364 }
365 
366 static int start_handshake(struct ldc_channel *lp)
367 {
368 	struct ldc_packet *p;
369 	struct ldc_version *ver;
370 	unsigned long new_tail;
371 
372 	ver = &ver_arr[0];
373 
374 	ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n",
375 	       ver->major, ver->minor);
376 
377 	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
378 				   ver, sizeof(*ver), &new_tail);
379 	if (p) {
380 		int err = send_tx_packet(lp, p, new_tail);
381 		if (!err)
382 			lp->flags &= ~LDC_FLAG_RESET;
383 		return err;
384 	}
385 	return -EBUSY;
386 }
387 
388 static int send_version_nack(struct ldc_channel *lp,
389 			     u16 major, u16 minor)
390 {
391 	struct ldc_packet *p;
392 	struct ldc_version ver;
393 	unsigned long new_tail;
394 
395 	ver.major = major;
396 	ver.minor = minor;
397 
398 	p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS,
399 				   &ver, sizeof(ver), &new_tail);
400 	if (p) {
401 		ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n",
402 		       ver.major, ver.minor);
403 
404 		return send_tx_packet(lp, p, new_tail);
405 	}
406 	return -EBUSY;
407 }
408 
409 static int send_version_ack(struct ldc_channel *lp,
410 			    struct ldc_version *vp)
411 {
412 	struct ldc_packet *p;
413 	unsigned long new_tail;
414 
415 	p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS,
416 				   vp, sizeof(*vp), &new_tail);
417 	if (p) {
418 		ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n",
419 		       vp->major, vp->minor);
420 
421 		return send_tx_packet(lp, p, new_tail);
422 	}
423 	return -EBUSY;
424 }
425 
426 static int send_rts(struct ldc_channel *lp)
427 {
428 	struct ldc_packet *p;
429 	unsigned long new_tail;
430 
431 	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0,
432 				   &new_tail);
433 	if (p) {
434 		p->env = lp->cfg.mode;
435 		p->seqid = 0;
436 		lp->rcv_nxt = 0;
437 
438 		ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n",
439 		       p->env, p->seqid);
440 
441 		return send_tx_packet(lp, p, new_tail);
442 	}
443 	return -EBUSY;
444 }
445 
446 static int send_rtr(struct ldc_channel *lp)
447 {
448 	struct ldc_packet *p;
449 	unsigned long new_tail;
450 
451 	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0,
452 				   &new_tail);
453 	if (p) {
454 		p->env = lp->cfg.mode;
455 		p->seqid = 0;
456 
457 		ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n",
458 		       p->env, p->seqid);
459 
460 		return send_tx_packet(lp, p, new_tail);
461 	}
462 	return -EBUSY;
463 }
464 
465 static int send_rdx(struct ldc_channel *lp)
466 {
467 	struct ldc_packet *p;
468 	unsigned long new_tail;
469 
470 	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0,
471 				   &new_tail);
472 	if (p) {
473 		p->env = 0;
474 		p->seqid = ++lp->snd_nxt;
475 		p->u.r.ackid = lp->rcv_nxt;
476 
477 		ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n",
478 		       p->env, p->seqid, p->u.r.ackid);
479 
480 		return send_tx_packet(lp, p, new_tail);
481 	}
482 	return -EBUSY;
483 }
484 
485 static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt)
486 {
487 	struct ldc_packet *p;
488 	unsigned long new_tail;
489 	int err;
490 
491 	p = data_get_tx_packet(lp, &new_tail);
492 	if (!p)
493 		return -EBUSY;
494 	memset(p, 0, sizeof(*p));
495 	p->type = data_pkt->type;
496 	p->stype = LDC_NACK;
497 	p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK;
498 	p->seqid = lp->snd_nxt + 1;
499 	p->u.r.ackid = lp->rcv_nxt;
500 
501 	ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n",
502 	       p->type, p->ctrl, p->seqid, p->u.r.ackid);
503 
504 	err = send_tx_packet(lp, p, new_tail);
505 	if (!err)
506 		lp->snd_nxt++;
507 
508 	return err;
509 }
510 
511 static int ldc_abort(struct ldc_channel *lp, const char *msg)
512 {
513 	unsigned long hv_err;
514 
515 	ldcdbg(STATE, "ABORT[%s]\n", msg);
516 	ldc_print(lp);
517 
518 	/* We report but do not act upon the hypervisor errors because
519 	 * there really isn't much we can do if they fail at this point.
520 	 */
521 	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
522 	if (hv_err)
523 		printk(KERN_ERR PFX "ldc_abort: "
524 		       "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
525 		       lp->id, lp->tx_ra, lp->tx_num_entries, hv_err);
526 
527 	hv_err = sun4v_ldc_tx_get_state(lp->id,
528 					&lp->tx_head,
529 					&lp->tx_tail,
530 					&lp->chan_state);
531 	if (hv_err)
532 		printk(KERN_ERR PFX "ldc_abort: "
533 		       "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n",
534 		       lp->id, hv_err);
535 
536 	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
537 	if (hv_err)
538 		printk(KERN_ERR PFX "ldc_abort: "
539 		       "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
540 		       lp->id, lp->rx_ra, lp->rx_num_entries, hv_err);
541 
542 	/* Refetch the RX queue state as well, because we could be invoked
543 	 * here in the queue processing context.
544 	 */
545 	hv_err = sun4v_ldc_rx_get_state(lp->id,
546 					&lp->rx_head,
547 					&lp->rx_tail,
548 					&lp->chan_state);
549 	if (hv_err)
550 		printk(KERN_ERR PFX "ldc_abort: "
551 		       "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n",
552 		       lp->id, hv_err);
553 
554 	return -ECONNRESET;
555 }
556 
557 static struct ldc_version *find_by_major(u16 major)
558 {
559 	struct ldc_version *ret = NULL;
560 	int i;
561 
562 	for (i = 0; i < ARRAY_SIZE(ver_arr); i++) {
563 		struct ldc_version *v = &ver_arr[i];
564 		if (v->major <= major) {
565 			ret = v;
566 			break;
567 		}
568 	}
569 	return ret;
570 }
571 
572 static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp)
573 {
574 	struct ldc_version *vap;
575 	int err;
576 
577 	ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n",
578 	       vp->major, vp->minor);
579 
580 	if (lp->hs_state == LDC_HS_GOTVERS) {
581 		lp->hs_state = LDC_HS_OPEN;
582 		memset(&lp->ver, 0, sizeof(lp->ver));
583 	}
584 
585 	vap = find_by_major(vp->major);
586 	if (!vap) {
587 		err = send_version_nack(lp, 0, 0);
588 	} else if (vap->major != vp->major) {
589 		err = send_version_nack(lp, vap->major, vap->minor);
590 	} else {
591 		struct ldc_version ver = *vp;
592 		if (ver.minor > vap->minor)
593 			ver.minor = vap->minor;
594 		err = send_version_ack(lp, &ver);
595 		if (!err) {
596 			lp->ver = ver;
597 			lp->hs_state = LDC_HS_GOTVERS;
598 		}
599 	}
600 	if (err)
601 		return LDC_ABORT(lp);
602 
603 	return 0;
604 }
605 
606 static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp)
607 {
608 	ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n",
609 	       vp->major, vp->minor);
610 
611 	if (lp->hs_state == LDC_HS_GOTVERS) {
612 		if (lp->ver.major != vp->major ||
613 		    lp->ver.minor != vp->minor)
614 			return LDC_ABORT(lp);
615 	} else {
616 		lp->ver = *vp;
617 		lp->hs_state = LDC_HS_GOTVERS;
618 	}
619 	if (send_rts(lp))
620 		return LDC_ABORT(lp);
621 	return 0;
622 }
623 
624 static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp)
625 {
626 	struct ldc_version *vap;
627 	struct ldc_packet *p;
628 	unsigned long new_tail;
629 
630 	if (vp->major == 0 && vp->minor == 0)
631 		return LDC_ABORT(lp);
632 
633 	vap = find_by_major(vp->major);
634 	if (!vap)
635 		return LDC_ABORT(lp);
636 
637 	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
638 					   vap, sizeof(*vap),
639 					   &new_tail);
640 	if (!p)
641 		return LDC_ABORT(lp);
642 
643 	return send_tx_packet(lp, p, new_tail);
644 }
645 
646 static int process_version(struct ldc_channel *lp,
647 			   struct ldc_packet *p)
648 {
649 	struct ldc_version *vp;
650 
651 	vp = (struct ldc_version *) p->u.u_data;
652 
653 	switch (p->stype) {
654 	case LDC_INFO:
655 		return process_ver_info(lp, vp);
656 
657 	case LDC_ACK:
658 		return process_ver_ack(lp, vp);
659 
660 	case LDC_NACK:
661 		return process_ver_nack(lp, vp);
662 
663 	default:
664 		return LDC_ABORT(lp);
665 	}
666 }
667 
668 static int process_rts(struct ldc_channel *lp,
669 		       struct ldc_packet *p)
670 {
671 	ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n",
672 	       p->stype, p->seqid, p->env);
673 
674 	if (p->stype     != LDC_INFO	   ||
675 	    lp->hs_state != LDC_HS_GOTVERS ||
676 	    p->env       != lp->cfg.mode)
677 		return LDC_ABORT(lp);
678 
679 	lp->snd_nxt = p->seqid;
680 	lp->rcv_nxt = p->seqid;
681 	lp->hs_state = LDC_HS_SENTRTR;
682 	if (send_rtr(lp))
683 		return LDC_ABORT(lp);
684 
685 	return 0;
686 }
687 
688 static int process_rtr(struct ldc_channel *lp,
689 		       struct ldc_packet *p)
690 {
691 	ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n",
692 	       p->stype, p->seqid, p->env);
693 
694 	if (p->stype     != LDC_INFO ||
695 	    p->env       != lp->cfg.mode)
696 		return LDC_ABORT(lp);
697 
698 	lp->snd_nxt = p->seqid;
699 	lp->hs_state = LDC_HS_COMPLETE;
700 	ldc_set_state(lp, LDC_STATE_CONNECTED);
701 	send_rdx(lp);
702 
703 	return LDC_EVENT_UP;
704 }
705 
706 static int rx_seq_ok(struct ldc_channel *lp, u32 seqid)
707 {
708 	return lp->rcv_nxt + 1 == seqid;
709 }
710 
711 static int process_rdx(struct ldc_channel *lp,
712 		       struct ldc_packet *p)
713 {
714 	ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n",
715 	       p->stype, p->seqid, p->env, p->u.r.ackid);
716 
717 	if (p->stype != LDC_INFO ||
718 	    !(rx_seq_ok(lp, p->seqid)))
719 		return LDC_ABORT(lp);
720 
721 	lp->rcv_nxt = p->seqid;
722 
723 	lp->hs_state = LDC_HS_COMPLETE;
724 	ldc_set_state(lp, LDC_STATE_CONNECTED);
725 
726 	return LDC_EVENT_UP;
727 }
728 
729 static int process_control_frame(struct ldc_channel *lp,
730 				 struct ldc_packet *p)
731 {
732 	switch (p->ctrl) {
733 	case LDC_VERS:
734 		return process_version(lp, p);
735 
736 	case LDC_RTS:
737 		return process_rts(lp, p);
738 
739 	case LDC_RTR:
740 		return process_rtr(lp, p);
741 
742 	case LDC_RDX:
743 		return process_rdx(lp, p);
744 
745 	default:
746 		return LDC_ABORT(lp);
747 	}
748 }
749 
750 static int process_error_frame(struct ldc_channel *lp,
751 			       struct ldc_packet *p)
752 {
753 	return LDC_ABORT(lp);
754 }
755 
756 static int process_data_ack(struct ldc_channel *lp,
757 			    struct ldc_packet *ack)
758 {
759 	unsigned long head = lp->tx_acked;
760 	u32 ackid = ack->u.r.ackid;
761 
762 	while (1) {
763 		struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE);
764 
765 		head = tx_advance(lp, head);
766 
767 		if (p->seqid == ackid) {
768 			lp->tx_acked = head;
769 			return 0;
770 		}
771 		if (head == lp->tx_tail)
772 			return LDC_ABORT(lp);
773 	}
774 
775 	return 0;
776 }
777 
778 static void send_events(struct ldc_channel *lp, unsigned int event_mask)
779 {
780 	if (event_mask & LDC_EVENT_RESET)
781 		lp->cfg.event(lp->event_arg, LDC_EVENT_RESET);
782 	if (event_mask & LDC_EVENT_UP)
783 		lp->cfg.event(lp->event_arg, LDC_EVENT_UP);
784 	if (event_mask & LDC_EVENT_DATA_READY)
785 		lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY);
786 }
787 
788 static irqreturn_t ldc_rx(int irq, void *dev_id)
789 {
790 	struct ldc_channel *lp = dev_id;
791 	unsigned long orig_state, flags;
792 	unsigned int event_mask;
793 
794 	spin_lock_irqsave(&lp->lock, flags);
795 
796 	orig_state = lp->chan_state;
797 
798 	/* We should probably check for hypervisor errors here and
799 	 * reset the LDC channel if we get one.
800 	 */
801 	sun4v_ldc_rx_get_state(lp->id,
802 			       &lp->rx_head,
803 			       &lp->rx_tail,
804 			       &lp->chan_state);
805 
806 	ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
807 	       orig_state, lp->chan_state, lp->rx_head, lp->rx_tail);
808 
809 	event_mask = 0;
810 
811 	if (lp->cfg.mode == LDC_MODE_RAW &&
812 	    lp->chan_state == LDC_CHANNEL_UP) {
813 		lp->hs_state = LDC_HS_COMPLETE;
814 		ldc_set_state(lp, LDC_STATE_CONNECTED);
815 
816 		/*
817 		 * Generate an LDC_EVENT_UP event if the channel
818 		 * was not already up.
819 		 */
820 		if (orig_state != LDC_CHANNEL_UP) {
821 			event_mask |= LDC_EVENT_UP;
822 			orig_state = lp->chan_state;
823 		}
824 	}
825 
826 	/* If we are in reset state, flush the RX queue and ignore
827 	 * everything.
828 	 */
829 	if (lp->flags & LDC_FLAG_RESET) {
830 		(void) ldc_rx_reset(lp);
831 		goto out;
832 	}
833 
834 	/* Once we finish the handshake, we let the ldc_read()
835 	 * paths do all of the control frame and state management.
836 	 * Just trigger the callback.
837 	 */
838 	if (lp->hs_state == LDC_HS_COMPLETE) {
839 handshake_complete:
840 		if (lp->chan_state != orig_state) {
841 			unsigned int event = LDC_EVENT_RESET;
842 
843 			if (lp->chan_state == LDC_CHANNEL_UP)
844 				event = LDC_EVENT_UP;
845 
846 			event_mask |= event;
847 		}
848 		if (lp->rx_head != lp->rx_tail)
849 			event_mask |= LDC_EVENT_DATA_READY;
850 
851 		goto out;
852 	}
853 
854 	if (lp->chan_state != orig_state)
855 		goto out;
856 
857 	while (lp->rx_head != lp->rx_tail) {
858 		struct ldc_packet *p;
859 		unsigned long new;
860 		int err;
861 
862 		p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
863 
864 		switch (p->type) {
865 		case LDC_CTRL:
866 			err = process_control_frame(lp, p);
867 			if (err > 0)
868 				event_mask |= err;
869 			break;
870 
871 		case LDC_DATA:
872 			event_mask |= LDC_EVENT_DATA_READY;
873 			err = 0;
874 			break;
875 
876 		case LDC_ERR:
877 			err = process_error_frame(lp, p);
878 			break;
879 
880 		default:
881 			err = LDC_ABORT(lp);
882 			break;
883 		}
884 
885 		if (err < 0)
886 			break;
887 
888 		new = lp->rx_head;
889 		new += LDC_PACKET_SIZE;
890 		if (new == (lp->rx_num_entries * LDC_PACKET_SIZE))
891 			new = 0;
892 		lp->rx_head = new;
893 
894 		err = __set_rx_head(lp, new);
895 		if (err < 0) {
896 			(void) LDC_ABORT(lp);
897 			break;
898 		}
899 		if (lp->hs_state == LDC_HS_COMPLETE)
900 			goto handshake_complete;
901 	}
902 
903 out:
904 	spin_unlock_irqrestore(&lp->lock, flags);
905 
906 	send_events(lp, event_mask);
907 
908 	return IRQ_HANDLED;
909 }
910 
911 static irqreturn_t ldc_tx(int irq, void *dev_id)
912 {
913 	struct ldc_channel *lp = dev_id;
914 	unsigned long flags, orig_state;
915 	unsigned int event_mask = 0;
916 
917 	spin_lock_irqsave(&lp->lock, flags);
918 
919 	orig_state = lp->chan_state;
920 
921 	/* We should probably check for hypervisor errors here and
922 	 * reset the LDC channel if we get one.
923 	 */
924 	sun4v_ldc_tx_get_state(lp->id,
925 			       &lp->tx_head,
926 			       &lp->tx_tail,
927 			       &lp->chan_state);
928 
929 	ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
930 	       orig_state, lp->chan_state, lp->tx_head, lp->tx_tail);
931 
932 	if (lp->cfg.mode == LDC_MODE_RAW &&
933 	    lp->chan_state == LDC_CHANNEL_UP) {
934 		lp->hs_state = LDC_HS_COMPLETE;
935 		ldc_set_state(lp, LDC_STATE_CONNECTED);
936 
937 		/*
938 		 * Generate an LDC_EVENT_UP event if the channel
939 		 * was not already up.
940 		 */
941 		if (orig_state != LDC_CHANNEL_UP) {
942 			event_mask |= LDC_EVENT_UP;
943 			orig_state = lp->chan_state;
944 		}
945 	}
946 
947 	spin_unlock_irqrestore(&lp->lock, flags);
948 
949 	send_events(lp, event_mask);
950 
951 	return IRQ_HANDLED;
952 }
953 
954 /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so
955  * XXX that addition and removal from the ldc_channel_list has
956  * XXX atomicity, otherwise the __ldc_channel_exists() check is
957  * XXX totally pointless as another thread can slip into ldc_alloc()
958  * XXX and add a channel with the same ID.  There also needs to be
959  * XXX a spinlock for ldc_channel_list.
960  */
961 static HLIST_HEAD(ldc_channel_list);
962 
963 static int __ldc_channel_exists(unsigned long id)
964 {
965 	struct ldc_channel *lp;
966 
967 	hlist_for_each_entry(lp, &ldc_channel_list, list) {
968 		if (lp->id == id)
969 			return 1;
970 	}
971 	return 0;
972 }
973 
974 static int alloc_queue(const char *name, unsigned long num_entries,
975 		       struct ldc_packet **base, unsigned long *ra)
976 {
977 	unsigned long size, order;
978 	void *q;
979 
980 	size = num_entries * LDC_PACKET_SIZE;
981 	order = get_order(size);
982 
983 	q = (void *) __get_free_pages(GFP_KERNEL, order);
984 	if (!q) {
985 		printk(KERN_ERR PFX "Alloc of %s queue failed with "
986 		       "size=%lu order=%lu\n", name, size, order);
987 		return -ENOMEM;
988 	}
989 
990 	memset(q, 0, PAGE_SIZE << order);
991 
992 	*base = q;
993 	*ra = __pa(q);
994 
995 	return 0;
996 }
997 
998 static void free_queue(unsigned long num_entries, struct ldc_packet *q)
999 {
1000 	unsigned long size, order;
1001 
1002 	if (!q)
1003 		return;
1004 
1005 	size = num_entries * LDC_PACKET_SIZE;
1006 	order = get_order(size);
1007 
1008 	free_pages((unsigned long)q, order);
1009 }
1010 
1011 static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
1012 {
1013 	u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
1014 	/* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */
1015 
1016 	cookie &= ~COOKIE_PGSZ_CODE;
1017 
1018 	return (cookie >> (13ULL + (szcode * 3ULL)));
1019 }
1020 
1021 static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,
1022 		      unsigned long entry, unsigned long npages)
1023 {
1024 	struct ldc_mtable_entry *base;
1025 	unsigned long i, shift;
1026 
1027 	shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
1028 	base = iommu->page_table + entry;
1029 	for (i = 0; i < npages; i++) {
1030 		if (base->cookie)
1031 			sun4v_ldc_revoke(id, cookie + (i << shift),
1032 					 base->cookie);
1033 		base->mte = 0;
1034 	}
1035 }
1036 
1037 /* XXX Make this configurable... XXX */
1038 #define LDC_IOTABLE_SIZE	(8 * 1024)
1039 
1040 static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
1041 {
1042 	unsigned long sz, num_tsb_entries, tsbsize, order;
1043 	struct ldc_iommu *ldc_iommu = &lp->iommu;
1044 	struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
1045 	struct ldc_mtable_entry *table;
1046 	unsigned long hv_err;
1047 	int err;
1048 
1049 	num_tsb_entries = LDC_IOTABLE_SIZE;
1050 	tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1051 	spin_lock_init(&ldc_iommu->lock);
1052 
1053 	sz = num_tsb_entries / 8;
1054 	sz = (sz + 7UL) & ~7UL;
1055 	iommu->map = kzalloc(sz, GFP_KERNEL);
1056 	if (!iommu->map) {
1057 		printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
1058 		return -ENOMEM;
1059 	}
1060 	iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
1061 			    NULL, false /* no large pool */,
1062 			    1 /* npools */,
1063 			    true /* skip span boundary check */);
1064 
1065 	order = get_order(tsbsize);
1066 
1067 	table = (struct ldc_mtable_entry *)
1068 		__get_free_pages(GFP_KERNEL, order);
1069 	err = -ENOMEM;
1070 	if (!table) {
1071 		printk(KERN_ERR PFX "Alloc of MTE table failed, "
1072 		       "size=%lu order=%lu\n", tsbsize, order);
1073 		goto out_free_map;
1074 	}
1075 
1076 	memset(table, 0, PAGE_SIZE << order);
1077 
1078 	ldc_iommu->page_table = table;
1079 
1080 	hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
1081 					 num_tsb_entries);
1082 	err = -EINVAL;
1083 	if (hv_err)
1084 		goto out_free_table;
1085 
1086 	return 0;
1087 
1088 out_free_table:
1089 	free_pages((unsigned long) table, order);
1090 	ldc_iommu->page_table = NULL;
1091 
1092 out_free_map:
1093 	kfree(iommu->map);
1094 	iommu->map = NULL;
1095 
1096 	return err;
1097 }
1098 
1099 static void ldc_iommu_release(struct ldc_channel *lp)
1100 {
1101 	struct ldc_iommu *ldc_iommu = &lp->iommu;
1102 	struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
1103 	unsigned long num_tsb_entries, tsbsize, order;
1104 
1105 	(void) sun4v_ldc_set_map_table(lp->id, 0, 0);
1106 
1107 	num_tsb_entries = iommu->poolsize * iommu->nr_pools;
1108 	tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1109 	order = get_order(tsbsize);
1110 
1111 	free_pages((unsigned long) ldc_iommu->page_table, order);
1112 	ldc_iommu->page_table = NULL;
1113 
1114 	kfree(iommu->map);
1115 	iommu->map = NULL;
1116 }
1117 
1118 struct ldc_channel *ldc_alloc(unsigned long id,
1119 			      const struct ldc_channel_config *cfgp,
1120 			      void *event_arg,
1121 			      const char *name)
1122 {
1123 	struct ldc_channel *lp;
1124 	const struct ldc_mode_ops *mops;
1125 	unsigned long dummy1, dummy2, hv_err;
1126 	u8 mss, *mssbuf;
1127 	int err;
1128 
1129 	err = -ENODEV;
1130 	if (!ldom_domaining_enabled)
1131 		goto out_err;
1132 
1133 	err = -EINVAL;
1134 	if (!cfgp)
1135 		goto out_err;
1136 	if (!name)
1137 		goto out_err;
1138 
1139 	switch (cfgp->mode) {
1140 	case LDC_MODE_RAW:
1141 		mops = &raw_ops;
1142 		mss = LDC_PACKET_SIZE;
1143 		break;
1144 
1145 	case LDC_MODE_UNRELIABLE:
1146 		mops = &nonraw_ops;
1147 		mss = LDC_PACKET_SIZE - 8;
1148 		break;
1149 
1150 	case LDC_MODE_STREAM:
1151 		mops = &stream_ops;
1152 		mss = LDC_PACKET_SIZE - 8 - 8;
1153 		break;
1154 
1155 	default:
1156 		goto out_err;
1157 	}
1158 
1159 	if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq)
1160 		goto out_err;
1161 
1162 	hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2);
1163 	err = -ENODEV;
1164 	if (hv_err == HV_ECHANNEL)
1165 		goto out_err;
1166 
1167 	err = -EEXIST;
1168 	if (__ldc_channel_exists(id))
1169 		goto out_err;
1170 
1171 	mssbuf = NULL;
1172 
1173 	lp = kzalloc(sizeof(*lp), GFP_KERNEL);
1174 	err = -ENOMEM;
1175 	if (!lp)
1176 		goto out_err;
1177 
1178 	spin_lock_init(&lp->lock);
1179 
1180 	lp->id = id;
1181 
1182 	err = ldc_iommu_init(name, lp);
1183 	if (err)
1184 		goto out_free_ldc;
1185 
1186 	lp->mops = mops;
1187 	lp->mss = mss;
1188 
1189 	lp->cfg = *cfgp;
1190 	if (!lp->cfg.mtu)
1191 		lp->cfg.mtu = LDC_DEFAULT_MTU;
1192 
1193 	if (lp->cfg.mode == LDC_MODE_STREAM) {
1194 		mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL);
1195 		if (!mssbuf) {
1196 			err = -ENOMEM;
1197 			goto out_free_iommu;
1198 		}
1199 		lp->mssbuf = mssbuf;
1200 	}
1201 
1202 	lp->event_arg = event_arg;
1203 
1204 	/* XXX allow setting via ldc_channel_config to override defaults
1205 	 * XXX or use some formula based upon mtu
1206 	 */
1207 	lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
1208 	lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
1209 
1210 	err = alloc_queue("TX", lp->tx_num_entries,
1211 			  &lp->tx_base, &lp->tx_ra);
1212 	if (err)
1213 		goto out_free_mssbuf;
1214 
1215 	err = alloc_queue("RX", lp->rx_num_entries,
1216 			  &lp->rx_base, &lp->rx_ra);
1217 	if (err)
1218 		goto out_free_txq;
1219 
1220 	lp->flags |= LDC_FLAG_ALLOCED_QUEUES;
1221 
1222 	lp->hs_state = LDC_HS_CLOSED;
1223 	ldc_set_state(lp, LDC_STATE_INIT);
1224 
1225 	INIT_HLIST_NODE(&lp->list);
1226 	hlist_add_head(&lp->list, &ldc_channel_list);
1227 
1228 	INIT_HLIST_HEAD(&lp->mh_list);
1229 
1230 	snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
1231 	snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
1232 
1233 	err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
1234 			  lp->rx_irq_name, lp);
1235 	if (err)
1236 		goto out_free_txq;
1237 
1238 	err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
1239 			  lp->tx_irq_name, lp);
1240 	if (err) {
1241 		free_irq(lp->cfg.rx_irq, lp);
1242 		goto out_free_txq;
1243 	}
1244 
1245 	return lp;
1246 
1247 out_free_txq:
1248 	free_queue(lp->tx_num_entries, lp->tx_base);
1249 
1250 out_free_mssbuf:
1251 	kfree(mssbuf);
1252 
1253 out_free_iommu:
1254 	ldc_iommu_release(lp);
1255 
1256 out_free_ldc:
1257 	kfree(lp);
1258 
1259 out_err:
1260 	return ERR_PTR(err);
1261 }
1262 EXPORT_SYMBOL(ldc_alloc);
1263 
1264 void ldc_unbind(struct ldc_channel *lp)
1265 {
1266 	if (lp->flags & LDC_FLAG_REGISTERED_IRQS) {
1267 		free_irq(lp->cfg.rx_irq, lp);
1268 		free_irq(lp->cfg.tx_irq, lp);
1269 		lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
1270 	}
1271 
1272 	if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) {
1273 		sun4v_ldc_tx_qconf(lp->id, 0, 0);
1274 		sun4v_ldc_rx_qconf(lp->id, 0, 0);
1275 		lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
1276 	}
1277 	if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) {
1278 		free_queue(lp->tx_num_entries, lp->tx_base);
1279 		free_queue(lp->rx_num_entries, lp->rx_base);
1280 		lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES;
1281 	}
1282 
1283 	ldc_set_state(lp, LDC_STATE_INIT);
1284 }
1285 EXPORT_SYMBOL(ldc_unbind);
1286 
1287 void ldc_free(struct ldc_channel *lp)
1288 {
1289 	ldc_unbind(lp);
1290 	hlist_del(&lp->list);
1291 	kfree(lp->mssbuf);
1292 	ldc_iommu_release(lp);
1293 
1294 	kfree(lp);
1295 }
1296 EXPORT_SYMBOL(ldc_free);
1297 
1298 /* Bind the channel.  This registers the LDC queues with
1299  * the hypervisor and puts the channel into a pseudo-listening
1300  * state.  This does not initiate a handshake, ldc_connect() does
1301  * that.
1302  */
1303 int ldc_bind(struct ldc_channel *lp)
1304 {
1305 	unsigned long hv_err, flags;
1306 	int err = -EINVAL;
1307 
1308 	if (lp->state != LDC_STATE_INIT)
1309 		return -EINVAL;
1310 
1311 	spin_lock_irqsave(&lp->lock, flags);
1312 
1313 	enable_irq(lp->cfg.rx_irq);
1314 	enable_irq(lp->cfg.tx_irq);
1315 
1316 	lp->flags |= LDC_FLAG_REGISTERED_IRQS;
1317 
1318 	err = -ENODEV;
1319 	hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
1320 	if (hv_err)
1321 		goto out_free_irqs;
1322 
1323 	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
1324 	if (hv_err)
1325 		goto out_free_irqs;
1326 
1327 	hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
1328 	if (hv_err)
1329 		goto out_unmap_tx;
1330 
1331 	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
1332 	if (hv_err)
1333 		goto out_unmap_tx;
1334 
1335 	lp->flags |= LDC_FLAG_REGISTERED_QUEUES;
1336 
1337 	hv_err = sun4v_ldc_tx_get_state(lp->id,
1338 					&lp->tx_head,
1339 					&lp->tx_tail,
1340 					&lp->chan_state);
1341 	err = -EBUSY;
1342 	if (hv_err)
1343 		goto out_unmap_rx;
1344 
1345 	lp->tx_acked = lp->tx_head;
1346 
1347 	lp->hs_state = LDC_HS_OPEN;
1348 	ldc_set_state(lp, LDC_STATE_BOUND);
1349 
1350 	if (lp->cfg.mode == LDC_MODE_RAW) {
1351 		/*
1352 		 * There is no handshake in RAW mode, so handshake
1353 		 * is completed.
1354 		 */
1355 		lp->hs_state = LDC_HS_COMPLETE;
1356 	}
1357 
1358 	spin_unlock_irqrestore(&lp->lock, flags);
1359 
1360 	return 0;
1361 
1362 out_unmap_rx:
1363 	lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
1364 	sun4v_ldc_rx_qconf(lp->id, 0, 0);
1365 
1366 out_unmap_tx:
1367 	sun4v_ldc_tx_qconf(lp->id, 0, 0);
1368 
1369 out_free_irqs:
1370 	lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
1371 	free_irq(lp->cfg.tx_irq, lp);
1372 	free_irq(lp->cfg.rx_irq, lp);
1373 
1374 	spin_unlock_irqrestore(&lp->lock, flags);
1375 
1376 	return err;
1377 }
1378 EXPORT_SYMBOL(ldc_bind);
1379 
1380 int ldc_connect(struct ldc_channel *lp)
1381 {
1382 	unsigned long flags;
1383 	int err;
1384 
1385 	if (lp->cfg.mode == LDC_MODE_RAW)
1386 		return -EINVAL;
1387 
1388 	spin_lock_irqsave(&lp->lock, flags);
1389 
1390 	if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
1391 	    !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) ||
1392 	    lp->hs_state != LDC_HS_OPEN)
1393 		err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL);
1394 	else
1395 		err = start_handshake(lp);
1396 
1397 	spin_unlock_irqrestore(&lp->lock, flags);
1398 
1399 	return err;
1400 }
1401 EXPORT_SYMBOL(ldc_connect);
1402 
1403 int ldc_disconnect(struct ldc_channel *lp)
1404 {
1405 	unsigned long hv_err, flags;
1406 	int err;
1407 
1408 	if (lp->cfg.mode == LDC_MODE_RAW)
1409 		return -EINVAL;
1410 
1411 	if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
1412 	    !(lp->flags & LDC_FLAG_REGISTERED_QUEUES))
1413 		return -EINVAL;
1414 
1415 	spin_lock_irqsave(&lp->lock, flags);
1416 
1417 	err = -ENODEV;
1418 	hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
1419 	if (hv_err)
1420 		goto out_err;
1421 
1422 	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
1423 	if (hv_err)
1424 		goto out_err;
1425 
1426 	hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
1427 	if (hv_err)
1428 		goto out_err;
1429 
1430 	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
1431 	if (hv_err)
1432 		goto out_err;
1433 
1434 	ldc_set_state(lp, LDC_STATE_BOUND);
1435 	lp->hs_state = LDC_HS_OPEN;
1436 	lp->flags |= LDC_FLAG_RESET;
1437 
1438 	spin_unlock_irqrestore(&lp->lock, flags);
1439 
1440 	return 0;
1441 
1442 out_err:
1443 	sun4v_ldc_tx_qconf(lp->id, 0, 0);
1444 	sun4v_ldc_rx_qconf(lp->id, 0, 0);
1445 	free_irq(lp->cfg.tx_irq, lp);
1446 	free_irq(lp->cfg.rx_irq, lp);
1447 	lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS |
1448 		       LDC_FLAG_REGISTERED_QUEUES);
1449 	ldc_set_state(lp, LDC_STATE_INIT);
1450 
1451 	spin_unlock_irqrestore(&lp->lock, flags);
1452 
1453 	return err;
1454 }
1455 EXPORT_SYMBOL(ldc_disconnect);
1456 
1457 int ldc_state(struct ldc_channel *lp)
1458 {
1459 	return lp->state;
1460 }
1461 EXPORT_SYMBOL(ldc_state);
1462 
1463 void ldc_set_state(struct ldc_channel *lp, u8 state)
1464 {
1465 	ldcdbg(STATE, "STATE (%s) --> (%s)\n",
1466 	       state_to_str(lp->state),
1467 	       state_to_str(state));
1468 
1469 	lp->state = state;
1470 }
1471 EXPORT_SYMBOL(ldc_set_state);
1472 
1473 int ldc_mode(struct ldc_channel *lp)
1474 {
1475 	return lp->cfg.mode;
1476 }
1477 EXPORT_SYMBOL(ldc_mode);
1478 
1479 int ldc_rx_reset(struct ldc_channel *lp)
1480 {
1481 	return __set_rx_head(lp, lp->rx_tail);
1482 }
1483 EXPORT_SYMBOL(ldc_rx_reset);
1484 
1485 void __ldc_print(struct ldc_channel *lp, const char *caller)
1486 {
1487 	pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n"
1488 		"\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n"
1489 		"\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n"
1490 		"\trcv_nxt=%u snd_nxt=%u\n",
1491 		caller, lp->id, lp->flags, state_to_str(lp->state),
1492 		lp->chan_state, lp->hs_state,
1493 		lp->rx_head, lp->rx_tail, lp->rx_num_entries,
1494 		lp->tx_head, lp->tx_tail, lp->tx_num_entries,
1495 		lp->rcv_nxt, lp->snd_nxt);
1496 }
1497 EXPORT_SYMBOL(__ldc_print);
1498 
1499 static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size)
1500 {
1501 	struct ldc_packet *p;
1502 	unsigned long new_tail, hv_err;
1503 	int err;
1504 
1505 	hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
1506 					&lp->chan_state);
1507 	if (unlikely(hv_err))
1508 		return -EBUSY;
1509 
1510 	if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
1511 		return LDC_ABORT(lp);
1512 
1513 	if (size > LDC_PACKET_SIZE)
1514 		return -EMSGSIZE;
1515 
1516 	p = data_get_tx_packet(lp, &new_tail);
1517 	if (!p)
1518 		return -EAGAIN;
1519 
1520 	memcpy(p, buf, size);
1521 
1522 	err = send_tx_packet(lp, p, new_tail);
1523 	if (!err)
1524 		err = size;
1525 
1526 	return err;
1527 }
1528 
1529 static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size)
1530 {
1531 	struct ldc_packet *p;
1532 	unsigned long hv_err, new;
1533 	int err;
1534 
1535 	if (size < LDC_PACKET_SIZE)
1536 		return -EINVAL;
1537 
1538 	hv_err = sun4v_ldc_rx_get_state(lp->id,
1539 					&lp->rx_head,
1540 					&lp->rx_tail,
1541 					&lp->chan_state);
1542 	if (hv_err)
1543 		return LDC_ABORT(lp);
1544 
1545 	if (lp->chan_state == LDC_CHANNEL_DOWN ||
1546 	    lp->chan_state == LDC_CHANNEL_RESETTING)
1547 		return -ECONNRESET;
1548 
1549 	if (lp->rx_head == lp->rx_tail)
1550 		return 0;
1551 
1552 	p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
1553 	memcpy(buf, p, LDC_PACKET_SIZE);
1554 
1555 	new = rx_advance(lp, lp->rx_head);
1556 	lp->rx_head = new;
1557 
1558 	err = __set_rx_head(lp, new);
1559 	if (err < 0)
1560 		err = -ECONNRESET;
1561 	else
1562 		err = LDC_PACKET_SIZE;
1563 
1564 	return err;
1565 }
1566 
1567 static const struct ldc_mode_ops raw_ops = {
1568 	.write		=	write_raw,
1569 	.read		=	read_raw,
1570 };
1571 
1572 static int write_nonraw(struct ldc_channel *lp, const void *buf,
1573 			unsigned int size)
1574 {
1575 	unsigned long hv_err, tail;
1576 	unsigned int copied;
1577 	u32 seq;
1578 	int err;
1579 
1580 	hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
1581 					&lp->chan_state);
1582 	if (unlikely(hv_err))
1583 		return -EBUSY;
1584 
1585 	if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
1586 		return LDC_ABORT(lp);
1587 
1588 	if (!tx_has_space_for(lp, size))
1589 		return -EAGAIN;
1590 
1591 	seq = lp->snd_nxt;
1592 	copied = 0;
1593 	tail = lp->tx_tail;
1594 	while (copied < size) {
1595 		struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE);
1596 		u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ?
1597 			    p->u.u_data :
1598 			    p->u.r.r_data);
1599 		int data_len;
1600 
1601 		p->type = LDC_DATA;
1602 		p->stype = LDC_INFO;
1603 		p->ctrl = 0;
1604 
1605 		data_len = size - copied;
1606 		if (data_len > lp->mss)
1607 			data_len = lp->mss;
1608 
1609 		BUG_ON(data_len > LDC_LEN);
1610 
1611 		p->env = (data_len |
1612 			  (copied == 0 ? LDC_START : 0) |
1613 			  (data_len == size - copied ? LDC_STOP : 0));
1614 
1615 		p->seqid = ++seq;
1616 
1617 		ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n",
1618 		       p->type,
1619 		       p->stype,
1620 		       p->ctrl,
1621 		       p->env,
1622 		       p->seqid);
1623 
1624 		memcpy(data, buf, data_len);
1625 		buf += data_len;
1626 		copied += data_len;
1627 
1628 		tail = tx_advance(lp, tail);
1629 	}
1630 
1631 	err = set_tx_tail(lp, tail);
1632 	if (!err) {
1633 		lp->snd_nxt = seq;
1634 		err = size;
1635 	}
1636 
1637 	return err;
1638 }
1639 
1640 static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p,
1641 		      struct ldc_packet *first_frag)
1642 {
1643 	int err;
1644 
1645 	if (first_frag)
1646 		lp->rcv_nxt = first_frag->seqid - 1;
1647 
1648 	err = send_data_nack(lp, p);
1649 	if (err)
1650 		return err;
1651 
1652 	err = ldc_rx_reset(lp);
1653 	if (err < 0)
1654 		return LDC_ABORT(lp);
1655 
1656 	return 0;
1657 }
1658 
1659 static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p)
1660 {
1661 	if (p->stype & LDC_ACK) {
1662 		int err = process_data_ack(lp, p);
1663 		if (err)
1664 			return err;
1665 	}
1666 	if (p->stype & LDC_NACK)
1667 		return LDC_ABORT(lp);
1668 
1669 	return 0;
1670 }
1671 
1672 static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head)
1673 {
1674 	unsigned long dummy;
1675 	int limit = 1000;
1676 
1677 	ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n",
1678 	       cur_head, lp->rx_head, lp->rx_tail);
1679 	while (limit-- > 0) {
1680 		unsigned long hv_err;
1681 
1682 		hv_err = sun4v_ldc_rx_get_state(lp->id,
1683 						&dummy,
1684 						&lp->rx_tail,
1685 						&lp->chan_state);
1686 		if (hv_err)
1687 			return LDC_ABORT(lp);
1688 
1689 		if (lp->chan_state == LDC_CHANNEL_DOWN ||
1690 		    lp->chan_state == LDC_CHANNEL_RESETTING)
1691 			return -ECONNRESET;
1692 
1693 		if (cur_head != lp->rx_tail) {
1694 			ldcdbg(DATA, "DATA WAIT DONE "
1695 			       "head[%lx] tail[%lx] chan_state[%lx]\n",
1696 			       dummy, lp->rx_tail, lp->chan_state);
1697 			return 0;
1698 		}
1699 
1700 		udelay(1);
1701 	}
1702 	return -EAGAIN;
1703 }
1704 
1705 static int rx_set_head(struct ldc_channel *lp, unsigned long head)
1706 {
1707 	int err = __set_rx_head(lp, head);
1708 
1709 	if (err < 0)
1710 		return LDC_ABORT(lp);
1711 
1712 	lp->rx_head = head;
1713 	return 0;
1714 }
1715 
1716 static void send_data_ack(struct ldc_channel *lp)
1717 {
1718 	unsigned long new_tail;
1719 	struct ldc_packet *p;
1720 
1721 	p = data_get_tx_packet(lp, &new_tail);
1722 	if (likely(p)) {
1723 		int err;
1724 
1725 		memset(p, 0, sizeof(*p));
1726 		p->type = LDC_DATA;
1727 		p->stype = LDC_ACK;
1728 		p->ctrl = 0;
1729 		p->seqid = lp->snd_nxt + 1;
1730 		p->u.r.ackid = lp->rcv_nxt;
1731 
1732 		err = send_tx_packet(lp, p, new_tail);
1733 		if (!err)
1734 			lp->snd_nxt++;
1735 	}
1736 }
1737 
1738 static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size)
1739 {
1740 	struct ldc_packet *first_frag;
1741 	unsigned long hv_err, new;
1742 	int err, copied;
1743 
1744 	hv_err = sun4v_ldc_rx_get_state(lp->id,
1745 					&lp->rx_head,
1746 					&lp->rx_tail,
1747 					&lp->chan_state);
1748 	if (hv_err)
1749 		return LDC_ABORT(lp);
1750 
1751 	if (lp->chan_state == LDC_CHANNEL_DOWN ||
1752 	    lp->chan_state == LDC_CHANNEL_RESETTING)
1753 		return -ECONNRESET;
1754 
1755 	if (lp->rx_head == lp->rx_tail)
1756 		return 0;
1757 
1758 	first_frag = NULL;
1759 	copied = err = 0;
1760 	new = lp->rx_head;
1761 	while (1) {
1762 		struct ldc_packet *p;
1763 		int pkt_len;
1764 
1765 		BUG_ON(new == lp->rx_tail);
1766 		p = lp->rx_base + (new / LDC_PACKET_SIZE);
1767 
1768 		ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] "
1769 		       "rcv_nxt[%08x]\n",
1770 		       p->type,
1771 		       p->stype,
1772 		       p->ctrl,
1773 		       p->env,
1774 		       p->seqid,
1775 		       p->u.r.ackid,
1776 		       lp->rcv_nxt);
1777 
1778 		if (unlikely(!rx_seq_ok(lp, p->seqid))) {
1779 			err = rx_bad_seq(lp, p, first_frag);
1780 			copied = 0;
1781 			break;
1782 		}
1783 
1784 		if (p->type & LDC_CTRL) {
1785 			err = process_control_frame(lp, p);
1786 			if (err < 0)
1787 				break;
1788 			err = 0;
1789 		}
1790 
1791 		lp->rcv_nxt = p->seqid;
1792 
1793 		/*
1794 		 * If this is a control-only packet, there is nothing
1795 		 * else to do but advance the rx queue since the packet
1796 		 * was already processed above.
1797 		 */
1798 		if (!(p->type & LDC_DATA)) {
1799 			new = rx_advance(lp, new);
1800 			break;
1801 		}
1802 		if (p->stype & (LDC_ACK | LDC_NACK)) {
1803 			err = data_ack_nack(lp, p);
1804 			if (err)
1805 				break;
1806 		}
1807 		if (!(p->stype & LDC_INFO)) {
1808 			new = rx_advance(lp, new);
1809 			err = rx_set_head(lp, new);
1810 			if (err)
1811 				break;
1812 			goto no_data;
1813 		}
1814 
1815 		pkt_len = p->env & LDC_LEN;
1816 
1817 		/* Every initial packet starts with the START bit set.
1818 		 *
1819 		 * Singleton packets will have both START+STOP set.
1820 		 *
1821 		 * Fragments will have START set in the first frame, STOP
1822 		 * set in the last frame, and neither bit set in middle
1823 		 * frames of the packet.
1824 		 *
1825 		 * Therefore if we are at the beginning of a packet and
1826 		 * we don't see START, or we are in the middle of a fragmented
1827 		 * packet and do see START, we are unsynchronized and should
1828 		 * flush the RX queue.
1829 		 */
1830 		if ((first_frag == NULL && !(p->env & LDC_START)) ||
1831 		    (first_frag != NULL &&  (p->env & LDC_START))) {
1832 			if (!first_frag)
1833 				new = rx_advance(lp, new);
1834 
1835 			err = rx_set_head(lp, new);
1836 			if (err)
1837 				break;
1838 
1839 			if (!first_frag)
1840 				goto no_data;
1841 		}
1842 		if (!first_frag)
1843 			first_frag = p;
1844 
1845 		if (pkt_len > size - copied) {
1846 			/* User didn't give us a big enough buffer,
1847 			 * what to do?  This is a pretty serious error.
1848 			 *
1849 			 * Since we haven't updated the RX ring head to
1850 			 * consume any of the packets, signal the error
1851 			 * to the user and just leave the RX ring alone.
1852 			 *
1853 			 * This seems the best behavior because this allows
1854 			 * a user of the LDC layer to start with a small
1855 			 * RX buffer for ldc_read() calls and use -EMSGSIZE
1856 			 * as a cue to enlarge it's read buffer.
1857 			 */
1858 			err = -EMSGSIZE;
1859 			break;
1860 		}
1861 
1862 		/* Ok, we are gonna eat this one.  */
1863 		new = rx_advance(lp, new);
1864 
1865 		memcpy(buf,
1866 		       (lp->cfg.mode == LDC_MODE_UNRELIABLE ?
1867 			p->u.u_data : p->u.r.r_data), pkt_len);
1868 		buf += pkt_len;
1869 		copied += pkt_len;
1870 
1871 		if (p->env & LDC_STOP)
1872 			break;
1873 
1874 no_data:
1875 		if (new == lp->rx_tail) {
1876 			err = rx_data_wait(lp, new);
1877 			if (err)
1878 				break;
1879 		}
1880 	}
1881 
1882 	if (!err)
1883 		err = rx_set_head(lp, new);
1884 
1885 	if (err && first_frag)
1886 		lp->rcv_nxt = first_frag->seqid - 1;
1887 
1888 	if (!err) {
1889 		err = copied;
1890 		if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE)
1891 			send_data_ack(lp);
1892 	}
1893 
1894 	return err;
1895 }
1896 
1897 static const struct ldc_mode_ops nonraw_ops = {
1898 	.write		=	write_nonraw,
1899 	.read		=	read_nonraw,
1900 };
1901 
1902 static int write_stream(struct ldc_channel *lp, const void *buf,
1903 			unsigned int size)
1904 {
1905 	if (size > lp->cfg.mtu)
1906 		size = lp->cfg.mtu;
1907 	return write_nonraw(lp, buf, size);
1908 }
1909 
1910 static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size)
1911 {
1912 	if (!lp->mssbuf_len) {
1913 		int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu);
1914 		if (err < 0)
1915 			return err;
1916 
1917 		lp->mssbuf_len = err;
1918 		lp->mssbuf_off = 0;
1919 	}
1920 
1921 	if (size > lp->mssbuf_len)
1922 		size = lp->mssbuf_len;
1923 	memcpy(buf, lp->mssbuf + lp->mssbuf_off, size);
1924 
1925 	lp->mssbuf_off += size;
1926 	lp->mssbuf_len -= size;
1927 
1928 	return size;
1929 }
1930 
1931 static const struct ldc_mode_ops stream_ops = {
1932 	.write		=	write_stream,
1933 	.read		=	read_stream,
1934 };
1935 
1936 int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size)
1937 {
1938 	unsigned long flags;
1939 	int err;
1940 
1941 	if (!buf)
1942 		return -EINVAL;
1943 
1944 	if (!size)
1945 		return 0;
1946 
1947 	spin_lock_irqsave(&lp->lock, flags);
1948 
1949 	if (lp->hs_state != LDC_HS_COMPLETE)
1950 		err = -ENOTCONN;
1951 	else
1952 		err = lp->mops->write(lp, buf, size);
1953 
1954 	spin_unlock_irqrestore(&lp->lock, flags);
1955 
1956 	return err;
1957 }
1958 EXPORT_SYMBOL(ldc_write);
1959 
1960 int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
1961 {
1962 	unsigned long flags;
1963 	int err;
1964 
1965 	ldcdbg(RX, "%s: entered size=%d\n", __func__, size);
1966 
1967 	if (!buf)
1968 		return -EINVAL;
1969 
1970 	if (!size)
1971 		return 0;
1972 
1973 	spin_lock_irqsave(&lp->lock, flags);
1974 
1975 	if (lp->hs_state != LDC_HS_COMPLETE)
1976 		err = -ENOTCONN;
1977 	else
1978 		err = lp->mops->read(lp, buf, size);
1979 
1980 	spin_unlock_irqrestore(&lp->lock, flags);
1981 
1982 	ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__,
1983 	       lp->cfg.mode, lp->rx_head, lp->rx_tail, err);
1984 
1985 	return err;
1986 }
1987 EXPORT_SYMBOL(ldc_read);
1988 
1989 static u64 pagesize_code(void)
1990 {
1991 	switch (PAGE_SIZE) {
1992 	default:
1993 	case (8ULL * 1024ULL):
1994 		return 0;
1995 	case (64ULL * 1024ULL):
1996 		return 1;
1997 	case (512ULL * 1024ULL):
1998 		return 2;
1999 	case (4ULL * 1024ULL * 1024ULL):
2000 		return 3;
2001 	case (32ULL * 1024ULL * 1024ULL):
2002 		return 4;
2003 	case (256ULL * 1024ULL * 1024ULL):
2004 		return 5;
2005 	}
2006 }
2007 
2008 static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
2009 {
2010 	return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) |
2011 		(index << PAGE_SHIFT) |
2012 		page_offset);
2013 }
2014 
2015 
2016 static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
2017 					     unsigned long npages)
2018 {
2019 	long entry;
2020 
2021 	entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,
2022 				      npages, NULL, (unsigned long)-1, 0);
2023 	if (unlikely(entry == IOMMU_ERROR_CODE))
2024 		return NULL;
2025 
2026 	return iommu->page_table + entry;
2027 }
2028 
2029 static u64 perm_to_mte(unsigned int map_perm)
2030 {
2031 	u64 mte_base;
2032 
2033 	mte_base = pagesize_code();
2034 
2035 	if (map_perm & LDC_MAP_SHADOW) {
2036 		if (map_perm & LDC_MAP_R)
2037 			mte_base |= LDC_MTE_COPY_R;
2038 		if (map_perm & LDC_MAP_W)
2039 			mte_base |= LDC_MTE_COPY_W;
2040 	}
2041 	if (map_perm & LDC_MAP_DIRECT) {
2042 		if (map_perm & LDC_MAP_R)
2043 			mte_base |= LDC_MTE_READ;
2044 		if (map_perm & LDC_MAP_W)
2045 			mte_base |= LDC_MTE_WRITE;
2046 		if (map_perm & LDC_MAP_X)
2047 			mte_base |= LDC_MTE_EXEC;
2048 	}
2049 	if (map_perm & LDC_MAP_IO) {
2050 		if (map_perm & LDC_MAP_R)
2051 			mte_base |= LDC_MTE_IOMMU_R;
2052 		if (map_perm & LDC_MAP_W)
2053 			mte_base |= LDC_MTE_IOMMU_W;
2054 	}
2055 
2056 	return mte_base;
2057 }
2058 
2059 static int pages_in_region(unsigned long base, long len)
2060 {
2061 	int count = 0;
2062 
2063 	do {
2064 		unsigned long new = (base + PAGE_SIZE) & PAGE_MASK;
2065 
2066 		len -= (new - base);
2067 		base = new;
2068 		count++;
2069 	} while (len > 0);
2070 
2071 	return count;
2072 }
2073 
2074 struct cookie_state {
2075 	struct ldc_mtable_entry		*page_table;
2076 	struct ldc_trans_cookie		*cookies;
2077 	u64				mte_base;
2078 	u64				prev_cookie;
2079 	u32				pte_idx;
2080 	u32				nc;
2081 };
2082 
2083 static void fill_cookies(struct cookie_state *sp, unsigned long pa,
2084 			 unsigned long off, unsigned long len)
2085 {
2086 	do {
2087 		unsigned long tlen, new = pa + PAGE_SIZE;
2088 		u64 this_cookie;
2089 
2090 		sp->page_table[sp->pte_idx].mte = sp->mte_base | pa;
2091 
2092 		tlen = PAGE_SIZE;
2093 		if (off)
2094 			tlen = PAGE_SIZE - off;
2095 		if (tlen > len)
2096 			tlen = len;
2097 
2098 		this_cookie = make_cookie(sp->pte_idx,
2099 					  pagesize_code(), off);
2100 
2101 		off = 0;
2102 
2103 		if (this_cookie == sp->prev_cookie) {
2104 			sp->cookies[sp->nc - 1].cookie_size += tlen;
2105 		} else {
2106 			sp->cookies[sp->nc].cookie_addr = this_cookie;
2107 			sp->cookies[sp->nc].cookie_size = tlen;
2108 			sp->nc++;
2109 		}
2110 		sp->prev_cookie = this_cookie + tlen;
2111 
2112 		sp->pte_idx++;
2113 
2114 		len -= tlen;
2115 		pa = new;
2116 	} while (len > 0);
2117 }
2118 
2119 static int sg_count_one(struct scatterlist *sg)
2120 {
2121 	unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT;
2122 	long len = sg->length;
2123 
2124 	if ((sg->offset | len) & (8UL - 1))
2125 		return -EFAULT;
2126 
2127 	return pages_in_region(base + sg->offset, len);
2128 }
2129 
2130 static int sg_count_pages(struct scatterlist *sg, int num_sg)
2131 {
2132 	int count;
2133 	int i;
2134 
2135 	count = 0;
2136 	for (i = 0; i < num_sg; i++) {
2137 		int err = sg_count_one(sg + i);
2138 		if (err < 0)
2139 			return err;
2140 		count += err;
2141 	}
2142 
2143 	return count;
2144 }
2145 
2146 int ldc_map_sg(struct ldc_channel *lp,
2147 	       struct scatterlist *sg, int num_sg,
2148 	       struct ldc_trans_cookie *cookies, int ncookies,
2149 	       unsigned int map_perm)
2150 {
2151 	unsigned long i, npages;
2152 	struct ldc_mtable_entry *base;
2153 	struct cookie_state state;
2154 	struct ldc_iommu *iommu;
2155 	int err;
2156 	struct scatterlist *s;
2157 
2158 	if (map_perm & ~LDC_MAP_ALL)
2159 		return -EINVAL;
2160 
2161 	err = sg_count_pages(sg, num_sg);
2162 	if (err < 0)
2163 		return err;
2164 
2165 	npages = err;
2166 	if (err > ncookies)
2167 		return -EMSGSIZE;
2168 
2169 	iommu = &lp->iommu;
2170 
2171 	base = alloc_npages(iommu, npages);
2172 
2173 	if (!base)
2174 		return -ENOMEM;
2175 
2176 	state.page_table = iommu->page_table;
2177 	state.cookies = cookies;
2178 	state.mte_base = perm_to_mte(map_perm);
2179 	state.prev_cookie = ~(u64)0;
2180 	state.pte_idx = (base - iommu->page_table);
2181 	state.nc = 0;
2182 
2183 	for_each_sg(sg, s, num_sg, i) {
2184 		fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT,
2185 			     s->offset, s->length);
2186 	}
2187 
2188 	return state.nc;
2189 }
2190 EXPORT_SYMBOL(ldc_map_sg);
2191 
2192 int ldc_map_single(struct ldc_channel *lp,
2193 		   void *buf, unsigned int len,
2194 		   struct ldc_trans_cookie *cookies, int ncookies,
2195 		   unsigned int map_perm)
2196 {
2197 	unsigned long npages, pa;
2198 	struct ldc_mtable_entry *base;
2199 	struct cookie_state state;
2200 	struct ldc_iommu *iommu;
2201 
2202 	if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1))
2203 		return -EINVAL;
2204 
2205 	pa = __pa(buf);
2206 	if ((pa | len) & (8UL - 1))
2207 		return -EFAULT;
2208 
2209 	npages = pages_in_region(pa, len);
2210 
2211 	iommu = &lp->iommu;
2212 
2213 	base = alloc_npages(iommu, npages);
2214 
2215 	if (!base)
2216 		return -ENOMEM;
2217 
2218 	state.page_table = iommu->page_table;
2219 	state.cookies = cookies;
2220 	state.mte_base = perm_to_mte(map_perm);
2221 	state.prev_cookie = ~(u64)0;
2222 	state.pte_idx = (base - iommu->page_table);
2223 	state.nc = 0;
2224 	fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len);
2225 	BUG_ON(state.nc > ncookies);
2226 
2227 	return state.nc;
2228 }
2229 EXPORT_SYMBOL(ldc_map_single);
2230 
2231 
2232 static void free_npages(unsigned long id, struct ldc_iommu *iommu,
2233 			u64 cookie, u64 size)
2234 {
2235 	unsigned long npages, entry;
2236 
2237 	npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;
2238 
2239 	entry = ldc_cookie_to_index(cookie, iommu);
2240 	ldc_demap(iommu, id, cookie, entry, npages);
2241 	iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);
2242 }
2243 
2244 void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
2245 	       int ncookies)
2246 {
2247 	struct ldc_iommu *iommu = &lp->iommu;
2248 	int i;
2249 	unsigned long flags;
2250 
2251 	spin_lock_irqsave(&iommu->lock, flags);
2252 	for (i = 0; i < ncookies; i++) {
2253 		u64 addr = cookies[i].cookie_addr;
2254 		u64 size = cookies[i].cookie_size;
2255 
2256 		free_npages(lp->id, iommu, addr, size);
2257 	}
2258 	spin_unlock_irqrestore(&iommu->lock, flags);
2259 }
2260 EXPORT_SYMBOL(ldc_unmap);
2261 
2262 int ldc_copy(struct ldc_channel *lp, int copy_dir,
2263 	     void *buf, unsigned int len, unsigned long offset,
2264 	     struct ldc_trans_cookie *cookies, int ncookies)
2265 {
2266 	unsigned int orig_len;
2267 	unsigned long ra;
2268 	int i;
2269 
2270 	if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) {
2271 		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n",
2272 		       lp->id, copy_dir);
2273 		return -EINVAL;
2274 	}
2275 
2276 	ra = __pa(buf);
2277 	if ((ra | len | offset) & (8UL - 1)) {
2278 		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer "
2279 		       "ra[%lx] len[%x] offset[%lx]\n",
2280 		       lp->id, ra, len, offset);
2281 		return -EFAULT;
2282 	}
2283 
2284 	if (lp->hs_state != LDC_HS_COMPLETE ||
2285 	    (lp->flags & LDC_FLAG_RESET)) {
2286 		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] "
2287 		       "flags[%x]\n", lp->id, lp->hs_state, lp->flags);
2288 		return -ECONNRESET;
2289 	}
2290 
2291 	orig_len = len;
2292 	for (i = 0; i < ncookies; i++) {
2293 		unsigned long cookie_raddr = cookies[i].cookie_addr;
2294 		unsigned long this_len = cookies[i].cookie_size;
2295 		unsigned long actual_len;
2296 
2297 		if (unlikely(offset)) {
2298 			unsigned long this_off = offset;
2299 
2300 			if (this_off > this_len)
2301 				this_off = this_len;
2302 
2303 			offset -= this_off;
2304 			this_len -= this_off;
2305 			if (!this_len)
2306 				continue;
2307 			cookie_raddr += this_off;
2308 		}
2309 
2310 		if (this_len > len)
2311 			this_len = len;
2312 
2313 		while (1) {
2314 			unsigned long hv_err;
2315 
2316 			hv_err = sun4v_ldc_copy(lp->id, copy_dir,
2317 						cookie_raddr, ra,
2318 						this_len, &actual_len);
2319 			if (unlikely(hv_err)) {
2320 				printk(KERN_ERR PFX "ldc_copy: ID[%lu] "
2321 				       "HV error %lu\n",
2322 				       lp->id, hv_err);
2323 				if (lp->hs_state != LDC_HS_COMPLETE ||
2324 				    (lp->flags & LDC_FLAG_RESET))
2325 					return -ECONNRESET;
2326 				else
2327 					return -EFAULT;
2328 			}
2329 
2330 			cookie_raddr += actual_len;
2331 			ra += actual_len;
2332 			len -= actual_len;
2333 			if (actual_len == this_len)
2334 				break;
2335 
2336 			this_len -= actual_len;
2337 		}
2338 
2339 		if (!len)
2340 			break;
2341 	}
2342 
2343 	/* It is caller policy what to do about short copies.
2344 	 * For example, a networking driver can declare the
2345 	 * packet a runt and drop it.
2346 	 */
2347 
2348 	return orig_len - len;
2349 }
2350 EXPORT_SYMBOL(ldc_copy);
2351 
2352 void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len,
2353 			  struct ldc_trans_cookie *cookies, int *ncookies,
2354 			  unsigned int map_perm)
2355 {
2356 	void *buf;
2357 	int err;
2358 
2359 	if (len & (8UL - 1))
2360 		return ERR_PTR(-EINVAL);
2361 
2362 	buf = kzalloc(len, GFP_ATOMIC);
2363 	if (!buf)
2364 		return ERR_PTR(-ENOMEM);
2365 
2366 	err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm);
2367 	if (err < 0) {
2368 		kfree(buf);
2369 		return ERR_PTR(err);
2370 	}
2371 	*ncookies = err;
2372 
2373 	return buf;
2374 }
2375 EXPORT_SYMBOL(ldc_alloc_exp_dring);
2376 
2377 void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len,
2378 			struct ldc_trans_cookie *cookies, int ncookies)
2379 {
2380 	ldc_unmap(lp, cookies, ncookies);
2381 	kfree(buf);
2382 }
2383 EXPORT_SYMBOL(ldc_free_exp_dring);
2384 
2385 static int __init ldc_init(void)
2386 {
2387 	unsigned long major, minor;
2388 	struct mdesc_handle *hp;
2389 	const u64 *v;
2390 	int err;
2391 	u64 mp;
2392 
2393 	hp = mdesc_grab();
2394 	if (!hp)
2395 		return -ENODEV;
2396 
2397 	mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
2398 	err = -ENODEV;
2399 	if (mp == MDESC_NODE_NULL)
2400 		goto out;
2401 
2402 	v = mdesc_get_property(hp, mp, "domaining-enabled", NULL);
2403 	if (!v)
2404 		goto out;
2405 
2406 	major = 1;
2407 	minor = 0;
2408 	if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) {
2409 		printk(KERN_INFO PFX "Could not register LDOM hvapi.\n");
2410 		goto out;
2411 	}
2412 
2413 	printk(KERN_INFO "%s", version);
2414 
2415 	if (!*v) {
2416 		printk(KERN_INFO PFX "Domaining disabled.\n");
2417 		goto out;
2418 	}
2419 	ldom_domaining_enabled = 1;
2420 	err = 0;
2421 
2422 out:
2423 	mdesc_release(hp);
2424 	return err;
2425 }
2426 
2427 core_initcall(ldc_init);
2428