1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2a88b5ba8SSam Ravnborg /* ldc.c: Logical Domain Channel link-layer protocol driver.
3a88b5ba8SSam Ravnborg *
4a88b5ba8SSam Ravnborg * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
5a88b5ba8SSam Ravnborg */
6a88b5ba8SSam Ravnborg
7a88b5ba8SSam Ravnborg #include <linux/kernel.h>
8066bcacaSPaul Gortmaker #include <linux/export.h>
9a88b5ba8SSam Ravnborg #include <linux/slab.h>
10a88b5ba8SSam Ravnborg #include <linux/spinlock.h>
11a88b5ba8SSam Ravnborg #include <linux/delay.h>
12a88b5ba8SSam Ravnborg #include <linux/errno.h>
13a88b5ba8SSam Ravnborg #include <linux/string.h>
14a88b5ba8SSam Ravnborg #include <linux/scatterlist.h>
15a88b5ba8SSam Ravnborg #include <linux/interrupt.h>
16a88b5ba8SSam Ravnborg #include <linux/list.h>
17a88b5ba8SSam Ravnborg #include <linux/init.h>
18e756fd80SAkinobu Mita #include <linux/bitmap.h>
190d3fdb15SChristoph Hellwig #include <asm/iommu-common.h>
20a88b5ba8SSam Ravnborg
21a88b5ba8SSam Ravnborg #include <asm/hypervisor.h>
22a88b5ba8SSam Ravnborg #include <asm/iommu.h>
23a88b5ba8SSam Ravnborg #include <asm/page.h>
24a88b5ba8SSam Ravnborg #include <asm/ldc.h>
25a88b5ba8SSam Ravnborg #include <asm/mdesc.h>
26a88b5ba8SSam Ravnborg
27a88b5ba8SSam Ravnborg #define DRV_MODULE_NAME "ldc"
28a88b5ba8SSam Ravnborg #define PFX DRV_MODULE_NAME ": "
29a88b5ba8SSam Ravnborg #define DRV_MODULE_VERSION "1.1"
30a88b5ba8SSam Ravnborg #define DRV_MODULE_RELDATE "July 22, 2008"
31a88b5ba8SSam Ravnborg
320ae53ed1SSowmini Varadhan #define COOKIE_PGSZ_CODE 0xf000000000000000ULL
330ae53ed1SSowmini Varadhan #define COOKIE_PGSZ_CODE_SHIFT 60ULL
340ae53ed1SSowmini Varadhan
350ae53ed1SSowmini Varadhan
367c9503b8SGreg Kroah-Hartman static char version[] =
37a88b5ba8SSam Ravnborg DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
38a88b5ba8SSam Ravnborg
39a88b5ba8SSam Ravnborg /* Packet header layout for unreliable and reliable mode frames.
40a88b5ba8SSam Ravnborg * When in RAW mode, packets are simply straight 64-byte payloads
41a88b5ba8SSam Ravnborg * with no headers.
42a88b5ba8SSam Ravnborg */
43a88b5ba8SSam Ravnborg struct ldc_packet {
44a88b5ba8SSam Ravnborg u8 type;
45a88b5ba8SSam Ravnborg #define LDC_CTRL 0x01
46a88b5ba8SSam Ravnborg #define LDC_DATA 0x02
47a88b5ba8SSam Ravnborg #define LDC_ERR 0x10
48a88b5ba8SSam Ravnborg
49a88b5ba8SSam Ravnborg u8 stype;
50a88b5ba8SSam Ravnborg #define LDC_INFO 0x01
51a88b5ba8SSam Ravnborg #define LDC_ACK 0x02
52a88b5ba8SSam Ravnborg #define LDC_NACK 0x04
53a88b5ba8SSam Ravnborg
54a88b5ba8SSam Ravnborg u8 ctrl;
55a88b5ba8SSam Ravnborg #define LDC_VERS 0x01 /* Link Version */
56a88b5ba8SSam Ravnborg #define LDC_RTS 0x02 /* Request To Send */
57a88b5ba8SSam Ravnborg #define LDC_RTR 0x03 /* Ready To Receive */
58a88b5ba8SSam Ravnborg #define LDC_RDX 0x04 /* Ready for Data eXchange */
59a88b5ba8SSam Ravnborg #define LDC_CTRL_MSK 0x0f
60a88b5ba8SSam Ravnborg
61a88b5ba8SSam Ravnborg u8 env;
62a88b5ba8SSam Ravnborg #define LDC_LEN 0x3f
63a88b5ba8SSam Ravnborg #define LDC_FRAG_MASK 0xc0
64a88b5ba8SSam Ravnborg #define LDC_START 0x40
65a88b5ba8SSam Ravnborg #define LDC_STOP 0x80
66a88b5ba8SSam Ravnborg
67a88b5ba8SSam Ravnborg u32 seqid;
68a88b5ba8SSam Ravnborg
69a88b5ba8SSam Ravnborg union {
70a88b5ba8SSam Ravnborg u8 u_data[LDC_PACKET_SIZE - 8];
71a88b5ba8SSam Ravnborg struct {
72a88b5ba8SSam Ravnborg u32 pad;
73a88b5ba8SSam Ravnborg u32 ackid;
74a88b5ba8SSam Ravnborg u8 r_data[LDC_PACKET_SIZE - 8 - 8];
75a88b5ba8SSam Ravnborg } r;
76a88b5ba8SSam Ravnborg } u;
77a88b5ba8SSam Ravnborg };
78a88b5ba8SSam Ravnborg
79a88b5ba8SSam Ravnborg struct ldc_version {
80a88b5ba8SSam Ravnborg u16 major;
81a88b5ba8SSam Ravnborg u16 minor;
82a88b5ba8SSam Ravnborg };
83a88b5ba8SSam Ravnborg
84a88b5ba8SSam Ravnborg /* Ordered from largest major to lowest. */
85a88b5ba8SSam Ravnborg static struct ldc_version ver_arr[] = {
86a88b5ba8SSam Ravnborg { .major = 1, .minor = 0 },
87a88b5ba8SSam Ravnborg };
88a88b5ba8SSam Ravnborg
89a88b5ba8SSam Ravnborg #define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE)
90a88b5ba8SSam Ravnborg #define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE)
91a88b5ba8SSam Ravnborg
92a88b5ba8SSam Ravnborg struct ldc_channel;
93a88b5ba8SSam Ravnborg
94a88b5ba8SSam Ravnborg struct ldc_mode_ops {
95a88b5ba8SSam Ravnborg int (*write)(struct ldc_channel *, const void *, unsigned int);
96a88b5ba8SSam Ravnborg int (*read)(struct ldc_channel *, void *, unsigned int);
97a88b5ba8SSam Ravnborg };
98a88b5ba8SSam Ravnborg
99a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops;
100a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops;
101a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops;
102a88b5ba8SSam Ravnborg
103a88b5ba8SSam Ravnborg int ldom_domaining_enabled;
104a88b5ba8SSam Ravnborg
105a88b5ba8SSam Ravnborg struct ldc_iommu {
1060ae53ed1SSowmini Varadhan /* Protects ldc_unmap. */
107a88b5ba8SSam Ravnborg spinlock_t lock;
108a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table;
1090ae53ed1SSowmini Varadhan struct iommu_map_table iommu_map_table;
110a88b5ba8SSam Ravnborg };
111a88b5ba8SSam Ravnborg
112a88b5ba8SSam Ravnborg struct ldc_channel {
113a88b5ba8SSam Ravnborg /* Protects all operations that depend upon channel state. */
114a88b5ba8SSam Ravnborg spinlock_t lock;
115a88b5ba8SSam Ravnborg
116a88b5ba8SSam Ravnborg unsigned long id;
117a88b5ba8SSam Ravnborg
118a88b5ba8SSam Ravnborg u8 *mssbuf;
119a88b5ba8SSam Ravnborg u32 mssbuf_len;
120a88b5ba8SSam Ravnborg u32 mssbuf_off;
121a88b5ba8SSam Ravnborg
122a88b5ba8SSam Ravnborg struct ldc_packet *tx_base;
123a88b5ba8SSam Ravnborg unsigned long tx_head;
124a88b5ba8SSam Ravnborg unsigned long tx_tail;
125a88b5ba8SSam Ravnborg unsigned long tx_num_entries;
126a88b5ba8SSam Ravnborg unsigned long tx_ra;
127a88b5ba8SSam Ravnborg
128a88b5ba8SSam Ravnborg unsigned long tx_acked;
129a88b5ba8SSam Ravnborg
130a88b5ba8SSam Ravnborg struct ldc_packet *rx_base;
131a88b5ba8SSam Ravnborg unsigned long rx_head;
132a88b5ba8SSam Ravnborg unsigned long rx_tail;
133a88b5ba8SSam Ravnborg unsigned long rx_num_entries;
134a88b5ba8SSam Ravnborg unsigned long rx_ra;
135a88b5ba8SSam Ravnborg
136a88b5ba8SSam Ravnborg u32 rcv_nxt;
137a88b5ba8SSam Ravnborg u32 snd_nxt;
138a88b5ba8SSam Ravnborg
139a88b5ba8SSam Ravnborg unsigned long chan_state;
140a88b5ba8SSam Ravnborg
141a88b5ba8SSam Ravnborg struct ldc_channel_config cfg;
142a88b5ba8SSam Ravnborg void *event_arg;
143a88b5ba8SSam Ravnborg
144a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops;
145a88b5ba8SSam Ravnborg
146a88b5ba8SSam Ravnborg struct ldc_iommu iommu;
147a88b5ba8SSam Ravnborg
148a88b5ba8SSam Ravnborg struct ldc_version ver;
149a88b5ba8SSam Ravnborg
150a88b5ba8SSam Ravnborg u8 hs_state;
151a88b5ba8SSam Ravnborg #define LDC_HS_CLOSED 0x00
152a88b5ba8SSam Ravnborg #define LDC_HS_OPEN 0x01
153a88b5ba8SSam Ravnborg #define LDC_HS_GOTVERS 0x02
154a88b5ba8SSam Ravnborg #define LDC_HS_SENTRTR 0x03
155a88b5ba8SSam Ravnborg #define LDC_HS_GOTRTR 0x04
156a88b5ba8SSam Ravnborg #define LDC_HS_COMPLETE 0x10
157a88b5ba8SSam Ravnborg
158a88b5ba8SSam Ravnborg u8 flags;
159a88b5ba8SSam Ravnborg #define LDC_FLAG_ALLOCED_QUEUES 0x01
160a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_QUEUES 0x02
161a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_IRQS 0x04
162a88b5ba8SSam Ravnborg #define LDC_FLAG_RESET 0x10
163a88b5ba8SSam Ravnborg
164a88b5ba8SSam Ravnborg u8 mss;
165a88b5ba8SSam Ravnborg u8 state;
166a88b5ba8SSam Ravnborg
167a88b5ba8SSam Ravnborg #define LDC_IRQ_NAME_MAX 32
168a88b5ba8SSam Ravnborg char rx_irq_name[LDC_IRQ_NAME_MAX];
169a88b5ba8SSam Ravnborg char tx_irq_name[LDC_IRQ_NAME_MAX];
170a88b5ba8SSam Ravnborg
171a88b5ba8SSam Ravnborg struct hlist_head mh_list;
172a88b5ba8SSam Ravnborg
173a88b5ba8SSam Ravnborg struct hlist_node list;
174a88b5ba8SSam Ravnborg };
175a88b5ba8SSam Ravnborg
176a88b5ba8SSam Ravnborg #define ldcdbg(TYPE, f, a...) \
177a88b5ba8SSam Ravnborg do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
178a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
179a88b5ba8SSam Ravnborg } while (0)
180a88b5ba8SSam Ravnborg
18129693e75SJag Raman #define LDC_ABORT(lp) ldc_abort((lp), __func__)
18229693e75SJag Raman
state_to_str(u8 state)183a88b5ba8SSam Ravnborg static const char *state_to_str(u8 state)
184a88b5ba8SSam Ravnborg {
185a88b5ba8SSam Ravnborg switch (state) {
186a88b5ba8SSam Ravnborg case LDC_STATE_INVALID:
187a88b5ba8SSam Ravnborg return "INVALID";
188a88b5ba8SSam Ravnborg case LDC_STATE_INIT:
189a88b5ba8SSam Ravnborg return "INIT";
190a88b5ba8SSam Ravnborg case LDC_STATE_BOUND:
191a88b5ba8SSam Ravnborg return "BOUND";
192a88b5ba8SSam Ravnborg case LDC_STATE_READY:
193a88b5ba8SSam Ravnborg return "READY";
194a88b5ba8SSam Ravnborg case LDC_STATE_CONNECTED:
195a88b5ba8SSam Ravnborg return "CONNECTED";
196a88b5ba8SSam Ravnborg default:
197a88b5ba8SSam Ravnborg return "<UNKNOWN>";
198a88b5ba8SSam Ravnborg }
199a88b5ba8SSam Ravnborg }
200a88b5ba8SSam Ravnborg
__advance(unsigned long off,unsigned long num_entries)201a88b5ba8SSam Ravnborg static unsigned long __advance(unsigned long off, unsigned long num_entries)
202a88b5ba8SSam Ravnborg {
203a88b5ba8SSam Ravnborg off += LDC_PACKET_SIZE;
204a88b5ba8SSam Ravnborg if (off == (num_entries * LDC_PACKET_SIZE))
205a88b5ba8SSam Ravnborg off = 0;
206a88b5ba8SSam Ravnborg
207a88b5ba8SSam Ravnborg return off;
208a88b5ba8SSam Ravnborg }
209a88b5ba8SSam Ravnborg
rx_advance(struct ldc_channel * lp,unsigned long off)210a88b5ba8SSam Ravnborg static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off)
211a88b5ba8SSam Ravnborg {
212a88b5ba8SSam Ravnborg return __advance(off, lp->rx_num_entries);
213a88b5ba8SSam Ravnborg }
214a88b5ba8SSam Ravnborg
tx_advance(struct ldc_channel * lp,unsigned long off)215a88b5ba8SSam Ravnborg static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off)
216a88b5ba8SSam Ravnborg {
217a88b5ba8SSam Ravnborg return __advance(off, lp->tx_num_entries);
218a88b5ba8SSam Ravnborg }
219a88b5ba8SSam Ravnborg
handshake_get_tx_packet(struct ldc_channel * lp,unsigned long * new_tail)220a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp,
221a88b5ba8SSam Ravnborg unsigned long *new_tail)
222a88b5ba8SSam Ravnborg {
223a88b5ba8SSam Ravnborg struct ldc_packet *p;
224a88b5ba8SSam Ravnborg unsigned long t;
225a88b5ba8SSam Ravnborg
226a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail);
227a88b5ba8SSam Ravnborg if (t == lp->tx_head)
228a88b5ba8SSam Ravnborg return NULL;
229a88b5ba8SSam Ravnborg
230a88b5ba8SSam Ravnborg *new_tail = t;
231a88b5ba8SSam Ravnborg
232a88b5ba8SSam Ravnborg p = lp->tx_base;
233a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE);
234a88b5ba8SSam Ravnborg }
235a88b5ba8SSam Ravnborg
236a88b5ba8SSam Ravnborg /* When we are in reliable or stream mode, have to track the next packet
237a88b5ba8SSam Ravnborg * we haven't gotten an ACK for in the TX queue using tx_acked. We have
238a88b5ba8SSam Ravnborg * to be careful not to stomp over the queue past that point. During
239a88b5ba8SSam Ravnborg * the handshake, we don't have TX data packets pending in the queue
240a88b5ba8SSam Ravnborg * and that's why handshake_get_tx_packet() need not be mindful of
241a88b5ba8SSam Ravnborg * lp->tx_acked.
242a88b5ba8SSam Ravnborg */
head_for_data(struct ldc_channel * lp)243a88b5ba8SSam Ravnborg static unsigned long head_for_data(struct ldc_channel *lp)
244a88b5ba8SSam Ravnborg {
245a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM)
246a88b5ba8SSam Ravnborg return lp->tx_acked;
247a88b5ba8SSam Ravnborg return lp->tx_head;
248a88b5ba8SSam Ravnborg }
249a88b5ba8SSam Ravnborg
tx_has_space_for(struct ldc_channel * lp,unsigned int size)250a88b5ba8SSam Ravnborg static int tx_has_space_for(struct ldc_channel *lp, unsigned int size)
251a88b5ba8SSam Ravnborg {
252a88b5ba8SSam Ravnborg unsigned long limit, tail, new_tail, diff;
253a88b5ba8SSam Ravnborg unsigned int mss;
254a88b5ba8SSam Ravnborg
255a88b5ba8SSam Ravnborg limit = head_for_data(lp);
256a88b5ba8SSam Ravnborg tail = lp->tx_tail;
257a88b5ba8SSam Ravnborg new_tail = tx_advance(lp, tail);
258a88b5ba8SSam Ravnborg if (new_tail == limit)
259a88b5ba8SSam Ravnborg return 0;
260a88b5ba8SSam Ravnborg
261a88b5ba8SSam Ravnborg if (limit > new_tail)
262a88b5ba8SSam Ravnborg diff = limit - new_tail;
263a88b5ba8SSam Ravnborg else
264a88b5ba8SSam Ravnborg diff = (limit +
265a88b5ba8SSam Ravnborg ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail));
266a88b5ba8SSam Ravnborg diff /= LDC_PACKET_SIZE;
267a88b5ba8SSam Ravnborg mss = lp->mss;
268a88b5ba8SSam Ravnborg
269a88b5ba8SSam Ravnborg if (diff * mss < size)
270a88b5ba8SSam Ravnborg return 0;
271a88b5ba8SSam Ravnborg
272a88b5ba8SSam Ravnborg return 1;
273a88b5ba8SSam Ravnborg }
274a88b5ba8SSam Ravnborg
data_get_tx_packet(struct ldc_channel * lp,unsigned long * new_tail)275a88b5ba8SSam Ravnborg static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp,
276a88b5ba8SSam Ravnborg unsigned long *new_tail)
277a88b5ba8SSam Ravnborg {
278a88b5ba8SSam Ravnborg struct ldc_packet *p;
279a88b5ba8SSam Ravnborg unsigned long h, t;
280a88b5ba8SSam Ravnborg
281a88b5ba8SSam Ravnborg h = head_for_data(lp);
282a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail);
283a88b5ba8SSam Ravnborg if (t == h)
284a88b5ba8SSam Ravnborg return NULL;
285a88b5ba8SSam Ravnborg
286a88b5ba8SSam Ravnborg *new_tail = t;
287a88b5ba8SSam Ravnborg
288a88b5ba8SSam Ravnborg p = lp->tx_base;
289a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE);
290a88b5ba8SSam Ravnborg }
291a88b5ba8SSam Ravnborg
set_tx_tail(struct ldc_channel * lp,unsigned long tail)292a88b5ba8SSam Ravnborg static int set_tx_tail(struct ldc_channel *lp, unsigned long tail)
293a88b5ba8SSam Ravnborg {
294a88b5ba8SSam Ravnborg unsigned long orig_tail = lp->tx_tail;
295a88b5ba8SSam Ravnborg int limit = 1000;
296a88b5ba8SSam Ravnborg
297a88b5ba8SSam Ravnborg lp->tx_tail = tail;
298a88b5ba8SSam Ravnborg while (limit-- > 0) {
299a88b5ba8SSam Ravnborg unsigned long err;
300a88b5ba8SSam Ravnborg
301a88b5ba8SSam Ravnborg err = sun4v_ldc_tx_set_qtail(lp->id, tail);
302a88b5ba8SSam Ravnborg if (!err)
303a88b5ba8SSam Ravnborg return 0;
304a88b5ba8SSam Ravnborg
305a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) {
306a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail;
307a88b5ba8SSam Ravnborg return -EINVAL;
308a88b5ba8SSam Ravnborg }
309a88b5ba8SSam Ravnborg udelay(1);
310a88b5ba8SSam Ravnborg }
311a88b5ba8SSam Ravnborg
312a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail;
313a88b5ba8SSam Ravnborg return -EBUSY;
314a88b5ba8SSam Ravnborg }
315a88b5ba8SSam Ravnborg
316a88b5ba8SSam Ravnborg /* This just updates the head value in the hypervisor using
317a88b5ba8SSam Ravnborg * a polling loop with a timeout. The caller takes care of
318a88b5ba8SSam Ravnborg * upating software state representing the head change, if any.
319a88b5ba8SSam Ravnborg */
__set_rx_head(struct ldc_channel * lp,unsigned long head)320a88b5ba8SSam Ravnborg static int __set_rx_head(struct ldc_channel *lp, unsigned long head)
321a88b5ba8SSam Ravnborg {
322a88b5ba8SSam Ravnborg int limit = 1000;
323a88b5ba8SSam Ravnborg
324a88b5ba8SSam Ravnborg while (limit-- > 0) {
325a88b5ba8SSam Ravnborg unsigned long err;
326a88b5ba8SSam Ravnborg
327a88b5ba8SSam Ravnborg err = sun4v_ldc_rx_set_qhead(lp->id, head);
328a88b5ba8SSam Ravnborg if (!err)
329a88b5ba8SSam Ravnborg return 0;
330a88b5ba8SSam Ravnborg
331a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK)
332a88b5ba8SSam Ravnborg return -EINVAL;
333a88b5ba8SSam Ravnborg
334a88b5ba8SSam Ravnborg udelay(1);
335a88b5ba8SSam Ravnborg }
336a88b5ba8SSam Ravnborg
337a88b5ba8SSam Ravnborg return -EBUSY;
338a88b5ba8SSam Ravnborg }
339a88b5ba8SSam Ravnborg
send_tx_packet(struct ldc_channel * lp,struct ldc_packet * p,unsigned long new_tail)340a88b5ba8SSam Ravnborg static int send_tx_packet(struct ldc_channel *lp,
341a88b5ba8SSam Ravnborg struct ldc_packet *p,
342a88b5ba8SSam Ravnborg unsigned long new_tail)
343a88b5ba8SSam Ravnborg {
344a88b5ba8SSam Ravnborg BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE)));
345a88b5ba8SSam Ravnborg
346a88b5ba8SSam Ravnborg return set_tx_tail(lp, new_tail);
347a88b5ba8SSam Ravnborg }
348a88b5ba8SSam Ravnborg
handshake_compose_ctrl(struct ldc_channel * lp,u8 stype,u8 ctrl,void * data,int dlen,unsigned long * new_tail)349a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp,
350a88b5ba8SSam Ravnborg u8 stype, u8 ctrl,
351a88b5ba8SSam Ravnborg void *data, int dlen,
352a88b5ba8SSam Ravnborg unsigned long *new_tail)
353a88b5ba8SSam Ravnborg {
354a88b5ba8SSam Ravnborg struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail);
355a88b5ba8SSam Ravnborg
356a88b5ba8SSam Ravnborg if (p) {
357a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p));
358a88b5ba8SSam Ravnborg p->type = LDC_CTRL;
359a88b5ba8SSam Ravnborg p->stype = stype;
360a88b5ba8SSam Ravnborg p->ctrl = ctrl;
361a88b5ba8SSam Ravnborg if (data)
362a88b5ba8SSam Ravnborg memcpy(p->u.u_data, data, dlen);
363a88b5ba8SSam Ravnborg }
364a88b5ba8SSam Ravnborg return p;
365a88b5ba8SSam Ravnborg }
366a88b5ba8SSam Ravnborg
start_handshake(struct ldc_channel * lp)367a88b5ba8SSam Ravnborg static int start_handshake(struct ldc_channel *lp)
368a88b5ba8SSam Ravnborg {
369a88b5ba8SSam Ravnborg struct ldc_packet *p;
370a88b5ba8SSam Ravnborg struct ldc_version *ver;
371a88b5ba8SSam Ravnborg unsigned long new_tail;
372a88b5ba8SSam Ravnborg
373a88b5ba8SSam Ravnborg ver = &ver_arr[0];
374a88b5ba8SSam Ravnborg
375a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n",
376a88b5ba8SSam Ravnborg ver->major, ver->minor);
377a88b5ba8SSam Ravnborg
378a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
379a88b5ba8SSam Ravnborg ver, sizeof(*ver), &new_tail);
380a88b5ba8SSam Ravnborg if (p) {
381a88b5ba8SSam Ravnborg int err = send_tx_packet(lp, p, new_tail);
382a88b5ba8SSam Ravnborg if (!err)
383a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_RESET;
384a88b5ba8SSam Ravnborg return err;
385a88b5ba8SSam Ravnborg }
386a88b5ba8SSam Ravnborg return -EBUSY;
387a88b5ba8SSam Ravnborg }
388a88b5ba8SSam Ravnborg
send_version_nack(struct ldc_channel * lp,u16 major,u16 minor)389a88b5ba8SSam Ravnborg static int send_version_nack(struct ldc_channel *lp,
390a88b5ba8SSam Ravnborg u16 major, u16 minor)
391a88b5ba8SSam Ravnborg {
392a88b5ba8SSam Ravnborg struct ldc_packet *p;
393a88b5ba8SSam Ravnborg struct ldc_version ver;
394a88b5ba8SSam Ravnborg unsigned long new_tail;
395a88b5ba8SSam Ravnborg
396a88b5ba8SSam Ravnborg ver.major = major;
397a88b5ba8SSam Ravnborg ver.minor = minor;
398a88b5ba8SSam Ravnborg
399a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS,
400a88b5ba8SSam Ravnborg &ver, sizeof(ver), &new_tail);
401a88b5ba8SSam Ravnborg if (p) {
402a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n",
403a88b5ba8SSam Ravnborg ver.major, ver.minor);
404a88b5ba8SSam Ravnborg
405a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail);
406a88b5ba8SSam Ravnborg }
407a88b5ba8SSam Ravnborg return -EBUSY;
408a88b5ba8SSam Ravnborg }
409a88b5ba8SSam Ravnborg
send_version_ack(struct ldc_channel * lp,struct ldc_version * vp)410a88b5ba8SSam Ravnborg static int send_version_ack(struct ldc_channel *lp,
411a88b5ba8SSam Ravnborg struct ldc_version *vp)
412a88b5ba8SSam Ravnborg {
413a88b5ba8SSam Ravnborg struct ldc_packet *p;
414a88b5ba8SSam Ravnborg unsigned long new_tail;
415a88b5ba8SSam Ravnborg
416a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS,
417a88b5ba8SSam Ravnborg vp, sizeof(*vp), &new_tail);
418a88b5ba8SSam Ravnborg if (p) {
419a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n",
420a88b5ba8SSam Ravnborg vp->major, vp->minor);
421a88b5ba8SSam Ravnborg
422a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail);
423a88b5ba8SSam Ravnborg }
424a88b5ba8SSam Ravnborg return -EBUSY;
425a88b5ba8SSam Ravnborg }
426a88b5ba8SSam Ravnborg
send_rts(struct ldc_channel * lp)427a88b5ba8SSam Ravnborg static int send_rts(struct ldc_channel *lp)
428a88b5ba8SSam Ravnborg {
429a88b5ba8SSam Ravnborg struct ldc_packet *p;
430a88b5ba8SSam Ravnborg unsigned long new_tail;
431a88b5ba8SSam Ravnborg
432a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0,
433a88b5ba8SSam Ravnborg &new_tail);
434a88b5ba8SSam Ravnborg if (p) {
435a88b5ba8SSam Ravnborg p->env = lp->cfg.mode;
436a88b5ba8SSam Ravnborg p->seqid = 0;
437a88b5ba8SSam Ravnborg lp->rcv_nxt = 0;
438a88b5ba8SSam Ravnborg
439a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n",
440a88b5ba8SSam Ravnborg p->env, p->seqid);
441a88b5ba8SSam Ravnborg
442a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail);
443a88b5ba8SSam Ravnborg }
444a88b5ba8SSam Ravnborg return -EBUSY;
445a88b5ba8SSam Ravnborg }
446a88b5ba8SSam Ravnborg
send_rtr(struct ldc_channel * lp)447a88b5ba8SSam Ravnborg static int send_rtr(struct ldc_channel *lp)
448a88b5ba8SSam Ravnborg {
449a88b5ba8SSam Ravnborg struct ldc_packet *p;
450a88b5ba8SSam Ravnborg unsigned long new_tail;
451a88b5ba8SSam Ravnborg
452a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0,
453a88b5ba8SSam Ravnborg &new_tail);
454a88b5ba8SSam Ravnborg if (p) {
455a88b5ba8SSam Ravnborg p->env = lp->cfg.mode;
456a88b5ba8SSam Ravnborg p->seqid = 0;
457a88b5ba8SSam Ravnborg
458a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n",
459a88b5ba8SSam Ravnborg p->env, p->seqid);
460a88b5ba8SSam Ravnborg
461a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail);
462a88b5ba8SSam Ravnborg }
463a88b5ba8SSam Ravnborg return -EBUSY;
464a88b5ba8SSam Ravnborg }
465a88b5ba8SSam Ravnborg
send_rdx(struct ldc_channel * lp)466a88b5ba8SSam Ravnborg static int send_rdx(struct ldc_channel *lp)
467a88b5ba8SSam Ravnborg {
468a88b5ba8SSam Ravnborg struct ldc_packet *p;
469a88b5ba8SSam Ravnborg unsigned long new_tail;
470a88b5ba8SSam Ravnborg
471a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0,
472a88b5ba8SSam Ravnborg &new_tail);
473a88b5ba8SSam Ravnborg if (p) {
474a88b5ba8SSam Ravnborg p->env = 0;
475a88b5ba8SSam Ravnborg p->seqid = ++lp->snd_nxt;
476a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt;
477a88b5ba8SSam Ravnborg
478a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n",
479a88b5ba8SSam Ravnborg p->env, p->seqid, p->u.r.ackid);
480a88b5ba8SSam Ravnborg
481a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail);
482a88b5ba8SSam Ravnborg }
483a88b5ba8SSam Ravnborg return -EBUSY;
484a88b5ba8SSam Ravnborg }
485a88b5ba8SSam Ravnborg
send_data_nack(struct ldc_channel * lp,struct ldc_packet * data_pkt)486a88b5ba8SSam Ravnborg static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt)
487a88b5ba8SSam Ravnborg {
488a88b5ba8SSam Ravnborg struct ldc_packet *p;
489a88b5ba8SSam Ravnborg unsigned long new_tail;
490a88b5ba8SSam Ravnborg int err;
491a88b5ba8SSam Ravnborg
492a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail);
493a88b5ba8SSam Ravnborg if (!p)
494a88b5ba8SSam Ravnborg return -EBUSY;
495a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p));
496a88b5ba8SSam Ravnborg p->type = data_pkt->type;
497a88b5ba8SSam Ravnborg p->stype = LDC_NACK;
498a88b5ba8SSam Ravnborg p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK;
499a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1;
500a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt;
501a88b5ba8SSam Ravnborg
502a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n",
503a88b5ba8SSam Ravnborg p->type, p->ctrl, p->seqid, p->u.r.ackid);
504a88b5ba8SSam Ravnborg
505a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail);
506a88b5ba8SSam Ravnborg if (!err)
507a88b5ba8SSam Ravnborg lp->snd_nxt++;
508a88b5ba8SSam Ravnborg
509a88b5ba8SSam Ravnborg return err;
510a88b5ba8SSam Ravnborg }
511a88b5ba8SSam Ravnborg
ldc_abort(struct ldc_channel * lp,const char * msg)51229693e75SJag Raman static int ldc_abort(struct ldc_channel *lp, const char *msg)
513a88b5ba8SSam Ravnborg {
514a88b5ba8SSam Ravnborg unsigned long hv_err;
515a88b5ba8SSam Ravnborg
51629693e75SJag Raman ldcdbg(STATE, "ABORT[%s]\n", msg);
51729693e75SJag Raman ldc_print(lp);
518a88b5ba8SSam Ravnborg
519a88b5ba8SSam Ravnborg /* We report but do not act upon the hypervisor errors because
520a88b5ba8SSam Ravnborg * there really isn't much we can do if they fail at this point.
521a88b5ba8SSam Ravnborg */
522a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
523a88b5ba8SSam Ravnborg if (hv_err)
524a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: "
525a88b5ba8SSam Ravnborg "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
526a88b5ba8SSam Ravnborg lp->id, lp->tx_ra, lp->tx_num_entries, hv_err);
527a88b5ba8SSam Ravnborg
528a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id,
529a88b5ba8SSam Ravnborg &lp->tx_head,
530a88b5ba8SSam Ravnborg &lp->tx_tail,
531a88b5ba8SSam Ravnborg &lp->chan_state);
532a88b5ba8SSam Ravnborg if (hv_err)
533a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: "
534a88b5ba8SSam Ravnborg "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n",
535a88b5ba8SSam Ravnborg lp->id, hv_err);
536a88b5ba8SSam Ravnborg
537a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
538a88b5ba8SSam Ravnborg if (hv_err)
539a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: "
540a88b5ba8SSam Ravnborg "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
541a88b5ba8SSam Ravnborg lp->id, lp->rx_ra, lp->rx_num_entries, hv_err);
542a88b5ba8SSam Ravnborg
543a88b5ba8SSam Ravnborg /* Refetch the RX queue state as well, because we could be invoked
544a88b5ba8SSam Ravnborg * here in the queue processing context.
545a88b5ba8SSam Ravnborg */
546a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id,
547a88b5ba8SSam Ravnborg &lp->rx_head,
548a88b5ba8SSam Ravnborg &lp->rx_tail,
549a88b5ba8SSam Ravnborg &lp->chan_state);
550a88b5ba8SSam Ravnborg if (hv_err)
551a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: "
552a88b5ba8SSam Ravnborg "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n",
553a88b5ba8SSam Ravnborg lp->id, hv_err);
554a88b5ba8SSam Ravnborg
555a88b5ba8SSam Ravnborg return -ECONNRESET;
556a88b5ba8SSam Ravnborg }
557a88b5ba8SSam Ravnborg
find_by_major(u16 major)558a88b5ba8SSam Ravnborg static struct ldc_version *find_by_major(u16 major)
559a88b5ba8SSam Ravnborg {
560a88b5ba8SSam Ravnborg struct ldc_version *ret = NULL;
561a88b5ba8SSam Ravnborg int i;
562a88b5ba8SSam Ravnborg
563a88b5ba8SSam Ravnborg for (i = 0; i < ARRAY_SIZE(ver_arr); i++) {
564a88b5ba8SSam Ravnborg struct ldc_version *v = &ver_arr[i];
565a88b5ba8SSam Ravnborg if (v->major <= major) {
566a88b5ba8SSam Ravnborg ret = v;
567a88b5ba8SSam Ravnborg break;
568a88b5ba8SSam Ravnborg }
569a88b5ba8SSam Ravnborg }
570a88b5ba8SSam Ravnborg return ret;
571a88b5ba8SSam Ravnborg }
572a88b5ba8SSam Ravnborg
process_ver_info(struct ldc_channel * lp,struct ldc_version * vp)573a88b5ba8SSam Ravnborg static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp)
574a88b5ba8SSam Ravnborg {
575a88b5ba8SSam Ravnborg struct ldc_version *vap;
576a88b5ba8SSam Ravnborg int err;
577a88b5ba8SSam Ravnborg
578a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n",
579a88b5ba8SSam Ravnborg vp->major, vp->minor);
580a88b5ba8SSam Ravnborg
581a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) {
582a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN;
583a88b5ba8SSam Ravnborg memset(&lp->ver, 0, sizeof(lp->ver));
584a88b5ba8SSam Ravnborg }
585a88b5ba8SSam Ravnborg
586a88b5ba8SSam Ravnborg vap = find_by_major(vp->major);
587a88b5ba8SSam Ravnborg if (!vap) {
588a88b5ba8SSam Ravnborg err = send_version_nack(lp, 0, 0);
589a88b5ba8SSam Ravnborg } else if (vap->major != vp->major) {
590a88b5ba8SSam Ravnborg err = send_version_nack(lp, vap->major, vap->minor);
591a88b5ba8SSam Ravnborg } else {
592a88b5ba8SSam Ravnborg struct ldc_version ver = *vp;
593a88b5ba8SSam Ravnborg if (ver.minor > vap->minor)
594a88b5ba8SSam Ravnborg ver.minor = vap->minor;
595a88b5ba8SSam Ravnborg err = send_version_ack(lp, &ver);
596a88b5ba8SSam Ravnborg if (!err) {
597a88b5ba8SSam Ravnborg lp->ver = ver;
598a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS;
599a88b5ba8SSam Ravnborg }
600a88b5ba8SSam Ravnborg }
601a88b5ba8SSam Ravnborg if (err)
60229693e75SJag Raman return LDC_ABORT(lp);
603a88b5ba8SSam Ravnborg
604a88b5ba8SSam Ravnborg return 0;
605a88b5ba8SSam Ravnborg }
606a88b5ba8SSam Ravnborg
process_ver_ack(struct ldc_channel * lp,struct ldc_version * vp)607a88b5ba8SSam Ravnborg static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp)
608a88b5ba8SSam Ravnborg {
609a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n",
610a88b5ba8SSam Ravnborg vp->major, vp->minor);
611a88b5ba8SSam Ravnborg
612a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) {
613a88b5ba8SSam Ravnborg if (lp->ver.major != vp->major ||
614a88b5ba8SSam Ravnborg lp->ver.minor != vp->minor)
61529693e75SJag Raman return LDC_ABORT(lp);
616a88b5ba8SSam Ravnborg } else {
617a88b5ba8SSam Ravnborg lp->ver = *vp;
618a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS;
619a88b5ba8SSam Ravnborg }
620a88b5ba8SSam Ravnborg if (send_rts(lp))
62129693e75SJag Raman return LDC_ABORT(lp);
622a88b5ba8SSam Ravnborg return 0;
623a88b5ba8SSam Ravnborg }
624a88b5ba8SSam Ravnborg
process_ver_nack(struct ldc_channel * lp,struct ldc_version * vp)625a88b5ba8SSam Ravnborg static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp)
626a88b5ba8SSam Ravnborg {
627a88b5ba8SSam Ravnborg struct ldc_version *vap;
628a88b5ba8SSam Ravnborg struct ldc_packet *p;
629a88b5ba8SSam Ravnborg unsigned long new_tail;
630a88b5ba8SSam Ravnborg
631b2c0805fSSteven Rostedt if (vp->major == 0 && vp->minor == 0)
63229693e75SJag Raman return LDC_ABORT(lp);
633b2c0805fSSteven Rostedt
634b2c0805fSSteven Rostedt vap = find_by_major(vp->major);
635b2c0805fSSteven Rostedt if (!vap)
63629693e75SJag Raman return LDC_ABORT(lp);
637b2c0805fSSteven Rostedt
638a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
639a88b5ba8SSam Ravnborg vap, sizeof(*vap),
640a88b5ba8SSam Ravnborg &new_tail);
641b2c0805fSSteven Rostedt if (!p)
64229693e75SJag Raman return LDC_ABORT(lp);
643b2c0805fSSteven Rostedt
644b2c0805fSSteven Rostedt return send_tx_packet(lp, p, new_tail);
645a88b5ba8SSam Ravnborg }
646a88b5ba8SSam Ravnborg
process_version(struct ldc_channel * lp,struct ldc_packet * p)647a88b5ba8SSam Ravnborg static int process_version(struct ldc_channel *lp,
648a88b5ba8SSam Ravnborg struct ldc_packet *p)
649a88b5ba8SSam Ravnborg {
650a88b5ba8SSam Ravnborg struct ldc_version *vp;
651a88b5ba8SSam Ravnborg
652a88b5ba8SSam Ravnborg vp = (struct ldc_version *) p->u.u_data;
653a88b5ba8SSam Ravnborg
654a88b5ba8SSam Ravnborg switch (p->stype) {
655a88b5ba8SSam Ravnborg case LDC_INFO:
656a88b5ba8SSam Ravnborg return process_ver_info(lp, vp);
657a88b5ba8SSam Ravnborg
658a88b5ba8SSam Ravnborg case LDC_ACK:
659a88b5ba8SSam Ravnborg return process_ver_ack(lp, vp);
660a88b5ba8SSam Ravnborg
661a88b5ba8SSam Ravnborg case LDC_NACK:
662a88b5ba8SSam Ravnborg return process_ver_nack(lp, vp);
663a88b5ba8SSam Ravnborg
664a88b5ba8SSam Ravnborg default:
66529693e75SJag Raman return LDC_ABORT(lp);
666a88b5ba8SSam Ravnborg }
667a88b5ba8SSam Ravnborg }
668a88b5ba8SSam Ravnborg
process_rts(struct ldc_channel * lp,struct ldc_packet * p)669a88b5ba8SSam Ravnborg static int process_rts(struct ldc_channel *lp,
670a88b5ba8SSam Ravnborg struct ldc_packet *p)
671a88b5ba8SSam Ravnborg {
672a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n",
673a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env);
674a88b5ba8SSam Ravnborg
675a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO ||
676a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_GOTVERS ||
677a88b5ba8SSam Ravnborg p->env != lp->cfg.mode)
67829693e75SJag Raman return LDC_ABORT(lp);
679a88b5ba8SSam Ravnborg
680a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid;
681a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid;
682a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_SENTRTR;
683a88b5ba8SSam Ravnborg if (send_rtr(lp))
68429693e75SJag Raman return LDC_ABORT(lp);
685a88b5ba8SSam Ravnborg
686a88b5ba8SSam Ravnborg return 0;
687a88b5ba8SSam Ravnborg }
688a88b5ba8SSam Ravnborg
process_rtr(struct ldc_channel * lp,struct ldc_packet * p)689a88b5ba8SSam Ravnborg static int process_rtr(struct ldc_channel *lp,
690a88b5ba8SSam Ravnborg struct ldc_packet *p)
691a88b5ba8SSam Ravnborg {
692a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n",
693a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env);
694a88b5ba8SSam Ravnborg
695a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO ||
696a88b5ba8SSam Ravnborg p->env != lp->cfg.mode)
69729693e75SJag Raman return LDC_ABORT(lp);
698a88b5ba8SSam Ravnborg
699a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid;
700a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE;
701a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED);
702a88b5ba8SSam Ravnborg send_rdx(lp);
703a88b5ba8SSam Ravnborg
704a88b5ba8SSam Ravnborg return LDC_EVENT_UP;
705a88b5ba8SSam Ravnborg }
706a88b5ba8SSam Ravnborg
rx_seq_ok(struct ldc_channel * lp,u32 seqid)707a88b5ba8SSam Ravnborg static int rx_seq_ok(struct ldc_channel *lp, u32 seqid)
708a88b5ba8SSam Ravnborg {
709a88b5ba8SSam Ravnborg return lp->rcv_nxt + 1 == seqid;
710a88b5ba8SSam Ravnborg }
711a88b5ba8SSam Ravnborg
process_rdx(struct ldc_channel * lp,struct ldc_packet * p)712a88b5ba8SSam Ravnborg static int process_rdx(struct ldc_channel *lp,
713a88b5ba8SSam Ravnborg struct ldc_packet *p)
714a88b5ba8SSam Ravnborg {
715a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n",
716a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env, p->u.r.ackid);
717a88b5ba8SSam Ravnborg
718a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO ||
719a88b5ba8SSam Ravnborg !(rx_seq_ok(lp, p->seqid)))
72029693e75SJag Raman return LDC_ABORT(lp);
721a88b5ba8SSam Ravnborg
722a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid;
723a88b5ba8SSam Ravnborg
724a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE;
725a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED);
726a88b5ba8SSam Ravnborg
727a88b5ba8SSam Ravnborg return LDC_EVENT_UP;
728a88b5ba8SSam Ravnborg }
729a88b5ba8SSam Ravnborg
process_control_frame(struct ldc_channel * lp,struct ldc_packet * p)730a88b5ba8SSam Ravnborg static int process_control_frame(struct ldc_channel *lp,
731a88b5ba8SSam Ravnborg struct ldc_packet *p)
732a88b5ba8SSam Ravnborg {
733a88b5ba8SSam Ravnborg switch (p->ctrl) {
734a88b5ba8SSam Ravnborg case LDC_VERS:
735a88b5ba8SSam Ravnborg return process_version(lp, p);
736a88b5ba8SSam Ravnborg
737a88b5ba8SSam Ravnborg case LDC_RTS:
738a88b5ba8SSam Ravnborg return process_rts(lp, p);
739a88b5ba8SSam Ravnborg
740a88b5ba8SSam Ravnborg case LDC_RTR:
741a88b5ba8SSam Ravnborg return process_rtr(lp, p);
742a88b5ba8SSam Ravnborg
743a88b5ba8SSam Ravnborg case LDC_RDX:
744a88b5ba8SSam Ravnborg return process_rdx(lp, p);
745a88b5ba8SSam Ravnborg
746a88b5ba8SSam Ravnborg default:
74729693e75SJag Raman return LDC_ABORT(lp);
748a88b5ba8SSam Ravnborg }
749a88b5ba8SSam Ravnborg }
750a88b5ba8SSam Ravnborg
process_error_frame(struct ldc_channel * lp,struct ldc_packet * p)751a88b5ba8SSam Ravnborg static int process_error_frame(struct ldc_channel *lp,
752a88b5ba8SSam Ravnborg struct ldc_packet *p)
753a88b5ba8SSam Ravnborg {
75429693e75SJag Raman return LDC_ABORT(lp);
755a88b5ba8SSam Ravnborg }
756a88b5ba8SSam Ravnborg
process_data_ack(struct ldc_channel * lp,struct ldc_packet * ack)757a88b5ba8SSam Ravnborg static int process_data_ack(struct ldc_channel *lp,
758a88b5ba8SSam Ravnborg struct ldc_packet *ack)
759a88b5ba8SSam Ravnborg {
760a88b5ba8SSam Ravnborg unsigned long head = lp->tx_acked;
761a88b5ba8SSam Ravnborg u32 ackid = ack->u.r.ackid;
762a88b5ba8SSam Ravnborg
763a88b5ba8SSam Ravnborg while (1) {
764a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE);
765a88b5ba8SSam Ravnborg
766a88b5ba8SSam Ravnborg head = tx_advance(lp, head);
767a88b5ba8SSam Ravnborg
768a88b5ba8SSam Ravnborg if (p->seqid == ackid) {
769a88b5ba8SSam Ravnborg lp->tx_acked = head;
770a88b5ba8SSam Ravnborg return 0;
771a88b5ba8SSam Ravnborg }
772a88b5ba8SSam Ravnborg if (head == lp->tx_tail)
77329693e75SJag Raman return LDC_ABORT(lp);
774a88b5ba8SSam Ravnborg }
775a88b5ba8SSam Ravnborg
776a88b5ba8SSam Ravnborg return 0;
777a88b5ba8SSam Ravnborg }
778a88b5ba8SSam Ravnborg
send_events(struct ldc_channel * lp,unsigned int event_mask)779a88b5ba8SSam Ravnborg static void send_events(struct ldc_channel *lp, unsigned int event_mask)
780a88b5ba8SSam Ravnborg {
781a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_RESET)
782a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_RESET);
783a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_UP)
784a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_UP);
785a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_DATA_READY)
786a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY);
787a88b5ba8SSam Ravnborg }
788a88b5ba8SSam Ravnborg
ldc_rx(int irq,void * dev_id)789a88b5ba8SSam Ravnborg static irqreturn_t ldc_rx(int irq, void *dev_id)
790a88b5ba8SSam Ravnborg {
791a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id;
792c6fee081SDavid S. Miller unsigned long orig_state, flags;
793a88b5ba8SSam Ravnborg unsigned int event_mask;
794a88b5ba8SSam Ravnborg
795a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags);
796a88b5ba8SSam Ravnborg
797a88b5ba8SSam Ravnborg orig_state = lp->chan_state;
798c6fee081SDavid S. Miller
799c6fee081SDavid S. Miller /* We should probably check for hypervisor errors here and
800c6fee081SDavid S. Miller * reset the LDC channel if we get one.
801c6fee081SDavid S. Miller */
802c6fee081SDavid S. Miller sun4v_ldc_rx_get_state(lp->id,
803a88b5ba8SSam Ravnborg &lp->rx_head,
804a88b5ba8SSam Ravnborg &lp->rx_tail,
805a88b5ba8SSam Ravnborg &lp->chan_state);
806a88b5ba8SSam Ravnborg
807a88b5ba8SSam Ravnborg ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
808a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->rx_head, lp->rx_tail);
809a88b5ba8SSam Ravnborg
810a88b5ba8SSam Ravnborg event_mask = 0;
811a88b5ba8SSam Ravnborg
812a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW &&
813a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) {
814a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE;
815a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED);
816a88b5ba8SSam Ravnborg
817fc43b978SJag Raman /*
818fc43b978SJag Raman * Generate an LDC_EVENT_UP event if the channel
819fc43b978SJag Raman * was not already up.
820fc43b978SJag Raman */
821fc43b978SJag Raman if (orig_state != LDC_CHANNEL_UP) {
822a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP;
823a88b5ba8SSam Ravnborg orig_state = lp->chan_state;
824a88b5ba8SSam Ravnborg }
825fc43b978SJag Raman }
826a88b5ba8SSam Ravnborg
827a88b5ba8SSam Ravnborg /* If we are in reset state, flush the RX queue and ignore
828a88b5ba8SSam Ravnborg * everything.
829a88b5ba8SSam Ravnborg */
830a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_RESET) {
83185762a65SJag Raman (void) ldc_rx_reset(lp);
832a88b5ba8SSam Ravnborg goto out;
833a88b5ba8SSam Ravnborg }
834a88b5ba8SSam Ravnborg
835a88b5ba8SSam Ravnborg /* Once we finish the handshake, we let the ldc_read()
836a88b5ba8SSam Ravnborg * paths do all of the control frame and state management.
837a88b5ba8SSam Ravnborg * Just trigger the callback.
838a88b5ba8SSam Ravnborg */
839a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) {
840a88b5ba8SSam Ravnborg handshake_complete:
841a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) {
842a88b5ba8SSam Ravnborg unsigned int event = LDC_EVENT_RESET;
843a88b5ba8SSam Ravnborg
844a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_UP)
845a88b5ba8SSam Ravnborg event = LDC_EVENT_UP;
846a88b5ba8SSam Ravnborg
847a88b5ba8SSam Ravnborg event_mask |= event;
848a88b5ba8SSam Ravnborg }
849a88b5ba8SSam Ravnborg if (lp->rx_head != lp->rx_tail)
850a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY;
851a88b5ba8SSam Ravnborg
852a88b5ba8SSam Ravnborg goto out;
853a88b5ba8SSam Ravnborg }
854a88b5ba8SSam Ravnborg
855a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state)
856a88b5ba8SSam Ravnborg goto out;
857a88b5ba8SSam Ravnborg
858a88b5ba8SSam Ravnborg while (lp->rx_head != lp->rx_tail) {
859a88b5ba8SSam Ravnborg struct ldc_packet *p;
860a88b5ba8SSam Ravnborg unsigned long new;
861a88b5ba8SSam Ravnborg int err;
862a88b5ba8SSam Ravnborg
863a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
864a88b5ba8SSam Ravnborg
865a88b5ba8SSam Ravnborg switch (p->type) {
866a88b5ba8SSam Ravnborg case LDC_CTRL:
867a88b5ba8SSam Ravnborg err = process_control_frame(lp, p);
868a88b5ba8SSam Ravnborg if (err > 0)
869a88b5ba8SSam Ravnborg event_mask |= err;
870a88b5ba8SSam Ravnborg break;
871a88b5ba8SSam Ravnborg
872a88b5ba8SSam Ravnborg case LDC_DATA:
873a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY;
874a88b5ba8SSam Ravnborg err = 0;
875a88b5ba8SSam Ravnborg break;
876a88b5ba8SSam Ravnborg
877a88b5ba8SSam Ravnborg case LDC_ERR:
878a88b5ba8SSam Ravnborg err = process_error_frame(lp, p);
879a88b5ba8SSam Ravnborg break;
880a88b5ba8SSam Ravnborg
881a88b5ba8SSam Ravnborg default:
88229693e75SJag Raman err = LDC_ABORT(lp);
883a88b5ba8SSam Ravnborg break;
884a88b5ba8SSam Ravnborg }
885a88b5ba8SSam Ravnborg
886a88b5ba8SSam Ravnborg if (err < 0)
887a88b5ba8SSam Ravnborg break;
888a88b5ba8SSam Ravnborg
889a88b5ba8SSam Ravnborg new = lp->rx_head;
890a88b5ba8SSam Ravnborg new += LDC_PACKET_SIZE;
891a88b5ba8SSam Ravnborg if (new == (lp->rx_num_entries * LDC_PACKET_SIZE))
892a88b5ba8SSam Ravnborg new = 0;
893a88b5ba8SSam Ravnborg lp->rx_head = new;
894a88b5ba8SSam Ravnborg
895a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new);
896a88b5ba8SSam Ravnborg if (err < 0) {
89729693e75SJag Raman (void) LDC_ABORT(lp);
898a88b5ba8SSam Ravnborg break;
899a88b5ba8SSam Ravnborg }
900a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE)
901a88b5ba8SSam Ravnborg goto handshake_complete;
902a88b5ba8SSam Ravnborg }
903a88b5ba8SSam Ravnborg
904a88b5ba8SSam Ravnborg out:
905a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags);
906a88b5ba8SSam Ravnborg
907a88b5ba8SSam Ravnborg send_events(lp, event_mask);
908a88b5ba8SSam Ravnborg
909a88b5ba8SSam Ravnborg return IRQ_HANDLED;
910a88b5ba8SSam Ravnborg }
911a88b5ba8SSam Ravnborg
ldc_tx(int irq,void * dev_id)912a88b5ba8SSam Ravnborg static irqreturn_t ldc_tx(int irq, void *dev_id)
913a88b5ba8SSam Ravnborg {
914a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id;
915c6fee081SDavid S. Miller unsigned long flags, orig_state;
916a88b5ba8SSam Ravnborg unsigned int event_mask = 0;
917a88b5ba8SSam Ravnborg
918a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags);
919a88b5ba8SSam Ravnborg
920a88b5ba8SSam Ravnborg orig_state = lp->chan_state;
921c6fee081SDavid S. Miller
922c6fee081SDavid S. Miller /* We should probably check for hypervisor errors here and
923c6fee081SDavid S. Miller * reset the LDC channel if we get one.
924c6fee081SDavid S. Miller */
925c6fee081SDavid S. Miller sun4v_ldc_tx_get_state(lp->id,
926a88b5ba8SSam Ravnborg &lp->tx_head,
927a88b5ba8SSam Ravnborg &lp->tx_tail,
928a88b5ba8SSam Ravnborg &lp->chan_state);
929a88b5ba8SSam Ravnborg
930a88b5ba8SSam Ravnborg ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
931a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->tx_head, lp->tx_tail);
932a88b5ba8SSam Ravnborg
933a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW &&
934a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) {
935a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE;
936a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED);
937a88b5ba8SSam Ravnborg
938fc43b978SJag Raman /*
939fc43b978SJag Raman * Generate an LDC_EVENT_UP event if the channel
940fc43b978SJag Raman * was not already up.
941fc43b978SJag Raman */
942fc43b978SJag Raman if (orig_state != LDC_CHANNEL_UP) {
943a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP;
944fc43b978SJag Raman orig_state = lp->chan_state;
945fc43b978SJag Raman }
946a88b5ba8SSam Ravnborg }
947a88b5ba8SSam Ravnborg
948a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags);
949a88b5ba8SSam Ravnborg
950a88b5ba8SSam Ravnborg send_events(lp, event_mask);
951a88b5ba8SSam Ravnborg
952a88b5ba8SSam Ravnborg return IRQ_HANDLED;
953a88b5ba8SSam Ravnborg }
954a88b5ba8SSam Ravnborg
955a88b5ba8SSam Ravnborg /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so
956a88b5ba8SSam Ravnborg * XXX that addition and removal from the ldc_channel_list has
957a88b5ba8SSam Ravnborg * XXX atomicity, otherwise the __ldc_channel_exists() check is
958a88b5ba8SSam Ravnborg * XXX totally pointless as another thread can slip into ldc_alloc()
959a88b5ba8SSam Ravnborg * XXX and add a channel with the same ID. There also needs to be
960a88b5ba8SSam Ravnborg * XXX a spinlock for ldc_channel_list.
961a88b5ba8SSam Ravnborg */
962a88b5ba8SSam Ravnborg static HLIST_HEAD(ldc_channel_list);
963a88b5ba8SSam Ravnborg
__ldc_channel_exists(unsigned long id)964a88b5ba8SSam Ravnborg static int __ldc_channel_exists(unsigned long id)
965a88b5ba8SSam Ravnborg {
966a88b5ba8SSam Ravnborg struct ldc_channel *lp;
967a88b5ba8SSam Ravnborg
968b67bfe0dSSasha Levin hlist_for_each_entry(lp, &ldc_channel_list, list) {
969a88b5ba8SSam Ravnborg if (lp->id == id)
970a88b5ba8SSam Ravnborg return 1;
971a88b5ba8SSam Ravnborg }
972a88b5ba8SSam Ravnborg return 0;
973a88b5ba8SSam Ravnborg }
974a88b5ba8SSam Ravnborg
alloc_queue(const char * name,unsigned long num_entries,struct ldc_packet ** base,unsigned long * ra)975a88b5ba8SSam Ravnborg static int alloc_queue(const char *name, unsigned long num_entries,
976a88b5ba8SSam Ravnborg struct ldc_packet **base, unsigned long *ra)
977a88b5ba8SSam Ravnborg {
978a88b5ba8SSam Ravnborg unsigned long size, order;
979a88b5ba8SSam Ravnborg void *q;
980a88b5ba8SSam Ravnborg
981a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE;
982a88b5ba8SSam Ravnborg order = get_order(size);
983a88b5ba8SSam Ravnborg
984a88b5ba8SSam Ravnborg q = (void *) __get_free_pages(GFP_KERNEL, order);
985a88b5ba8SSam Ravnborg if (!q) {
986a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of %s queue failed with "
987a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", name, size, order);
988a88b5ba8SSam Ravnborg return -ENOMEM;
989a88b5ba8SSam Ravnborg }
990a88b5ba8SSam Ravnborg
991a88b5ba8SSam Ravnborg memset(q, 0, PAGE_SIZE << order);
992a88b5ba8SSam Ravnborg
993a88b5ba8SSam Ravnborg *base = q;
994a88b5ba8SSam Ravnborg *ra = __pa(q);
995a88b5ba8SSam Ravnborg
996a88b5ba8SSam Ravnborg return 0;
997a88b5ba8SSam Ravnborg }
998a88b5ba8SSam Ravnborg
free_queue(unsigned long num_entries,struct ldc_packet * q)999a88b5ba8SSam Ravnborg static void free_queue(unsigned long num_entries, struct ldc_packet *q)
1000a88b5ba8SSam Ravnborg {
1001a88b5ba8SSam Ravnborg unsigned long size, order;
1002a88b5ba8SSam Ravnborg
1003a88b5ba8SSam Ravnborg if (!q)
1004a88b5ba8SSam Ravnborg return;
1005a88b5ba8SSam Ravnborg
1006a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE;
1007a88b5ba8SSam Ravnborg order = get_order(size);
1008a88b5ba8SSam Ravnborg
1009a88b5ba8SSam Ravnborg free_pages((unsigned long)q, order);
1010a88b5ba8SSam Ravnborg }
1011a88b5ba8SSam Ravnborg
ldc_cookie_to_index(u64 cookie,void * arg)10120ae53ed1SSowmini Varadhan static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
10130ae53ed1SSowmini Varadhan {
10140ae53ed1SSowmini Varadhan u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
10150ae53ed1SSowmini Varadhan /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */
10160ae53ed1SSowmini Varadhan
10170ae53ed1SSowmini Varadhan cookie &= ~COOKIE_PGSZ_CODE;
10180ae53ed1SSowmini Varadhan
10190ae53ed1SSowmini Varadhan return (cookie >> (13ULL + (szcode * 3ULL)));
10200ae53ed1SSowmini Varadhan }
10210ae53ed1SSowmini Varadhan
ldc_demap(struct ldc_iommu * iommu,unsigned long id,u64 cookie,unsigned long entry,unsigned long npages)10220ae53ed1SSowmini Varadhan static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,
10230ae53ed1SSowmini Varadhan unsigned long entry, unsigned long npages)
10240ae53ed1SSowmini Varadhan {
10250ae53ed1SSowmini Varadhan struct ldc_mtable_entry *base;
10260ae53ed1SSowmini Varadhan unsigned long i, shift;
10270ae53ed1SSowmini Varadhan
10280ae53ed1SSowmini Varadhan shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
10290ae53ed1SSowmini Varadhan base = iommu->page_table + entry;
10300ae53ed1SSowmini Varadhan for (i = 0; i < npages; i++) {
10310ae53ed1SSowmini Varadhan if (base->cookie)
10320ae53ed1SSowmini Varadhan sun4v_ldc_revoke(id, cookie + (i << shift),
10330ae53ed1SSowmini Varadhan base->cookie);
10340ae53ed1SSowmini Varadhan base->mte = 0;
10350ae53ed1SSowmini Varadhan }
10360ae53ed1SSowmini Varadhan }
10370ae53ed1SSowmini Varadhan
1038a88b5ba8SSam Ravnborg /* XXX Make this configurable... XXX */
1039a88b5ba8SSam Ravnborg #define LDC_IOTABLE_SIZE (8 * 1024)
1040a88b5ba8SSam Ravnborg
ldc_iommu_init(const char * name,struct ldc_channel * lp)10410ae53ed1SSowmini Varadhan static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
1042a88b5ba8SSam Ravnborg {
1043a88b5ba8SSam Ravnborg unsigned long sz, num_tsb_entries, tsbsize, order;
10440ae53ed1SSowmini Varadhan struct ldc_iommu *ldc_iommu = &lp->iommu;
10450ae53ed1SSowmini Varadhan struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
1046a88b5ba8SSam Ravnborg struct ldc_mtable_entry *table;
1047a88b5ba8SSam Ravnborg unsigned long hv_err;
1048a88b5ba8SSam Ravnborg int err;
1049a88b5ba8SSam Ravnborg
1050a88b5ba8SSam Ravnborg num_tsb_entries = LDC_IOTABLE_SIZE;
1051a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
10520ae53ed1SSowmini Varadhan spin_lock_init(&ldc_iommu->lock);
1053a88b5ba8SSam Ravnborg
1054a88b5ba8SSam Ravnborg sz = num_tsb_entries / 8;
1055a88b5ba8SSam Ravnborg sz = (sz + 7UL) & ~7UL;
10560ae53ed1SSowmini Varadhan iommu->map = kzalloc(sz, GFP_KERNEL);
10570ae53ed1SSowmini Varadhan if (!iommu->map) {
1058a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
1059a88b5ba8SSam Ravnborg return -ENOMEM;
1060a88b5ba8SSam Ravnborg }
10610ae53ed1SSowmini Varadhan iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
10620ae53ed1SSowmini Varadhan NULL, false /* no large pool */,
10630ae53ed1SSowmini Varadhan 1 /* npools */,
10640ae53ed1SSowmini Varadhan true /* skip span boundary check */);
1065a88b5ba8SSam Ravnborg
1066a88b5ba8SSam Ravnborg order = get_order(tsbsize);
1067a88b5ba8SSam Ravnborg
1068a88b5ba8SSam Ravnborg table = (struct ldc_mtable_entry *)
1069a88b5ba8SSam Ravnborg __get_free_pages(GFP_KERNEL, order);
1070a88b5ba8SSam Ravnborg err = -ENOMEM;
1071a88b5ba8SSam Ravnborg if (!table) {
1072a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of MTE table failed, "
1073a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", tsbsize, order);
1074a88b5ba8SSam Ravnborg goto out_free_map;
1075a88b5ba8SSam Ravnborg }
1076a88b5ba8SSam Ravnborg
1077a88b5ba8SSam Ravnborg memset(table, 0, PAGE_SIZE << order);
1078a88b5ba8SSam Ravnborg
10790ae53ed1SSowmini Varadhan ldc_iommu->page_table = table;
1080a88b5ba8SSam Ravnborg
1081a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
1082a88b5ba8SSam Ravnborg num_tsb_entries);
1083a88b5ba8SSam Ravnborg err = -EINVAL;
1084a88b5ba8SSam Ravnborg if (hv_err)
1085a88b5ba8SSam Ravnborg goto out_free_table;
1086a88b5ba8SSam Ravnborg
1087a88b5ba8SSam Ravnborg return 0;
1088a88b5ba8SSam Ravnborg
1089a88b5ba8SSam Ravnborg out_free_table:
1090a88b5ba8SSam Ravnborg free_pages((unsigned long) table, order);
10910ae53ed1SSowmini Varadhan ldc_iommu->page_table = NULL;
1092a88b5ba8SSam Ravnborg
1093a88b5ba8SSam Ravnborg out_free_map:
10940ae53ed1SSowmini Varadhan kfree(iommu->map);
10950ae53ed1SSowmini Varadhan iommu->map = NULL;
1096a88b5ba8SSam Ravnborg
1097a88b5ba8SSam Ravnborg return err;
1098a88b5ba8SSam Ravnborg }
1099a88b5ba8SSam Ravnborg
ldc_iommu_release(struct ldc_channel * lp)1100a88b5ba8SSam Ravnborg static void ldc_iommu_release(struct ldc_channel *lp)
1101a88b5ba8SSam Ravnborg {
11020ae53ed1SSowmini Varadhan struct ldc_iommu *ldc_iommu = &lp->iommu;
11030ae53ed1SSowmini Varadhan struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
1104a88b5ba8SSam Ravnborg unsigned long num_tsb_entries, tsbsize, order;
1105a88b5ba8SSam Ravnborg
1106a88b5ba8SSam Ravnborg (void) sun4v_ldc_set_map_table(lp->id, 0, 0);
1107a88b5ba8SSam Ravnborg
11080ae53ed1SSowmini Varadhan num_tsb_entries = iommu->poolsize * iommu->nr_pools;
1109a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1110a88b5ba8SSam Ravnborg order = get_order(tsbsize);
1111a88b5ba8SSam Ravnborg
11120ae53ed1SSowmini Varadhan free_pages((unsigned long) ldc_iommu->page_table, order);
11130ae53ed1SSowmini Varadhan ldc_iommu->page_table = NULL;
1114a88b5ba8SSam Ravnborg
11150ae53ed1SSowmini Varadhan kfree(iommu->map);
11160ae53ed1SSowmini Varadhan iommu->map = NULL;
1117a88b5ba8SSam Ravnborg }
1118a88b5ba8SSam Ravnborg
ldc_alloc(unsigned long id,const struct ldc_channel_config * cfgp,void * event_arg,const char * name)1119a88b5ba8SSam Ravnborg struct ldc_channel *ldc_alloc(unsigned long id,
1120a88b5ba8SSam Ravnborg const struct ldc_channel_config *cfgp,
1121c21c4ab0SSowmini Varadhan void *event_arg,
1122c21c4ab0SSowmini Varadhan const char *name)
1123a88b5ba8SSam Ravnborg {
1124a88b5ba8SSam Ravnborg struct ldc_channel *lp;
1125a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops;
1126a88b5ba8SSam Ravnborg unsigned long dummy1, dummy2, hv_err;
1127a88b5ba8SSam Ravnborg u8 mss, *mssbuf;
1128a88b5ba8SSam Ravnborg int err;
1129a88b5ba8SSam Ravnborg
1130a88b5ba8SSam Ravnborg err = -ENODEV;
1131a88b5ba8SSam Ravnborg if (!ldom_domaining_enabled)
1132a88b5ba8SSam Ravnborg goto out_err;
1133a88b5ba8SSam Ravnborg
1134a88b5ba8SSam Ravnborg err = -EINVAL;
1135a88b5ba8SSam Ravnborg if (!cfgp)
1136a88b5ba8SSam Ravnborg goto out_err;
1137c21c4ab0SSowmini Varadhan if (!name)
1138c21c4ab0SSowmini Varadhan goto out_err;
1139a88b5ba8SSam Ravnborg
1140a88b5ba8SSam Ravnborg switch (cfgp->mode) {
1141a88b5ba8SSam Ravnborg case LDC_MODE_RAW:
1142a88b5ba8SSam Ravnborg mops = &raw_ops;
1143a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE;
1144a88b5ba8SSam Ravnborg break;
1145a88b5ba8SSam Ravnborg
1146a88b5ba8SSam Ravnborg case LDC_MODE_UNRELIABLE:
1147a88b5ba8SSam Ravnborg mops = &nonraw_ops;
1148a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8;
1149a88b5ba8SSam Ravnborg break;
1150a88b5ba8SSam Ravnborg
1151a88b5ba8SSam Ravnborg case LDC_MODE_STREAM:
1152a88b5ba8SSam Ravnborg mops = &stream_ops;
1153a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8 - 8;
1154a88b5ba8SSam Ravnborg break;
1155a88b5ba8SSam Ravnborg
1156a88b5ba8SSam Ravnborg default:
1157a88b5ba8SSam Ravnborg goto out_err;
1158a88b5ba8SSam Ravnborg }
1159a88b5ba8SSam Ravnborg
1160a88b5ba8SSam Ravnborg if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq)
1161a88b5ba8SSam Ravnborg goto out_err;
1162a88b5ba8SSam Ravnborg
1163a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2);
1164a88b5ba8SSam Ravnborg err = -ENODEV;
1165a88b5ba8SSam Ravnborg if (hv_err == HV_ECHANNEL)
1166a88b5ba8SSam Ravnborg goto out_err;
1167a88b5ba8SSam Ravnborg
1168a88b5ba8SSam Ravnborg err = -EEXIST;
1169a88b5ba8SSam Ravnborg if (__ldc_channel_exists(id))
1170a88b5ba8SSam Ravnborg goto out_err;
1171a88b5ba8SSam Ravnborg
1172a88b5ba8SSam Ravnborg mssbuf = NULL;
1173a88b5ba8SSam Ravnborg
1174a88b5ba8SSam Ravnborg lp = kzalloc(sizeof(*lp), GFP_KERNEL);
1175a88b5ba8SSam Ravnborg err = -ENOMEM;
1176a88b5ba8SSam Ravnborg if (!lp)
1177a88b5ba8SSam Ravnborg goto out_err;
1178a88b5ba8SSam Ravnborg
1179a88b5ba8SSam Ravnborg spin_lock_init(&lp->lock);
1180a88b5ba8SSam Ravnborg
1181a88b5ba8SSam Ravnborg lp->id = id;
1182a88b5ba8SSam Ravnborg
11830ae53ed1SSowmini Varadhan err = ldc_iommu_init(name, lp);
1184a88b5ba8SSam Ravnborg if (err)
1185a88b5ba8SSam Ravnborg goto out_free_ldc;
1186a88b5ba8SSam Ravnborg
1187a88b5ba8SSam Ravnborg lp->mops = mops;
1188a88b5ba8SSam Ravnborg lp->mss = mss;
1189a88b5ba8SSam Ravnborg
1190a88b5ba8SSam Ravnborg lp->cfg = *cfgp;
1191a88b5ba8SSam Ravnborg if (!lp->cfg.mtu)
1192a88b5ba8SSam Ravnborg lp->cfg.mtu = LDC_DEFAULT_MTU;
1193a88b5ba8SSam Ravnborg
1194a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) {
1195a88b5ba8SSam Ravnborg mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL);
1196a88b5ba8SSam Ravnborg if (!mssbuf) {
1197a88b5ba8SSam Ravnborg err = -ENOMEM;
1198a88b5ba8SSam Ravnborg goto out_free_iommu;
1199a88b5ba8SSam Ravnborg }
1200a88b5ba8SSam Ravnborg lp->mssbuf = mssbuf;
1201a88b5ba8SSam Ravnborg }
1202a88b5ba8SSam Ravnborg
1203a88b5ba8SSam Ravnborg lp->event_arg = event_arg;
1204a88b5ba8SSam Ravnborg
1205a88b5ba8SSam Ravnborg /* XXX allow setting via ldc_channel_config to override defaults
1206a88b5ba8SSam Ravnborg * XXX or use some formula based upon mtu
1207a88b5ba8SSam Ravnborg */
1208a88b5ba8SSam Ravnborg lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
1209a88b5ba8SSam Ravnborg lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
1210a88b5ba8SSam Ravnborg
1211a88b5ba8SSam Ravnborg err = alloc_queue("TX", lp->tx_num_entries,
1212a88b5ba8SSam Ravnborg &lp->tx_base, &lp->tx_ra);
1213a88b5ba8SSam Ravnborg if (err)
1214a88b5ba8SSam Ravnborg goto out_free_mssbuf;
1215a88b5ba8SSam Ravnborg
1216a88b5ba8SSam Ravnborg err = alloc_queue("RX", lp->rx_num_entries,
1217a88b5ba8SSam Ravnborg &lp->rx_base, &lp->rx_ra);
1218a88b5ba8SSam Ravnborg if (err)
1219a88b5ba8SSam Ravnborg goto out_free_txq;
1220a88b5ba8SSam Ravnborg
1221a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_ALLOCED_QUEUES;
1222a88b5ba8SSam Ravnborg
1223a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_CLOSED;
1224a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT);
1225a88b5ba8SSam Ravnborg
1226a88b5ba8SSam Ravnborg INIT_HLIST_NODE(&lp->list);
1227a88b5ba8SSam Ravnborg hlist_add_head(&lp->list, &ldc_channel_list);
1228a88b5ba8SSam Ravnborg
1229a88b5ba8SSam Ravnborg INIT_HLIST_HEAD(&lp->mh_list);
1230a88b5ba8SSam Ravnborg
1231c21c4ab0SSowmini Varadhan snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
1232c21c4ab0SSowmini Varadhan snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
1233c21c4ab0SSowmini Varadhan
1234c21c4ab0SSowmini Varadhan err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
1235c21c4ab0SSowmini Varadhan lp->rx_irq_name, lp);
1236c21c4ab0SSowmini Varadhan if (err)
1237c21c4ab0SSowmini Varadhan goto out_free_txq;
1238c21c4ab0SSowmini Varadhan
1239c21c4ab0SSowmini Varadhan err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
1240c21c4ab0SSowmini Varadhan lp->tx_irq_name, lp);
1241c21c4ab0SSowmini Varadhan if (err) {
1242c21c4ab0SSowmini Varadhan free_irq(lp->cfg.rx_irq, lp);
1243c21c4ab0SSowmini Varadhan goto out_free_txq;
1244c21c4ab0SSowmini Varadhan }
1245c21c4ab0SSowmini Varadhan
1246a88b5ba8SSam Ravnborg return lp;
1247a88b5ba8SSam Ravnborg
1248a88b5ba8SSam Ravnborg out_free_txq:
1249a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base);
1250a88b5ba8SSam Ravnborg
1251a88b5ba8SSam Ravnborg out_free_mssbuf:
1252a88b5ba8SSam Ravnborg kfree(mssbuf);
1253a88b5ba8SSam Ravnborg
1254a88b5ba8SSam Ravnborg out_free_iommu:
1255a88b5ba8SSam Ravnborg ldc_iommu_release(lp);
1256a88b5ba8SSam Ravnborg
1257a88b5ba8SSam Ravnborg out_free_ldc:
1258a88b5ba8SSam Ravnborg kfree(lp);
1259a88b5ba8SSam Ravnborg
1260a88b5ba8SSam Ravnborg out_err:
1261a88b5ba8SSam Ravnborg return ERR_PTR(err);
1262a88b5ba8SSam Ravnborg }
1263a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc);
1264a88b5ba8SSam Ravnborg
ldc_unbind(struct ldc_channel * lp)12651678c2bdSDwight Engen void ldc_unbind(struct ldc_channel *lp)
1266a88b5ba8SSam Ravnborg {
1267a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_IRQS) {
1268a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp);
1269a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp);
12701678c2bdSDwight Engen lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
1271a88b5ba8SSam Ravnborg }
1272a88b5ba8SSam Ravnborg
1273a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) {
1274a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0);
1275a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0);
1276a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
1277a88b5ba8SSam Ravnborg }
1278a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) {
1279a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base);
1280a88b5ba8SSam Ravnborg free_queue(lp->rx_num_entries, lp->rx_base);
1281a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES;
1282a88b5ba8SSam Ravnborg }
1283a88b5ba8SSam Ravnborg
12841678c2bdSDwight Engen ldc_set_state(lp, LDC_STATE_INIT);
12851678c2bdSDwight Engen }
12861678c2bdSDwight Engen EXPORT_SYMBOL(ldc_unbind);
12871678c2bdSDwight Engen
ldc_free(struct ldc_channel * lp)12881678c2bdSDwight Engen void ldc_free(struct ldc_channel *lp)
12891678c2bdSDwight Engen {
12901678c2bdSDwight Engen ldc_unbind(lp);
1291a88b5ba8SSam Ravnborg hlist_del(&lp->list);
1292a88b5ba8SSam Ravnborg kfree(lp->mssbuf);
1293a88b5ba8SSam Ravnborg ldc_iommu_release(lp);
1294a88b5ba8SSam Ravnborg
1295a88b5ba8SSam Ravnborg kfree(lp);
1296a88b5ba8SSam Ravnborg }
1297a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free);
1298a88b5ba8SSam Ravnborg
1299a88b5ba8SSam Ravnborg /* Bind the channel. This registers the LDC queues with
1300a88b5ba8SSam Ravnborg * the hypervisor and puts the channel into a pseudo-listening
1301a88b5ba8SSam Ravnborg * state. This does not initiate a handshake, ldc_connect() does
1302a88b5ba8SSam Ravnborg * that.
1303a88b5ba8SSam Ravnborg */
ldc_bind(struct ldc_channel * lp)1304c21c4ab0SSowmini Varadhan int ldc_bind(struct ldc_channel *lp)
1305a88b5ba8SSam Ravnborg {
1306a88b5ba8SSam Ravnborg unsigned long hv_err, flags;
1307a88b5ba8SSam Ravnborg int err = -EINVAL;
1308a88b5ba8SSam Ravnborg
1309c21c4ab0SSowmini Varadhan if (lp->state != LDC_STATE_INIT)
1310a88b5ba8SSam Ravnborg return -EINVAL;
1311a88b5ba8SSam Ravnborg
1312a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags);
1313a88b5ba8SSam Ravnborg
1314a88b5ba8SSam Ravnborg enable_irq(lp->cfg.rx_irq);
1315a88b5ba8SSam Ravnborg enable_irq(lp->cfg.tx_irq);
1316a88b5ba8SSam Ravnborg
1317a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_IRQS;
1318a88b5ba8SSam Ravnborg
1319a88b5ba8SSam Ravnborg err = -ENODEV;
1320a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
1321a88b5ba8SSam Ravnborg if (hv_err)
1322a88b5ba8SSam Ravnborg goto out_free_irqs;
1323a88b5ba8SSam Ravnborg
1324a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
1325a88b5ba8SSam Ravnborg if (hv_err)
1326a88b5ba8SSam Ravnborg goto out_free_irqs;
1327a88b5ba8SSam Ravnborg
1328a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
1329a88b5ba8SSam Ravnborg if (hv_err)
1330a88b5ba8SSam Ravnborg goto out_unmap_tx;
1331a88b5ba8SSam Ravnborg
1332a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
1333a88b5ba8SSam Ravnborg if (hv_err)
1334a88b5ba8SSam Ravnborg goto out_unmap_tx;
1335a88b5ba8SSam Ravnborg
1336a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_QUEUES;
1337a88b5ba8SSam Ravnborg
1338a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id,
1339a88b5ba8SSam Ravnborg &lp->tx_head,
1340a88b5ba8SSam Ravnborg &lp->tx_tail,
1341a88b5ba8SSam Ravnborg &lp->chan_state);
1342a88b5ba8SSam Ravnborg err = -EBUSY;
1343a88b5ba8SSam Ravnborg if (hv_err)
1344a88b5ba8SSam Ravnborg goto out_unmap_rx;
1345a88b5ba8SSam Ravnborg
1346a88b5ba8SSam Ravnborg lp->tx_acked = lp->tx_head;
1347a88b5ba8SSam Ravnborg
1348a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN;
1349a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND);
1350a88b5ba8SSam Ravnborg
135101b7a471SJag Raman if (lp->cfg.mode == LDC_MODE_RAW) {
135201b7a471SJag Raman /*
135301b7a471SJag Raman * There is no handshake in RAW mode, so handshake
135401b7a471SJag Raman * is completed.
135501b7a471SJag Raman */
135601b7a471SJag Raman lp->hs_state = LDC_HS_COMPLETE;
135701b7a471SJag Raman }
135801b7a471SJag Raman
1359a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags);
1360a88b5ba8SSam Ravnborg
1361a88b5ba8SSam Ravnborg return 0;
1362a88b5ba8SSam Ravnborg
1363a88b5ba8SSam Ravnborg out_unmap_rx:
1364a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
1365a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0);
1366a88b5ba8SSam Ravnborg
1367a88b5ba8SSam Ravnborg out_unmap_tx:
1368a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0);
1369a88b5ba8SSam Ravnborg
1370a88b5ba8SSam Ravnborg out_free_irqs:
1371a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
1372a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp);
1373a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp);
1374a88b5ba8SSam Ravnborg
1375a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags);
1376a88b5ba8SSam Ravnborg
1377a88b5ba8SSam Ravnborg return err;
1378a88b5ba8SSam Ravnborg }
1379a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_bind);
1380a88b5ba8SSam Ravnborg
ldc_connect(struct ldc_channel * lp)1381a88b5ba8SSam Ravnborg int ldc_connect(struct ldc_channel *lp)
1382a88b5ba8SSam Ravnborg {
1383a88b5ba8SSam Ravnborg unsigned long flags;
1384a88b5ba8SSam Ravnborg int err;
1385a88b5ba8SSam Ravnborg
1386a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW)
1387a88b5ba8SSam Ravnborg return -EINVAL;
1388a88b5ba8SSam Ravnborg
1389a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags);
1390a88b5ba8SSam Ravnborg
1391a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
1392a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) ||
1393a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_OPEN)
13944ec1b010SSowmini Varadhan err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL);
1395a88b5ba8SSam Ravnborg else
1396a88b5ba8SSam Ravnborg err = start_handshake(lp);
1397a88b5ba8SSam Ravnborg
1398a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags);
1399a88b5ba8SSam Ravnborg
1400a88b5ba8SSam Ravnborg return err;
1401a88b5ba8SSam Ravnborg }
1402a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_connect);
1403a88b5ba8SSam Ravnborg
ldc_disconnect(struct ldc_channel * lp)1404a88b5ba8SSam Ravnborg int ldc_disconnect(struct ldc_channel *lp)
1405a88b5ba8SSam Ravnborg {
1406a88b5ba8SSam Ravnborg unsigned long hv_err, flags;
1407a88b5ba8SSam Ravnborg int err;
1408a88b5ba8SSam Ravnborg
1409a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW)
1410a88b5ba8SSam Ravnborg return -EINVAL;
1411a88b5ba8SSam Ravnborg
1412a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
1413a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES))
1414a88b5ba8SSam Ravnborg return -EINVAL;
1415a88b5ba8SSam Ravnborg
1416a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags);
1417a88b5ba8SSam Ravnborg
1418a88b5ba8SSam Ravnborg err = -ENODEV;
1419a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
1420a88b5ba8SSam Ravnborg if (hv_err)
1421a88b5ba8SSam Ravnborg goto out_err;
1422a88b5ba8SSam Ravnborg
1423a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
1424a88b5ba8SSam Ravnborg if (hv_err)
1425a88b5ba8SSam Ravnborg goto out_err;
1426a88b5ba8SSam Ravnborg
1427a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
1428a88b5ba8SSam Ravnborg if (hv_err)
1429a88b5ba8SSam Ravnborg goto out_err;
1430a88b5ba8SSam Ravnborg
1431a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
1432a88b5ba8SSam Ravnborg if (hv_err)
1433a88b5ba8SSam Ravnborg goto out_err;
1434a88b5ba8SSam Ravnborg
1435a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND);
1436a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN;
1437a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_RESET;
1438a88b5ba8SSam Ravnborg
1439a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags);
1440a88b5ba8SSam Ravnborg
1441a88b5ba8SSam Ravnborg return 0;
1442a88b5ba8SSam Ravnborg
1443a88b5ba8SSam Ravnborg out_err:
1444a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0);
1445a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0);
1446a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp);
1447a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp);
1448a88b5ba8SSam Ravnborg lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS |
1449a88b5ba8SSam Ravnborg LDC_FLAG_REGISTERED_QUEUES);
1450a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT);
1451a88b5ba8SSam Ravnborg
1452a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags);
1453a88b5ba8SSam Ravnborg
1454a88b5ba8SSam Ravnborg return err;
1455a88b5ba8SSam Ravnborg }
1456a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_disconnect);
1457a88b5ba8SSam Ravnborg
ldc_state(struct ldc_channel * lp)1458a88b5ba8SSam Ravnborg int ldc_state(struct ldc_channel *lp)
1459a88b5ba8SSam Ravnborg {
1460a88b5ba8SSam Ravnborg return lp->state;
1461a88b5ba8SSam Ravnborg }
1462a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_state);
1463a88b5ba8SSam Ravnborg
ldc_set_state(struct ldc_channel * lp,u8 state)146485762a65SJag Raman void ldc_set_state(struct ldc_channel *lp, u8 state)
146585762a65SJag Raman {
146685762a65SJag Raman ldcdbg(STATE, "STATE (%s) --> (%s)\n",
146785762a65SJag Raman state_to_str(lp->state),
146885762a65SJag Raman state_to_str(state));
146985762a65SJag Raman
147085762a65SJag Raman lp->state = state;
147185762a65SJag Raman }
147201b7a471SJag Raman EXPORT_SYMBOL(ldc_set_state);
147385762a65SJag Raman
ldc_mode(struct ldc_channel * lp)147485762a65SJag Raman int ldc_mode(struct ldc_channel *lp)
147585762a65SJag Raman {
147685762a65SJag Raman return lp->cfg.mode;
147785762a65SJag Raman }
147801b7a471SJag Raman EXPORT_SYMBOL(ldc_mode);
147985762a65SJag Raman
ldc_rx_reset(struct ldc_channel * lp)148085762a65SJag Raman int ldc_rx_reset(struct ldc_channel *lp)
148185762a65SJag Raman {
148285762a65SJag Raman return __set_rx_head(lp, lp->rx_tail);
148385762a65SJag Raman }
1484f8c55335SJag Raman EXPORT_SYMBOL(ldc_rx_reset);
148585762a65SJag Raman
__ldc_print(struct ldc_channel * lp,const char * caller)148685762a65SJag Raman void __ldc_print(struct ldc_channel *lp, const char *caller)
148785762a65SJag Raman {
148885762a65SJag Raman pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n"
148985762a65SJag Raman "\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n"
149085762a65SJag Raman "\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n"
149185762a65SJag Raman "\trcv_nxt=%u snd_nxt=%u\n",
149285762a65SJag Raman caller, lp->id, lp->flags, state_to_str(lp->state),
149385762a65SJag Raman lp->chan_state, lp->hs_state,
149485762a65SJag Raman lp->rx_head, lp->rx_tail, lp->rx_num_entries,
149585762a65SJag Raman lp->tx_head, lp->tx_tail, lp->tx_num_entries,
149685762a65SJag Raman lp->rcv_nxt, lp->snd_nxt);
149785762a65SJag Raman }
1498f283ebd5SJag Raman EXPORT_SYMBOL(__ldc_print);
149985762a65SJag Raman
write_raw(struct ldc_channel * lp,const void * buf,unsigned int size)1500a88b5ba8SSam Ravnborg static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size)
1501a88b5ba8SSam Ravnborg {
1502a88b5ba8SSam Ravnborg struct ldc_packet *p;
1503fc43b978SJag Raman unsigned long new_tail, hv_err;
1504a88b5ba8SSam Ravnborg int err;
1505a88b5ba8SSam Ravnborg
1506fc43b978SJag Raman hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
1507fc43b978SJag Raman &lp->chan_state);
1508fc43b978SJag Raman if (unlikely(hv_err))
1509fc43b978SJag Raman return -EBUSY;
1510fc43b978SJag Raman
1511fc43b978SJag Raman if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
1512fc43b978SJag Raman return LDC_ABORT(lp);
1513fc43b978SJag Raman
1514a88b5ba8SSam Ravnborg if (size > LDC_PACKET_SIZE)
1515a88b5ba8SSam Ravnborg return -EMSGSIZE;
1516a88b5ba8SSam Ravnborg
1517a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail);
1518a88b5ba8SSam Ravnborg if (!p)
1519a88b5ba8SSam Ravnborg return -EAGAIN;
1520a88b5ba8SSam Ravnborg
1521a88b5ba8SSam Ravnborg memcpy(p, buf, size);
1522a88b5ba8SSam Ravnborg
1523a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail);
1524a88b5ba8SSam Ravnborg if (!err)
1525a88b5ba8SSam Ravnborg err = size;
1526a88b5ba8SSam Ravnborg
1527a88b5ba8SSam Ravnborg return err;
1528a88b5ba8SSam Ravnborg }
1529a88b5ba8SSam Ravnborg
read_raw(struct ldc_channel * lp,void * buf,unsigned int size)1530a88b5ba8SSam Ravnborg static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size)
1531a88b5ba8SSam Ravnborg {
1532a88b5ba8SSam Ravnborg struct ldc_packet *p;
1533a88b5ba8SSam Ravnborg unsigned long hv_err, new;
1534a88b5ba8SSam Ravnborg int err;
1535a88b5ba8SSam Ravnborg
1536a88b5ba8SSam Ravnborg if (size < LDC_PACKET_SIZE)
1537a88b5ba8SSam Ravnborg return -EINVAL;
1538a88b5ba8SSam Ravnborg
1539a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id,
1540a88b5ba8SSam Ravnborg &lp->rx_head,
1541a88b5ba8SSam Ravnborg &lp->rx_tail,
1542a88b5ba8SSam Ravnborg &lp->chan_state);
1543a88b5ba8SSam Ravnborg if (hv_err)
154429693e75SJag Raman return LDC_ABORT(lp);
1545a88b5ba8SSam Ravnborg
1546a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN ||
1547a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING)
1548a88b5ba8SSam Ravnborg return -ECONNRESET;
1549a88b5ba8SSam Ravnborg
1550a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail)
1551a88b5ba8SSam Ravnborg return 0;
1552a88b5ba8SSam Ravnborg
1553a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
1554a88b5ba8SSam Ravnborg memcpy(buf, p, LDC_PACKET_SIZE);
1555a88b5ba8SSam Ravnborg
1556a88b5ba8SSam Ravnborg new = rx_advance(lp, lp->rx_head);
1557a88b5ba8SSam Ravnborg lp->rx_head = new;
1558a88b5ba8SSam Ravnborg
1559a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new);
1560a88b5ba8SSam Ravnborg if (err < 0)
1561a88b5ba8SSam Ravnborg err = -ECONNRESET;
1562a88b5ba8SSam Ravnborg else
1563a88b5ba8SSam Ravnborg err = LDC_PACKET_SIZE;
1564a88b5ba8SSam Ravnborg
1565a88b5ba8SSam Ravnborg return err;
1566a88b5ba8SSam Ravnborg }
1567a88b5ba8SSam Ravnborg
1568a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops = {
1569a88b5ba8SSam Ravnborg .write = write_raw,
1570a88b5ba8SSam Ravnborg .read = read_raw,
1571a88b5ba8SSam Ravnborg };
1572a88b5ba8SSam Ravnborg
write_nonraw(struct ldc_channel * lp,const void * buf,unsigned int size)1573a88b5ba8SSam Ravnborg static int write_nonraw(struct ldc_channel *lp, const void *buf,
1574a88b5ba8SSam Ravnborg unsigned int size)
1575a88b5ba8SSam Ravnborg {
1576a88b5ba8SSam Ravnborg unsigned long hv_err, tail;
1577a88b5ba8SSam Ravnborg unsigned int copied;
1578a88b5ba8SSam Ravnborg u32 seq;
1579a88b5ba8SSam Ravnborg int err;
1580a88b5ba8SSam Ravnborg
1581a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
1582a88b5ba8SSam Ravnborg &lp->chan_state);
1583a88b5ba8SSam Ravnborg if (unlikely(hv_err))
1584a88b5ba8SSam Ravnborg return -EBUSY;
1585a88b5ba8SSam Ravnborg
1586a88b5ba8SSam Ravnborg if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
158729693e75SJag Raman return LDC_ABORT(lp);
1588a88b5ba8SSam Ravnborg
1589a88b5ba8SSam Ravnborg if (!tx_has_space_for(lp, size))
1590a88b5ba8SSam Ravnborg return -EAGAIN;
1591a88b5ba8SSam Ravnborg
1592a88b5ba8SSam Ravnborg seq = lp->snd_nxt;
1593a88b5ba8SSam Ravnborg copied = 0;
1594a88b5ba8SSam Ravnborg tail = lp->tx_tail;
1595a88b5ba8SSam Ravnborg while (copied < size) {
1596a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE);
1597a88b5ba8SSam Ravnborg u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ?
1598a88b5ba8SSam Ravnborg p->u.u_data :
1599a88b5ba8SSam Ravnborg p->u.r.r_data);
1600a88b5ba8SSam Ravnborg int data_len;
1601a88b5ba8SSam Ravnborg
1602a88b5ba8SSam Ravnborg p->type = LDC_DATA;
1603a88b5ba8SSam Ravnborg p->stype = LDC_INFO;
1604a88b5ba8SSam Ravnborg p->ctrl = 0;
1605a88b5ba8SSam Ravnborg
1606a88b5ba8SSam Ravnborg data_len = size - copied;
1607a88b5ba8SSam Ravnborg if (data_len > lp->mss)
1608a88b5ba8SSam Ravnborg data_len = lp->mss;
1609a88b5ba8SSam Ravnborg
1610a88b5ba8SSam Ravnborg BUG_ON(data_len > LDC_LEN);
1611a88b5ba8SSam Ravnborg
1612a88b5ba8SSam Ravnborg p->env = (data_len |
1613a88b5ba8SSam Ravnborg (copied == 0 ? LDC_START : 0) |
1614a88b5ba8SSam Ravnborg (data_len == size - copied ? LDC_STOP : 0));
1615a88b5ba8SSam Ravnborg
1616a88b5ba8SSam Ravnborg p->seqid = ++seq;
1617a88b5ba8SSam Ravnborg
1618a88b5ba8SSam Ravnborg ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n",
1619a88b5ba8SSam Ravnborg p->type,
1620a88b5ba8SSam Ravnborg p->stype,
1621a88b5ba8SSam Ravnborg p->ctrl,
1622a88b5ba8SSam Ravnborg p->env,
1623a88b5ba8SSam Ravnborg p->seqid);
1624a88b5ba8SSam Ravnborg
1625a88b5ba8SSam Ravnborg memcpy(data, buf, data_len);
1626a88b5ba8SSam Ravnborg buf += data_len;
1627a88b5ba8SSam Ravnborg copied += data_len;
1628a88b5ba8SSam Ravnborg
1629a88b5ba8SSam Ravnborg tail = tx_advance(lp, tail);
1630a88b5ba8SSam Ravnborg }
1631a88b5ba8SSam Ravnborg
1632a88b5ba8SSam Ravnborg err = set_tx_tail(lp, tail);
1633a88b5ba8SSam Ravnborg if (!err) {
1634a88b5ba8SSam Ravnborg lp->snd_nxt = seq;
1635a88b5ba8SSam Ravnborg err = size;
1636a88b5ba8SSam Ravnborg }
1637a88b5ba8SSam Ravnborg
1638a88b5ba8SSam Ravnborg return err;
1639a88b5ba8SSam Ravnborg }
1640a88b5ba8SSam Ravnborg
rx_bad_seq(struct ldc_channel * lp,struct ldc_packet * p,struct ldc_packet * first_frag)1641a88b5ba8SSam Ravnborg static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p,
1642a88b5ba8SSam Ravnborg struct ldc_packet *first_frag)
1643a88b5ba8SSam Ravnborg {
1644a88b5ba8SSam Ravnborg int err;
1645a88b5ba8SSam Ravnborg
1646a88b5ba8SSam Ravnborg if (first_frag)
1647a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1;
1648a88b5ba8SSam Ravnborg
1649a88b5ba8SSam Ravnborg err = send_data_nack(lp, p);
1650a88b5ba8SSam Ravnborg if (err)
1651a88b5ba8SSam Ravnborg return err;
1652a88b5ba8SSam Ravnborg
165385762a65SJag Raman err = ldc_rx_reset(lp);
1654a88b5ba8SSam Ravnborg if (err < 0)
165529693e75SJag Raman return LDC_ABORT(lp);
1656a88b5ba8SSam Ravnborg
1657a88b5ba8SSam Ravnborg return 0;
1658a88b5ba8SSam Ravnborg }
1659a88b5ba8SSam Ravnborg
data_ack_nack(struct ldc_channel * lp,struct ldc_packet * p)1660a88b5ba8SSam Ravnborg static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p)
1661a88b5ba8SSam Ravnborg {
1662a88b5ba8SSam Ravnborg if (p->stype & LDC_ACK) {
1663a88b5ba8SSam Ravnborg int err = process_data_ack(lp, p);
1664a88b5ba8SSam Ravnborg if (err)
1665a88b5ba8SSam Ravnborg return err;
1666a88b5ba8SSam Ravnborg }
1667a88b5ba8SSam Ravnborg if (p->stype & LDC_NACK)
166829693e75SJag Raman return LDC_ABORT(lp);
1669a88b5ba8SSam Ravnborg
1670a88b5ba8SSam Ravnborg return 0;
1671a88b5ba8SSam Ravnborg }
1672a88b5ba8SSam Ravnborg
rx_data_wait(struct ldc_channel * lp,unsigned long cur_head)1673a88b5ba8SSam Ravnborg static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head)
1674a88b5ba8SSam Ravnborg {
1675a88b5ba8SSam Ravnborg unsigned long dummy;
1676a88b5ba8SSam Ravnborg int limit = 1000;
1677a88b5ba8SSam Ravnborg
1678a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n",
1679a88b5ba8SSam Ravnborg cur_head, lp->rx_head, lp->rx_tail);
1680a88b5ba8SSam Ravnborg while (limit-- > 0) {
1681a88b5ba8SSam Ravnborg unsigned long hv_err;
1682a88b5ba8SSam Ravnborg
1683a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id,
1684a88b5ba8SSam Ravnborg &dummy,
1685a88b5ba8SSam Ravnborg &lp->rx_tail,
1686a88b5ba8SSam Ravnborg &lp->chan_state);
1687a88b5ba8SSam Ravnborg if (hv_err)
168829693e75SJag Raman return LDC_ABORT(lp);
1689a88b5ba8SSam Ravnborg
1690a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN ||
1691a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING)
1692a88b5ba8SSam Ravnborg return -ECONNRESET;
1693a88b5ba8SSam Ravnborg
1694a88b5ba8SSam Ravnborg if (cur_head != lp->rx_tail) {
1695a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT DONE "
1696a88b5ba8SSam Ravnborg "head[%lx] tail[%lx] chan_state[%lx]\n",
1697a88b5ba8SSam Ravnborg dummy, lp->rx_tail, lp->chan_state);
1698a88b5ba8SSam Ravnborg return 0;
1699a88b5ba8SSam Ravnborg }
1700a88b5ba8SSam Ravnborg
1701a88b5ba8SSam Ravnborg udelay(1);
1702a88b5ba8SSam Ravnborg }
1703a88b5ba8SSam Ravnborg return -EAGAIN;
1704a88b5ba8SSam Ravnborg }
1705a88b5ba8SSam Ravnborg
rx_set_head(struct ldc_channel * lp,unsigned long head)1706a88b5ba8SSam Ravnborg static int rx_set_head(struct ldc_channel *lp, unsigned long head)
1707a88b5ba8SSam Ravnborg {
1708a88b5ba8SSam Ravnborg int err = __set_rx_head(lp, head);
1709a88b5ba8SSam Ravnborg
1710a88b5ba8SSam Ravnborg if (err < 0)
171129693e75SJag Raman return LDC_ABORT(lp);
1712a88b5ba8SSam Ravnborg
1713a88b5ba8SSam Ravnborg lp->rx_head = head;
1714a88b5ba8SSam Ravnborg return 0;
1715a88b5ba8SSam Ravnborg }
1716a88b5ba8SSam Ravnborg
send_data_ack(struct ldc_channel * lp)1717a88b5ba8SSam Ravnborg static void send_data_ack(struct ldc_channel *lp)
1718a88b5ba8SSam Ravnborg {
1719a88b5ba8SSam Ravnborg unsigned long new_tail;
1720a88b5ba8SSam Ravnborg struct ldc_packet *p;
1721a88b5ba8SSam Ravnborg
1722a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail);
1723a88b5ba8SSam Ravnborg if (likely(p)) {
1724a88b5ba8SSam Ravnborg int err;
1725a88b5ba8SSam Ravnborg
1726a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p));
1727a88b5ba8SSam Ravnborg p->type = LDC_DATA;
1728a88b5ba8SSam Ravnborg p->stype = LDC_ACK;
1729a88b5ba8SSam Ravnborg p->ctrl = 0;
1730a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1;
1731a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt;
1732a88b5ba8SSam Ravnborg
1733a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail);
1734a88b5ba8SSam Ravnborg if (!err)
1735a88b5ba8SSam Ravnborg lp->snd_nxt++;
1736a88b5ba8SSam Ravnborg }
1737a88b5ba8SSam Ravnborg }
1738a88b5ba8SSam Ravnborg
read_nonraw(struct ldc_channel * lp,void * buf,unsigned int size)1739a88b5ba8SSam Ravnborg static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size)
1740a88b5ba8SSam Ravnborg {
1741a88b5ba8SSam Ravnborg struct ldc_packet *first_frag;
1742a88b5ba8SSam Ravnborg unsigned long hv_err, new;
1743a88b5ba8SSam Ravnborg int err, copied;
1744a88b5ba8SSam Ravnborg
1745a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id,
1746a88b5ba8SSam Ravnborg &lp->rx_head,
1747a88b5ba8SSam Ravnborg &lp->rx_tail,
1748a88b5ba8SSam Ravnborg &lp->chan_state);
1749a88b5ba8SSam Ravnborg if (hv_err)
175029693e75SJag Raman return LDC_ABORT(lp);
1751a88b5ba8SSam Ravnborg
1752a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN ||
1753a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING)
1754a88b5ba8SSam Ravnborg return -ECONNRESET;
1755a88b5ba8SSam Ravnborg
1756a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail)
1757a88b5ba8SSam Ravnborg return 0;
1758a88b5ba8SSam Ravnborg
1759a88b5ba8SSam Ravnborg first_frag = NULL;
1760a88b5ba8SSam Ravnborg copied = err = 0;
1761a88b5ba8SSam Ravnborg new = lp->rx_head;
1762a88b5ba8SSam Ravnborg while (1) {
1763a88b5ba8SSam Ravnborg struct ldc_packet *p;
1764a88b5ba8SSam Ravnborg int pkt_len;
1765a88b5ba8SSam Ravnborg
1766a88b5ba8SSam Ravnborg BUG_ON(new == lp->rx_tail);
1767a88b5ba8SSam Ravnborg p = lp->rx_base + (new / LDC_PACKET_SIZE);
1768a88b5ba8SSam Ravnborg
1769a88b5ba8SSam Ravnborg ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] "
1770a88b5ba8SSam Ravnborg "rcv_nxt[%08x]\n",
1771a88b5ba8SSam Ravnborg p->type,
1772a88b5ba8SSam Ravnborg p->stype,
1773a88b5ba8SSam Ravnborg p->ctrl,
1774a88b5ba8SSam Ravnborg p->env,
1775a88b5ba8SSam Ravnborg p->seqid,
1776a88b5ba8SSam Ravnborg p->u.r.ackid,
1777a88b5ba8SSam Ravnborg lp->rcv_nxt);
1778a88b5ba8SSam Ravnborg
1779a88b5ba8SSam Ravnborg if (unlikely(!rx_seq_ok(lp, p->seqid))) {
1780a88b5ba8SSam Ravnborg err = rx_bad_seq(lp, p, first_frag);
1781a88b5ba8SSam Ravnborg copied = 0;
1782a88b5ba8SSam Ravnborg break;
1783a88b5ba8SSam Ravnborg }
1784a88b5ba8SSam Ravnborg
1785a88b5ba8SSam Ravnborg if (p->type & LDC_CTRL) {
1786a88b5ba8SSam Ravnborg err = process_control_frame(lp, p);
1787a88b5ba8SSam Ravnborg if (err < 0)
1788a88b5ba8SSam Ravnborg break;
1789a88b5ba8SSam Ravnborg err = 0;
1790a88b5ba8SSam Ravnborg }
1791a88b5ba8SSam Ravnborg
1792a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid;
1793a88b5ba8SSam Ravnborg
17946c95483bSJag Raman /*
17956c95483bSJag Raman * If this is a control-only packet, there is nothing
17966c95483bSJag Raman * else to do but advance the rx queue since the packet
17976c95483bSJag Raman * was already processed above.
17986c95483bSJag Raman */
1799a88b5ba8SSam Ravnborg if (!(p->type & LDC_DATA)) {
1800a88b5ba8SSam Ravnborg new = rx_advance(lp, new);
18016c95483bSJag Raman break;
1802a88b5ba8SSam Ravnborg }
1803a88b5ba8SSam Ravnborg if (p->stype & (LDC_ACK | LDC_NACK)) {
1804a88b5ba8SSam Ravnborg err = data_ack_nack(lp, p);
1805a88b5ba8SSam Ravnborg if (err)
1806a88b5ba8SSam Ravnborg break;
1807a88b5ba8SSam Ravnborg }
1808a88b5ba8SSam Ravnborg if (!(p->stype & LDC_INFO)) {
1809a88b5ba8SSam Ravnborg new = rx_advance(lp, new);
1810a88b5ba8SSam Ravnborg err = rx_set_head(lp, new);
1811a88b5ba8SSam Ravnborg if (err)
1812a88b5ba8SSam Ravnborg break;
1813a88b5ba8SSam Ravnborg goto no_data;
1814a88b5ba8SSam Ravnborg }
1815a88b5ba8SSam Ravnborg
1816a88b5ba8SSam Ravnborg pkt_len = p->env & LDC_LEN;
1817a88b5ba8SSam Ravnborg
1818a88b5ba8SSam Ravnborg /* Every initial packet starts with the START bit set.
1819a88b5ba8SSam Ravnborg *
1820a88b5ba8SSam Ravnborg * Singleton packets will have both START+STOP set.
1821a88b5ba8SSam Ravnborg *
1822a88b5ba8SSam Ravnborg * Fragments will have START set in the first frame, STOP
1823a88b5ba8SSam Ravnborg * set in the last frame, and neither bit set in middle
1824a88b5ba8SSam Ravnborg * frames of the packet.
1825a88b5ba8SSam Ravnborg *
1826a88b5ba8SSam Ravnborg * Therefore if we are at the beginning of a packet and
1827a88b5ba8SSam Ravnborg * we don't see START, or we are in the middle of a fragmented
1828a88b5ba8SSam Ravnborg * packet and do see START, we are unsynchronized and should
1829a88b5ba8SSam Ravnborg * flush the RX queue.
1830a88b5ba8SSam Ravnborg */
1831a88b5ba8SSam Ravnborg if ((first_frag == NULL && !(p->env & LDC_START)) ||
1832a88b5ba8SSam Ravnborg (first_frag != NULL && (p->env & LDC_START))) {
1833a88b5ba8SSam Ravnborg if (!first_frag)
1834a88b5ba8SSam Ravnborg new = rx_advance(lp, new);
1835a88b5ba8SSam Ravnborg
1836a88b5ba8SSam Ravnborg err = rx_set_head(lp, new);
1837a88b5ba8SSam Ravnborg if (err)
1838a88b5ba8SSam Ravnborg break;
1839a88b5ba8SSam Ravnborg
1840a88b5ba8SSam Ravnborg if (!first_frag)
1841a88b5ba8SSam Ravnborg goto no_data;
1842a88b5ba8SSam Ravnborg }
1843a88b5ba8SSam Ravnborg if (!first_frag)
1844a88b5ba8SSam Ravnborg first_frag = p;
1845a88b5ba8SSam Ravnborg
1846a88b5ba8SSam Ravnborg if (pkt_len > size - copied) {
1847a88b5ba8SSam Ravnborg /* User didn't give us a big enough buffer,
1848a88b5ba8SSam Ravnborg * what to do? This is a pretty serious error.
1849a88b5ba8SSam Ravnborg *
1850a88b5ba8SSam Ravnborg * Since we haven't updated the RX ring head to
1851a88b5ba8SSam Ravnborg * consume any of the packets, signal the error
1852a88b5ba8SSam Ravnborg * to the user and just leave the RX ring alone.
1853a88b5ba8SSam Ravnborg *
1854a88b5ba8SSam Ravnborg * This seems the best behavior because this allows
1855a88b5ba8SSam Ravnborg * a user of the LDC layer to start with a small
1856a88b5ba8SSam Ravnborg * RX buffer for ldc_read() calls and use -EMSGSIZE
1857a88b5ba8SSam Ravnborg * as a cue to enlarge it's read buffer.
1858a88b5ba8SSam Ravnborg */
1859a88b5ba8SSam Ravnborg err = -EMSGSIZE;
1860a88b5ba8SSam Ravnborg break;
1861a88b5ba8SSam Ravnborg }
1862a88b5ba8SSam Ravnborg
1863a88b5ba8SSam Ravnborg /* Ok, we are gonna eat this one. */
1864a88b5ba8SSam Ravnborg new = rx_advance(lp, new);
1865a88b5ba8SSam Ravnborg
1866a88b5ba8SSam Ravnborg memcpy(buf,
1867a88b5ba8SSam Ravnborg (lp->cfg.mode == LDC_MODE_UNRELIABLE ?
1868a88b5ba8SSam Ravnborg p->u.u_data : p->u.r.r_data), pkt_len);
1869a88b5ba8SSam Ravnborg buf += pkt_len;
1870a88b5ba8SSam Ravnborg copied += pkt_len;
1871a88b5ba8SSam Ravnborg
1872a88b5ba8SSam Ravnborg if (p->env & LDC_STOP)
1873a88b5ba8SSam Ravnborg break;
1874a88b5ba8SSam Ravnborg
1875a88b5ba8SSam Ravnborg no_data:
1876a88b5ba8SSam Ravnborg if (new == lp->rx_tail) {
1877a88b5ba8SSam Ravnborg err = rx_data_wait(lp, new);
1878a88b5ba8SSam Ravnborg if (err)
1879a88b5ba8SSam Ravnborg break;
1880a88b5ba8SSam Ravnborg }
1881a88b5ba8SSam Ravnborg }
1882a88b5ba8SSam Ravnborg
1883a88b5ba8SSam Ravnborg if (!err)
1884a88b5ba8SSam Ravnborg err = rx_set_head(lp, new);
1885a88b5ba8SSam Ravnborg
1886a88b5ba8SSam Ravnborg if (err && first_frag)
1887a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1;
1888a88b5ba8SSam Ravnborg
1889a88b5ba8SSam Ravnborg if (!err) {
1890a88b5ba8SSam Ravnborg err = copied;
1891a88b5ba8SSam Ravnborg if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE)
1892a88b5ba8SSam Ravnborg send_data_ack(lp);
1893a88b5ba8SSam Ravnborg }
1894a88b5ba8SSam Ravnborg
1895a88b5ba8SSam Ravnborg return err;
1896a88b5ba8SSam Ravnborg }
1897a88b5ba8SSam Ravnborg
1898a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops = {
1899a88b5ba8SSam Ravnborg .write = write_nonraw,
1900a88b5ba8SSam Ravnborg .read = read_nonraw,
1901a88b5ba8SSam Ravnborg };
1902a88b5ba8SSam Ravnborg
write_stream(struct ldc_channel * lp,const void * buf,unsigned int size)1903a88b5ba8SSam Ravnborg static int write_stream(struct ldc_channel *lp, const void *buf,
1904a88b5ba8SSam Ravnborg unsigned int size)
1905a88b5ba8SSam Ravnborg {
1906a88b5ba8SSam Ravnborg if (size > lp->cfg.mtu)
1907a88b5ba8SSam Ravnborg size = lp->cfg.mtu;
1908a88b5ba8SSam Ravnborg return write_nonraw(lp, buf, size);
1909a88b5ba8SSam Ravnborg }
1910a88b5ba8SSam Ravnborg
read_stream(struct ldc_channel * lp,void * buf,unsigned int size)1911a88b5ba8SSam Ravnborg static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size)
1912a88b5ba8SSam Ravnborg {
1913a88b5ba8SSam Ravnborg if (!lp->mssbuf_len) {
1914a88b5ba8SSam Ravnborg int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu);
1915a88b5ba8SSam Ravnborg if (err < 0)
1916a88b5ba8SSam Ravnborg return err;
1917a88b5ba8SSam Ravnborg
1918a88b5ba8SSam Ravnborg lp->mssbuf_len = err;
1919a88b5ba8SSam Ravnborg lp->mssbuf_off = 0;
1920a88b5ba8SSam Ravnborg }
1921a88b5ba8SSam Ravnborg
1922a88b5ba8SSam Ravnborg if (size > lp->mssbuf_len)
1923a88b5ba8SSam Ravnborg size = lp->mssbuf_len;
1924a88b5ba8SSam Ravnborg memcpy(buf, lp->mssbuf + lp->mssbuf_off, size);
1925a88b5ba8SSam Ravnborg
1926a88b5ba8SSam Ravnborg lp->mssbuf_off += size;
1927a88b5ba8SSam Ravnborg lp->mssbuf_len -= size;
1928a88b5ba8SSam Ravnborg
1929a88b5ba8SSam Ravnborg return size;
1930a88b5ba8SSam Ravnborg }
1931a88b5ba8SSam Ravnborg
1932a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops = {
1933a88b5ba8SSam Ravnborg .write = write_stream,
1934a88b5ba8SSam Ravnborg .read = read_stream,
1935a88b5ba8SSam Ravnborg };
1936a88b5ba8SSam Ravnborg
ldc_write(struct ldc_channel * lp,const void * buf,unsigned int size)1937a88b5ba8SSam Ravnborg int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size)
1938a88b5ba8SSam Ravnborg {
1939a88b5ba8SSam Ravnborg unsigned long flags;
1940a88b5ba8SSam Ravnborg int err;
1941a88b5ba8SSam Ravnborg
1942a88b5ba8SSam Ravnborg if (!buf)
1943a88b5ba8SSam Ravnborg return -EINVAL;
1944a88b5ba8SSam Ravnborg
1945a88b5ba8SSam Ravnborg if (!size)
1946a88b5ba8SSam Ravnborg return 0;
1947a88b5ba8SSam Ravnborg
1948a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags);
1949a88b5ba8SSam Ravnborg
1950a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE)
1951a88b5ba8SSam Ravnborg err = -ENOTCONN;
1952a88b5ba8SSam Ravnborg else
1953a88b5ba8SSam Ravnborg err = lp->mops->write(lp, buf, size);
1954a88b5ba8SSam Ravnborg
1955a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags);
1956a88b5ba8SSam Ravnborg
1957a88b5ba8SSam Ravnborg return err;
1958a88b5ba8SSam Ravnborg }
1959a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_write);
1960a88b5ba8SSam Ravnborg
ldc_read(struct ldc_channel * lp,void * buf,unsigned int size)1961a88b5ba8SSam Ravnborg int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
1962a88b5ba8SSam Ravnborg {
1963a88b5ba8SSam Ravnborg unsigned long flags;
1964a88b5ba8SSam Ravnborg int err;
1965a88b5ba8SSam Ravnborg
196693ec4a82SJag Raman ldcdbg(RX, "%s: entered size=%d\n", __func__, size);
196793ec4a82SJag Raman
1968a88b5ba8SSam Ravnborg if (!buf)
1969a88b5ba8SSam Ravnborg return -EINVAL;
1970a88b5ba8SSam Ravnborg
1971a88b5ba8SSam Ravnborg if (!size)
1972a88b5ba8SSam Ravnborg return 0;
1973a88b5ba8SSam Ravnborg
1974a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags);
1975a88b5ba8SSam Ravnborg
1976a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE)
1977a88b5ba8SSam Ravnborg err = -ENOTCONN;
1978a88b5ba8SSam Ravnborg else
1979a88b5ba8SSam Ravnborg err = lp->mops->read(lp, buf, size);
1980a88b5ba8SSam Ravnborg
1981a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags);
1982a88b5ba8SSam Ravnborg
198393ec4a82SJag Raman ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__,
198493ec4a82SJag Raman lp->cfg.mode, lp->rx_head, lp->rx_tail, err);
198593ec4a82SJag Raman
1986a88b5ba8SSam Ravnborg return err;
1987a88b5ba8SSam Ravnborg }
1988a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_read);
1989a88b5ba8SSam Ravnborg
pagesize_code(void)1990a88b5ba8SSam Ravnborg static u64 pagesize_code(void)
1991a88b5ba8SSam Ravnborg {
1992a88b5ba8SSam Ravnborg switch (PAGE_SIZE) {
1993a88b5ba8SSam Ravnborg default:
1994a88b5ba8SSam Ravnborg case (8ULL * 1024ULL):
1995a88b5ba8SSam Ravnborg return 0;
1996a88b5ba8SSam Ravnborg case (64ULL * 1024ULL):
1997a88b5ba8SSam Ravnborg return 1;
1998a88b5ba8SSam Ravnborg case (512ULL * 1024ULL):
1999a88b5ba8SSam Ravnborg return 2;
2000a88b5ba8SSam Ravnborg case (4ULL * 1024ULL * 1024ULL):
2001a88b5ba8SSam Ravnborg return 3;
2002a88b5ba8SSam Ravnborg case (32ULL * 1024ULL * 1024ULL):
2003a88b5ba8SSam Ravnborg return 4;
2004a88b5ba8SSam Ravnborg case (256ULL * 1024ULL * 1024ULL):
2005a88b5ba8SSam Ravnborg return 5;
2006a88b5ba8SSam Ravnborg }
2007a88b5ba8SSam Ravnborg }
2008a88b5ba8SSam Ravnborg
make_cookie(u64 index,u64 pgsz_code,u64 page_offset)2009a88b5ba8SSam Ravnborg static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
2010a88b5ba8SSam Ravnborg {
2011a88b5ba8SSam Ravnborg return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) |
2012a88b5ba8SSam Ravnborg (index << PAGE_SHIFT) |
2013a88b5ba8SSam Ravnborg page_offset);
2014a88b5ba8SSam Ravnborg }
2015a88b5ba8SSam Ravnborg
2016a88b5ba8SSam Ravnborg
alloc_npages(struct ldc_iommu * iommu,unsigned long npages)2017a88b5ba8SSam Ravnborg static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
2018a88b5ba8SSam Ravnborg unsigned long npages)
2019a88b5ba8SSam Ravnborg {
2020a88b5ba8SSam Ravnborg long entry;
2021a88b5ba8SSam Ravnborg
20220ae53ed1SSowmini Varadhan entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,
20230ae53ed1SSowmini Varadhan npages, NULL, (unsigned long)-1, 0);
2024d618382bSDavid S. Miller if (unlikely(entry == IOMMU_ERROR_CODE))
2025a88b5ba8SSam Ravnborg return NULL;
2026a88b5ba8SSam Ravnborg
2027a88b5ba8SSam Ravnborg return iommu->page_table + entry;
2028a88b5ba8SSam Ravnborg }
2029a88b5ba8SSam Ravnborg
perm_to_mte(unsigned int map_perm)2030a88b5ba8SSam Ravnborg static u64 perm_to_mte(unsigned int map_perm)
2031a88b5ba8SSam Ravnborg {
2032a88b5ba8SSam Ravnborg u64 mte_base;
2033a88b5ba8SSam Ravnborg
2034a88b5ba8SSam Ravnborg mte_base = pagesize_code();
2035a88b5ba8SSam Ravnborg
2036a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_SHADOW) {
2037a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R)
2038a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_R;
2039a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W)
2040a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_W;
2041a88b5ba8SSam Ravnborg }
2042a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_DIRECT) {
2043a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R)
2044a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_READ;
2045a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W)
2046a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_WRITE;
2047a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_X)
2048a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_EXEC;
2049a88b5ba8SSam Ravnborg }
2050a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_IO) {
2051a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R)
2052a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_R;
2053a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W)
2054a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_W;
2055a88b5ba8SSam Ravnborg }
2056a88b5ba8SSam Ravnborg
2057a88b5ba8SSam Ravnborg return mte_base;
2058a88b5ba8SSam Ravnborg }
2059a88b5ba8SSam Ravnborg
pages_in_region(unsigned long base,long len)2060a88b5ba8SSam Ravnborg static int pages_in_region(unsigned long base, long len)
2061a88b5ba8SSam Ravnborg {
2062a88b5ba8SSam Ravnborg int count = 0;
2063a88b5ba8SSam Ravnborg
2064a88b5ba8SSam Ravnborg do {
2065a88b5ba8SSam Ravnborg unsigned long new = (base + PAGE_SIZE) & PAGE_MASK;
2066a88b5ba8SSam Ravnborg
2067a88b5ba8SSam Ravnborg len -= (new - base);
2068a88b5ba8SSam Ravnborg base = new;
2069a88b5ba8SSam Ravnborg count++;
2070a88b5ba8SSam Ravnborg } while (len > 0);
2071a88b5ba8SSam Ravnborg
2072a88b5ba8SSam Ravnborg return count;
2073a88b5ba8SSam Ravnborg }
2074a88b5ba8SSam Ravnborg
2075a88b5ba8SSam Ravnborg struct cookie_state {
2076a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table;
2077a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies;
2078a88b5ba8SSam Ravnborg u64 mte_base;
2079a88b5ba8SSam Ravnborg u64 prev_cookie;
2080a88b5ba8SSam Ravnborg u32 pte_idx;
2081a88b5ba8SSam Ravnborg u32 nc;
2082a88b5ba8SSam Ravnborg };
2083a88b5ba8SSam Ravnborg
fill_cookies(struct cookie_state * sp,unsigned long pa,unsigned long off,unsigned long len)2084a88b5ba8SSam Ravnborg static void fill_cookies(struct cookie_state *sp, unsigned long pa,
2085a88b5ba8SSam Ravnborg unsigned long off, unsigned long len)
2086a88b5ba8SSam Ravnborg {
2087a88b5ba8SSam Ravnborg do {
2088a88b5ba8SSam Ravnborg unsigned long tlen, new = pa + PAGE_SIZE;
2089a88b5ba8SSam Ravnborg u64 this_cookie;
2090a88b5ba8SSam Ravnborg
2091a88b5ba8SSam Ravnborg sp->page_table[sp->pte_idx].mte = sp->mte_base | pa;
2092a88b5ba8SSam Ravnborg
2093a88b5ba8SSam Ravnborg tlen = PAGE_SIZE;
2094a88b5ba8SSam Ravnborg if (off)
2095a88b5ba8SSam Ravnborg tlen = PAGE_SIZE - off;
2096a88b5ba8SSam Ravnborg if (tlen > len)
2097a88b5ba8SSam Ravnborg tlen = len;
2098a88b5ba8SSam Ravnborg
2099a88b5ba8SSam Ravnborg this_cookie = make_cookie(sp->pte_idx,
2100a88b5ba8SSam Ravnborg pagesize_code(), off);
2101a88b5ba8SSam Ravnborg
2102a88b5ba8SSam Ravnborg off = 0;
2103a88b5ba8SSam Ravnborg
2104a88b5ba8SSam Ravnborg if (this_cookie == sp->prev_cookie) {
2105a88b5ba8SSam Ravnborg sp->cookies[sp->nc - 1].cookie_size += tlen;
2106a88b5ba8SSam Ravnborg } else {
2107a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_addr = this_cookie;
2108a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_size = tlen;
2109a88b5ba8SSam Ravnborg sp->nc++;
2110a88b5ba8SSam Ravnborg }
2111a88b5ba8SSam Ravnborg sp->prev_cookie = this_cookie + tlen;
2112a88b5ba8SSam Ravnborg
2113a88b5ba8SSam Ravnborg sp->pte_idx++;
2114a88b5ba8SSam Ravnborg
2115a88b5ba8SSam Ravnborg len -= tlen;
2116a88b5ba8SSam Ravnborg pa = new;
2117a88b5ba8SSam Ravnborg } while (len > 0);
2118a88b5ba8SSam Ravnborg }
2119a88b5ba8SSam Ravnborg
sg_count_one(struct scatterlist * sg)2120a88b5ba8SSam Ravnborg static int sg_count_one(struct scatterlist *sg)
2121a88b5ba8SSam Ravnborg {
2122a88b5ba8SSam Ravnborg unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT;
2123a88b5ba8SSam Ravnborg long len = sg->length;
2124a88b5ba8SSam Ravnborg
2125a88b5ba8SSam Ravnborg if ((sg->offset | len) & (8UL - 1))
2126a88b5ba8SSam Ravnborg return -EFAULT;
2127a88b5ba8SSam Ravnborg
2128a88b5ba8SSam Ravnborg return pages_in_region(base + sg->offset, len);
2129a88b5ba8SSam Ravnborg }
2130a88b5ba8SSam Ravnborg
sg_count_pages(struct scatterlist * sg,int num_sg)2131a88b5ba8SSam Ravnborg static int sg_count_pages(struct scatterlist *sg, int num_sg)
2132a88b5ba8SSam Ravnborg {
2133a88b5ba8SSam Ravnborg int count;
2134a88b5ba8SSam Ravnborg int i;
2135a88b5ba8SSam Ravnborg
2136a88b5ba8SSam Ravnborg count = 0;
2137a88b5ba8SSam Ravnborg for (i = 0; i < num_sg; i++) {
2138a88b5ba8SSam Ravnborg int err = sg_count_one(sg + i);
2139a88b5ba8SSam Ravnborg if (err < 0)
2140a88b5ba8SSam Ravnborg return err;
2141a88b5ba8SSam Ravnborg count += err;
2142a88b5ba8SSam Ravnborg }
2143a88b5ba8SSam Ravnborg
2144a88b5ba8SSam Ravnborg return count;
2145a88b5ba8SSam Ravnborg }
2146a88b5ba8SSam Ravnborg
ldc_map_sg(struct ldc_channel * lp,struct scatterlist * sg,int num_sg,struct ldc_trans_cookie * cookies,int ncookies,unsigned int map_perm)2147a88b5ba8SSam Ravnborg int ldc_map_sg(struct ldc_channel *lp,
2148a88b5ba8SSam Ravnborg struct scatterlist *sg, int num_sg,
2149a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies,
2150a88b5ba8SSam Ravnborg unsigned int map_perm)
2151a88b5ba8SSam Ravnborg {
21520ae53ed1SSowmini Varadhan unsigned long i, npages;
2153a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base;
2154a88b5ba8SSam Ravnborg struct cookie_state state;
2155a88b5ba8SSam Ravnborg struct ldc_iommu *iommu;
2156a88b5ba8SSam Ravnborg int err;
21578c07a308SAkinobu Mita struct scatterlist *s;
2158a88b5ba8SSam Ravnborg
2159a88b5ba8SSam Ravnborg if (map_perm & ~LDC_MAP_ALL)
2160a88b5ba8SSam Ravnborg return -EINVAL;
2161a88b5ba8SSam Ravnborg
2162a88b5ba8SSam Ravnborg err = sg_count_pages(sg, num_sg);
2163a88b5ba8SSam Ravnborg if (err < 0)
2164a88b5ba8SSam Ravnborg return err;
2165a88b5ba8SSam Ravnborg
2166a88b5ba8SSam Ravnborg npages = err;
2167a88b5ba8SSam Ravnborg if (err > ncookies)
2168a88b5ba8SSam Ravnborg return -EMSGSIZE;
2169a88b5ba8SSam Ravnborg
2170a88b5ba8SSam Ravnborg iommu = &lp->iommu;
2171a88b5ba8SSam Ravnborg
2172a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages);
2173a88b5ba8SSam Ravnborg
2174a88b5ba8SSam Ravnborg if (!base)
2175a88b5ba8SSam Ravnborg return -ENOMEM;
2176a88b5ba8SSam Ravnborg
2177a88b5ba8SSam Ravnborg state.page_table = iommu->page_table;
2178a88b5ba8SSam Ravnborg state.cookies = cookies;
2179a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm);
2180a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0;
2181a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table);
2182a88b5ba8SSam Ravnborg state.nc = 0;
2183a88b5ba8SSam Ravnborg
21848c07a308SAkinobu Mita for_each_sg(sg, s, num_sg, i) {
21858c07a308SAkinobu Mita fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT,
21868c07a308SAkinobu Mita s->offset, s->length);
21878c07a308SAkinobu Mita }
2188a88b5ba8SSam Ravnborg
2189a88b5ba8SSam Ravnborg return state.nc;
2190a88b5ba8SSam Ravnborg }
2191a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_sg);
2192a88b5ba8SSam Ravnborg
ldc_map_single(struct ldc_channel * lp,void * buf,unsigned int len,struct ldc_trans_cookie * cookies,int ncookies,unsigned int map_perm)2193a88b5ba8SSam Ravnborg int ldc_map_single(struct ldc_channel *lp,
2194a88b5ba8SSam Ravnborg void *buf, unsigned int len,
2195a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies,
2196a88b5ba8SSam Ravnborg unsigned int map_perm)
2197a88b5ba8SSam Ravnborg {
21980ae53ed1SSowmini Varadhan unsigned long npages, pa;
2199a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base;
2200a88b5ba8SSam Ravnborg struct cookie_state state;
2201a88b5ba8SSam Ravnborg struct ldc_iommu *iommu;
2202a88b5ba8SSam Ravnborg
2203a88b5ba8SSam Ravnborg if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1))
2204a88b5ba8SSam Ravnborg return -EINVAL;
2205a88b5ba8SSam Ravnborg
2206a88b5ba8SSam Ravnborg pa = __pa(buf);
2207a88b5ba8SSam Ravnborg if ((pa | len) & (8UL - 1))
2208a88b5ba8SSam Ravnborg return -EFAULT;
2209a88b5ba8SSam Ravnborg
2210a88b5ba8SSam Ravnborg npages = pages_in_region(pa, len);
2211a88b5ba8SSam Ravnborg
2212a88b5ba8SSam Ravnborg iommu = &lp->iommu;
2213a88b5ba8SSam Ravnborg
2214a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages);
2215a88b5ba8SSam Ravnborg
2216a88b5ba8SSam Ravnborg if (!base)
2217a88b5ba8SSam Ravnborg return -ENOMEM;
2218a88b5ba8SSam Ravnborg
2219a88b5ba8SSam Ravnborg state.page_table = iommu->page_table;
2220a88b5ba8SSam Ravnborg state.cookies = cookies;
2221a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm);
2222a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0;
2223a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table);
2224a88b5ba8SSam Ravnborg state.nc = 0;
2225a88b5ba8SSam Ravnborg fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len);
222642db672dSDavid L Stevens BUG_ON(state.nc > ncookies);
2227a88b5ba8SSam Ravnborg
2228a88b5ba8SSam Ravnborg return state.nc;
2229a88b5ba8SSam Ravnborg }
2230a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_single);
2231a88b5ba8SSam Ravnborg
22320ae53ed1SSowmini Varadhan
free_npages(unsigned long id,struct ldc_iommu * iommu,u64 cookie,u64 size)2233a88b5ba8SSam Ravnborg static void free_npages(unsigned long id, struct ldc_iommu *iommu,
2234a88b5ba8SSam Ravnborg u64 cookie, u64 size)
2235a88b5ba8SSam Ravnborg {
22360ae53ed1SSowmini Varadhan unsigned long npages, entry;
2237a88b5ba8SSam Ravnborg
2238a88b5ba8SSam Ravnborg npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;
2239a88b5ba8SSam Ravnborg
22400ae53ed1SSowmini Varadhan entry = ldc_cookie_to_index(cookie, iommu);
22410ae53ed1SSowmini Varadhan ldc_demap(iommu, id, cookie, entry, npages);
22420ae53ed1SSowmini Varadhan iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);
2243a88b5ba8SSam Ravnborg }
2244a88b5ba8SSam Ravnborg
ldc_unmap(struct ldc_channel * lp,struct ldc_trans_cookie * cookies,int ncookies)2245a88b5ba8SSam Ravnborg void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
2246a88b5ba8SSam Ravnborg int ncookies)
2247a88b5ba8SSam Ravnborg {
2248a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu;
2249c12f048fSDavid S. Miller int i;
22500ae53ed1SSowmini Varadhan unsigned long flags;
2251a88b5ba8SSam Ravnborg
2252a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags);
2253a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) {
2254a88b5ba8SSam Ravnborg u64 addr = cookies[i].cookie_addr;
2255a88b5ba8SSam Ravnborg u64 size = cookies[i].cookie_size;
2256a88b5ba8SSam Ravnborg
2257a88b5ba8SSam Ravnborg free_npages(lp->id, iommu, addr, size);
2258a88b5ba8SSam Ravnborg }
2259a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags);
2260a88b5ba8SSam Ravnborg }
2261a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_unmap);
2262a88b5ba8SSam Ravnborg
ldc_copy(struct ldc_channel * lp,int copy_dir,void * buf,unsigned int len,unsigned long offset,struct ldc_trans_cookie * cookies,int ncookies)2263a88b5ba8SSam Ravnborg int ldc_copy(struct ldc_channel *lp, int copy_dir,
2264a88b5ba8SSam Ravnborg void *buf, unsigned int len, unsigned long offset,
2265a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies)
2266a88b5ba8SSam Ravnborg {
2267a88b5ba8SSam Ravnborg unsigned int orig_len;
2268a88b5ba8SSam Ravnborg unsigned long ra;
2269a88b5ba8SSam Ravnborg int i;
2270a88b5ba8SSam Ravnborg
2271a88b5ba8SSam Ravnborg if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) {
2272a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n",
2273a88b5ba8SSam Ravnborg lp->id, copy_dir);
2274a88b5ba8SSam Ravnborg return -EINVAL;
2275a88b5ba8SSam Ravnborg }
2276a88b5ba8SSam Ravnborg
2277a88b5ba8SSam Ravnborg ra = __pa(buf);
2278a88b5ba8SSam Ravnborg if ((ra | len | offset) & (8UL - 1)) {
2279a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer "
2280a88b5ba8SSam Ravnborg "ra[%lx] len[%x] offset[%lx]\n",
2281a88b5ba8SSam Ravnborg lp->id, ra, len, offset);
2282a88b5ba8SSam Ravnborg return -EFAULT;
2283a88b5ba8SSam Ravnborg }
2284a88b5ba8SSam Ravnborg
2285a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE ||
2286a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) {
2287a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] "
2288a88b5ba8SSam Ravnborg "flags[%x]\n", lp->id, lp->hs_state, lp->flags);
2289a88b5ba8SSam Ravnborg return -ECONNRESET;
2290a88b5ba8SSam Ravnborg }
2291a88b5ba8SSam Ravnborg
2292a88b5ba8SSam Ravnborg orig_len = len;
2293a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) {
2294a88b5ba8SSam Ravnborg unsigned long cookie_raddr = cookies[i].cookie_addr;
2295a88b5ba8SSam Ravnborg unsigned long this_len = cookies[i].cookie_size;
2296a88b5ba8SSam Ravnborg unsigned long actual_len;
2297a88b5ba8SSam Ravnborg
2298a88b5ba8SSam Ravnborg if (unlikely(offset)) {
2299a88b5ba8SSam Ravnborg unsigned long this_off = offset;
2300a88b5ba8SSam Ravnborg
2301a88b5ba8SSam Ravnborg if (this_off > this_len)
2302a88b5ba8SSam Ravnborg this_off = this_len;
2303a88b5ba8SSam Ravnborg
2304a88b5ba8SSam Ravnborg offset -= this_off;
2305a88b5ba8SSam Ravnborg this_len -= this_off;
2306a88b5ba8SSam Ravnborg if (!this_len)
2307a88b5ba8SSam Ravnborg continue;
2308a88b5ba8SSam Ravnborg cookie_raddr += this_off;
2309a88b5ba8SSam Ravnborg }
2310a88b5ba8SSam Ravnborg
2311a88b5ba8SSam Ravnborg if (this_len > len)
2312a88b5ba8SSam Ravnborg this_len = len;
2313a88b5ba8SSam Ravnborg
2314a88b5ba8SSam Ravnborg while (1) {
2315a88b5ba8SSam Ravnborg unsigned long hv_err;
2316a88b5ba8SSam Ravnborg
2317a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_copy(lp->id, copy_dir,
2318a88b5ba8SSam Ravnborg cookie_raddr, ra,
2319a88b5ba8SSam Ravnborg this_len, &actual_len);
2320a88b5ba8SSam Ravnborg if (unlikely(hv_err)) {
2321a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] "
2322a88b5ba8SSam Ravnborg "HV error %lu\n",
2323a88b5ba8SSam Ravnborg lp->id, hv_err);
2324a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE ||
2325a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET))
2326a88b5ba8SSam Ravnborg return -ECONNRESET;
2327a88b5ba8SSam Ravnborg else
2328a88b5ba8SSam Ravnborg return -EFAULT;
2329a88b5ba8SSam Ravnborg }
2330a88b5ba8SSam Ravnborg
2331a88b5ba8SSam Ravnborg cookie_raddr += actual_len;
2332a88b5ba8SSam Ravnborg ra += actual_len;
2333a88b5ba8SSam Ravnborg len -= actual_len;
2334a88b5ba8SSam Ravnborg if (actual_len == this_len)
2335a88b5ba8SSam Ravnborg break;
2336a88b5ba8SSam Ravnborg
2337a88b5ba8SSam Ravnborg this_len -= actual_len;
2338a88b5ba8SSam Ravnborg }
2339a88b5ba8SSam Ravnborg
2340a88b5ba8SSam Ravnborg if (!len)
2341a88b5ba8SSam Ravnborg break;
2342a88b5ba8SSam Ravnborg }
2343a88b5ba8SSam Ravnborg
2344a88b5ba8SSam Ravnborg /* It is caller policy what to do about short copies.
2345a88b5ba8SSam Ravnborg * For example, a networking driver can declare the
2346a88b5ba8SSam Ravnborg * packet a runt and drop it.
2347a88b5ba8SSam Ravnborg */
2348a88b5ba8SSam Ravnborg
2349a88b5ba8SSam Ravnborg return orig_len - len;
2350a88b5ba8SSam Ravnborg }
2351a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_copy);
2352a88b5ba8SSam Ravnborg
ldc_alloc_exp_dring(struct ldc_channel * lp,unsigned int len,struct ldc_trans_cookie * cookies,int * ncookies,unsigned int map_perm)2353a88b5ba8SSam Ravnborg void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len,
2354a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int *ncookies,
2355a88b5ba8SSam Ravnborg unsigned int map_perm)
2356a88b5ba8SSam Ravnborg {
2357a88b5ba8SSam Ravnborg void *buf;
2358a88b5ba8SSam Ravnborg int err;
2359a88b5ba8SSam Ravnborg
2360a88b5ba8SSam Ravnborg if (len & (8UL - 1))
2361a88b5ba8SSam Ravnborg return ERR_PTR(-EINVAL);
2362a88b5ba8SSam Ravnborg
23630edfad59SSowmini Varadhan buf = kzalloc(len, GFP_ATOMIC);
2364a88b5ba8SSam Ravnborg if (!buf)
2365a88b5ba8SSam Ravnborg return ERR_PTR(-ENOMEM);
2366a88b5ba8SSam Ravnborg
2367a88b5ba8SSam Ravnborg err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm);
2368a88b5ba8SSam Ravnborg if (err < 0) {
2369a88b5ba8SSam Ravnborg kfree(buf);
2370a88b5ba8SSam Ravnborg return ERR_PTR(err);
2371a88b5ba8SSam Ravnborg }
2372a88b5ba8SSam Ravnborg *ncookies = err;
2373a88b5ba8SSam Ravnborg
2374a88b5ba8SSam Ravnborg return buf;
2375a88b5ba8SSam Ravnborg }
2376a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc_exp_dring);
2377a88b5ba8SSam Ravnborg
ldc_free_exp_dring(struct ldc_channel * lp,void * buf,unsigned int len,struct ldc_trans_cookie * cookies,int ncookies)2378a88b5ba8SSam Ravnborg void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len,
2379a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies)
2380a88b5ba8SSam Ravnborg {
2381a88b5ba8SSam Ravnborg ldc_unmap(lp, cookies, ncookies);
2382a88b5ba8SSam Ravnborg kfree(buf);
2383a88b5ba8SSam Ravnborg }
2384a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free_exp_dring);
2385a88b5ba8SSam Ravnborg
ldc_init(void)2386a88b5ba8SSam Ravnborg static int __init ldc_init(void)
2387a88b5ba8SSam Ravnborg {
2388a88b5ba8SSam Ravnborg unsigned long major, minor;
2389a88b5ba8SSam Ravnborg struct mdesc_handle *hp;
2390a88b5ba8SSam Ravnborg const u64 *v;
2391a88b5ba8SSam Ravnborg int err;
2392a88b5ba8SSam Ravnborg u64 mp;
2393a88b5ba8SSam Ravnborg
2394a88b5ba8SSam Ravnborg hp = mdesc_grab();
2395a88b5ba8SSam Ravnborg if (!hp)
2396a88b5ba8SSam Ravnborg return -ENODEV;
2397a88b5ba8SSam Ravnborg
2398a88b5ba8SSam Ravnborg mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
2399a88b5ba8SSam Ravnborg err = -ENODEV;
2400a88b5ba8SSam Ravnborg if (mp == MDESC_NODE_NULL)
2401a88b5ba8SSam Ravnborg goto out;
2402a88b5ba8SSam Ravnborg
2403a88b5ba8SSam Ravnborg v = mdesc_get_property(hp, mp, "domaining-enabled", NULL);
2404a88b5ba8SSam Ravnborg if (!v)
2405a88b5ba8SSam Ravnborg goto out;
2406a88b5ba8SSam Ravnborg
2407a88b5ba8SSam Ravnborg major = 1;
2408a88b5ba8SSam Ravnborg minor = 0;
2409a88b5ba8SSam Ravnborg if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) {
2410a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Could not register LDOM hvapi.\n");
2411a88b5ba8SSam Ravnborg goto out;
2412a88b5ba8SSam Ravnborg }
2413a88b5ba8SSam Ravnborg
2414a88b5ba8SSam Ravnborg printk(KERN_INFO "%s", version);
2415a88b5ba8SSam Ravnborg
2416a88b5ba8SSam Ravnborg if (!*v) {
2417a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Domaining disabled.\n");
2418a88b5ba8SSam Ravnborg goto out;
2419a88b5ba8SSam Ravnborg }
2420a88b5ba8SSam Ravnborg ldom_domaining_enabled = 1;
2421a88b5ba8SSam Ravnborg err = 0;
2422a88b5ba8SSam Ravnborg
2423a88b5ba8SSam Ravnborg out:
2424a88b5ba8SSam Ravnborg mdesc_release(hp);
2425a88b5ba8SSam Ravnborg return err;
2426a88b5ba8SSam Ravnborg }
2427a88b5ba8SSam Ravnborg
2428a88b5ba8SSam Ravnborg core_initcall(ldc_init);
2429