1 /*
2  * linux/drivers/misc/xillybus_core.c
3  *
4  * Copyright 2011 Xillybus Ltd, http://xillybus.com
5  *
6  * Driver for the Xillybus FPGA/host framework.
7  *
8  * This driver interfaces with a special IP core in an FPGA, setting up
9  * a pipe between a hardware FIFO in the programmable logic and a device
10  * file in the host. The number of such pipes and their attributes are
11  * set up on the logic. This driver detects these automatically and
12  * creates the device files accordingly.
13  *
14  * This program is free software; you can redistribute it and/or modify
15  * it under the smems of the GNU General Public License as published by
16  * the Free Software Foundation; version 2 of the License.
17  */
18 
19 #include <linux/list.h>
20 #include <linux/device.h>
21 #include <linux/module.h>
22 #include <linux/io.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/interrupt.h>
25 #include <linux/sched.h>
26 #include <linux/fs.h>
27 #include <linux/cdev.h>
28 #include <linux/spinlock.h>
29 #include <linux/mutex.h>
30 #include <linux/crc32.h>
31 #include <linux/poll.h>
32 #include <linux/delay.h>
33 #include <linux/slab.h>
34 #include <linux/workqueue.h>
35 #include "xillybus.h"
36 
37 MODULE_DESCRIPTION("Xillybus core functions");
38 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
39 MODULE_VERSION("1.07");
40 MODULE_ALIAS("xillybus_core");
41 MODULE_LICENSE("GPL v2");
42 
43 /* General timeout is 100 ms, rx timeout is 10 ms */
44 #define XILLY_RX_TIMEOUT (10*HZ/1000)
45 #define XILLY_TIMEOUT (100*HZ/1000)
46 
47 #define fpga_msg_ctrl_reg              0x0008
48 #define fpga_dma_control_reg           0x0020
49 #define fpga_dma_bufno_reg             0x0024
50 #define fpga_dma_bufaddr_lowaddr_reg   0x0028
51 #define fpga_dma_bufaddr_highaddr_reg  0x002c
52 #define fpga_buf_ctrl_reg              0x0030
53 #define fpga_buf_offset_reg            0x0034
54 #define fpga_endian_reg                0x0040
55 
56 #define XILLYMSG_OPCODE_RELEASEBUF 1
57 #define XILLYMSG_OPCODE_QUIESCEACK 2
58 #define XILLYMSG_OPCODE_FIFOEOF 3
59 #define XILLYMSG_OPCODE_FATAL_ERROR 4
60 #define XILLYMSG_OPCODE_NONEMPTY 5
61 
62 static const char xillyname[] = "xillybus";
63 
64 static struct class *xillybus_class;
65 
66 /*
67  * ep_list_lock is the last lock to be taken; No other lock requests are
68  * allowed while holding it. It merely protects list_of_endpoints, and not
69  * the endpoints listed in it.
70  */
71 
72 static LIST_HEAD(list_of_endpoints);
73 static struct mutex ep_list_lock;
74 static struct workqueue_struct *xillybus_wq;
75 
76 /*
77  * Locking scheme: Mutexes protect invocations of character device methods.
78  * If both locks are taken, wr_mutex is taken first, rd_mutex second.
79  *
80  * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
81  * buffers' end_offset fields against changes made by IRQ handler (and in
82  * theory, other file request handlers, but the mutex handles that). Nothing
83  * else.
84  * They are held for short direct memory manipulations. Needless to say,
85  * no mutex locking is allowed when a spinlock is held.
86  *
87  * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
88  *
89  * register_mutex is endpoint-specific, and is held when non-atomic
90  * register operations are performed. wr_mutex and rd_mutex may be
91  * held when register_mutex is taken, but none of the spinlocks. Note that
92  * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
93  * which are unrelated to buf_offset_reg, since they are harmless.
94  *
95  * Blocking on the wait queues is allowed with mutexes held, but not with
96  * spinlocks.
97  *
98  * Only interruptible blocking is allowed on mutexes and wait queues.
99  *
100  * All in all, the locking order goes (with skips allowed, of course):
101  * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
102  */
103 
104 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
105 {
106 	int opcode;
107 	int msg_channel, msg_bufno, msg_data, msg_dir;
108 
109 	opcode = (buf[0] >> 24) & 0xff;
110 	msg_dir = buf[0] & 1;
111 	msg_channel = (buf[0] >> 1) & 0x7ff;
112 	msg_bufno = (buf[0] >> 12) & 0x3ff;
113 	msg_data = buf[1] & 0xfffffff;
114 
115 	dev_warn(endpoint->dev,
116 		 "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
117 		 opcode, msg_channel, msg_dir, msg_bufno, msg_data);
118 }
119 
120 /*
121  * xillybus_isr assumes the interrupt is allocated exclusively to it,
122  * which is the natural case MSI and several other hardware-oriented
123  * interrupts. Sharing is not allowed.
124  */
125 
126 irqreturn_t xillybus_isr(int irq, void *data)
127 {
128 	struct xilly_endpoint *ep = data;
129 	u32 *buf;
130 	unsigned int buf_size;
131 	int i;
132 	int opcode;
133 	unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
134 	struct xilly_channel *channel;
135 
136 	buf = ep->msgbuf_addr;
137 	buf_size = ep->msg_buf_size/sizeof(u32);
138 
139 	ep->ephw->hw_sync_sgl_for_cpu(ep,
140 				      ep->msgbuf_dma_addr,
141 				      ep->msg_buf_size,
142 				      DMA_FROM_DEVICE);
143 
144 	for (i = 0; i < buf_size; i += 2) {
145 		if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
146 			malformed_message(ep, &buf[i]);
147 			dev_warn(ep->dev,
148 				 "Sending a NACK on counter %x (instead of %x) on entry %d\n",
149 				 ((buf[i+1] >> 28) & 0xf),
150 				 ep->msg_counter,
151 				 i/2);
152 
153 			if (++ep->failed_messages > 10) {
154 				dev_err(ep->dev,
155 					"Lost sync with interrupt messages. Stopping.\n");
156 			} else {
157 				ep->ephw->hw_sync_sgl_for_device(
158 					ep,
159 					ep->msgbuf_dma_addr,
160 					ep->msg_buf_size,
161 					DMA_FROM_DEVICE);
162 
163 				iowrite32(0x01,  /* Message NACK */
164 					  ep->registers + fpga_msg_ctrl_reg);
165 			}
166 			return IRQ_HANDLED;
167 		} else if (buf[i] & (1 << 22)) /* Last message */
168 			break;
169 	}
170 
171 	if (i >= buf_size) {
172 		dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
173 		return IRQ_HANDLED;
174 	}
175 
176 	buf_size = i + 2;
177 
178 	for (i = 0; i < buf_size; i += 2) { /* Scan through messages */
179 		opcode = (buf[i] >> 24) & 0xff;
180 
181 		msg_dir = buf[i] & 1;
182 		msg_channel = (buf[i] >> 1) & 0x7ff;
183 		msg_bufno = (buf[i] >> 12) & 0x3ff;
184 		msg_data = buf[i+1] & 0xfffffff;
185 
186 		switch (opcode) {
187 		case XILLYMSG_OPCODE_RELEASEBUF:
188 			if ((msg_channel > ep->num_channels) ||
189 			    (msg_channel == 0)) {
190 				malformed_message(ep, &buf[i]);
191 				break;
192 			}
193 
194 			channel = ep->channels[msg_channel];
195 
196 			if (msg_dir) { /* Write channel */
197 				if (msg_bufno >= channel->num_wr_buffers) {
198 					malformed_message(ep, &buf[i]);
199 					break;
200 				}
201 				spin_lock(&channel->wr_spinlock);
202 				channel->wr_buffers[msg_bufno]->end_offset =
203 					msg_data;
204 				channel->wr_fpga_buf_idx = msg_bufno;
205 				channel->wr_empty = 0;
206 				channel->wr_sleepy = 0;
207 				spin_unlock(&channel->wr_spinlock);
208 
209 				wake_up_interruptible(&channel->wr_wait);
210 
211 			} else {
212 				/* Read channel */
213 
214 				if (msg_bufno >= channel->num_rd_buffers) {
215 					malformed_message(ep, &buf[i]);
216 					break;
217 				}
218 
219 				spin_lock(&channel->rd_spinlock);
220 				channel->rd_fpga_buf_idx = msg_bufno;
221 				channel->rd_full = 0;
222 				spin_unlock(&channel->rd_spinlock);
223 
224 				wake_up_interruptible(&channel->rd_wait);
225 				if (!channel->rd_synchronous)
226 					queue_delayed_work(
227 						xillybus_wq,
228 						&channel->rd_workitem,
229 						XILLY_RX_TIMEOUT);
230 			}
231 
232 			break;
233 		case XILLYMSG_OPCODE_NONEMPTY:
234 			if ((msg_channel > ep->num_channels) ||
235 			    (msg_channel == 0) || (!msg_dir) ||
236 			    !ep->channels[msg_channel]->wr_supports_nonempty) {
237 				malformed_message(ep, &buf[i]);
238 				break;
239 			}
240 
241 			channel = ep->channels[msg_channel];
242 
243 			if (msg_bufno >= channel->num_wr_buffers) {
244 				malformed_message(ep, &buf[i]);
245 				break;
246 			}
247 			spin_lock(&channel->wr_spinlock);
248 			if (msg_bufno == channel->wr_host_buf_idx)
249 				channel->wr_ready = 1;
250 			spin_unlock(&channel->wr_spinlock);
251 
252 			wake_up_interruptible(&channel->wr_ready_wait);
253 
254 			break;
255 		case XILLYMSG_OPCODE_QUIESCEACK:
256 			ep->idtlen = msg_data;
257 			wake_up_interruptible(&ep->ep_wait);
258 
259 			break;
260 		case XILLYMSG_OPCODE_FIFOEOF:
261 			if ((msg_channel > ep->num_channels) ||
262 			    (msg_channel == 0) || (!msg_dir) ||
263 			    !ep->channels[msg_channel]->num_wr_buffers) {
264 				malformed_message(ep, &buf[i]);
265 				break;
266 			}
267 			channel = ep->channels[msg_channel];
268 			spin_lock(&channel->wr_spinlock);
269 			channel->wr_eof = msg_bufno;
270 			channel->wr_sleepy = 0;
271 
272 			channel->wr_hangup = channel->wr_empty &&
273 				(channel->wr_host_buf_idx == msg_bufno);
274 
275 			spin_unlock(&channel->wr_spinlock);
276 
277 			wake_up_interruptible(&channel->wr_wait);
278 
279 			break;
280 		case XILLYMSG_OPCODE_FATAL_ERROR:
281 			ep->fatal_error = 1;
282 			wake_up_interruptible(&ep->ep_wait); /* For select() */
283 			dev_err(ep->dev,
284 				"FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
285 			break;
286 		default:
287 			malformed_message(ep, &buf[i]);
288 			break;
289 		}
290 	}
291 
292 	ep->ephw->hw_sync_sgl_for_device(ep,
293 					 ep->msgbuf_dma_addr,
294 					 ep->msg_buf_size,
295 					 DMA_FROM_DEVICE);
296 
297 	ep->msg_counter = (ep->msg_counter + 1) & 0xf;
298 	ep->failed_messages = 0;
299 	iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */
300 
301 	return IRQ_HANDLED;
302 }
303 EXPORT_SYMBOL(xillybus_isr);
304 
305 /*
306  * A few trivial memory management functions.
307  * NOTE: These functions are used only on probe and remove, and therefore
308  * no locks are applied!
309  */
310 
311 static void xillybus_autoflush(struct work_struct *work);
312 
313 struct xilly_alloc_state {
314 	void *salami;
315 	int left_of_salami;
316 	int nbuffer;
317 	enum dma_data_direction direction;
318 	u32 regdirection;
319 };
320 
321 static int xilly_get_dma_buffers(struct xilly_endpoint *ep,
322 				 struct xilly_alloc_state *s,
323 				 struct xilly_buffer **buffers,
324 				 int bufnum, int bytebufsize)
325 {
326 	int i, rc;
327 	dma_addr_t dma_addr;
328 	struct device *dev = ep->dev;
329 	struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
330 
331 	if (buffers) { /* Not the message buffer */
332 		this_buffer = devm_kcalloc(dev, bufnum,
333 					   sizeof(struct xilly_buffer),
334 					   GFP_KERNEL);
335 		if (!this_buffer)
336 			return -ENOMEM;
337 	}
338 
339 	for (i = 0; i < bufnum; i++) {
340 		/*
341 		 * Buffers are expected in descending size order, so there
342 		 * is either enough space for this buffer or none at all.
343 		 */
344 
345 		if ((s->left_of_salami < bytebufsize) &&
346 		    (s->left_of_salami > 0)) {
347 			dev_err(ep->dev,
348 				"Corrupt buffer allocation in IDT. Aborting.\n");
349 			return -ENODEV;
350 		}
351 
352 		if (s->left_of_salami == 0) {
353 			int allocorder, allocsize;
354 
355 			allocsize = PAGE_SIZE;
356 			allocorder = 0;
357 			while (bytebufsize > allocsize) {
358 				allocsize *= 2;
359 				allocorder++;
360 			}
361 
362 			s->salami = (void *) devm_get_free_pages(
363 				dev,
364 				GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,
365 				allocorder);
366 			if (!s->salami)
367 				return -ENOMEM;
368 
369 			s->left_of_salami = allocsize;
370 		}
371 
372 		rc = ep->ephw->map_single(ep, s->salami,
373 					  bytebufsize, s->direction,
374 					  &dma_addr);
375 		if (rc)
376 			return rc;
377 
378 		iowrite32((u32) (dma_addr & 0xffffffff),
379 			  ep->registers + fpga_dma_bufaddr_lowaddr_reg);
380 		iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),
381 			  ep->registers + fpga_dma_bufaddr_highaddr_reg);
382 
383 		if (buffers) { /* Not the message buffer */
384 			this_buffer->addr = s->salami;
385 			this_buffer->dma_addr = dma_addr;
386 			buffers[i] = this_buffer++;
387 
388 			iowrite32(s->regdirection | s->nbuffer++,
389 				  ep->registers + fpga_dma_bufno_reg);
390 		} else {
391 			ep->msgbuf_addr = s->salami;
392 			ep->msgbuf_dma_addr = dma_addr;
393 			ep->msg_buf_size = bytebufsize;
394 
395 			iowrite32(s->regdirection,
396 				  ep->registers + fpga_dma_bufno_reg);
397 		}
398 
399 		s->left_of_salami -= bytebufsize;
400 		s->salami += bytebufsize;
401 	}
402 	return 0;
403 }
404 
405 static int xilly_setupchannels(struct xilly_endpoint *ep,
406 			       unsigned char *chandesc,
407 			       int entries)
408 {
409 	struct device *dev = ep->dev;
410 	int i, entry, rc;
411 	struct xilly_channel *channel;
412 	int channelnum, bufnum, bufsize, format, is_writebuf;
413 	int bytebufsize;
414 	int synchronous, allowpartial, exclusive_open, seekable;
415 	int supports_nonempty;
416 	int msg_buf_done = 0;
417 
418 	struct xilly_alloc_state rd_alloc = {
419 		.salami = NULL,
420 		.left_of_salami = 0,
421 		.nbuffer = 1,
422 		.direction = DMA_TO_DEVICE,
423 		.regdirection = 0,
424 	};
425 
426 	struct xilly_alloc_state wr_alloc = {
427 		.salami = NULL,
428 		.left_of_salami = 0,
429 		.nbuffer = 1,
430 		.direction = DMA_FROM_DEVICE,
431 		.regdirection = 0x80000000,
432 	};
433 
434 	channel = devm_kcalloc(dev, ep->num_channels,
435 			       sizeof(struct xilly_channel), GFP_KERNEL);
436 	if (!channel)
437 		return -ENOMEM;
438 
439 	ep->channels = devm_kcalloc(dev, ep->num_channels + 1,
440 				    sizeof(struct xilly_channel *),
441 				    GFP_KERNEL);
442 	if (!ep->channels)
443 		return -ENOMEM;
444 
445 	ep->channels[0] = NULL; /* Channel 0 is message buf. */
446 
447 	/* Initialize all channels with defaults */
448 
449 	for (i = 1; i <= ep->num_channels; i++) {
450 		channel->wr_buffers = NULL;
451 		channel->rd_buffers = NULL;
452 		channel->num_wr_buffers = 0;
453 		channel->num_rd_buffers = 0;
454 		channel->wr_fpga_buf_idx = -1;
455 		channel->wr_host_buf_idx = 0;
456 		channel->wr_host_buf_pos = 0;
457 		channel->wr_empty = 1;
458 		channel->wr_ready = 0;
459 		channel->wr_sleepy = 1;
460 		channel->rd_fpga_buf_idx = 0;
461 		channel->rd_host_buf_idx = 0;
462 		channel->rd_host_buf_pos = 0;
463 		channel->rd_full = 0;
464 		channel->wr_ref_count = 0;
465 		channel->rd_ref_count = 0;
466 
467 		spin_lock_init(&channel->wr_spinlock);
468 		spin_lock_init(&channel->rd_spinlock);
469 		mutex_init(&channel->wr_mutex);
470 		mutex_init(&channel->rd_mutex);
471 		init_waitqueue_head(&channel->rd_wait);
472 		init_waitqueue_head(&channel->wr_wait);
473 		init_waitqueue_head(&channel->wr_ready_wait);
474 
475 		INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
476 
477 		channel->endpoint = ep;
478 		channel->chan_num = i;
479 
480 		channel->log2_element_size = 0;
481 
482 		ep->channels[i] = channel++;
483 	}
484 
485 	for (entry = 0; entry < entries; entry++, chandesc += 4) {
486 		struct xilly_buffer **buffers = NULL;
487 
488 		is_writebuf = chandesc[0] & 0x01;
489 		channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
490 		format = (chandesc[1] >> 4) & 0x03;
491 		allowpartial = (chandesc[1] >> 6) & 0x01;
492 		synchronous = (chandesc[1] >> 7) & 0x01;
493 		bufsize = 1 << (chandesc[2] & 0x1f);
494 		bufnum = 1 << (chandesc[3] & 0x0f);
495 		exclusive_open = (chandesc[2] >> 7) & 0x01;
496 		seekable = (chandesc[2] >> 6) & 0x01;
497 		supports_nonempty = (chandesc[2] >> 5) & 0x01;
498 
499 		if ((channelnum > ep->num_channels) ||
500 		    ((channelnum == 0) && !is_writebuf)) {
501 			dev_err(ep->dev,
502 				"IDT requests channel out of range. Aborting.\n");
503 			return -ENODEV;
504 		}
505 
506 		channel = ep->channels[channelnum]; /* NULL for msg channel */
507 
508 		if (!is_writebuf || channelnum > 0) {
509 			channel->log2_element_size = ((format > 2) ?
510 						      2 : format);
511 
512 			bytebufsize = bufsize *
513 				(1 << channel->log2_element_size);
514 
515 			buffers = devm_kcalloc(dev, bufnum,
516 					       sizeof(struct xilly_buffer *),
517 					       GFP_KERNEL);
518 			if (!buffers)
519 				return -ENOMEM;
520 		} else {
521 			bytebufsize = bufsize << 2;
522 		}
523 
524 		if (!is_writebuf) {
525 			channel->num_rd_buffers = bufnum;
526 			channel->rd_buf_size = bytebufsize;
527 			channel->rd_allow_partial = allowpartial;
528 			channel->rd_synchronous = synchronous;
529 			channel->rd_exclusive_open = exclusive_open;
530 			channel->seekable = seekable;
531 
532 			channel->rd_buffers = buffers;
533 			rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers,
534 						   bufnum, bytebufsize);
535 		} else if (channelnum > 0) {
536 			channel->num_wr_buffers = bufnum;
537 			channel->wr_buf_size = bytebufsize;
538 
539 			channel->seekable = seekable;
540 			channel->wr_supports_nonempty = supports_nonempty;
541 
542 			channel->wr_allow_partial = allowpartial;
543 			channel->wr_synchronous = synchronous;
544 			channel->wr_exclusive_open = exclusive_open;
545 
546 			channel->wr_buffers = buffers;
547 			rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers,
548 						   bufnum, bytebufsize);
549 		} else {
550 			rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL,
551 						   bufnum, bytebufsize);
552 			msg_buf_done++;
553 		}
554 
555 		if (rc)
556 			return -ENOMEM;
557 	}
558 
559 	if (!msg_buf_done) {
560 		dev_err(ep->dev,
561 			"Corrupt IDT: No message buffer. Aborting.\n");
562 		return -ENODEV;
563 	}
564 	return 0;
565 }
566 
567 static int xilly_scan_idt(struct xilly_endpoint *endpoint,
568 			  struct xilly_idt_handle *idt_handle)
569 {
570 	int count = 0;
571 	unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
572 	unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
573 	unsigned char *scan;
574 	int len;
575 
576 	scan = idt;
577 	idt_handle->idt = idt;
578 
579 	scan++; /* Skip version number */
580 
581 	while ((scan <= end_of_idt) && *scan) {
582 		while ((scan <= end_of_idt) && *scan++)
583 			/* Do nothing, just scan thru string */;
584 		count++;
585 	}
586 
587 	scan++;
588 
589 	if (scan > end_of_idt) {
590 		dev_err(endpoint->dev,
591 			"IDT device name list overflow. Aborting.\n");
592 		return -ENODEV;
593 	}
594 	idt_handle->chandesc = scan;
595 
596 	len = endpoint->idtlen - (3 + ((int) (scan - idt)));
597 
598 	if (len & 0x03) {
599 		dev_err(endpoint->dev,
600 			"Corrupt IDT device name list. Aborting.\n");
601 		return -ENODEV;
602 	}
603 
604 	idt_handle->entries = len >> 2;
605 	endpoint->num_channels = count;
606 
607 	return 0;
608 }
609 
610 static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
611 {
612 	struct xilly_channel *channel;
613 	unsigned char *version;
614 	long t;
615 
616 	channel = endpoint->channels[1]; /* This should be generated ad-hoc */
617 
618 	channel->wr_sleepy = 1;
619 
620 	iowrite32(1 |
621 		  (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
622 		  endpoint->registers + fpga_buf_ctrl_reg);
623 
624 	t = wait_event_interruptible_timeout(channel->wr_wait,
625 					     (!channel->wr_sleepy),
626 					     XILLY_TIMEOUT);
627 
628 	if (t <= 0) {
629 		dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
630 
631 		if (endpoint->fatal_error)
632 			return -EIO;
633 
634 		return -ENODEV;
635 	}
636 
637 	endpoint->ephw->hw_sync_sgl_for_cpu(
638 		channel->endpoint,
639 		channel->wr_buffers[0]->dma_addr,
640 		channel->wr_buf_size,
641 		DMA_FROM_DEVICE);
642 
643 	if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
644 		dev_err(endpoint->dev,
645 			"IDT length mismatch (%d != %d). Aborting.\n",
646 			channel->wr_buffers[0]->end_offset, endpoint->idtlen);
647 		return -ENODEV;
648 	}
649 
650 	if (crc32_le(~0, channel->wr_buffers[0]->addr,
651 		     endpoint->idtlen+1) != 0) {
652 		dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
653 		return -ENODEV;
654 	}
655 
656 	version = channel->wr_buffers[0]->addr;
657 
658 	/* Check version number. Reject anything above 0x82. */
659 	if (*version > 0x82) {
660 		dev_err(endpoint->dev,
661 			"No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n",
662 			*version);
663 		return -ENODEV;
664 	}
665 
666 	return 0;
667 }
668 
669 static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
670 			     size_t count, loff_t *f_pos)
671 {
672 	ssize_t rc;
673 	unsigned long flags;
674 	int bytes_done = 0;
675 	int no_time_left = 0;
676 	long deadline, left_to_sleep;
677 	struct xilly_channel *channel = filp->private_data;
678 
679 	int empty, reached_eof, exhausted, ready;
680 	/* Initializations are there only to silence warnings */
681 
682 	int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
683 	int waiting_bufidx;
684 
685 	if (channel->endpoint->fatal_error)
686 		return -EIO;
687 
688 	deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
689 
690 	rc = mutex_lock_interruptible(&channel->wr_mutex);
691 	if (rc)
692 		return rc;
693 
694 	while (1) { /* Note that we may drop mutex within this loop */
695 		int bytes_to_do = count - bytes_done;
696 
697 		spin_lock_irqsave(&channel->wr_spinlock, flags);
698 
699 		empty = channel->wr_empty;
700 		ready = !empty || channel->wr_ready;
701 
702 		if (!empty) {
703 			bufidx = channel->wr_host_buf_idx;
704 			bufpos = channel->wr_host_buf_pos;
705 			howmany = ((channel->wr_buffers[bufidx]->end_offset
706 				    + 1) << channel->log2_element_size)
707 				- bufpos;
708 
709 			/* Update wr_host_* to its post-operation state */
710 			if (howmany > bytes_to_do) {
711 				bufferdone = 0;
712 
713 				howmany = bytes_to_do;
714 				channel->wr_host_buf_pos += howmany;
715 			} else {
716 				bufferdone = 1;
717 
718 				channel->wr_host_buf_pos = 0;
719 
720 				if (bufidx == channel->wr_fpga_buf_idx) {
721 					channel->wr_empty = 1;
722 					channel->wr_sleepy = 1;
723 					channel->wr_ready = 0;
724 				}
725 
726 				if (bufidx >= (channel->num_wr_buffers - 1))
727 					channel->wr_host_buf_idx = 0;
728 				else
729 					channel->wr_host_buf_idx++;
730 			}
731 		}
732 
733 		/*
734 		 * Marking our situation after the possible changes above,
735 		 * for use after releasing the spinlock.
736 		 *
737 		 * empty = empty before change
738 		 * exhasted = empty after possible change
739 		 */
740 
741 		reached_eof = channel->wr_empty &&
742 			(channel->wr_host_buf_idx == channel->wr_eof);
743 		channel->wr_hangup = reached_eof;
744 		exhausted = channel->wr_empty;
745 		waiting_bufidx = channel->wr_host_buf_idx;
746 
747 		spin_unlock_irqrestore(&channel->wr_spinlock, flags);
748 
749 		if (!empty) { /* Go on, now without the spinlock */
750 
751 			if (bufpos == 0) /* Position zero means it's virgin */
752 				channel->endpoint->ephw->hw_sync_sgl_for_cpu(
753 					channel->endpoint,
754 					channel->wr_buffers[bufidx]->dma_addr,
755 					channel->wr_buf_size,
756 					DMA_FROM_DEVICE);
757 
758 			if (copy_to_user(
759 				    userbuf,
760 				    channel->wr_buffers[bufidx]->addr
761 				    + bufpos, howmany))
762 				rc = -EFAULT;
763 
764 			userbuf += howmany;
765 			bytes_done += howmany;
766 
767 			if (bufferdone) {
768 				channel->endpoint->ephw->hw_sync_sgl_for_device(
769 					channel->endpoint,
770 					channel->wr_buffers[bufidx]->dma_addr,
771 					channel->wr_buf_size,
772 					DMA_FROM_DEVICE);
773 
774 				/*
775 				 * Tell FPGA the buffer is done with. It's an
776 				 * atomic operation to the FPGA, so what
777 				 * happens with other channels doesn't matter,
778 				 * and the certain channel is protected with
779 				 * the channel-specific mutex.
780 				 */
781 
782 				iowrite32(1 | (channel->chan_num << 1) |
783 					  (bufidx << 12),
784 					  channel->endpoint->registers +
785 					  fpga_buf_ctrl_reg);
786 			}
787 
788 			if (rc) {
789 				mutex_unlock(&channel->wr_mutex);
790 				return rc;
791 			}
792 		}
793 
794 		/* This includes a zero-count return = EOF */
795 		if ((bytes_done >= count) || reached_eof)
796 			break;
797 
798 		if (!exhausted)
799 			continue; /* More in RAM buffer(s)? Just go on. */
800 
801 		if ((bytes_done > 0) &&
802 		    (no_time_left ||
803 		     (channel->wr_synchronous && channel->wr_allow_partial)))
804 			break;
805 
806 		/*
807 		 * Nonblocking read: The "ready" flag tells us that the FPGA
808 		 * has data to send. In non-blocking mode, if it isn't on,
809 		 * just return. But if there is, we jump directly to the point
810 		 * where we ask for the FPGA to send all it has, and wait
811 		 * until that data arrives. So in a sense, we *do* block in
812 		 * nonblocking mode, but only for a very short time.
813 		 */
814 
815 		if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
816 			if (bytes_done > 0)
817 				break;
818 
819 			if (ready)
820 				goto desperate;
821 
822 			rc = -EAGAIN;
823 			break;
824 		}
825 
826 		if (!no_time_left || (bytes_done > 0)) {
827 			/*
828 			 * Note that in case of an element-misaligned read
829 			 * request, offsetlimit will include the last element,
830 			 * which will be partially read from.
831 			 */
832 			int offsetlimit = ((count - bytes_done) - 1) >>
833 				channel->log2_element_size;
834 			int buf_elements = channel->wr_buf_size >>
835 				channel->log2_element_size;
836 
837 			/*
838 			 * In synchronous mode, always send an offset limit.
839 			 * Just don't send a value too big.
840 			 */
841 
842 			if (channel->wr_synchronous) {
843 				/* Don't request more than one buffer */
844 				if (channel->wr_allow_partial &&
845 				    (offsetlimit >= buf_elements))
846 					offsetlimit = buf_elements - 1;
847 
848 				/* Don't request more than all buffers */
849 				if (!channel->wr_allow_partial &&
850 				    (offsetlimit >=
851 				     (buf_elements * channel->num_wr_buffers)))
852 					offsetlimit = buf_elements *
853 						channel->num_wr_buffers - 1;
854 			}
855 
856 			/*
857 			 * In asynchronous mode, force early flush of a buffer
858 			 * only if that will allow returning a full count. The
859 			 * "offsetlimit < ( ... )" rather than "<=" excludes
860 			 * requesting a full buffer, which would obviously
861 			 * cause a buffer transmission anyhow
862 			 */
863 
864 			if (channel->wr_synchronous ||
865 			    (offsetlimit < (buf_elements - 1))) {
866 				mutex_lock(&channel->endpoint->register_mutex);
867 
868 				iowrite32(offsetlimit,
869 					  channel->endpoint->registers +
870 					  fpga_buf_offset_reg);
871 
872 				iowrite32(1 | (channel->chan_num << 1) |
873 					  (2 << 24) |  /* 2 = offset limit */
874 					  (waiting_bufidx << 12),
875 					  channel->endpoint->registers +
876 					  fpga_buf_ctrl_reg);
877 
878 				mutex_unlock(&channel->endpoint->
879 					     register_mutex);
880 			}
881 		}
882 
883 		/*
884 		 * If partial completion is disallowed, there is no point in
885 		 * timeout sleeping. Neither if no_time_left is set and
886 		 * there's no data.
887 		 */
888 
889 		if (!channel->wr_allow_partial ||
890 		    (no_time_left && (bytes_done == 0))) {
891 			/*
892 			 * This do-loop will run more than once if another
893 			 * thread reasserted wr_sleepy before we got the mutex
894 			 * back, so we try again.
895 			 */
896 
897 			do {
898 				mutex_unlock(&channel->wr_mutex);
899 
900 				if (wait_event_interruptible(
901 					    channel->wr_wait,
902 					    (!channel->wr_sleepy)))
903 					goto interrupted;
904 
905 				if (mutex_lock_interruptible(
906 					    &channel->wr_mutex))
907 					goto interrupted;
908 			} while (channel->wr_sleepy);
909 
910 			continue;
911 
912 interrupted: /* Mutex is not held if got here */
913 			if (channel->endpoint->fatal_error)
914 				return -EIO;
915 			if (bytes_done)
916 				return bytes_done;
917 			if (filp->f_flags & O_NONBLOCK)
918 				return -EAGAIN; /* Don't admit snoozing */
919 			return -EINTR;
920 		}
921 
922 		left_to_sleep = deadline - ((long) jiffies);
923 
924 		/*
925 		 * If our time is out, skip the waiting. We may miss wr_sleepy
926 		 * being deasserted but hey, almost missing the train is like
927 		 * missing it.
928 		 */
929 
930 		if (left_to_sleep > 0) {
931 			left_to_sleep =
932 				wait_event_interruptible_timeout(
933 					channel->wr_wait,
934 					(!channel->wr_sleepy),
935 					left_to_sleep);
936 
937 			if (left_to_sleep > 0) /* wr_sleepy deasserted */
938 				continue;
939 
940 			if (left_to_sleep < 0) { /* Interrupt */
941 				mutex_unlock(&channel->wr_mutex);
942 				if (channel->endpoint->fatal_error)
943 					return -EIO;
944 				if (bytes_done)
945 					return bytes_done;
946 				return -EINTR;
947 			}
948 		}
949 
950 desperate:
951 		no_time_left = 1; /* We're out of sleeping time. Desperate! */
952 
953 		if (bytes_done == 0) {
954 			/*
955 			 * Reaching here means that we allow partial return,
956 			 * that we've run out of time, and that we have
957 			 * nothing to return.
958 			 * So tell the FPGA to send anything it has or gets.
959 			 */
960 
961 			iowrite32(1 | (channel->chan_num << 1) |
962 				  (3 << 24) |  /* Opcode 3, flush it all! */
963 				  (waiting_bufidx << 12),
964 				  channel->endpoint->registers +
965 				  fpga_buf_ctrl_reg);
966 		}
967 
968 		/*
969 		 * Reaching here means that we *do* have data in the buffer,
970 		 * but the "partial" flag disallows returning less than
971 		 * required. And we don't have as much. So loop again,
972 		 * which is likely to end up blocking indefinitely until
973 		 * enough data has arrived.
974 		 */
975 	}
976 
977 	mutex_unlock(&channel->wr_mutex);
978 
979 	if (channel->endpoint->fatal_error)
980 		return -EIO;
981 
982 	if (rc)
983 		return rc;
984 
985 	return bytes_done;
986 }
987 
988 /*
989  * The timeout argument takes values as follows:
990  *  >0 : Flush with timeout
991  * ==0 : Flush, and wait idefinitely for the flush to complete
992  *  <0 : Autoflush: Flush only if there's a single buffer occupied
993  */
994 
995 static int xillybus_myflush(struct xilly_channel *channel, long timeout)
996 {
997 	int rc;
998 	unsigned long flags;
999 
1000 	int end_offset_plus1;
1001 	int bufidx, bufidx_minus1;
1002 	int i;
1003 	int empty;
1004 	int new_rd_host_buf_pos;
1005 
1006 	if (channel->endpoint->fatal_error)
1007 		return -EIO;
1008 	rc = mutex_lock_interruptible(&channel->rd_mutex);
1009 	if (rc)
1010 		return rc;
1011 
1012 	/*
1013 	 * Don't flush a closed channel. This can happen when the work queued
1014 	 * autoflush thread fires off after the file has closed. This is not
1015 	 * an error, just something to dismiss.
1016 	 */
1017 
1018 	if (!channel->rd_ref_count)
1019 		goto done;
1020 
1021 	bufidx = channel->rd_host_buf_idx;
1022 
1023 	bufidx_minus1 = (bufidx == 0) ?
1024 		channel->num_rd_buffers - 1 :
1025 		bufidx - 1;
1026 
1027 	end_offset_plus1 = channel->rd_host_buf_pos >>
1028 		channel->log2_element_size;
1029 
1030 	new_rd_host_buf_pos = channel->rd_host_buf_pos -
1031 		(end_offset_plus1 << channel->log2_element_size);
1032 
1033 	/* Submit the current buffer if it's nonempty */
1034 	if (end_offset_plus1) {
1035 		unsigned char *tail = channel->rd_buffers[bufidx]->addr +
1036 			(end_offset_plus1 << channel->log2_element_size);
1037 
1038 		/* Copy  unflushed data, so we can put it in next buffer */
1039 		for (i = 0; i < new_rd_host_buf_pos; i++)
1040 			channel->rd_leftovers[i] = *tail++;
1041 
1042 		spin_lock_irqsave(&channel->rd_spinlock, flags);
1043 
1044 		/* Autoflush only if a single buffer is occupied */
1045 
1046 		if ((timeout < 0) &&
1047 		    (channel->rd_full ||
1048 		     (bufidx_minus1 != channel->rd_fpga_buf_idx))) {
1049 			spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1050 			/*
1051 			 * A new work item may be queued by the ISR exactly
1052 			 * now, since the execution of a work item allows the
1053 			 * queuing of a new one while it's running.
1054 			 */
1055 			goto done;
1056 		}
1057 
1058 		/* The 4th element is never needed for data, so it's a flag */
1059 		channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
1060 
1061 		/* Set up rd_full to reflect a certain moment's state */
1062 
1063 		if (bufidx == channel->rd_fpga_buf_idx)
1064 			channel->rd_full = 1;
1065 		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1066 
1067 		if (bufidx >= (channel->num_rd_buffers - 1))
1068 			channel->rd_host_buf_idx = 0;
1069 		else
1070 			channel->rd_host_buf_idx++;
1071 
1072 		channel->endpoint->ephw->hw_sync_sgl_for_device(
1073 			channel->endpoint,
1074 			channel->rd_buffers[bufidx]->dma_addr,
1075 			channel->rd_buf_size,
1076 			DMA_TO_DEVICE);
1077 
1078 		mutex_lock(&channel->endpoint->register_mutex);
1079 
1080 		iowrite32(end_offset_plus1 - 1,
1081 			  channel->endpoint->registers + fpga_buf_offset_reg);
1082 
1083 		iowrite32((channel->chan_num << 1) | /* Channel ID */
1084 			  (2 << 24) |  /* Opcode 2, submit buffer */
1085 			  (bufidx << 12),
1086 			  channel->endpoint->registers + fpga_buf_ctrl_reg);
1087 
1088 		mutex_unlock(&channel->endpoint->register_mutex);
1089 	} else if (bufidx == 0) {
1090 		bufidx = channel->num_rd_buffers - 1;
1091 	} else {
1092 		bufidx--;
1093 	}
1094 
1095 	channel->rd_host_buf_pos = new_rd_host_buf_pos;
1096 
1097 	if (timeout < 0)
1098 		goto done; /* Autoflush */
1099 
1100 	/*
1101 	 * bufidx is now the last buffer written to (or equal to
1102 	 * rd_fpga_buf_idx if buffer was never written to), and
1103 	 * channel->rd_host_buf_idx the one after it.
1104 	 *
1105 	 * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
1106 	 */
1107 
1108 	while (1) { /* Loop waiting for draining of buffers */
1109 		spin_lock_irqsave(&channel->rd_spinlock, flags);
1110 
1111 		if (bufidx != channel->rd_fpga_buf_idx)
1112 			channel->rd_full = 1; /*
1113 					       * Not really full,
1114 					       * but needs waiting.
1115 					       */
1116 
1117 		empty = !channel->rd_full;
1118 
1119 		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1120 
1121 		if (empty)
1122 			break;
1123 
1124 		/*
1125 		 * Indefinite sleep with mutex taken. With data waiting for
1126 		 * flushing user should not be surprised if open() for write
1127 		 * sleeps.
1128 		 */
1129 		if (timeout == 0)
1130 			wait_event_interruptible(channel->rd_wait,
1131 						 (!channel->rd_full));
1132 
1133 		else if (wait_event_interruptible_timeout(
1134 				 channel->rd_wait,
1135 				 (!channel->rd_full),
1136 				 timeout) == 0) {
1137 			dev_warn(channel->endpoint->dev,
1138 				 "Timed out while flushing. Output data may be lost.\n");
1139 
1140 			rc = -ETIMEDOUT;
1141 			break;
1142 		}
1143 
1144 		if (channel->rd_full) {
1145 			rc = -EINTR;
1146 			break;
1147 		}
1148 	}
1149 
1150 done:
1151 	mutex_unlock(&channel->rd_mutex);
1152 
1153 	if (channel->endpoint->fatal_error)
1154 		return -EIO;
1155 
1156 	return rc;
1157 }
1158 
1159 static int xillybus_flush(struct file *filp, fl_owner_t id)
1160 {
1161 	if (!(filp->f_mode & FMODE_WRITE))
1162 		return 0;
1163 
1164 	return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */
1165 }
1166 
1167 static void xillybus_autoflush(struct work_struct *work)
1168 {
1169 	struct delayed_work *workitem = container_of(
1170 		work, struct delayed_work, work);
1171 	struct xilly_channel *channel = container_of(
1172 		workitem, struct xilly_channel, rd_workitem);
1173 	int rc;
1174 
1175 	rc = xillybus_myflush(channel, -1);
1176 	if (rc == -EINTR)
1177 		dev_warn(channel->endpoint->dev,
1178 			 "Autoflush failed because work queue thread got a signal.\n");
1179 	else if (rc)
1180 		dev_err(channel->endpoint->dev,
1181 			"Autoflush failed under weird circumstances.\n");
1182 }
1183 
1184 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
1185 			      size_t count, loff_t *f_pos)
1186 {
1187 	ssize_t rc;
1188 	unsigned long flags;
1189 	int bytes_done = 0;
1190 	struct xilly_channel *channel = filp->private_data;
1191 
1192 	int full, exhausted;
1193 	/* Initializations are there only to silence warnings */
1194 
1195 	int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
1196 	int end_offset_plus1 = 0;
1197 
1198 	if (channel->endpoint->fatal_error)
1199 		return -EIO;
1200 
1201 	rc = mutex_lock_interruptible(&channel->rd_mutex);
1202 	if (rc)
1203 		return rc;
1204 
1205 	while (1) {
1206 		int bytes_to_do = count - bytes_done;
1207 
1208 		spin_lock_irqsave(&channel->rd_spinlock, flags);
1209 
1210 		full = channel->rd_full;
1211 
1212 		if (!full) {
1213 			bufidx = channel->rd_host_buf_idx;
1214 			bufpos = channel->rd_host_buf_pos;
1215 			howmany = channel->rd_buf_size - bufpos;
1216 
1217 			/*
1218 			 * Update rd_host_* to its state after this operation.
1219 			 * count=0 means committing the buffer immediately,
1220 			 * which is like flushing, but not necessarily block.
1221 			 */
1222 
1223 			if ((howmany > bytes_to_do) &&
1224 			    (count ||
1225 			     ((bufpos >> channel->log2_element_size) == 0))) {
1226 				bufferdone = 0;
1227 
1228 				howmany = bytes_to_do;
1229 				channel->rd_host_buf_pos += howmany;
1230 			} else {
1231 				bufferdone = 1;
1232 
1233 				if (count) {
1234 					end_offset_plus1 =
1235 						channel->rd_buf_size >>
1236 						channel->log2_element_size;
1237 					channel->rd_host_buf_pos = 0;
1238 				} else {
1239 					unsigned char *tail;
1240 					int i;
1241 
1242 					howmany = 0;
1243 
1244 					end_offset_plus1 = bufpos >>
1245 						channel->log2_element_size;
1246 
1247 					channel->rd_host_buf_pos -=
1248 						end_offset_plus1 <<
1249 						channel->log2_element_size;
1250 
1251 					tail = channel->
1252 						rd_buffers[bufidx]->addr +
1253 						(end_offset_plus1 <<
1254 						 channel->log2_element_size);
1255 
1256 					for (i = 0;
1257 					     i < channel->rd_host_buf_pos;
1258 					     i++)
1259 						channel->rd_leftovers[i] =
1260 							*tail++;
1261 				}
1262 
1263 				if (bufidx == channel->rd_fpga_buf_idx)
1264 					channel->rd_full = 1;
1265 
1266 				if (bufidx >= (channel->num_rd_buffers - 1))
1267 					channel->rd_host_buf_idx = 0;
1268 				else
1269 					channel->rd_host_buf_idx++;
1270 			}
1271 		}
1272 
1273 		/*
1274 		 * Marking our situation after the possible changes above,
1275 		 * for use  after releasing the spinlock.
1276 		 *
1277 		 * full = full before change
1278 		 * exhasted = full after possible change
1279 		 */
1280 
1281 		exhausted = channel->rd_full;
1282 
1283 		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1284 
1285 		if (!full) { /* Go on, now without the spinlock */
1286 			unsigned char *head =
1287 				channel->rd_buffers[bufidx]->addr;
1288 			int i;
1289 
1290 			if ((bufpos == 0) || /* Zero means it's virgin */
1291 			    (channel->rd_leftovers[3] != 0)) {
1292 				channel->endpoint->ephw->hw_sync_sgl_for_cpu(
1293 					channel->endpoint,
1294 					channel->rd_buffers[bufidx]->dma_addr,
1295 					channel->rd_buf_size,
1296 					DMA_TO_DEVICE);
1297 
1298 				/* Virgin, but leftovers are due */
1299 				for (i = 0; i < bufpos; i++)
1300 					*head++ = channel->rd_leftovers[i];
1301 
1302 				channel->rd_leftovers[3] = 0; /* Clear flag */
1303 			}
1304 
1305 			if (copy_from_user(
1306 				    channel->rd_buffers[bufidx]->addr + bufpos,
1307 				    userbuf, howmany))
1308 				rc = -EFAULT;
1309 
1310 			userbuf += howmany;
1311 			bytes_done += howmany;
1312 
1313 			if (bufferdone) {
1314 				channel->endpoint->ephw->hw_sync_sgl_for_device(
1315 					channel->endpoint,
1316 					channel->rd_buffers[bufidx]->dma_addr,
1317 					channel->rd_buf_size,
1318 					DMA_TO_DEVICE);
1319 
1320 				mutex_lock(&channel->endpoint->register_mutex);
1321 
1322 				iowrite32(end_offset_plus1 - 1,
1323 					  channel->endpoint->registers +
1324 					  fpga_buf_offset_reg);
1325 
1326 				iowrite32((channel->chan_num << 1) |
1327 					  (2 << 24) |  /* 2 = submit buffer */
1328 					  (bufidx << 12),
1329 					  channel->endpoint->registers +
1330 					  fpga_buf_ctrl_reg);
1331 
1332 				mutex_unlock(&channel->endpoint->
1333 					     register_mutex);
1334 
1335 				channel->rd_leftovers[3] =
1336 					(channel->rd_host_buf_pos != 0);
1337 			}
1338 
1339 			if (rc) {
1340 				mutex_unlock(&channel->rd_mutex);
1341 
1342 				if (channel->endpoint->fatal_error)
1343 					return -EIO;
1344 
1345 				if (!channel->rd_synchronous)
1346 					queue_delayed_work(
1347 						xillybus_wq,
1348 						&channel->rd_workitem,
1349 						XILLY_RX_TIMEOUT);
1350 
1351 				return rc;
1352 			}
1353 		}
1354 
1355 		if (bytes_done >= count)
1356 			break;
1357 
1358 		if (!exhausted)
1359 			continue; /* If there's more space, just go on */
1360 
1361 		if ((bytes_done > 0) && channel->rd_allow_partial)
1362 			break;
1363 
1364 		/*
1365 		 * Indefinite sleep with mutex taken. With data waiting for
1366 		 * flushing, user should not be surprised if open() for write
1367 		 * sleeps.
1368 		 */
1369 
1370 		if (filp->f_flags & O_NONBLOCK) {
1371 			rc = -EAGAIN;
1372 			break;
1373 		}
1374 
1375 		if (wait_event_interruptible(channel->rd_wait,
1376 					     (!channel->rd_full))) {
1377 			mutex_unlock(&channel->rd_mutex);
1378 
1379 			if (channel->endpoint->fatal_error)
1380 				return -EIO;
1381 
1382 			if (bytes_done)
1383 				return bytes_done;
1384 			return -EINTR;
1385 		}
1386 	}
1387 
1388 	mutex_unlock(&channel->rd_mutex);
1389 
1390 	if (!channel->rd_synchronous)
1391 		queue_delayed_work(xillybus_wq,
1392 				   &channel->rd_workitem,
1393 				   XILLY_RX_TIMEOUT);
1394 
1395 	if (channel->endpoint->fatal_error)
1396 		return -EIO;
1397 
1398 	if (rc)
1399 		return rc;
1400 
1401 	if ((channel->rd_synchronous) && (bytes_done > 0)) {
1402 		rc = xillybus_myflush(filp->private_data, 0); /* No timeout */
1403 
1404 		if (rc && (rc != -EINTR))
1405 			return rc;
1406 	}
1407 
1408 	return bytes_done;
1409 }
1410 
1411 static int xillybus_open(struct inode *inode, struct file *filp)
1412 {
1413 	int rc = 0;
1414 	unsigned long flags;
1415 	int minor = iminor(inode);
1416 	int major = imajor(inode);
1417 	struct xilly_endpoint *ep_iter, *endpoint = NULL;
1418 	struct xilly_channel *channel;
1419 
1420 	mutex_lock(&ep_list_lock);
1421 
1422 	list_for_each_entry(ep_iter, &list_of_endpoints, ep_list) {
1423 		if ((ep_iter->major == major) &&
1424 		    (minor >= ep_iter->lowest_minor) &&
1425 		    (minor < (ep_iter->lowest_minor +
1426 			      ep_iter->num_channels))) {
1427 			endpoint = ep_iter;
1428 			break;
1429 		}
1430 	}
1431 	mutex_unlock(&ep_list_lock);
1432 
1433 	if (!endpoint) {
1434 		pr_err("xillybus: open() failed to find a device for major=%d and minor=%d\n",
1435 		       major, minor);
1436 		return -ENODEV;
1437 	}
1438 
1439 	if (endpoint->fatal_error)
1440 		return -EIO;
1441 
1442 	channel = endpoint->channels[1 + minor - endpoint->lowest_minor];
1443 	filp->private_data = channel;
1444 
1445 	/*
1446 	 * It gets complicated because:
1447 	 * 1. We don't want to take a mutex we don't have to
1448 	 * 2. We don't want to open one direction if the other will fail.
1449 	 */
1450 
1451 	if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
1452 		return -ENODEV;
1453 
1454 	if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
1455 		return -ENODEV;
1456 
1457 	if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
1458 	    (channel->wr_synchronous || !channel->wr_allow_partial ||
1459 	     !channel->wr_supports_nonempty)) {
1460 		dev_err(endpoint->dev,
1461 			"open() failed: O_NONBLOCK not allowed for read on this device\n");
1462 		return -ENODEV;
1463 	}
1464 
1465 	if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
1466 	    (channel->rd_synchronous || !channel->rd_allow_partial)) {
1467 		dev_err(endpoint->dev,
1468 			"open() failed: O_NONBLOCK not allowed for write on this device\n");
1469 		return -ENODEV;
1470 	}
1471 
1472 	/*
1473 	 * Note: open() may block on getting mutexes despite O_NONBLOCK.
1474 	 * This shouldn't occur normally, since multiple open of the same
1475 	 * file descriptor is almost always prohibited anyhow
1476 	 * (*_exclusive_open is normally set in real-life systems).
1477 	 */
1478 
1479 	if (filp->f_mode & FMODE_READ) {
1480 		rc = mutex_lock_interruptible(&channel->wr_mutex);
1481 		if (rc)
1482 			return rc;
1483 	}
1484 
1485 	if (filp->f_mode & FMODE_WRITE) {
1486 		rc = mutex_lock_interruptible(&channel->rd_mutex);
1487 		if (rc)
1488 			goto unlock_wr;
1489 	}
1490 
1491 	if ((filp->f_mode & FMODE_READ) &&
1492 	    (channel->wr_ref_count != 0) &&
1493 	    (channel->wr_exclusive_open)) {
1494 		rc = -EBUSY;
1495 		goto unlock;
1496 	}
1497 
1498 	if ((filp->f_mode & FMODE_WRITE) &&
1499 	    (channel->rd_ref_count != 0) &&
1500 	    (channel->rd_exclusive_open)) {
1501 		rc = -EBUSY;
1502 		goto unlock;
1503 	}
1504 
1505 	if (filp->f_mode & FMODE_READ) {
1506 		if (channel->wr_ref_count == 0) { /* First open of file */
1507 			/* Move the host to first buffer */
1508 			spin_lock_irqsave(&channel->wr_spinlock, flags);
1509 			channel->wr_host_buf_idx = 0;
1510 			channel->wr_host_buf_pos = 0;
1511 			channel->wr_fpga_buf_idx = -1;
1512 			channel->wr_empty = 1;
1513 			channel->wr_ready = 0;
1514 			channel->wr_sleepy = 1;
1515 			channel->wr_eof = -1;
1516 			channel->wr_hangup = 0;
1517 
1518 			spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1519 
1520 			iowrite32(1 | (channel->chan_num << 1) |
1521 				  (4 << 24) |  /* Opcode 4, open channel */
1522 				  ((channel->wr_synchronous & 1) << 23),
1523 				  channel->endpoint->registers +
1524 				  fpga_buf_ctrl_reg);
1525 		}
1526 
1527 		channel->wr_ref_count++;
1528 	}
1529 
1530 	if (filp->f_mode & FMODE_WRITE) {
1531 		if (channel->rd_ref_count == 0) { /* First open of file */
1532 			/* Move the host to first buffer */
1533 			spin_lock_irqsave(&channel->rd_spinlock, flags);
1534 			channel->rd_host_buf_idx = 0;
1535 			channel->rd_host_buf_pos = 0;
1536 			channel->rd_leftovers[3] = 0; /* No leftovers. */
1537 			channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
1538 			channel->rd_full = 0;
1539 
1540 			spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1541 
1542 			iowrite32((channel->chan_num << 1) |
1543 				  (4 << 24),   /* Opcode 4, open channel */
1544 				  channel->endpoint->registers +
1545 				  fpga_buf_ctrl_reg);
1546 		}
1547 
1548 		channel->rd_ref_count++;
1549 	}
1550 
1551 unlock:
1552 	if (filp->f_mode & FMODE_WRITE)
1553 		mutex_unlock(&channel->rd_mutex);
1554 unlock_wr:
1555 	if (filp->f_mode & FMODE_READ)
1556 		mutex_unlock(&channel->wr_mutex);
1557 
1558 	if (!rc && (!channel->seekable))
1559 		return nonseekable_open(inode, filp);
1560 
1561 	return rc;
1562 }
1563 
1564 static int xillybus_release(struct inode *inode, struct file *filp)
1565 {
1566 	unsigned long flags;
1567 	struct xilly_channel *channel = filp->private_data;
1568 
1569 	int buf_idx;
1570 	int eof;
1571 
1572 	if (channel->endpoint->fatal_error)
1573 		return -EIO;
1574 
1575 	if (filp->f_mode & FMODE_WRITE) {
1576 		mutex_lock(&channel->rd_mutex);
1577 
1578 		channel->rd_ref_count--;
1579 
1580 		if (channel->rd_ref_count == 0) {
1581 			/*
1582 			 * We rely on the kernel calling flush()
1583 			 * before we get here.
1584 			 */
1585 
1586 			iowrite32((channel->chan_num << 1) | /* Channel ID */
1587 				  (5 << 24),  /* Opcode 5, close channel */
1588 				  channel->endpoint->registers +
1589 				  fpga_buf_ctrl_reg);
1590 		}
1591 		mutex_unlock(&channel->rd_mutex);
1592 	}
1593 
1594 	if (filp->f_mode & FMODE_READ) {
1595 		mutex_lock(&channel->wr_mutex);
1596 
1597 		channel->wr_ref_count--;
1598 
1599 		if (channel->wr_ref_count == 0) {
1600 			iowrite32(1 | (channel->chan_num << 1) |
1601 				  (5 << 24),  /* Opcode 5, close channel */
1602 				  channel->endpoint->registers +
1603 				  fpga_buf_ctrl_reg);
1604 
1605 			/*
1606 			 * This is crazily cautious: We make sure that not
1607 			 * only that we got an EOF (be it because we closed
1608 			 * the channel or because of a user's EOF), but verify
1609 			 * that it's one beyond the last buffer arrived, so
1610 			 * we have no leftover buffers pending before wrapping
1611 			 * up (which can only happen in asynchronous channels,
1612 			 * BTW)
1613 			 */
1614 
1615 			while (1) {
1616 				spin_lock_irqsave(&channel->wr_spinlock,
1617 						  flags);
1618 				buf_idx = channel->wr_fpga_buf_idx;
1619 				eof = channel->wr_eof;
1620 				channel->wr_sleepy = 1;
1621 				spin_unlock_irqrestore(&channel->wr_spinlock,
1622 						       flags);
1623 
1624 				/*
1625 				 * Check if eof points at the buffer after
1626 				 * the last one the FPGA submitted. Note that
1627 				 * no EOF is marked by negative eof.
1628 				 */
1629 
1630 				buf_idx++;
1631 				if (buf_idx == channel->num_wr_buffers)
1632 					buf_idx = 0;
1633 
1634 				if (buf_idx == eof)
1635 					break;
1636 
1637 				/*
1638 				 * Steal extra 100 ms if awaken by interrupt.
1639 				 * This is a simple workaround for an
1640 				 * interrupt pending when entering, which would
1641 				 * otherwise result in declaring the hardware
1642 				 * non-responsive.
1643 				 */
1644 
1645 				if (wait_event_interruptible(
1646 					    channel->wr_wait,
1647 					    (!channel->wr_sleepy)))
1648 					msleep(100);
1649 
1650 				if (channel->wr_sleepy) {
1651 					mutex_unlock(&channel->wr_mutex);
1652 					dev_warn(channel->endpoint->dev,
1653 						 "Hardware failed to respond to close command, therefore left in messy state.\n");
1654 					return -EINTR;
1655 				}
1656 			}
1657 		}
1658 
1659 		mutex_unlock(&channel->wr_mutex);
1660 	}
1661 
1662 	return 0;
1663 }
1664 
1665 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
1666 {
1667 	struct xilly_channel *channel = filp->private_data;
1668 	loff_t pos = filp->f_pos;
1669 	int rc = 0;
1670 
1671 	/*
1672 	 * Take both mutexes not allowing interrupts, since it seems like
1673 	 * common applications don't expect an -EINTR here. Besides, multiple
1674 	 * access to a single file descriptor on seekable devices is a mess
1675 	 * anyhow.
1676 	 */
1677 
1678 	if (channel->endpoint->fatal_error)
1679 		return -EIO;
1680 
1681 	mutex_lock(&channel->wr_mutex);
1682 	mutex_lock(&channel->rd_mutex);
1683 
1684 	switch (whence) {
1685 	case SEEK_SET:
1686 		pos = offset;
1687 		break;
1688 	case SEEK_CUR:
1689 		pos += offset;
1690 		break;
1691 	case SEEK_END:
1692 		pos = offset; /* Going to the end => to the beginning */
1693 		break;
1694 	default:
1695 		rc = -EINVAL;
1696 		goto end;
1697 	}
1698 
1699 	/* In any case, we must finish on an element boundary */
1700 	if (pos & ((1 << channel->log2_element_size) - 1)) {
1701 		rc = -EINVAL;
1702 		goto end;
1703 	}
1704 
1705 	mutex_lock(&channel->endpoint->register_mutex);
1706 
1707 	iowrite32(pos >> channel->log2_element_size,
1708 		  channel->endpoint->registers + fpga_buf_offset_reg);
1709 
1710 	iowrite32((channel->chan_num << 1) |
1711 		  (6 << 24),  /* Opcode 6, set address */
1712 		  channel->endpoint->registers + fpga_buf_ctrl_reg);
1713 
1714 	mutex_unlock(&channel->endpoint->register_mutex);
1715 
1716 end:
1717 	mutex_unlock(&channel->rd_mutex);
1718 	mutex_unlock(&channel->wr_mutex);
1719 
1720 	if (rc) /* Return error after releasing mutexes */
1721 		return rc;
1722 
1723 	filp->f_pos = pos;
1724 
1725 	/*
1726 	 * Since seekable devices are allowed only when the channel is
1727 	 * synchronous, we assume that there is no data pending in either
1728 	 * direction (which holds true as long as no concurrent access on the
1729 	 * file descriptor takes place).
1730 	 * The only thing we may need to throw away is leftovers from partial
1731 	 * write() flush.
1732 	 */
1733 
1734 	channel->rd_leftovers[3] = 0;
1735 
1736 	return pos;
1737 }
1738 
1739 static __poll_t xillybus_poll(struct file *filp, poll_table *wait)
1740 {
1741 	struct xilly_channel *channel = filp->private_data;
1742 	__poll_t mask = 0;
1743 	unsigned long flags;
1744 
1745 	poll_wait(filp, &channel->endpoint->ep_wait, wait);
1746 
1747 	/*
1748 	 * poll() won't play ball regarding read() channels which
1749 	 * aren't asynchronous and support the nonempty message. Allowing
1750 	 * that will create situations where data has been delivered at
1751 	 * the FPGA, and users expecting select() to wake up, which it may
1752 	 * not.
1753 	 */
1754 
1755 	if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
1756 		poll_wait(filp, &channel->wr_wait, wait);
1757 		poll_wait(filp, &channel->wr_ready_wait, wait);
1758 
1759 		spin_lock_irqsave(&channel->wr_spinlock, flags);
1760 		if (!channel->wr_empty || channel->wr_ready)
1761 			mask |= EPOLLIN | EPOLLRDNORM;
1762 
1763 		if (channel->wr_hangup)
1764 			/*
1765 			 * Not EPOLLHUP, because its behavior is in the
1766 			 * mist, and EPOLLIN does what we want: Wake up
1767 			 * the read file descriptor so it sees EOF.
1768 			 */
1769 			mask |=  EPOLLIN | EPOLLRDNORM;
1770 		spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1771 	}
1772 
1773 	/*
1774 	 * If partial data write is disallowed on a write() channel,
1775 	 * it's pointless to ever signal OK to write, because is could
1776 	 * block despite some space being available.
1777 	 */
1778 
1779 	if (channel->rd_allow_partial) {
1780 		poll_wait(filp, &channel->rd_wait, wait);
1781 
1782 		spin_lock_irqsave(&channel->rd_spinlock, flags);
1783 		if (!channel->rd_full)
1784 			mask |= EPOLLOUT | EPOLLWRNORM;
1785 		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1786 	}
1787 
1788 	if (channel->endpoint->fatal_error)
1789 		mask |= EPOLLERR;
1790 
1791 	return mask;
1792 }
1793 
1794 static const struct file_operations xillybus_fops = {
1795 	.owner      = THIS_MODULE,
1796 	.read       = xillybus_read,
1797 	.write      = xillybus_write,
1798 	.open       = xillybus_open,
1799 	.flush      = xillybus_flush,
1800 	.release    = xillybus_release,
1801 	.llseek     = xillybus_llseek,
1802 	.poll       = xillybus_poll,
1803 };
1804 
1805 static int xillybus_init_chrdev(struct xilly_endpoint *endpoint,
1806 				const unsigned char *idt)
1807 {
1808 	int rc;
1809 	dev_t dev;
1810 	int devnum, i, minor, major;
1811 	char devname[48];
1812 	struct device *device;
1813 
1814 	rc = alloc_chrdev_region(&dev, 0, /* minor start */
1815 				 endpoint->num_channels,
1816 				 xillyname);
1817 	if (rc) {
1818 		dev_warn(endpoint->dev, "Failed to obtain major/minors");
1819 		return rc;
1820 	}
1821 
1822 	endpoint->major = major = MAJOR(dev);
1823 	endpoint->lowest_minor = minor = MINOR(dev);
1824 
1825 	cdev_init(&endpoint->cdev, &xillybus_fops);
1826 	endpoint->cdev.owner = endpoint->ephw->owner;
1827 	rc = cdev_add(&endpoint->cdev, MKDEV(major, minor),
1828 		      endpoint->num_channels);
1829 	if (rc) {
1830 		dev_warn(endpoint->dev, "Failed to add cdev. Aborting.\n");
1831 		goto unregister_chrdev;
1832 	}
1833 
1834 	idt++;
1835 
1836 	for (i = minor, devnum = 0;
1837 	     devnum < endpoint->num_channels;
1838 	     devnum++, i++) {
1839 		snprintf(devname, sizeof(devname)-1, "xillybus_%s", idt);
1840 
1841 		devname[sizeof(devname)-1] = 0; /* Should never matter */
1842 
1843 		while (*idt++)
1844 			/* Skip to next */;
1845 
1846 		device = device_create(xillybus_class,
1847 				       NULL,
1848 				       MKDEV(major, i),
1849 				       NULL,
1850 				       "%s", devname);
1851 
1852 		if (IS_ERR(device)) {
1853 			dev_warn(endpoint->dev,
1854 				 "Failed to create %s device. Aborting.\n",
1855 				 devname);
1856 			rc = -ENODEV;
1857 			goto unroll_device_create;
1858 		}
1859 	}
1860 
1861 	dev_info(endpoint->dev, "Created %d device files.\n",
1862 		 endpoint->num_channels);
1863 	return 0; /* succeed */
1864 
1865 unroll_device_create:
1866 	devnum--; i--;
1867 	for (; devnum >= 0; devnum--, i--)
1868 		device_destroy(xillybus_class, MKDEV(major, i));
1869 
1870 	cdev_del(&endpoint->cdev);
1871 unregister_chrdev:
1872 	unregister_chrdev_region(MKDEV(major, minor), endpoint->num_channels);
1873 
1874 	return rc;
1875 }
1876 
1877 static void xillybus_cleanup_chrdev(struct xilly_endpoint *endpoint)
1878 {
1879 	int minor;
1880 
1881 	for (minor = endpoint->lowest_minor;
1882 	     minor < (endpoint->lowest_minor + endpoint->num_channels);
1883 	     minor++)
1884 		device_destroy(xillybus_class, MKDEV(endpoint->major, minor));
1885 	cdev_del(&endpoint->cdev);
1886 	unregister_chrdev_region(MKDEV(endpoint->major,
1887 				       endpoint->lowest_minor),
1888 				 endpoint->num_channels);
1889 
1890 	dev_info(endpoint->dev, "Removed %d device files.\n",
1891 		 endpoint->num_channels);
1892 }
1893 
1894 struct xilly_endpoint *xillybus_init_endpoint(struct pci_dev *pdev,
1895 					      struct device *dev,
1896 					      struct xilly_endpoint_hardware
1897 					      *ephw)
1898 {
1899 	struct xilly_endpoint *endpoint;
1900 
1901 	endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL);
1902 	if (!endpoint)
1903 		return NULL;
1904 
1905 	endpoint->pdev = pdev;
1906 	endpoint->dev = dev;
1907 	endpoint->ephw = ephw;
1908 	endpoint->msg_counter = 0x0b;
1909 	endpoint->failed_messages = 0;
1910 	endpoint->fatal_error = 0;
1911 
1912 	init_waitqueue_head(&endpoint->ep_wait);
1913 	mutex_init(&endpoint->register_mutex);
1914 
1915 	return endpoint;
1916 }
1917 EXPORT_SYMBOL(xillybus_init_endpoint);
1918 
1919 static int xilly_quiesce(struct xilly_endpoint *endpoint)
1920 {
1921 	long t;
1922 
1923 	endpoint->idtlen = -1;
1924 
1925 	iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1926 		  endpoint->registers + fpga_dma_control_reg);
1927 
1928 	t = wait_event_interruptible_timeout(endpoint->ep_wait,
1929 					     (endpoint->idtlen >= 0),
1930 					     XILLY_TIMEOUT);
1931 	if (t <= 0) {
1932 		dev_err(endpoint->dev,
1933 			"Failed to quiesce the device on exit.\n");
1934 		return -ENODEV;
1935 	}
1936 	return 0;
1937 }
1938 
1939 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
1940 {
1941 	int rc;
1942 	long t;
1943 
1944 	void *bootstrap_resources;
1945 	int idtbuffersize = (1 << PAGE_SHIFT);
1946 	struct device *dev = endpoint->dev;
1947 
1948 	/*
1949 	 * The bogus IDT is used during bootstrap for allocating the initial
1950 	 * message buffer, and then the message buffer and space for the IDT
1951 	 * itself. The initial message buffer is of a single page's size, but
1952 	 * it's soon replaced with a more modest one (and memory is freed).
1953 	 */
1954 
1955 	unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
1956 				       3, 192, PAGE_SHIFT, 0 };
1957 	struct xilly_idt_handle idt_handle;
1958 
1959 	/*
1960 	 * Writing the value 0x00000001 to Endianness register signals which
1961 	 * endianness this processor is using, so the FPGA can swap words as
1962 	 * necessary.
1963 	 */
1964 
1965 	iowrite32(1, endpoint->registers + fpga_endian_reg);
1966 
1967 	/* Bootstrap phase I: Allocate temporary message buffer */
1968 
1969 	bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);
1970 	if (!bootstrap_resources)
1971 		return -ENOMEM;
1972 
1973 	endpoint->num_channels = 0;
1974 
1975 	rc = xilly_setupchannels(endpoint, bogus_idt, 1);
1976 	if (rc)
1977 		return rc;
1978 
1979 	/* Clear the message subsystem (and counter in particular) */
1980 	iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);
1981 
1982 	endpoint->idtlen = -1;
1983 
1984 	/*
1985 	 * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
1986 	 * buffer size.
1987 	 */
1988 	iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1989 		  endpoint->registers + fpga_dma_control_reg);
1990 
1991 	t = wait_event_interruptible_timeout(endpoint->ep_wait,
1992 					     (endpoint->idtlen >= 0),
1993 					     XILLY_TIMEOUT);
1994 	if (t <= 0) {
1995 		dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
1996 		return -ENODEV;
1997 	}
1998 
1999 	/* Enable DMA */
2000 	iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
2001 		  endpoint->registers + fpga_dma_control_reg);
2002 
2003 	/* Bootstrap phase II: Allocate buffer for IDT and obtain it */
2004 	while (endpoint->idtlen >= idtbuffersize) {
2005 		idtbuffersize *= 2;
2006 		bogus_idt[6]++;
2007 	}
2008 
2009 	endpoint->num_channels = 1;
2010 
2011 	rc = xilly_setupchannels(endpoint, bogus_idt, 2);
2012 	if (rc)
2013 		goto failed_idt;
2014 
2015 	rc = xilly_obtain_idt(endpoint);
2016 	if (rc)
2017 		goto failed_idt;
2018 
2019 	rc = xilly_scan_idt(endpoint, &idt_handle);
2020 	if (rc)
2021 		goto failed_idt;
2022 
2023 	devres_close_group(dev, bootstrap_resources);
2024 
2025 	/* Bootstrap phase III: Allocate buffers according to IDT */
2026 
2027 	rc = xilly_setupchannels(endpoint,
2028 				 idt_handle.chandesc,
2029 				 idt_handle.entries);
2030 	if (rc)
2031 		goto failed_idt;
2032 
2033 	/*
2034 	 * endpoint is now completely configured. We put it on the list
2035 	 * available to open() before registering the char device(s)
2036 	 */
2037 
2038 	mutex_lock(&ep_list_lock);
2039 	list_add_tail(&endpoint->ep_list, &list_of_endpoints);
2040 	mutex_unlock(&ep_list_lock);
2041 
2042 	rc = xillybus_init_chrdev(endpoint, idt_handle.idt);
2043 	if (rc)
2044 		goto failed_chrdevs;
2045 
2046 	devres_release_group(dev, bootstrap_resources);
2047 
2048 	return 0;
2049 
2050 failed_chrdevs:
2051 	mutex_lock(&ep_list_lock);
2052 	list_del(&endpoint->ep_list);
2053 	mutex_unlock(&ep_list_lock);
2054 
2055 failed_idt:
2056 	xilly_quiesce(endpoint);
2057 	flush_workqueue(xillybus_wq);
2058 
2059 	return rc;
2060 }
2061 EXPORT_SYMBOL(xillybus_endpoint_discovery);
2062 
2063 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
2064 {
2065 	xillybus_cleanup_chrdev(endpoint);
2066 
2067 	mutex_lock(&ep_list_lock);
2068 	list_del(&endpoint->ep_list);
2069 	mutex_unlock(&ep_list_lock);
2070 
2071 	xilly_quiesce(endpoint);
2072 
2073 	/*
2074 	 * Flushing is done upon endpoint release to prevent access to memory
2075 	 * just about to be released. This makes the quiesce complete.
2076 	 */
2077 	flush_workqueue(xillybus_wq);
2078 }
2079 EXPORT_SYMBOL(xillybus_endpoint_remove);
2080 
2081 static int __init xillybus_init(void)
2082 {
2083 	mutex_init(&ep_list_lock);
2084 
2085 	xillybus_class = class_create(THIS_MODULE, xillyname);
2086 	if (IS_ERR(xillybus_class))
2087 		return PTR_ERR(xillybus_class);
2088 
2089 	xillybus_wq = alloc_workqueue(xillyname, 0, 0);
2090 	if (!xillybus_wq) {
2091 		class_destroy(xillybus_class);
2092 		return -ENOMEM;
2093 	}
2094 
2095 	return 0;
2096 }
2097 
2098 static void __exit xillybus_exit(void)
2099 {
2100 	/* flush_workqueue() was called for each endpoint released */
2101 	destroy_workqueue(xillybus_wq);
2102 
2103 	class_destroy(xillybus_class);
2104 }
2105 
2106 module_init(xillybus_init);
2107 module_exit(xillybus_exit);
2108