// SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2020 Xillybus Ltd, http://xillybus.com * * Driver for the XillyUSB FPGA/host framework. * * This driver interfaces with a special IP core in an FPGA, setting up * a pipe between a hardware FIFO in the programmable logic and a device * file in the host. The number of such pipes and their attributes are * set up on the logic. This driver detects these automatically and * creates the device files accordingly. */ #include <linux/types.h> #include <linux/slab.h> #include <linux/list.h> #include <linux/device.h> #include <linux/module.h> #include <asm/byteorder.h> #include <linux/io.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <linux/fs.h> #include <linux/spinlock.h> #include <linux/mutex.h> #include <linux/workqueue.h> #include <linux/crc32.h> #include <linux/poll.h> #include <linux/delay.h> #include <linux/usb.h> #include "xillybus_class.h" MODULE_DESCRIPTION("Driver for XillyUSB FPGA IP Core"); MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); MODULE_ALIAS("xillyusb"); MODULE_LICENSE("GPL v2"); #define XILLY_RX_TIMEOUT (10 * HZ / 1000) #define XILLY_RESPONSE_TIMEOUT (500 * HZ / 1000) #define BUF_SIZE_ORDER 4 #define BUFNUM 8 #define LOG2_IDT_FIFO_SIZE 16 #define LOG2_INITIAL_FIFO_BUF_SIZE 16 #define MSG_EP_NUM 1 #define IN_EP_NUM 1 static const char xillyname[] = "xillyusb"; static unsigned int fifo_buf_order; #define USB_VENDOR_ID_XILINX 0x03fd #define USB_VENDOR_ID_ALTERA 0x09fb #define USB_PRODUCT_ID_XILLYUSB 0xebbe static const struct usb_device_id xillyusb_table[] = { { USB_DEVICE(USB_VENDOR_ID_XILINX, USB_PRODUCT_ID_XILLYUSB) }, { USB_DEVICE(USB_VENDOR_ID_ALTERA, USB_PRODUCT_ID_XILLYUSB) }, { } }; MODULE_DEVICE_TABLE(usb, xillyusb_table); struct xillyusb_dev; struct xillyfifo { unsigned int bufsize; /* In bytes, always a power of 2 */ unsigned int bufnum; unsigned int size; /* Lazy: Equals bufsize * bufnum */ unsigned int buf_order; int fill; /* Number of bytes in the FIFO */ spinlock_t lock; wait_queue_head_t waitq; unsigned int readpos; unsigned int readbuf; unsigned int writepos; unsigned int writebuf; char **mem; }; struct xillyusb_channel; struct xillyusb_endpoint { struct xillyusb_dev *xdev; struct mutex ep_mutex; /* serialize operations on endpoint */ struct list_head buffers; struct list_head filled_buffers; spinlock_t buffers_lock; /* protect these two lists */ unsigned int order; unsigned int buffer_size; unsigned int fill_mask; int outstanding_urbs; struct usb_anchor anchor; struct xillyfifo fifo; struct work_struct workitem; bool shutting_down; bool drained; bool wake_on_drain; u8 ep_num; }; struct xillyusb_channel { struct xillyusb_dev *xdev; struct xillyfifo *in_fifo; struct xillyusb_endpoint *out_ep; struct mutex lock; /* protect @out_ep, @in_fifo, bit fields below */ struct mutex in_mutex; /* serialize fops on FPGA to host stream */ struct mutex out_mutex; /* serialize fops on host to FPGA stream */ wait_queue_head_t flushq; int chan_idx; u32 in_consumed_bytes; u32 in_current_checkpoint; u32 out_bytes; unsigned int in_log2_element_size; unsigned int out_log2_element_size; unsigned int in_log2_fifo_size; unsigned int out_log2_fifo_size; unsigned int read_data_ok; /* EOF not arrived (yet) */ unsigned int poll_used; unsigned int flushing; unsigned int flushed; unsigned int canceled; /* Bit fields protected by @lock except for initialization */ unsigned readable:1; unsigned writable:1; unsigned open_for_read:1; unsigned open_for_write:1; unsigned in_synchronous:1; unsigned out_synchronous:1; unsigned in_seekable:1; unsigned out_seekable:1; }; struct xillybuffer { struct list_head entry; struct xillyusb_endpoint *ep; void *buf; unsigned int len; }; struct xillyusb_dev { struct xillyusb_channel *channels; struct usb_device *udev; struct device *dev; /* For dev_err() and such */ struct kref kref; struct workqueue_struct *workq; int error; spinlock_t error_lock; /* protect @error */ struct work_struct wakeup_workitem; int num_channels; struct xillyusb_endpoint *msg_ep; struct xillyusb_endpoint *in_ep; struct mutex msg_mutex; /* serialize opcode transmission */ int in_bytes_left; int leftover_chan_num; unsigned int in_counter; struct mutex process_in_mutex; /* synchronize wakeup_all() */ }; /* * kref_mutex is used in xillyusb_open() to prevent the xillyusb_dev * struct from being freed during the gap between being found by * xillybus_find_inode() and having its reference count incremented. */ static DEFINE_MUTEX(kref_mutex); /* FPGA to host opcodes */ enum { OPCODE_DATA = 0, OPCODE_QUIESCE_ACK = 1, OPCODE_EOF = 2, OPCODE_REACHED_CHECKPOINT = 3, OPCODE_CANCELED_CHECKPOINT = 4, }; /* Host to FPGA opcodes */ enum { OPCODE_QUIESCE = 0, OPCODE_REQ_IDT = 1, OPCODE_SET_CHECKPOINT = 2, OPCODE_CLOSE = 3, OPCODE_SET_PUSH = 4, OPCODE_UPDATE_PUSH = 5, OPCODE_CANCEL_CHECKPOINT = 6, OPCODE_SET_ADDR = 7, }; /* * fifo_write() and fifo_read() are NOT reentrant (i.e. concurrent multiple * calls to each on the same FIFO is not allowed) however it's OK to have * threads calling each of the two functions once on the same FIFO, and * at the same time. */ static int fifo_write(struct xillyfifo *fifo, const void *data, unsigned int len, int (*copier)(void *, const void *, int)) { unsigned int done = 0; unsigned int todo = len; unsigned int nmax; unsigned int writepos = fifo->writepos; unsigned int writebuf = fifo->writebuf; unsigned long flags; int rc; nmax = fifo->size - READ_ONCE(fifo->fill); while (1) { unsigned int nrail = fifo->bufsize - writepos; unsigned int n = min(todo, nmax); if (n == 0) { spin_lock_irqsave(&fifo->lock, flags); fifo->fill += done; spin_unlock_irqrestore(&fifo->lock, flags); fifo->writepos = writepos; fifo->writebuf = writebuf; return done; } if (n > nrail) n = nrail; rc = (*copier)(fifo->mem[writebuf] + writepos, data + done, n); if (rc) return rc; done += n; todo -= n; writepos += n; nmax -= n; if (writepos == fifo->bufsize) { writepos = 0; writebuf++; if (writebuf == fifo->bufnum) writebuf = 0; } } } static int fifo_read(struct xillyfifo *fifo, void *data, unsigned int len, int (*copier)(void *, const void *, int)) { unsigned int done = 0; unsigned int todo = len; unsigned int fill; unsigned int readpos = fifo->readpos; unsigned int readbuf = fifo->readbuf; unsigned long flags; int rc; /* * The spinlock here is necessary, because otherwise fifo->fill * could have been increased by fifo_write() after writing data * to the buffer, but this data would potentially not have been * visible on this thread at the time the updated fifo->fill was. * That could lead to reading invalid data. */ spin_lock_irqsave(&fifo->lock, flags); fill = fifo->fill; spin_unlock_irqrestore(&fifo->lock, flags); while (1) { unsigned int nrail = fifo->bufsize - readpos; unsigned int n = min(todo, fill); if (n == 0) { spin_lock_irqsave(&fifo->lock, flags); fifo->fill -= done; spin_unlock_irqrestore(&fifo->lock, flags); fifo->readpos = readpos; fifo->readbuf = readbuf; return done; } if (n > nrail) n = nrail; rc = (*copier)(data + done, fifo->mem[readbuf] + readpos, n); if (rc) return rc; done += n; todo -= n; readpos += n; fill -= n; if (readpos == fifo->bufsize) { readpos = 0; readbuf++; if (readbuf == fifo->bufnum) readbuf = 0; } } } /* * These three wrapper functions are used as the @copier argument to * fifo_write() and fifo_read(), so that they can work directly with * user memory as well. */ static int xilly_copy_from_user(void *dst, const void *src, int n) { if (copy_from_user(dst, (const void __user *)src, n)) return -EFAULT; return 0; } static int xilly_copy_to_user(void *dst, const void *src, int n) { if (copy_to_user((void __user *)dst, src, n)) return -EFAULT; return 0; } static int xilly_memcpy(void *dst, const void *src, int n) { memcpy(dst, src, n); return 0; } static int fifo_init(struct xillyfifo *fifo, unsigned int log2_size) { unsigned int log2_bufnum; unsigned int buf_order; int i; unsigned int log2_fifo_buf_size; retry: log2_fifo_buf_size = fifo_buf_order + PAGE_SHIFT; if (log2_size > log2_fifo_buf_size) { log2_bufnum = log2_size - log2_fifo_buf_size; buf_order = fifo_buf_order; fifo->bufsize = 1 << log2_fifo_buf_size; } else { log2_bufnum = 0; buf_order = (log2_size > PAGE_SHIFT) ? log2_size - PAGE_SHIFT : 0; fifo->bufsize = 1 << log2_size; } fifo->bufnum = 1 << log2_bufnum; fifo->size = fifo->bufnum * fifo->bufsize; fifo->buf_order = buf_order; fifo->mem = kmalloc_array(fifo->bufnum, sizeof(void *), GFP_KERNEL); if (!fifo->mem) return -ENOMEM; for (i = 0; i < fifo->bufnum; i++) { fifo->mem[i] = (void *) __get_free_pages(GFP_KERNEL, buf_order); if (!fifo->mem[i]) goto memfail; } fifo->fill = 0; fifo->readpos = 0; fifo->readbuf = 0; fifo->writepos = 0; fifo->writebuf = 0; spin_lock_init(&fifo->lock); init_waitqueue_head(&fifo->waitq); return 0; memfail: for (i--; i >= 0; i--) free_pages((unsigned long)fifo->mem[i], buf_order); kfree(fifo->mem); fifo->mem = NULL; if (fifo_buf_order) { fifo_buf_order--; goto retry; } else { return -ENOMEM; } } static void fifo_mem_release(struct xillyfifo *fifo) { int i; if (!fifo->mem) return; for (i = 0; i < fifo->bufnum; i++) free_pages((unsigned long)fifo->mem[i], fifo->buf_order); kfree(fifo->mem); } /* * When endpoint_quiesce() returns, the endpoint has no URBs submitted, * won't accept any new URB submissions, and its related work item doesn't * and won't run anymore. */ static void endpoint_quiesce(struct xillyusb_endpoint *ep) { mutex_lock(&ep->ep_mutex); ep->shutting_down = true; mutex_unlock(&ep->ep_mutex); usb_kill_anchored_urbs(&ep->anchor); cancel_work_sync(&ep->workitem); } /* * Note that endpoint_dealloc() also frees fifo memory (if allocated), even * though endpoint_alloc doesn't allocate that memory. */ static void endpoint_dealloc(struct xillyusb_endpoint *ep) { struct list_head *this, *next; fifo_mem_release(&ep->fifo); /* Join @filled_buffers with @buffers to free these entries too */ list_splice(&ep->filled_buffers, &ep->buffers); list_for_each_safe(this, next, &ep->buffers) { struct xillybuffer *xb = list_entry(this, struct xillybuffer, entry); free_pages((unsigned long)xb->buf, ep->order); kfree(xb); } kfree(ep); } static struct xillyusb_endpoint *endpoint_alloc(struct xillyusb_dev *xdev, u8 ep_num, void (*work)(struct work_struct *), unsigned int order, int bufnum) { int i; struct xillyusb_endpoint *ep; ep = kzalloc(sizeof(*ep), GFP_KERNEL); if (!ep) return NULL; INIT_LIST_HEAD(&ep->buffers); INIT_LIST_HEAD(&ep->filled_buffers); spin_lock_init(&ep->buffers_lock); mutex_init(&ep->ep_mutex); init_usb_anchor(&ep->anchor); INIT_WORK(&ep->workitem, work); ep->order = order; ep->buffer_size = 1 << (PAGE_SHIFT + order); ep->outstanding_urbs = 0; ep->drained = true; ep->wake_on_drain = false; ep->xdev = xdev; ep->ep_num = ep_num; ep->shutting_down = false; for (i = 0; i < bufnum; i++) { struct xillybuffer *xb; unsigned long addr; xb = kzalloc(sizeof(*xb), GFP_KERNEL); if (!xb) { endpoint_dealloc(ep); return NULL; } addr = __get_free_pages(GFP_KERNEL, order); if (!addr) { kfree(xb); endpoint_dealloc(ep); return NULL; } xb->buf = (void *)addr; xb->ep = ep; list_add_tail(&xb->entry, &ep->buffers); } return ep; } static void cleanup_dev(struct kref *kref) { struct xillyusb_dev *xdev = container_of(kref, struct xillyusb_dev, kref); if (xdev->in_ep) endpoint_dealloc(xdev->in_ep); if (xdev->msg_ep) endpoint_dealloc(xdev->msg_ep); if (xdev->workq) destroy_workqueue(xdev->workq); usb_put_dev(xdev->udev); kfree(xdev->channels); /* Argument may be NULL, and that's fine */ kfree(xdev); } /* * @process_in_mutex is taken to ensure that bulk_in_work() won't call * process_bulk_in() after wakeup_all()'s execution: The latter zeroes all * @read_data_ok entries, which will make process_bulk_in() report false * errors if executed. The mechanism relies on that xdev->error is assigned * a non-zero value by report_io_error() prior to queueing wakeup_all(), * which prevents bulk_in_work() from calling process_bulk_in(). * * The fact that wakeup_all() and bulk_in_work() are queued on the same * workqueue makes their concurrent execution very unlikely, however the * kernel's API doesn't seem to ensure this strictly. */ static void wakeup_all(struct work_struct *work) { int i; struct xillyusb_dev *xdev = container_of(work, struct xillyusb_dev, wakeup_workitem); mutex_lock(&xdev->process_in_mutex); for (i = 0; i < xdev->num_channels; i++) { struct xillyusb_channel *chan = &xdev->channels[i]; mutex_lock(&chan->lock); if (chan->in_fifo) { /* * Fake an EOF: Even if such arrives, it won't be * processed. */ chan->read_data_ok = 0; wake_up_interruptible(&chan->in_fifo->waitq); } if (chan->out_ep) wake_up_interruptible(&chan->out_ep->fifo.waitq); mutex_unlock(&chan->lock); wake_up_interruptible(&chan->flushq); } mutex_unlock(&xdev->process_in_mutex); wake_up_interruptible(&xdev->msg_ep->fifo.waitq); kref_put(&xdev->kref, cleanup_dev); } static void report_io_error(struct xillyusb_dev *xdev, int errcode) { unsigned long flags; bool do_once = false; spin_lock_irqsave(&xdev->error_lock, flags); if (!xdev->error) { xdev->error = errcode; do_once = true; } spin_unlock_irqrestore(&xdev->error_lock, flags); if (do_once) { kref_get(&xdev->kref); /* xdev is used by work item */ queue_work(xdev->workq, &xdev->wakeup_workitem); } } /* * safely_assign_in_fifo() changes the value of chan->in_fifo and ensures * the previous pointer is never used after its return. */ static void safely_assign_in_fifo(struct xillyusb_channel *chan, struct xillyfifo *fifo) { mutex_lock(&chan->lock); chan->in_fifo = fifo; mutex_unlock(&chan->lock); flush_work(&chan->xdev->in_ep->workitem); } static void bulk_in_completer(struct urb *urb) { struct xillybuffer *xb = urb->context; struct xillyusb_endpoint *ep = xb->ep; unsigned long flags; if (urb->status) { if (!(urb->status == -ENOENT || urb->status == -ECONNRESET || urb->status == -ESHUTDOWN)) report_io_error(ep->xdev, -EIO); spin_lock_irqsave(&ep->buffers_lock, flags); list_add_tail(&xb->entry, &ep->buffers); ep->outstanding_urbs--; spin_unlock_irqrestore(&ep->buffers_lock, flags); return; } xb->len = urb->actual_length; spin_lock_irqsave(&ep->buffers_lock, flags); list_add_tail(&xb->entry, &ep->filled_buffers); spin_unlock_irqrestore(&ep->buffers_lock, flags); if (!ep->shutting_down) queue_work(ep->xdev->workq, &ep->workitem); } static void bulk_out_completer(struct urb *urb) { struct xillybuffer *xb = urb->context; struct xillyusb_endpoint *ep = xb->ep; unsigned long flags; if (urb->status && (!(urb->status == -ENOENT || urb->status == -ECONNRESET || urb->status == -ESHUTDOWN))) report_io_error(ep->xdev, -EIO); spin_lock_irqsave(&ep->buffers_lock, flags); list_add_tail(&xb->entry, &ep->buffers); ep->outstanding_urbs--; spin_unlock_irqrestore(&ep->buffers_lock, flags); if (!ep->shutting_down) queue_work(ep->xdev->workq, &ep->workitem); } static void try_queue_bulk_in(struct xillyusb_endpoint *ep) { struct xillyusb_dev *xdev = ep->xdev; struct xillybuffer *xb; struct urb *urb; int rc; unsigned long flags; unsigned int bufsize = ep->buffer_size; mutex_lock(&ep->ep_mutex); if (ep->shutting_down || xdev->error) goto done; while (1) { spin_lock_irqsave(&ep->buffers_lock, flags); if (list_empty(&ep->buffers)) { spin_unlock_irqrestore(&ep->buffers_lock, flags); goto done; } xb = list_first_entry(&ep->buffers, struct xillybuffer, entry); list_del(&xb->entry); ep->outstanding_urbs++; spin_unlock_irqrestore(&ep->buffers_lock, flags); urb = usb_alloc_urb(0, GFP_KERNEL); if (!urb) { report_io_error(xdev, -ENOMEM); goto relist; } usb_fill_bulk_urb(urb, xdev->udev, usb_rcvbulkpipe(xdev->udev, ep->ep_num), xb->buf, bufsize, bulk_in_completer, xb); usb_anchor_urb(urb, &ep->anchor); rc = usb_submit_urb(urb, GFP_KERNEL); if (rc) { report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM : -EIO); goto unanchor; } usb_free_urb(urb); /* This just decrements reference count */ } unanchor: usb_unanchor_urb(urb); usb_free_urb(urb); relist: spin_lock_irqsave(&ep->buffers_lock, flags); list_add_tail(&xb->entry, &ep->buffers); ep->outstanding_urbs--; spin_unlock_irqrestore(&ep->buffers_lock, flags); done: mutex_unlock(&ep->ep_mutex); } static void try_queue_bulk_out(struct xillyusb_endpoint *ep) { struct xillyfifo *fifo = &ep->fifo; struct xillyusb_dev *xdev = ep->xdev; struct xillybuffer *xb; struct urb *urb; int rc; unsigned int fill; unsigned long flags; bool do_wake = false; mutex_lock(&ep->ep_mutex); if (ep->shutting_down || xdev->error) goto done; fill = READ_ONCE(fifo->fill) & ep->fill_mask; while (1) { int count; unsigned int max_read; spin_lock_irqsave(&ep->buffers_lock, flags); /* * Race conditions might have the FIFO filled while the * endpoint is marked as drained here. That doesn't matter, * because the sole purpose of @drained is to ensure that * certain data has been sent on the USB channel before * shutting it down. Hence knowing that the FIFO appears * to be empty with no outstanding URBs at some moment * is good enough. */ if (!fill) { ep->drained = !ep->outstanding_urbs; if (ep->drained && ep->wake_on_drain) do_wake = true; spin_unlock_irqrestore(&ep->buffers_lock, flags); goto done; } ep->drained = false; if ((fill < ep->buffer_size && ep->outstanding_urbs) || list_empty(&ep->buffers)) { spin_unlock_irqrestore(&ep->buffers_lock, flags); goto done; } xb = list_first_entry(&ep->buffers, struct xillybuffer, entry); list_del(&xb->entry); ep->outstanding_urbs++; spin_unlock_irqrestore(&ep->buffers_lock, flags); max_read = min(fill, ep->buffer_size); count = fifo_read(&ep->fifo, xb->buf, max_read, xilly_memcpy); /* * xilly_memcpy always returns 0 => fifo_read can't fail => * count > 0 */ urb = usb_alloc_urb(0, GFP_KERNEL); if (!urb) { report_io_error(xdev, -ENOMEM); goto relist; } usb_fill_bulk_urb(urb, xdev->udev, usb_sndbulkpipe(xdev->udev, ep->ep_num), xb->buf, count, bulk_out_completer, xb); usb_anchor_urb(urb, &ep->anchor); rc = usb_submit_urb(urb, GFP_KERNEL); if (rc) { report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM : -EIO); goto unanchor; } usb_free_urb(urb); /* This just decrements reference count */ fill -= count; do_wake = true; } unanchor: usb_unanchor_urb(urb); usb_free_urb(urb); relist: spin_lock_irqsave(&ep->buffers_lock, flags); list_add_tail(&xb->entry, &ep->buffers); ep->outstanding_urbs--; spin_unlock_irqrestore(&ep->buffers_lock, flags); done: mutex_unlock(&ep->ep_mutex); if (do_wake) wake_up_interruptible(&fifo->waitq); } static void bulk_out_work(struct work_struct *work) { struct xillyusb_endpoint *ep = container_of(work, struct xillyusb_endpoint, workitem); try_queue_bulk_out(ep); } static int process_in_opcode(struct xillyusb_dev *xdev, int opcode, int chan_num) { struct xillyusb_channel *chan; struct device *dev = xdev->dev; int chan_idx = chan_num >> 1; if (chan_idx >= xdev->num_channels) { dev_err(dev, "Received illegal channel ID %d from FPGA\n", chan_num); return -EIO; } chan = &xdev->channels[chan_idx]; switch (opcode) { case OPCODE_EOF: if (!chan->read_data_ok) { dev_err(dev, "Received unexpected EOF for channel %d\n", chan_num); return -EIO; } /* * A write memory barrier ensures that the FIFO's fill level * is visible before read_data_ok turns zero, so the data in * the FIFO isn't missed by the consumer. */ smp_wmb(); WRITE_ONCE(chan->read_data_ok, 0); wake_up_interruptible(&chan->in_fifo->waitq); break; case OPCODE_REACHED_CHECKPOINT: chan->flushing = 0; wake_up_interruptible(&chan->flushq); break; case OPCODE_CANCELED_CHECKPOINT: chan->canceled = 1; wake_up_interruptible(&chan->flushq); break; default: dev_err(dev, "Received illegal opcode %d from FPGA\n", opcode); return -EIO; } return 0; } static int process_bulk_in(struct xillybuffer *xb) { struct xillyusb_endpoint *ep = xb->ep; struct xillyusb_dev *xdev = ep->xdev; struct device *dev = xdev->dev; int dws = xb->len >> 2; __le32 *p = xb->buf; u32 ctrlword; struct xillyusb_channel *chan; struct xillyfifo *fifo; int chan_num = 0, opcode; int chan_idx; int bytes, count, dwconsume; int in_bytes_left = 0; int rc; if ((dws << 2) != xb->len) { dev_err(dev, "Received BULK IN transfer with %d bytes, not a multiple of 4\n", xb->len); return -EIO; } if (xdev->in_bytes_left) { bytes = min(xdev->in_bytes_left, dws << 2); in_bytes_left = xdev->in_bytes_left - bytes; chan_num = xdev->leftover_chan_num; goto resume_leftovers; } while (dws) { ctrlword = le32_to_cpu(*p++); dws--; chan_num = ctrlword & 0xfff; count = (ctrlword >> 12) & 0x3ff; opcode = (ctrlword >> 24) & 0xf; if (opcode != OPCODE_DATA) { unsigned int in_counter = xdev->in_counter++ & 0x3ff; if (count != in_counter) { dev_err(dev, "Expected opcode counter %d, got %d\n", in_counter, count); return -EIO; } rc = process_in_opcode(xdev, opcode, chan_num); if (rc) return rc; continue; } bytes = min(count + 1, dws << 2); in_bytes_left = count + 1 - bytes; resume_leftovers: chan_idx = chan_num >> 1; if (!(chan_num & 1) || chan_idx >= xdev->num_channels || !xdev->channels[chan_idx].read_data_ok) { dev_err(dev, "Received illegal channel ID %d from FPGA\n", chan_num); return -EIO; } chan = &xdev->channels[chan_idx]; fifo = chan->in_fifo; if (unlikely(!fifo)) return -EIO; /* We got really unexpected data */ if (bytes != fifo_write(fifo, p, bytes, xilly_memcpy)) { dev_err(dev, "Misbehaving FPGA overflowed an upstream FIFO!\n"); return -EIO; } wake_up_interruptible(&fifo->waitq); dwconsume = (bytes + 3) >> 2; dws -= dwconsume; p += dwconsume; } xdev->in_bytes_left = in_bytes_left; xdev->leftover_chan_num = chan_num; return 0; } static void bulk_in_work(struct work_struct *work) { struct xillyusb_endpoint *ep = container_of(work, struct xillyusb_endpoint, workitem); struct xillyusb_dev *xdev = ep->xdev; unsigned long flags; struct xillybuffer *xb; bool consumed = false; int rc = 0; mutex_lock(&xdev->process_in_mutex); spin_lock_irqsave(&ep->buffers_lock, flags); while (1) { if (rc || list_empty(&ep->filled_buffers)) { spin_unlock_irqrestore(&ep->buffers_lock, flags); mutex_unlock(&xdev->process_in_mutex); if (rc) report_io_error(xdev, rc); else if (consumed) try_queue_bulk_in(ep); return; } xb = list_first_entry(&ep->filled_buffers, struct xillybuffer, entry); list_del(&xb->entry); spin_unlock_irqrestore(&ep->buffers_lock, flags); consumed = true; if (!xdev->error) rc = process_bulk_in(xb); spin_lock_irqsave(&ep->buffers_lock, flags); list_add_tail(&xb->entry, &ep->buffers); ep->outstanding_urbs--; } } static int xillyusb_send_opcode(struct xillyusb_dev *xdev, int chan_num, char opcode, u32 data) { struct xillyusb_endpoint *ep = xdev->msg_ep; struct xillyfifo *fifo = &ep->fifo; __le32 msg[2]; int rc = 0; msg[0] = cpu_to_le32((chan_num & 0xfff) | ((opcode & 0xf) << 24)); msg[1] = cpu_to_le32(data); mutex_lock(&xdev->msg_mutex); /* * The wait queue is woken with the interruptible variant, so the * wait function matches, however returning because of an interrupt * will mess things up considerably, in particular when the caller is * the release method. And the xdev->error part prevents being stuck * forever in the event of a bizarre hardware bug: Pull the USB plug. */ while (wait_event_interruptible(fifo->waitq, fifo->fill <= (fifo->size - 8) || xdev->error)) ; /* Empty loop */ if (xdev->error) { rc = xdev->error; goto unlock_done; } fifo_write(fifo, (void *)msg, 8, xilly_memcpy); try_queue_bulk_out(ep); unlock_done: mutex_unlock(&xdev->msg_mutex); return rc; } /* * Note that flush_downstream() merely waits for the data to arrive to * the application logic at the FPGA -- unlike PCIe Xillybus' counterpart, * it does nothing to make it happen (and neither is it necessary). * * This function is not reentrant for the same @chan, but this is covered * by the fact that for any given @chan, it's called either by the open, * write, llseek and flush fops methods, which can't run in parallel (and the * write + flush and llseek method handlers are protected with out_mutex). * * chan->flushed is there to avoid multiple flushes at the same position, * in particular as a result of programs that close the file descriptor * e.g. after a dup2() for redirection. */ static int flush_downstream(struct xillyusb_channel *chan, long timeout, bool interruptible) { struct xillyusb_dev *xdev = chan->xdev; int chan_num = chan->chan_idx << 1; long deadline, left_to_sleep; int rc; if (chan->flushed) return 0; deadline = jiffies + 1 + timeout; if (chan->flushing) { long cancel_deadline = jiffies + 1 + XILLY_RESPONSE_TIMEOUT; chan->canceled = 0; rc = xillyusb_send_opcode(xdev, chan_num, OPCODE_CANCEL_CHECKPOINT, 0); if (rc) return rc; /* Only real error, never -EINTR */ /* Ignoring interrupts. Cancellation must be handled */ while (!chan->canceled) { left_to_sleep = cancel_deadline - ((long)jiffies); if (left_to_sleep <= 0) { report_io_error(xdev, -EIO); return -EIO; } rc = wait_event_interruptible_timeout(chan->flushq, chan->canceled || xdev->error, left_to_sleep); if (xdev->error) return xdev->error; } } chan->flushing = 1; /* * The checkpoint is given in terms of data elements, not bytes. As * a result, if less than an element's worth of data is stored in the * FIFO, it's not flushed, including the flush before closing, which * means that such data is lost. This is consistent with PCIe Xillybus. */ rc = xillyusb_send_opcode(xdev, chan_num, OPCODE_SET_CHECKPOINT, chan->out_bytes >> chan->out_log2_element_size); if (rc) return rc; /* Only real error, never -EINTR */ if (!timeout) { while (chan->flushing) { rc = wait_event_interruptible(chan->flushq, !chan->flushing || xdev->error); if (xdev->error) return xdev->error; if (interruptible && rc) return -EINTR; } goto done; } while (chan->flushing) { left_to_sleep = deadline - ((long)jiffies); if (left_to_sleep <= 0) return -ETIMEDOUT; rc = wait_event_interruptible_timeout(chan->flushq, !chan->flushing || xdev->error, left_to_sleep); if (xdev->error) return xdev->error; if (interruptible && rc < 0) return -EINTR; } done: chan->flushed = 1; return 0; } /* request_read_anything(): Ask the FPGA for any little amount of data */ static int request_read_anything(struct xillyusb_channel *chan, char opcode) { struct xillyusb_dev *xdev = chan->xdev; unsigned int sh = chan->in_log2_element_size; int chan_num = (chan->chan_idx << 1) | 1; u32 mercy = chan->in_consumed_bytes + (2 << sh) - 1; return xillyusb_send_opcode(xdev, chan_num, opcode, mercy >> sh); } static int xillyusb_open(struct inode *inode, struct file *filp) { struct xillyusb_dev *xdev; struct xillyusb_channel *chan; struct xillyfifo *in_fifo = NULL; struct xillyusb_endpoint *out_ep = NULL; int rc; int index; mutex_lock(&kref_mutex); rc = xillybus_find_inode(inode, (void **)&xdev, &index); if (rc) { mutex_unlock(&kref_mutex); return rc; } kref_get(&xdev->kref); mutex_unlock(&kref_mutex); chan = &xdev->channels[index]; filp->private_data = chan; mutex_lock(&chan->lock); rc = -ENODEV; if (xdev->error) goto unmutex_fail; if (((filp->f_mode & FMODE_READ) && !chan->readable) || ((filp->f_mode & FMODE_WRITE) && !chan->writable)) goto unmutex_fail; if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_READ) && chan->in_synchronous) { dev_err(xdev->dev, "open() failed: O_NONBLOCK not allowed for read on this device\n"); goto unmutex_fail; } if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_WRITE) && chan->out_synchronous) { dev_err(xdev->dev, "open() failed: O_NONBLOCK not allowed for write on this device\n"); goto unmutex_fail; } rc = -EBUSY; if (((filp->f_mode & FMODE_READ) && chan->open_for_read) || ((filp->f_mode & FMODE_WRITE) && chan->open_for_write)) goto unmutex_fail; if (filp->f_mode & FMODE_READ) chan->open_for_read = 1; if (filp->f_mode & FMODE_WRITE) chan->open_for_write = 1; mutex_unlock(&chan->lock); if (filp->f_mode & FMODE_WRITE) { out_ep = endpoint_alloc(xdev, (chan->chan_idx + 2) | USB_DIR_OUT, bulk_out_work, BUF_SIZE_ORDER, BUFNUM); if (!out_ep) { rc = -ENOMEM; goto unopen; } rc = fifo_init(&out_ep->fifo, chan->out_log2_fifo_size); if (rc) goto late_unopen; out_ep->fill_mask = -(1 << chan->out_log2_element_size); chan->out_bytes = 0; chan->flushed = 0; /* * Sending a flush request to a previously closed stream * effectively opens it, and also waits until the command is * confirmed by the FPGA. The latter is necessary because the * data is sent through a separate BULK OUT endpoint, and the * xHCI controller is free to reorder transmissions. * * This can't go wrong unless there's a serious hardware error * (or the computer is stuck for 500 ms?) */ rc = flush_downstream(chan, XILLY_RESPONSE_TIMEOUT, false); if (rc == -ETIMEDOUT) { rc = -EIO; report_io_error(xdev, rc); } if (rc) goto late_unopen; } if (filp->f_mode & FMODE_READ) { in_fifo = kzalloc(sizeof(*in_fifo), GFP_KERNEL); if (!in_fifo) { rc = -ENOMEM; goto late_unopen; } rc = fifo_init(in_fifo, chan->in_log2_fifo_size); if (rc) { kfree(in_fifo); goto late_unopen; } } mutex_lock(&chan->lock); if (in_fifo) { chan->in_fifo = in_fifo; chan->read_data_ok = 1; } if (out_ep) chan->out_ep = out_ep; mutex_unlock(&chan->lock); if (in_fifo) { u32 in_checkpoint = 0; if (!chan->in_synchronous) in_checkpoint = in_fifo->size >> chan->in_log2_element_size; chan->in_consumed_bytes = 0; chan->poll_used = 0; chan->in_current_checkpoint = in_checkpoint; rc = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1, OPCODE_SET_CHECKPOINT, in_checkpoint); if (rc) /* Failure guarantees that opcode wasn't sent */ goto unfifo; /* * In non-blocking mode, request the FPGA to send any data it * has right away. Otherwise, the first read() will always * return -EAGAIN, which is OK strictly speaking, but ugly. * Checking and unrolling if this fails isn't worth the * effort -- the error is propagated to the first read() * anyhow. */ if (filp->f_flags & O_NONBLOCK) request_read_anything(chan, OPCODE_SET_PUSH); } return 0; unfifo: chan->read_data_ok = 0; safely_assign_in_fifo(chan, NULL); fifo_mem_release(in_fifo); kfree(in_fifo); if (out_ep) { mutex_lock(&chan->lock); chan->out_ep = NULL; mutex_unlock(&chan->lock); } late_unopen: if (out_ep) endpoint_dealloc(out_ep); unopen: mutex_lock(&chan->lock); if (filp->f_mode & FMODE_READ) chan->open_for_read = 0; if (filp->f_mode & FMODE_WRITE) chan->open_for_write = 0; mutex_unlock(&chan->lock); kref_put(&xdev->kref, cleanup_dev); return rc; unmutex_fail: kref_put(&xdev->kref, cleanup_dev); mutex_unlock(&chan->lock); return rc; } static ssize_t xillyusb_read(struct file *filp, char __user *userbuf, size_t count, loff_t *f_pos) { struct xillyusb_channel *chan = filp->private_data; struct xillyusb_dev *xdev = chan->xdev; struct xillyfifo *fifo = chan->in_fifo; int chan_num = (chan->chan_idx << 1) | 1; long deadline, left_to_sleep; int bytes_done = 0; bool sent_set_push = false; int rc; deadline = jiffies + 1 + XILLY_RX_TIMEOUT; rc = mutex_lock_interruptible(&chan->in_mutex); if (rc) return rc; while (1) { u32 fifo_checkpoint_bytes, complete_checkpoint_bytes; u32 complete_checkpoint, fifo_checkpoint; u32 checkpoint; s32 diff, leap; unsigned int sh = chan->in_log2_element_size; bool checkpoint_for_complete; rc = fifo_read(fifo, (__force void *)userbuf + bytes_done, count - bytes_done, xilly_copy_to_user); if (rc < 0) break; bytes_done += rc; chan->in_consumed_bytes += rc; left_to_sleep = deadline - ((long)jiffies); /* * Some 32-bit arithmetic that may wrap. Note that * complete_checkpoint is rounded up to the closest element * boundary, because the read() can't be completed otherwise. * fifo_checkpoint_bytes is rounded down, because it protects * in_fifo from overflowing. */ fifo_checkpoint_bytes = chan->in_consumed_bytes + fifo->size; complete_checkpoint_bytes = chan->in_consumed_bytes + count - bytes_done; fifo_checkpoint = fifo_checkpoint_bytes >> sh; complete_checkpoint = (complete_checkpoint_bytes + (1 << sh) - 1) >> sh; diff = (fifo_checkpoint - complete_checkpoint) << sh; if (chan->in_synchronous && diff >= 0) { checkpoint = complete_checkpoint; checkpoint_for_complete = true; } else { checkpoint = fifo_checkpoint; checkpoint_for_complete = false; } leap = (checkpoint - chan->in_current_checkpoint) << sh; /* * To prevent flooding of OPCODE_SET_CHECKPOINT commands as * data is consumed, it's issued only if it moves the * checkpoint by at least an 8th of the FIFO's size, or if * it's necessary to complete the number of bytes requested by * the read() call. * * chan->read_data_ok is checked to spare an unnecessary * submission after receiving EOF, however it's harmless if * such slips away. */ if (chan->read_data_ok && (leap > (fifo->size >> 3) || (checkpoint_for_complete && leap > 0))) { chan->in_current_checkpoint = checkpoint; rc = xillyusb_send_opcode(xdev, chan_num, OPCODE_SET_CHECKPOINT, checkpoint); if (rc) break; } if (bytes_done == count || (left_to_sleep <= 0 && bytes_done)) break; /* * Reaching here means that the FIFO was empty when * fifo_read() returned, but not necessarily right now. Error * and EOF are checked and reported only now, so that no data * that managed its way to the FIFO is lost. */ if (!READ_ONCE(chan->read_data_ok)) { /* FPGA has sent EOF */ /* Has data slipped into the FIFO since fifo_read()? */ smp_rmb(); if (READ_ONCE(fifo->fill)) continue; rc = 0; break; } if (xdev->error) { rc = xdev->error; break; } if (filp->f_flags & O_NONBLOCK) { rc = -EAGAIN; break; } if (!sent_set_push) { rc = xillyusb_send_opcode(xdev, chan_num, OPCODE_SET_PUSH, complete_checkpoint); if (rc) break; sent_set_push = true; } if (left_to_sleep > 0) { /* * Note that when xdev->error is set (e.g. when the * device is unplugged), read_data_ok turns zero and * fifo->waitq is awaken. * Therefore no special attention to xdev->error. */ rc = wait_event_interruptible_timeout (fifo->waitq, fifo->fill || !chan->read_data_ok, left_to_sleep); } else { /* bytes_done == 0 */ /* Tell FPGA to send anything it has */ rc = request_read_anything(chan, OPCODE_UPDATE_PUSH); if (rc) break; rc = wait_event_interruptible (fifo->waitq, fifo->fill || !chan->read_data_ok); } if (rc < 0) { rc = -EINTR; break; } } if (((filp->f_flags & O_NONBLOCK) || chan->poll_used) && !READ_ONCE(fifo->fill)) request_read_anything(chan, OPCODE_SET_PUSH); mutex_unlock(&chan->in_mutex); if (bytes_done) return bytes_done; return rc; } static int xillyusb_flush(struct file *filp, fl_owner_t id) { struct xillyusb_channel *chan = filp->private_data; int rc; if (!(filp->f_mode & FMODE_WRITE)) return 0; rc = mutex_lock_interruptible(&chan->out_mutex); if (rc) return rc; /* * One second's timeout on flushing. Interrupts are ignored, because if * the user pressed CTRL-C, that interrupt will still be in flight by * the time we reach here, and the opportunity to flush is lost. */ rc = flush_downstream(chan, HZ, false); mutex_unlock(&chan->out_mutex); if (rc == -ETIMEDOUT) { /* The things you do to use dev_warn() and not pr_warn() */ struct xillyusb_dev *xdev = chan->xdev; mutex_lock(&chan->lock); if (!xdev->error) dev_warn(xdev->dev, "Timed out while flushing. Output data may be lost.\n"); mutex_unlock(&chan->lock); } return rc; } static ssize_t xillyusb_write(struct file *filp, const char __user *userbuf, size_t count, loff_t *f_pos) { struct xillyusb_channel *chan = filp->private_data; struct xillyusb_dev *xdev = chan->xdev; struct xillyfifo *fifo = &chan->out_ep->fifo; int rc; rc = mutex_lock_interruptible(&chan->out_mutex); if (rc) return rc; while (1) { if (xdev->error) { rc = xdev->error; break; } if (count == 0) break; rc = fifo_write(fifo, (__force void *)userbuf, count, xilly_copy_from_user); if (rc != 0) break; if (filp->f_flags & O_NONBLOCK) { rc = -EAGAIN; break; } if (wait_event_interruptible (fifo->waitq, fifo->fill != fifo->size || xdev->error)) { rc = -EINTR; break; } } if (rc < 0) goto done; chan->out_bytes += rc; if (rc) { try_queue_bulk_out(chan->out_ep); chan->flushed = 0; } if (chan->out_synchronous) { int flush_rc = flush_downstream(chan, 0, true); if (flush_rc && !rc) rc = flush_rc; } done: mutex_unlock(&chan->out_mutex); return rc; } static int xillyusb_release(struct inode *inode, struct file *filp) { struct xillyusb_channel *chan = filp->private_data; struct xillyusb_dev *xdev = chan->xdev; int rc_read = 0, rc_write = 0; if (filp->f_mode & FMODE_READ) { struct xillyfifo *in_fifo = chan->in_fifo; rc_read = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1, OPCODE_CLOSE, 0); /* * If rc_read is nonzero, xdev->error indicates a global * device error. The error is reported later, so that * resources are freed. * * Looping on wait_event_interruptible() kinda breaks the idea * of being interruptible, and this should have been * wait_event(). Only it's being waken with * wake_up_interruptible() for the sake of other uses. If * there's a global device error, chan->read_data_ok is * deasserted and the wait queue is awaken, so this is covered. */ while (wait_event_interruptible(in_fifo->waitq, !chan->read_data_ok)) ; /* Empty loop */ safely_assign_in_fifo(chan, NULL); fifo_mem_release(in_fifo); kfree(in_fifo); mutex_lock(&chan->lock); chan->open_for_read = 0; mutex_unlock(&chan->lock); } if (filp->f_mode & FMODE_WRITE) { struct xillyusb_endpoint *ep = chan->out_ep; /* * chan->flushing isn't zeroed. If the pre-release flush timed * out, a cancel request will be sent before the next * OPCODE_SET_CHECKPOINT (i.e. when the file is opened again). * This is despite that the FPGA forgets about the checkpoint * request as the file closes. Still, in an exceptional race * condition, the FPGA could send an OPCODE_REACHED_CHECKPOINT * just before closing that would reach the host after the * file has re-opened. */ mutex_lock(&chan->lock); chan->out_ep = NULL; mutex_unlock(&chan->lock); endpoint_quiesce(ep); endpoint_dealloc(ep); /* See comments on rc_read above */ rc_write = xillyusb_send_opcode(xdev, chan->chan_idx << 1, OPCODE_CLOSE, 0); mutex_lock(&chan->lock); chan->open_for_write = 0; mutex_unlock(&chan->lock); } kref_put(&xdev->kref, cleanup_dev); return rc_read ? rc_read : rc_write; } /* * Xillybus' API allows device nodes to be seekable, giving the user * application access to a RAM array on the FPGA (or logic emulating it). */ static loff_t xillyusb_llseek(struct file *filp, loff_t offset, int whence) { struct xillyusb_channel *chan = filp->private_data; struct xillyusb_dev *xdev = chan->xdev; loff_t pos = filp->f_pos; int rc = 0; unsigned int log2_element_size = chan->readable ? chan->in_log2_element_size : chan->out_log2_element_size; /* * Take both mutexes not allowing interrupts, since it seems like * common applications don't expect an -EINTR here. Besides, multiple * access to a single file descriptor on seekable devices is a mess * anyhow. */ mutex_lock(&chan->out_mutex); mutex_lock(&chan->in_mutex); switch (whence) { case SEEK_SET: pos = offset; break; case SEEK_CUR: pos += offset; break; case SEEK_END: pos = offset; /* Going to the end => to the beginning */ break; default: rc = -EINVAL; goto end; } /* In any case, we must finish on an element boundary */ if (pos & ((1 << log2_element_size) - 1)) { rc = -EINVAL; goto end; } rc = xillyusb_send_opcode(xdev, chan->chan_idx << 1, OPCODE_SET_ADDR, pos >> log2_element_size); if (rc) goto end; if (chan->writable) { chan->flushed = 0; rc = flush_downstream(chan, HZ, false); } end: mutex_unlock(&chan->out_mutex); mutex_unlock(&chan->in_mutex); if (rc) /* Return error after releasing mutexes */ return rc; filp->f_pos = pos; return pos; } static __poll_t xillyusb_poll(struct file *filp, poll_table *wait) { struct xillyusb_channel *chan = filp->private_data; __poll_t mask = 0; if (chan->in_fifo) poll_wait(filp, &chan->in_fifo->waitq, wait); if (chan->out_ep) poll_wait(filp, &chan->out_ep->fifo.waitq, wait); /* * If this is the first time poll() is called, and the file is * readable, set the relevant flag. Also tell the FPGA to send all it * has, to kickstart the mechanism that ensures there's always some * data in in_fifo unless the stream is dry end-to-end. Note that the * first poll() may not return a EPOLLIN, even if there's data on the * FPGA. Rather, the data will arrive soon, and trigger the relevant * wait queue. */ if (!chan->poll_used && chan->in_fifo) { chan->poll_used = 1; request_read_anything(chan, OPCODE_SET_PUSH); } /* * poll() won't play ball regarding read() channels which * are synchronous. Allowing that will create situations where data has * been delivered at the FPGA, and users expecting select() to wake up, * which it may not. So make it never work. */ if (chan->in_fifo && !chan->in_synchronous && (READ_ONCE(chan->in_fifo->fill) || !chan->read_data_ok)) mask |= EPOLLIN | EPOLLRDNORM; if (chan->out_ep && (READ_ONCE(chan->out_ep->fifo.fill) != chan->out_ep->fifo.size)) mask |= EPOLLOUT | EPOLLWRNORM; if (chan->xdev->error) mask |= EPOLLERR; return mask; } static const struct file_operations xillyusb_fops = { .owner = THIS_MODULE, .read = xillyusb_read, .write = xillyusb_write, .open = xillyusb_open, .flush = xillyusb_flush, .release = xillyusb_release, .llseek = xillyusb_llseek, .poll = xillyusb_poll, }; static int xillyusb_setup_base_eps(struct xillyusb_dev *xdev) { xdev->msg_ep = endpoint_alloc(xdev, MSG_EP_NUM | USB_DIR_OUT, bulk_out_work, 1, 2); if (!xdev->msg_ep) return -ENOMEM; if (fifo_init(&xdev->msg_ep->fifo, 13)) /* 8 kiB */ goto dealloc; xdev->msg_ep->fill_mask = -8; /* 8 bytes granularity */ xdev->in_ep = endpoint_alloc(xdev, IN_EP_NUM | USB_DIR_IN, bulk_in_work, BUF_SIZE_ORDER, BUFNUM); if (!xdev->in_ep) goto dealloc; try_queue_bulk_in(xdev->in_ep); return 0; dealloc: endpoint_dealloc(xdev->msg_ep); /* Also frees FIFO mem if allocated */ xdev->msg_ep = NULL; return -ENOMEM; } static int setup_channels(struct xillyusb_dev *xdev, __le16 *chandesc, int num_channels) { struct xillyusb_channel *chan; int i; chan = kcalloc(num_channels, sizeof(*chan), GFP_KERNEL); if (!chan) return -ENOMEM; xdev->channels = chan; for (i = 0; i < num_channels; i++, chan++) { unsigned int in_desc = le16_to_cpu(*chandesc++); unsigned int out_desc = le16_to_cpu(*chandesc++); chan->xdev = xdev; mutex_init(&chan->in_mutex); mutex_init(&chan->out_mutex); mutex_init(&chan->lock); init_waitqueue_head(&chan->flushq); chan->chan_idx = i; if (in_desc & 0x80) { /* Entry is valid */ chan->readable = 1; chan->in_synchronous = !!(in_desc & 0x40); chan->in_seekable = !!(in_desc & 0x20); chan->in_log2_element_size = in_desc & 0x0f; chan->in_log2_fifo_size = ((in_desc >> 8) & 0x1f) + 16; } /* * A downstream channel should never exist above index 13, * as it would request a nonexistent BULK endpoint > 15. * In the peculiar case that it does, it's ignored silently. */ if ((out_desc & 0x80) && i < 14) { /* Entry is valid */ chan->writable = 1; chan->out_synchronous = !!(out_desc & 0x40); chan->out_seekable = !!(out_desc & 0x20); chan->out_log2_element_size = out_desc & 0x0f; chan->out_log2_fifo_size = ((out_desc >> 8) & 0x1f) + 16; } } return 0; } static int xillyusb_discovery(struct usb_interface *interface) { int rc; struct xillyusb_dev *xdev = usb_get_intfdata(interface); __le16 bogus_chandesc[2]; struct xillyfifo idt_fifo; struct xillyusb_channel *chan; unsigned int idt_len, names_offset; unsigned char *idt; int num_channels; rc = xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0); if (rc) { dev_err(&interface->dev, "Failed to send quiesce request. Aborting.\n"); return rc; } /* Phase I: Set up one fake upstream channel and obtain IDT */ /* Set up a fake IDT with one async IN stream */ bogus_chandesc[0] = cpu_to_le16(0x80); bogus_chandesc[1] = cpu_to_le16(0); rc = setup_channels(xdev, bogus_chandesc, 1); if (rc) return rc; rc = fifo_init(&idt_fifo, LOG2_IDT_FIFO_SIZE); if (rc) return rc; chan = xdev->channels; chan->in_fifo = &idt_fifo; chan->read_data_ok = 1; xdev->num_channels = 1; rc = xillyusb_send_opcode(xdev, ~0, OPCODE_REQ_IDT, 0); if (rc) { dev_err(&interface->dev, "Failed to send IDT request. Aborting.\n"); goto unfifo; } rc = wait_event_interruptible_timeout(idt_fifo.waitq, !chan->read_data_ok, XILLY_RESPONSE_TIMEOUT); if (xdev->error) { rc = xdev->error; goto unfifo; } if (rc < 0) { rc = -EINTR; /* Interrupt on probe method? Interesting. */ goto unfifo; } if (chan->read_data_ok) { rc = -ETIMEDOUT; dev_err(&interface->dev, "No response from FPGA. Aborting.\n"); goto unfifo; } idt_len = READ_ONCE(idt_fifo.fill); idt = kmalloc(idt_len, GFP_KERNEL); if (!idt) { rc = -ENOMEM; goto unfifo; } fifo_read(&idt_fifo, idt, idt_len, xilly_memcpy); if (crc32_le(~0, idt, idt_len) != 0) { dev_err(&interface->dev, "IDT failed CRC check. Aborting.\n"); rc = -ENODEV; goto unidt; } if (*idt > 0x90) { dev_err(&interface->dev, "No support for IDT version 0x%02x. Maybe the xillyusb driver needs an upgrade. Aborting.\n", (int)*idt); rc = -ENODEV; goto unidt; } /* Phase II: Set up the streams as defined in IDT */ num_channels = le16_to_cpu(*((__le16 *)(idt + 1))); names_offset = 3 + num_channels * 4; idt_len -= 4; /* Exclude CRC */ if (idt_len < names_offset) { dev_err(&interface->dev, "IDT too short. This is exceptionally weird, because its CRC is OK\n"); rc = -ENODEV; goto unidt; } rc = setup_channels(xdev, (void *)idt + 3, num_channels); if (rc) goto unidt; /* * Except for wildly misbehaving hardware, or if it was disconnected * just after responding with the IDT, there is no reason for any * work item to be running now. To be sure that xdev->channels * is updated on anything that might run in parallel, flush the * workqueue, which rarely does anything. */ flush_workqueue(xdev->workq); xdev->num_channels = num_channels; fifo_mem_release(&idt_fifo); kfree(chan); rc = xillybus_init_chrdev(&interface->dev, &xillyusb_fops, THIS_MODULE, xdev, idt + names_offset, idt_len - names_offset, num_channels, xillyname, true); kfree(idt); return rc; unidt: kfree(idt); unfifo: safely_assign_in_fifo(chan, NULL); fifo_mem_release(&idt_fifo); return rc; } static int xillyusb_probe(struct usb_interface *interface, const struct usb_device_id *id) { struct xillyusb_dev *xdev; int rc; xdev = kzalloc(sizeof(*xdev), GFP_KERNEL); if (!xdev) return -ENOMEM; kref_init(&xdev->kref); mutex_init(&xdev->process_in_mutex); mutex_init(&xdev->msg_mutex); xdev->udev = usb_get_dev(interface_to_usbdev(interface)); xdev->dev = &interface->dev; xdev->error = 0; spin_lock_init(&xdev->error_lock); xdev->in_counter = 0; xdev->in_bytes_left = 0; xdev->workq = alloc_workqueue(xillyname, WQ_HIGHPRI, 0); if (!xdev->workq) { dev_err(&interface->dev, "Failed to allocate work queue\n"); rc = -ENOMEM; goto fail; } INIT_WORK(&xdev->wakeup_workitem, wakeup_all); usb_set_intfdata(interface, xdev); rc = xillyusb_setup_base_eps(xdev); if (rc) goto fail; rc = xillyusb_discovery(interface); if (rc) goto latefail; return 0; latefail: endpoint_quiesce(xdev->in_ep); endpoint_quiesce(xdev->msg_ep); fail: usb_set_intfdata(interface, NULL); kref_put(&xdev->kref, cleanup_dev); return rc; } static void xillyusb_disconnect(struct usb_interface *interface) { struct xillyusb_dev *xdev = usb_get_intfdata(interface); struct xillyusb_endpoint *msg_ep = xdev->msg_ep; struct xillyfifo *fifo = &msg_ep->fifo; int rc; int i; xillybus_cleanup_chrdev(xdev, &interface->dev); /* * Try to send OPCODE_QUIESCE, which will fail silently if the device * was disconnected, but makes sense on module unload. */ msg_ep->wake_on_drain = true; xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0); /* * If the device has been disconnected, sending the opcode causes * a global device error with xdev->error, if such error didn't * occur earlier. Hence timing out means that the USB link is fine, * but somehow the message wasn't sent. Should never happen. */ rc = wait_event_interruptible_timeout(fifo->waitq, msg_ep->drained || xdev->error, XILLY_RESPONSE_TIMEOUT); if (!rc) dev_err(&interface->dev, "Weird timeout condition on sending quiesce request.\n"); report_io_error(xdev, -ENODEV); /* Discourage further activity */ /* * This device driver is declared with soft_unbind set, or else * sending OPCODE_QUIESCE above would always fail. The price is * that the USB framework didn't kill outstanding URBs, so it has * to be done explicitly before returning from this call. */ for (i = 0; i < xdev->num_channels; i++) { struct xillyusb_channel *chan = &xdev->channels[i]; /* * Lock taken to prevent chan->out_ep from changing. It also * ensures xillyusb_open() and xillyusb_flush() don't access * xdev->dev after being nullified below. */ mutex_lock(&chan->lock); if (chan->out_ep) endpoint_quiesce(chan->out_ep); mutex_unlock(&chan->lock); } endpoint_quiesce(xdev->in_ep); endpoint_quiesce(xdev->msg_ep); usb_set_intfdata(interface, NULL); xdev->dev = NULL; mutex_lock(&kref_mutex); kref_put(&xdev->kref, cleanup_dev); mutex_unlock(&kref_mutex); } static struct usb_driver xillyusb_driver = { .name = xillyname, .id_table = xillyusb_table, .probe = xillyusb_probe, .disconnect = xillyusb_disconnect, .soft_unbind = 1, }; static int __init xillyusb_init(void) { int rc = 0; if (LOG2_INITIAL_FIFO_BUF_SIZE > PAGE_SHIFT) fifo_buf_order = LOG2_INITIAL_FIFO_BUF_SIZE - PAGE_SHIFT; else fifo_buf_order = 0; rc = usb_register(&xillyusb_driver); return rc; } static void __exit xillyusb_exit(void) { usb_deregister(&xillyusb_driver); } module_init(xillyusb_init); module_exit(xillyusb_exit);