1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/drivers/misc/xillybus_core.c 4 * 5 * Copyright 2011 Xillybus Ltd, http://xillybus.com 6 * 7 * Driver for the Xillybus FPGA/host framework. 8 * 9 * This driver interfaces with a special IP core in an FPGA, setting up 10 * a pipe between a hardware FIFO in the programmable logic and a device 11 * file in the host. The number of such pipes and their attributes are 12 * set up on the logic. This driver detects these automatically and 13 * creates the device files accordingly. 14 */ 15 16 #include <linux/list.h> 17 #include <linux/device.h> 18 #include <linux/module.h> 19 #include <linux/io.h> 20 #include <linux/dma-mapping.h> 21 #include <linux/interrupt.h> 22 #include <linux/sched.h> 23 #include <linux/fs.h> 24 #include <linux/cdev.h> 25 #include <linux/spinlock.h> 26 #include <linux/mutex.h> 27 #include <linux/crc32.h> 28 #include <linux/poll.h> 29 #include <linux/delay.h> 30 #include <linux/slab.h> 31 #include <linux/workqueue.h> 32 #include "xillybus.h" 33 34 MODULE_DESCRIPTION("Xillybus core functions"); 35 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); 36 MODULE_VERSION("1.07"); 37 MODULE_ALIAS("xillybus_core"); 38 MODULE_LICENSE("GPL v2"); 39 40 /* General timeout is 100 ms, rx timeout is 10 ms */ 41 #define XILLY_RX_TIMEOUT (10*HZ/1000) 42 #define XILLY_TIMEOUT (100*HZ/1000) 43 44 #define fpga_msg_ctrl_reg 0x0008 45 #define fpga_dma_control_reg 0x0020 46 #define fpga_dma_bufno_reg 0x0024 47 #define fpga_dma_bufaddr_lowaddr_reg 0x0028 48 #define fpga_dma_bufaddr_highaddr_reg 0x002c 49 #define fpga_buf_ctrl_reg 0x0030 50 #define fpga_buf_offset_reg 0x0034 51 #define fpga_endian_reg 0x0040 52 53 #define XILLYMSG_OPCODE_RELEASEBUF 1 54 #define XILLYMSG_OPCODE_QUIESCEACK 2 55 #define XILLYMSG_OPCODE_FIFOEOF 3 56 #define XILLYMSG_OPCODE_FATAL_ERROR 4 57 #define XILLYMSG_OPCODE_NONEMPTY 5 58 59 static const char xillyname[] = "xillybus"; 60 61 static struct class *xillybus_class; 62 63 /* 64 * ep_list_lock is the last lock to be taken; No other lock requests are 65 * allowed while holding it. It merely protects list_of_endpoints, and not 66 * the endpoints listed in it. 67 */ 68 69 static LIST_HEAD(list_of_endpoints); 70 static struct mutex ep_list_lock; 71 static struct workqueue_struct *xillybus_wq; 72 73 /* 74 * Locking scheme: Mutexes protect invocations of character device methods. 75 * If both locks are taken, wr_mutex is taken first, rd_mutex second. 76 * 77 * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the 78 * buffers' end_offset fields against changes made by IRQ handler (and in 79 * theory, other file request handlers, but the mutex handles that). Nothing 80 * else. 81 * They are held for short direct memory manipulations. Needless to say, 82 * no mutex locking is allowed when a spinlock is held. 83 * 84 * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset. 85 * 86 * register_mutex is endpoint-specific, and is held when non-atomic 87 * register operations are performed. wr_mutex and rd_mutex may be 88 * held when register_mutex is taken, but none of the spinlocks. Note that 89 * register_mutex doesn't protect against sporadic buf_ctrl_reg writes 90 * which are unrelated to buf_offset_reg, since they are harmless. 91 * 92 * Blocking on the wait queues is allowed with mutexes held, but not with 93 * spinlocks. 94 * 95 * Only interruptible blocking is allowed on mutexes and wait queues. 96 * 97 * All in all, the locking order goes (with skips allowed, of course): 98 * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock 99 */ 100 101 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf) 102 { 103 int opcode; 104 int msg_channel, msg_bufno, msg_data, msg_dir; 105 106 opcode = (buf[0] >> 24) & 0xff; 107 msg_dir = buf[0] & 1; 108 msg_channel = (buf[0] >> 1) & 0x7ff; 109 msg_bufno = (buf[0] >> 12) & 0x3ff; 110 msg_data = buf[1] & 0xfffffff; 111 112 dev_warn(endpoint->dev, 113 "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n", 114 opcode, msg_channel, msg_dir, msg_bufno, msg_data); 115 } 116 117 /* 118 * xillybus_isr assumes the interrupt is allocated exclusively to it, 119 * which is the natural case MSI and several other hardware-oriented 120 * interrupts. Sharing is not allowed. 121 */ 122 123 irqreturn_t xillybus_isr(int irq, void *data) 124 { 125 struct xilly_endpoint *ep = data; 126 u32 *buf; 127 unsigned int buf_size; 128 int i; 129 int opcode; 130 unsigned int msg_channel, msg_bufno, msg_data, msg_dir; 131 struct xilly_channel *channel; 132 133 buf = ep->msgbuf_addr; 134 buf_size = ep->msg_buf_size/sizeof(u32); 135 136 ep->ephw->hw_sync_sgl_for_cpu(ep, 137 ep->msgbuf_dma_addr, 138 ep->msg_buf_size, 139 DMA_FROM_DEVICE); 140 141 for (i = 0; i < buf_size; i += 2) { 142 if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) { 143 malformed_message(ep, &buf[i]); 144 dev_warn(ep->dev, 145 "Sending a NACK on counter %x (instead of %x) on entry %d\n", 146 ((buf[i+1] >> 28) & 0xf), 147 ep->msg_counter, 148 i/2); 149 150 if (++ep->failed_messages > 10) { 151 dev_err(ep->dev, 152 "Lost sync with interrupt messages. Stopping.\n"); 153 } else { 154 ep->ephw->hw_sync_sgl_for_device( 155 ep, 156 ep->msgbuf_dma_addr, 157 ep->msg_buf_size, 158 DMA_FROM_DEVICE); 159 160 iowrite32(0x01, /* Message NACK */ 161 ep->registers + fpga_msg_ctrl_reg); 162 } 163 return IRQ_HANDLED; 164 } else if (buf[i] & (1 << 22)) /* Last message */ 165 break; 166 } 167 168 if (i >= buf_size) { 169 dev_err(ep->dev, "Bad interrupt message. Stopping.\n"); 170 return IRQ_HANDLED; 171 } 172 173 buf_size = i + 2; 174 175 for (i = 0; i < buf_size; i += 2) { /* Scan through messages */ 176 opcode = (buf[i] >> 24) & 0xff; 177 178 msg_dir = buf[i] & 1; 179 msg_channel = (buf[i] >> 1) & 0x7ff; 180 msg_bufno = (buf[i] >> 12) & 0x3ff; 181 msg_data = buf[i+1] & 0xfffffff; 182 183 switch (opcode) { 184 case XILLYMSG_OPCODE_RELEASEBUF: 185 if ((msg_channel > ep->num_channels) || 186 (msg_channel == 0)) { 187 malformed_message(ep, &buf[i]); 188 break; 189 } 190 191 channel = ep->channels[msg_channel]; 192 193 if (msg_dir) { /* Write channel */ 194 if (msg_bufno >= channel->num_wr_buffers) { 195 malformed_message(ep, &buf[i]); 196 break; 197 } 198 spin_lock(&channel->wr_spinlock); 199 channel->wr_buffers[msg_bufno]->end_offset = 200 msg_data; 201 channel->wr_fpga_buf_idx = msg_bufno; 202 channel->wr_empty = 0; 203 channel->wr_sleepy = 0; 204 spin_unlock(&channel->wr_spinlock); 205 206 wake_up_interruptible(&channel->wr_wait); 207 208 } else { 209 /* Read channel */ 210 211 if (msg_bufno >= channel->num_rd_buffers) { 212 malformed_message(ep, &buf[i]); 213 break; 214 } 215 216 spin_lock(&channel->rd_spinlock); 217 channel->rd_fpga_buf_idx = msg_bufno; 218 channel->rd_full = 0; 219 spin_unlock(&channel->rd_spinlock); 220 221 wake_up_interruptible(&channel->rd_wait); 222 if (!channel->rd_synchronous) 223 queue_delayed_work( 224 xillybus_wq, 225 &channel->rd_workitem, 226 XILLY_RX_TIMEOUT); 227 } 228 229 break; 230 case XILLYMSG_OPCODE_NONEMPTY: 231 if ((msg_channel > ep->num_channels) || 232 (msg_channel == 0) || (!msg_dir) || 233 !ep->channels[msg_channel]->wr_supports_nonempty) { 234 malformed_message(ep, &buf[i]); 235 break; 236 } 237 238 channel = ep->channels[msg_channel]; 239 240 if (msg_bufno >= channel->num_wr_buffers) { 241 malformed_message(ep, &buf[i]); 242 break; 243 } 244 spin_lock(&channel->wr_spinlock); 245 if (msg_bufno == channel->wr_host_buf_idx) 246 channel->wr_ready = 1; 247 spin_unlock(&channel->wr_spinlock); 248 249 wake_up_interruptible(&channel->wr_ready_wait); 250 251 break; 252 case XILLYMSG_OPCODE_QUIESCEACK: 253 ep->idtlen = msg_data; 254 wake_up_interruptible(&ep->ep_wait); 255 256 break; 257 case XILLYMSG_OPCODE_FIFOEOF: 258 if ((msg_channel > ep->num_channels) || 259 (msg_channel == 0) || (!msg_dir) || 260 !ep->channels[msg_channel]->num_wr_buffers) { 261 malformed_message(ep, &buf[i]); 262 break; 263 } 264 channel = ep->channels[msg_channel]; 265 spin_lock(&channel->wr_spinlock); 266 channel->wr_eof = msg_bufno; 267 channel->wr_sleepy = 0; 268 269 channel->wr_hangup = channel->wr_empty && 270 (channel->wr_host_buf_idx == msg_bufno); 271 272 spin_unlock(&channel->wr_spinlock); 273 274 wake_up_interruptible(&channel->wr_wait); 275 276 break; 277 case XILLYMSG_OPCODE_FATAL_ERROR: 278 ep->fatal_error = 1; 279 wake_up_interruptible(&ep->ep_wait); /* For select() */ 280 dev_err(ep->dev, 281 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n"); 282 break; 283 default: 284 malformed_message(ep, &buf[i]); 285 break; 286 } 287 } 288 289 ep->ephw->hw_sync_sgl_for_device(ep, 290 ep->msgbuf_dma_addr, 291 ep->msg_buf_size, 292 DMA_FROM_DEVICE); 293 294 ep->msg_counter = (ep->msg_counter + 1) & 0xf; 295 ep->failed_messages = 0; 296 iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */ 297 298 return IRQ_HANDLED; 299 } 300 EXPORT_SYMBOL(xillybus_isr); 301 302 /* 303 * A few trivial memory management functions. 304 * NOTE: These functions are used only on probe and remove, and therefore 305 * no locks are applied! 306 */ 307 308 static void xillybus_autoflush(struct work_struct *work); 309 310 struct xilly_alloc_state { 311 void *salami; 312 int left_of_salami; 313 int nbuffer; 314 enum dma_data_direction direction; 315 u32 regdirection; 316 }; 317 318 static int xilly_get_dma_buffers(struct xilly_endpoint *ep, 319 struct xilly_alloc_state *s, 320 struct xilly_buffer **buffers, 321 int bufnum, int bytebufsize) 322 { 323 int i, rc; 324 dma_addr_t dma_addr; 325 struct device *dev = ep->dev; 326 struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */ 327 328 if (buffers) { /* Not the message buffer */ 329 this_buffer = devm_kcalloc(dev, bufnum, 330 sizeof(struct xilly_buffer), 331 GFP_KERNEL); 332 if (!this_buffer) 333 return -ENOMEM; 334 } 335 336 for (i = 0; i < bufnum; i++) { 337 /* 338 * Buffers are expected in descending size order, so there 339 * is either enough space for this buffer or none at all. 340 */ 341 342 if ((s->left_of_salami < bytebufsize) && 343 (s->left_of_salami > 0)) { 344 dev_err(ep->dev, 345 "Corrupt buffer allocation in IDT. Aborting.\n"); 346 return -ENODEV; 347 } 348 349 if (s->left_of_salami == 0) { 350 int allocorder, allocsize; 351 352 allocsize = PAGE_SIZE; 353 allocorder = 0; 354 while (bytebufsize > allocsize) { 355 allocsize *= 2; 356 allocorder++; 357 } 358 359 s->salami = (void *) devm_get_free_pages( 360 dev, 361 GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO, 362 allocorder); 363 if (!s->salami) 364 return -ENOMEM; 365 366 s->left_of_salami = allocsize; 367 } 368 369 rc = ep->ephw->map_single(ep, s->salami, 370 bytebufsize, s->direction, 371 &dma_addr); 372 if (rc) 373 return rc; 374 375 iowrite32((u32) (dma_addr & 0xffffffff), 376 ep->registers + fpga_dma_bufaddr_lowaddr_reg); 377 iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)), 378 ep->registers + fpga_dma_bufaddr_highaddr_reg); 379 380 if (buffers) { /* Not the message buffer */ 381 this_buffer->addr = s->salami; 382 this_buffer->dma_addr = dma_addr; 383 buffers[i] = this_buffer++; 384 385 iowrite32(s->regdirection | s->nbuffer++, 386 ep->registers + fpga_dma_bufno_reg); 387 } else { 388 ep->msgbuf_addr = s->salami; 389 ep->msgbuf_dma_addr = dma_addr; 390 ep->msg_buf_size = bytebufsize; 391 392 iowrite32(s->regdirection, 393 ep->registers + fpga_dma_bufno_reg); 394 } 395 396 s->left_of_salami -= bytebufsize; 397 s->salami += bytebufsize; 398 } 399 return 0; 400 } 401 402 static int xilly_setupchannels(struct xilly_endpoint *ep, 403 unsigned char *chandesc, 404 int entries) 405 { 406 struct device *dev = ep->dev; 407 int i, entry, rc; 408 struct xilly_channel *channel; 409 int channelnum, bufnum, bufsize, format, is_writebuf; 410 int bytebufsize; 411 int synchronous, allowpartial, exclusive_open, seekable; 412 int supports_nonempty; 413 int msg_buf_done = 0; 414 415 struct xilly_alloc_state rd_alloc = { 416 .salami = NULL, 417 .left_of_salami = 0, 418 .nbuffer = 1, 419 .direction = DMA_TO_DEVICE, 420 .regdirection = 0, 421 }; 422 423 struct xilly_alloc_state wr_alloc = { 424 .salami = NULL, 425 .left_of_salami = 0, 426 .nbuffer = 1, 427 .direction = DMA_FROM_DEVICE, 428 .regdirection = 0x80000000, 429 }; 430 431 channel = devm_kcalloc(dev, ep->num_channels, 432 sizeof(struct xilly_channel), GFP_KERNEL); 433 if (!channel) 434 return -ENOMEM; 435 436 ep->channels = devm_kcalloc(dev, ep->num_channels + 1, 437 sizeof(struct xilly_channel *), 438 GFP_KERNEL); 439 if (!ep->channels) 440 return -ENOMEM; 441 442 ep->channels[0] = NULL; /* Channel 0 is message buf. */ 443 444 /* Initialize all channels with defaults */ 445 446 for (i = 1; i <= ep->num_channels; i++) { 447 channel->wr_buffers = NULL; 448 channel->rd_buffers = NULL; 449 channel->num_wr_buffers = 0; 450 channel->num_rd_buffers = 0; 451 channel->wr_fpga_buf_idx = -1; 452 channel->wr_host_buf_idx = 0; 453 channel->wr_host_buf_pos = 0; 454 channel->wr_empty = 1; 455 channel->wr_ready = 0; 456 channel->wr_sleepy = 1; 457 channel->rd_fpga_buf_idx = 0; 458 channel->rd_host_buf_idx = 0; 459 channel->rd_host_buf_pos = 0; 460 channel->rd_full = 0; 461 channel->wr_ref_count = 0; 462 channel->rd_ref_count = 0; 463 464 spin_lock_init(&channel->wr_spinlock); 465 spin_lock_init(&channel->rd_spinlock); 466 mutex_init(&channel->wr_mutex); 467 mutex_init(&channel->rd_mutex); 468 init_waitqueue_head(&channel->rd_wait); 469 init_waitqueue_head(&channel->wr_wait); 470 init_waitqueue_head(&channel->wr_ready_wait); 471 472 INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush); 473 474 channel->endpoint = ep; 475 channel->chan_num = i; 476 477 channel->log2_element_size = 0; 478 479 ep->channels[i] = channel++; 480 } 481 482 for (entry = 0; entry < entries; entry++, chandesc += 4) { 483 struct xilly_buffer **buffers = NULL; 484 485 is_writebuf = chandesc[0] & 0x01; 486 channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7); 487 format = (chandesc[1] >> 4) & 0x03; 488 allowpartial = (chandesc[1] >> 6) & 0x01; 489 synchronous = (chandesc[1] >> 7) & 0x01; 490 bufsize = 1 << (chandesc[2] & 0x1f); 491 bufnum = 1 << (chandesc[3] & 0x0f); 492 exclusive_open = (chandesc[2] >> 7) & 0x01; 493 seekable = (chandesc[2] >> 6) & 0x01; 494 supports_nonempty = (chandesc[2] >> 5) & 0x01; 495 496 if ((channelnum > ep->num_channels) || 497 ((channelnum == 0) && !is_writebuf)) { 498 dev_err(ep->dev, 499 "IDT requests channel out of range. Aborting.\n"); 500 return -ENODEV; 501 } 502 503 channel = ep->channels[channelnum]; /* NULL for msg channel */ 504 505 if (!is_writebuf || channelnum > 0) { 506 channel->log2_element_size = ((format > 2) ? 507 2 : format); 508 509 bytebufsize = bufsize * 510 (1 << channel->log2_element_size); 511 512 buffers = devm_kcalloc(dev, bufnum, 513 sizeof(struct xilly_buffer *), 514 GFP_KERNEL); 515 if (!buffers) 516 return -ENOMEM; 517 } else { 518 bytebufsize = bufsize << 2; 519 } 520 521 if (!is_writebuf) { 522 channel->num_rd_buffers = bufnum; 523 channel->rd_buf_size = bytebufsize; 524 channel->rd_allow_partial = allowpartial; 525 channel->rd_synchronous = synchronous; 526 channel->rd_exclusive_open = exclusive_open; 527 channel->seekable = seekable; 528 529 channel->rd_buffers = buffers; 530 rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers, 531 bufnum, bytebufsize); 532 } else if (channelnum > 0) { 533 channel->num_wr_buffers = bufnum; 534 channel->wr_buf_size = bytebufsize; 535 536 channel->seekable = seekable; 537 channel->wr_supports_nonempty = supports_nonempty; 538 539 channel->wr_allow_partial = allowpartial; 540 channel->wr_synchronous = synchronous; 541 channel->wr_exclusive_open = exclusive_open; 542 543 channel->wr_buffers = buffers; 544 rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers, 545 bufnum, bytebufsize); 546 } else { 547 rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL, 548 bufnum, bytebufsize); 549 msg_buf_done++; 550 } 551 552 if (rc) 553 return -ENOMEM; 554 } 555 556 if (!msg_buf_done) { 557 dev_err(ep->dev, 558 "Corrupt IDT: No message buffer. Aborting.\n"); 559 return -ENODEV; 560 } 561 return 0; 562 } 563 564 static int xilly_scan_idt(struct xilly_endpoint *endpoint, 565 struct xilly_idt_handle *idt_handle) 566 { 567 int count = 0; 568 unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr; 569 unsigned char *end_of_idt = idt + endpoint->idtlen - 4; 570 unsigned char *scan; 571 int len; 572 573 scan = idt; 574 idt_handle->idt = idt; 575 576 scan++; /* Skip version number */ 577 578 while ((scan <= end_of_idt) && *scan) { 579 while ((scan <= end_of_idt) && *scan++) 580 /* Do nothing, just scan thru string */; 581 count++; 582 } 583 584 scan++; 585 586 if (scan > end_of_idt) { 587 dev_err(endpoint->dev, 588 "IDT device name list overflow. Aborting.\n"); 589 return -ENODEV; 590 } 591 idt_handle->chandesc = scan; 592 593 len = endpoint->idtlen - (3 + ((int) (scan - idt))); 594 595 if (len & 0x03) { 596 dev_err(endpoint->dev, 597 "Corrupt IDT device name list. Aborting.\n"); 598 return -ENODEV; 599 } 600 601 idt_handle->entries = len >> 2; 602 endpoint->num_channels = count; 603 604 return 0; 605 } 606 607 static int xilly_obtain_idt(struct xilly_endpoint *endpoint) 608 { 609 struct xilly_channel *channel; 610 unsigned char *version; 611 long t; 612 613 channel = endpoint->channels[1]; /* This should be generated ad-hoc */ 614 615 channel->wr_sleepy = 1; 616 617 iowrite32(1 | 618 (3 << 24), /* Opcode 3 for channel 0 = Send IDT */ 619 endpoint->registers + fpga_buf_ctrl_reg); 620 621 t = wait_event_interruptible_timeout(channel->wr_wait, 622 (!channel->wr_sleepy), 623 XILLY_TIMEOUT); 624 625 if (t <= 0) { 626 dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n"); 627 628 if (endpoint->fatal_error) 629 return -EIO; 630 631 return -ENODEV; 632 } 633 634 endpoint->ephw->hw_sync_sgl_for_cpu( 635 channel->endpoint, 636 channel->wr_buffers[0]->dma_addr, 637 channel->wr_buf_size, 638 DMA_FROM_DEVICE); 639 640 if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) { 641 dev_err(endpoint->dev, 642 "IDT length mismatch (%d != %d). Aborting.\n", 643 channel->wr_buffers[0]->end_offset, endpoint->idtlen); 644 return -ENODEV; 645 } 646 647 if (crc32_le(~0, channel->wr_buffers[0]->addr, 648 endpoint->idtlen+1) != 0) { 649 dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n"); 650 return -ENODEV; 651 } 652 653 version = channel->wr_buffers[0]->addr; 654 655 /* Check version number. Reject anything above 0x82. */ 656 if (*version > 0x82) { 657 dev_err(endpoint->dev, 658 "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n", 659 *version); 660 return -ENODEV; 661 } 662 663 return 0; 664 } 665 666 static ssize_t xillybus_read(struct file *filp, char __user *userbuf, 667 size_t count, loff_t *f_pos) 668 { 669 ssize_t rc; 670 unsigned long flags; 671 int bytes_done = 0; 672 int no_time_left = 0; 673 long deadline, left_to_sleep; 674 struct xilly_channel *channel = filp->private_data; 675 676 int empty, reached_eof, exhausted, ready; 677 /* Initializations are there only to silence warnings */ 678 679 int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; 680 int waiting_bufidx; 681 682 if (channel->endpoint->fatal_error) 683 return -EIO; 684 685 deadline = jiffies + 1 + XILLY_RX_TIMEOUT; 686 687 rc = mutex_lock_interruptible(&channel->wr_mutex); 688 if (rc) 689 return rc; 690 691 while (1) { /* Note that we may drop mutex within this loop */ 692 int bytes_to_do = count - bytes_done; 693 694 spin_lock_irqsave(&channel->wr_spinlock, flags); 695 696 empty = channel->wr_empty; 697 ready = !empty || channel->wr_ready; 698 699 if (!empty) { 700 bufidx = channel->wr_host_buf_idx; 701 bufpos = channel->wr_host_buf_pos; 702 howmany = ((channel->wr_buffers[bufidx]->end_offset 703 + 1) << channel->log2_element_size) 704 - bufpos; 705 706 /* Update wr_host_* to its post-operation state */ 707 if (howmany > bytes_to_do) { 708 bufferdone = 0; 709 710 howmany = bytes_to_do; 711 channel->wr_host_buf_pos += howmany; 712 } else { 713 bufferdone = 1; 714 715 channel->wr_host_buf_pos = 0; 716 717 if (bufidx == channel->wr_fpga_buf_idx) { 718 channel->wr_empty = 1; 719 channel->wr_sleepy = 1; 720 channel->wr_ready = 0; 721 } 722 723 if (bufidx >= (channel->num_wr_buffers - 1)) 724 channel->wr_host_buf_idx = 0; 725 else 726 channel->wr_host_buf_idx++; 727 } 728 } 729 730 /* 731 * Marking our situation after the possible changes above, 732 * for use after releasing the spinlock. 733 * 734 * empty = empty before change 735 * exhasted = empty after possible change 736 */ 737 738 reached_eof = channel->wr_empty && 739 (channel->wr_host_buf_idx == channel->wr_eof); 740 channel->wr_hangup = reached_eof; 741 exhausted = channel->wr_empty; 742 waiting_bufidx = channel->wr_host_buf_idx; 743 744 spin_unlock_irqrestore(&channel->wr_spinlock, flags); 745 746 if (!empty) { /* Go on, now without the spinlock */ 747 748 if (bufpos == 0) /* Position zero means it's virgin */ 749 channel->endpoint->ephw->hw_sync_sgl_for_cpu( 750 channel->endpoint, 751 channel->wr_buffers[bufidx]->dma_addr, 752 channel->wr_buf_size, 753 DMA_FROM_DEVICE); 754 755 if (copy_to_user( 756 userbuf, 757 channel->wr_buffers[bufidx]->addr 758 + bufpos, howmany)) 759 rc = -EFAULT; 760 761 userbuf += howmany; 762 bytes_done += howmany; 763 764 if (bufferdone) { 765 channel->endpoint->ephw->hw_sync_sgl_for_device( 766 channel->endpoint, 767 channel->wr_buffers[bufidx]->dma_addr, 768 channel->wr_buf_size, 769 DMA_FROM_DEVICE); 770 771 /* 772 * Tell FPGA the buffer is done with. It's an 773 * atomic operation to the FPGA, so what 774 * happens with other channels doesn't matter, 775 * and the certain channel is protected with 776 * the channel-specific mutex. 777 */ 778 779 iowrite32(1 | (channel->chan_num << 1) | 780 (bufidx << 12), 781 channel->endpoint->registers + 782 fpga_buf_ctrl_reg); 783 } 784 785 if (rc) { 786 mutex_unlock(&channel->wr_mutex); 787 return rc; 788 } 789 } 790 791 /* This includes a zero-count return = EOF */ 792 if ((bytes_done >= count) || reached_eof) 793 break; 794 795 if (!exhausted) 796 continue; /* More in RAM buffer(s)? Just go on. */ 797 798 if ((bytes_done > 0) && 799 (no_time_left || 800 (channel->wr_synchronous && channel->wr_allow_partial))) 801 break; 802 803 /* 804 * Nonblocking read: The "ready" flag tells us that the FPGA 805 * has data to send. In non-blocking mode, if it isn't on, 806 * just return. But if there is, we jump directly to the point 807 * where we ask for the FPGA to send all it has, and wait 808 * until that data arrives. So in a sense, we *do* block in 809 * nonblocking mode, but only for a very short time. 810 */ 811 812 if (!no_time_left && (filp->f_flags & O_NONBLOCK)) { 813 if (bytes_done > 0) 814 break; 815 816 if (ready) 817 goto desperate; 818 819 rc = -EAGAIN; 820 break; 821 } 822 823 if (!no_time_left || (bytes_done > 0)) { 824 /* 825 * Note that in case of an element-misaligned read 826 * request, offsetlimit will include the last element, 827 * which will be partially read from. 828 */ 829 int offsetlimit = ((count - bytes_done) - 1) >> 830 channel->log2_element_size; 831 int buf_elements = channel->wr_buf_size >> 832 channel->log2_element_size; 833 834 /* 835 * In synchronous mode, always send an offset limit. 836 * Just don't send a value too big. 837 */ 838 839 if (channel->wr_synchronous) { 840 /* Don't request more than one buffer */ 841 if (channel->wr_allow_partial && 842 (offsetlimit >= buf_elements)) 843 offsetlimit = buf_elements - 1; 844 845 /* Don't request more than all buffers */ 846 if (!channel->wr_allow_partial && 847 (offsetlimit >= 848 (buf_elements * channel->num_wr_buffers))) 849 offsetlimit = buf_elements * 850 channel->num_wr_buffers - 1; 851 } 852 853 /* 854 * In asynchronous mode, force early flush of a buffer 855 * only if that will allow returning a full count. The 856 * "offsetlimit < ( ... )" rather than "<=" excludes 857 * requesting a full buffer, which would obviously 858 * cause a buffer transmission anyhow 859 */ 860 861 if (channel->wr_synchronous || 862 (offsetlimit < (buf_elements - 1))) { 863 mutex_lock(&channel->endpoint->register_mutex); 864 865 iowrite32(offsetlimit, 866 channel->endpoint->registers + 867 fpga_buf_offset_reg); 868 869 iowrite32(1 | (channel->chan_num << 1) | 870 (2 << 24) | /* 2 = offset limit */ 871 (waiting_bufidx << 12), 872 channel->endpoint->registers + 873 fpga_buf_ctrl_reg); 874 875 mutex_unlock(&channel->endpoint-> 876 register_mutex); 877 } 878 } 879 880 /* 881 * If partial completion is disallowed, there is no point in 882 * timeout sleeping. Neither if no_time_left is set and 883 * there's no data. 884 */ 885 886 if (!channel->wr_allow_partial || 887 (no_time_left && (bytes_done == 0))) { 888 /* 889 * This do-loop will run more than once if another 890 * thread reasserted wr_sleepy before we got the mutex 891 * back, so we try again. 892 */ 893 894 do { 895 mutex_unlock(&channel->wr_mutex); 896 897 if (wait_event_interruptible( 898 channel->wr_wait, 899 (!channel->wr_sleepy))) 900 goto interrupted; 901 902 if (mutex_lock_interruptible( 903 &channel->wr_mutex)) 904 goto interrupted; 905 } while (channel->wr_sleepy); 906 907 continue; 908 909 interrupted: /* Mutex is not held if got here */ 910 if (channel->endpoint->fatal_error) 911 return -EIO; 912 if (bytes_done) 913 return bytes_done; 914 if (filp->f_flags & O_NONBLOCK) 915 return -EAGAIN; /* Don't admit snoozing */ 916 return -EINTR; 917 } 918 919 left_to_sleep = deadline - ((long) jiffies); 920 921 /* 922 * If our time is out, skip the waiting. We may miss wr_sleepy 923 * being deasserted but hey, almost missing the train is like 924 * missing it. 925 */ 926 927 if (left_to_sleep > 0) { 928 left_to_sleep = 929 wait_event_interruptible_timeout( 930 channel->wr_wait, 931 (!channel->wr_sleepy), 932 left_to_sleep); 933 934 if (left_to_sleep > 0) /* wr_sleepy deasserted */ 935 continue; 936 937 if (left_to_sleep < 0) { /* Interrupt */ 938 mutex_unlock(&channel->wr_mutex); 939 if (channel->endpoint->fatal_error) 940 return -EIO; 941 if (bytes_done) 942 return bytes_done; 943 return -EINTR; 944 } 945 } 946 947 desperate: 948 no_time_left = 1; /* We're out of sleeping time. Desperate! */ 949 950 if (bytes_done == 0) { 951 /* 952 * Reaching here means that we allow partial return, 953 * that we've run out of time, and that we have 954 * nothing to return. 955 * So tell the FPGA to send anything it has or gets. 956 */ 957 958 iowrite32(1 | (channel->chan_num << 1) | 959 (3 << 24) | /* Opcode 3, flush it all! */ 960 (waiting_bufidx << 12), 961 channel->endpoint->registers + 962 fpga_buf_ctrl_reg); 963 } 964 965 /* 966 * Reaching here means that we *do* have data in the buffer, 967 * but the "partial" flag disallows returning less than 968 * required. And we don't have as much. So loop again, 969 * which is likely to end up blocking indefinitely until 970 * enough data has arrived. 971 */ 972 } 973 974 mutex_unlock(&channel->wr_mutex); 975 976 if (channel->endpoint->fatal_error) 977 return -EIO; 978 979 if (rc) 980 return rc; 981 982 return bytes_done; 983 } 984 985 /* 986 * The timeout argument takes values as follows: 987 * >0 : Flush with timeout 988 * ==0 : Flush, and wait idefinitely for the flush to complete 989 * <0 : Autoflush: Flush only if there's a single buffer occupied 990 */ 991 992 static int xillybus_myflush(struct xilly_channel *channel, long timeout) 993 { 994 int rc; 995 unsigned long flags; 996 997 int end_offset_plus1; 998 int bufidx, bufidx_minus1; 999 int i; 1000 int empty; 1001 int new_rd_host_buf_pos; 1002 1003 if (channel->endpoint->fatal_error) 1004 return -EIO; 1005 rc = mutex_lock_interruptible(&channel->rd_mutex); 1006 if (rc) 1007 return rc; 1008 1009 /* 1010 * Don't flush a closed channel. This can happen when the work queued 1011 * autoflush thread fires off after the file has closed. This is not 1012 * an error, just something to dismiss. 1013 */ 1014 1015 if (!channel->rd_ref_count) 1016 goto done; 1017 1018 bufidx = channel->rd_host_buf_idx; 1019 1020 bufidx_minus1 = (bufidx == 0) ? 1021 channel->num_rd_buffers - 1 : 1022 bufidx - 1; 1023 1024 end_offset_plus1 = channel->rd_host_buf_pos >> 1025 channel->log2_element_size; 1026 1027 new_rd_host_buf_pos = channel->rd_host_buf_pos - 1028 (end_offset_plus1 << channel->log2_element_size); 1029 1030 /* Submit the current buffer if it's nonempty */ 1031 if (end_offset_plus1) { 1032 unsigned char *tail = channel->rd_buffers[bufidx]->addr + 1033 (end_offset_plus1 << channel->log2_element_size); 1034 1035 /* Copy unflushed data, so we can put it in next buffer */ 1036 for (i = 0; i < new_rd_host_buf_pos; i++) 1037 channel->rd_leftovers[i] = *tail++; 1038 1039 spin_lock_irqsave(&channel->rd_spinlock, flags); 1040 1041 /* Autoflush only if a single buffer is occupied */ 1042 1043 if ((timeout < 0) && 1044 (channel->rd_full || 1045 (bufidx_minus1 != channel->rd_fpga_buf_idx))) { 1046 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1047 /* 1048 * A new work item may be queued by the ISR exactly 1049 * now, since the execution of a work item allows the 1050 * queuing of a new one while it's running. 1051 */ 1052 goto done; 1053 } 1054 1055 /* The 4th element is never needed for data, so it's a flag */ 1056 channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0); 1057 1058 /* Set up rd_full to reflect a certain moment's state */ 1059 1060 if (bufidx == channel->rd_fpga_buf_idx) 1061 channel->rd_full = 1; 1062 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1063 1064 if (bufidx >= (channel->num_rd_buffers - 1)) 1065 channel->rd_host_buf_idx = 0; 1066 else 1067 channel->rd_host_buf_idx++; 1068 1069 channel->endpoint->ephw->hw_sync_sgl_for_device( 1070 channel->endpoint, 1071 channel->rd_buffers[bufidx]->dma_addr, 1072 channel->rd_buf_size, 1073 DMA_TO_DEVICE); 1074 1075 mutex_lock(&channel->endpoint->register_mutex); 1076 1077 iowrite32(end_offset_plus1 - 1, 1078 channel->endpoint->registers + fpga_buf_offset_reg); 1079 1080 iowrite32((channel->chan_num << 1) | /* Channel ID */ 1081 (2 << 24) | /* Opcode 2, submit buffer */ 1082 (bufidx << 12), 1083 channel->endpoint->registers + fpga_buf_ctrl_reg); 1084 1085 mutex_unlock(&channel->endpoint->register_mutex); 1086 } else if (bufidx == 0) { 1087 bufidx = channel->num_rd_buffers - 1; 1088 } else { 1089 bufidx--; 1090 } 1091 1092 channel->rd_host_buf_pos = new_rd_host_buf_pos; 1093 1094 if (timeout < 0) 1095 goto done; /* Autoflush */ 1096 1097 /* 1098 * bufidx is now the last buffer written to (or equal to 1099 * rd_fpga_buf_idx if buffer was never written to), and 1100 * channel->rd_host_buf_idx the one after it. 1101 * 1102 * If bufidx == channel->rd_fpga_buf_idx we're either empty or full. 1103 */ 1104 1105 while (1) { /* Loop waiting for draining of buffers */ 1106 spin_lock_irqsave(&channel->rd_spinlock, flags); 1107 1108 if (bufidx != channel->rd_fpga_buf_idx) 1109 channel->rd_full = 1; /* 1110 * Not really full, 1111 * but needs waiting. 1112 */ 1113 1114 empty = !channel->rd_full; 1115 1116 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1117 1118 if (empty) 1119 break; 1120 1121 /* 1122 * Indefinite sleep with mutex taken. With data waiting for 1123 * flushing user should not be surprised if open() for write 1124 * sleeps. 1125 */ 1126 if (timeout == 0) 1127 wait_event_interruptible(channel->rd_wait, 1128 (!channel->rd_full)); 1129 1130 else if (wait_event_interruptible_timeout( 1131 channel->rd_wait, 1132 (!channel->rd_full), 1133 timeout) == 0) { 1134 dev_warn(channel->endpoint->dev, 1135 "Timed out while flushing. Output data may be lost.\n"); 1136 1137 rc = -ETIMEDOUT; 1138 break; 1139 } 1140 1141 if (channel->rd_full) { 1142 rc = -EINTR; 1143 break; 1144 } 1145 } 1146 1147 done: 1148 mutex_unlock(&channel->rd_mutex); 1149 1150 if (channel->endpoint->fatal_error) 1151 return -EIO; 1152 1153 return rc; 1154 } 1155 1156 static int xillybus_flush(struct file *filp, fl_owner_t id) 1157 { 1158 if (!(filp->f_mode & FMODE_WRITE)) 1159 return 0; 1160 1161 return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */ 1162 } 1163 1164 static void xillybus_autoflush(struct work_struct *work) 1165 { 1166 struct delayed_work *workitem = container_of( 1167 work, struct delayed_work, work); 1168 struct xilly_channel *channel = container_of( 1169 workitem, struct xilly_channel, rd_workitem); 1170 int rc; 1171 1172 rc = xillybus_myflush(channel, -1); 1173 if (rc == -EINTR) 1174 dev_warn(channel->endpoint->dev, 1175 "Autoflush failed because work queue thread got a signal.\n"); 1176 else if (rc) 1177 dev_err(channel->endpoint->dev, 1178 "Autoflush failed under weird circumstances.\n"); 1179 } 1180 1181 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf, 1182 size_t count, loff_t *f_pos) 1183 { 1184 ssize_t rc; 1185 unsigned long flags; 1186 int bytes_done = 0; 1187 struct xilly_channel *channel = filp->private_data; 1188 1189 int full, exhausted; 1190 /* Initializations are there only to silence warnings */ 1191 1192 int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; 1193 int end_offset_plus1 = 0; 1194 1195 if (channel->endpoint->fatal_error) 1196 return -EIO; 1197 1198 rc = mutex_lock_interruptible(&channel->rd_mutex); 1199 if (rc) 1200 return rc; 1201 1202 while (1) { 1203 int bytes_to_do = count - bytes_done; 1204 1205 spin_lock_irqsave(&channel->rd_spinlock, flags); 1206 1207 full = channel->rd_full; 1208 1209 if (!full) { 1210 bufidx = channel->rd_host_buf_idx; 1211 bufpos = channel->rd_host_buf_pos; 1212 howmany = channel->rd_buf_size - bufpos; 1213 1214 /* 1215 * Update rd_host_* to its state after this operation. 1216 * count=0 means committing the buffer immediately, 1217 * which is like flushing, but not necessarily block. 1218 */ 1219 1220 if ((howmany > bytes_to_do) && 1221 (count || 1222 ((bufpos >> channel->log2_element_size) == 0))) { 1223 bufferdone = 0; 1224 1225 howmany = bytes_to_do; 1226 channel->rd_host_buf_pos += howmany; 1227 } else { 1228 bufferdone = 1; 1229 1230 if (count) { 1231 end_offset_plus1 = 1232 channel->rd_buf_size >> 1233 channel->log2_element_size; 1234 channel->rd_host_buf_pos = 0; 1235 } else { 1236 unsigned char *tail; 1237 int i; 1238 1239 howmany = 0; 1240 1241 end_offset_plus1 = bufpos >> 1242 channel->log2_element_size; 1243 1244 channel->rd_host_buf_pos -= 1245 end_offset_plus1 << 1246 channel->log2_element_size; 1247 1248 tail = channel-> 1249 rd_buffers[bufidx]->addr + 1250 (end_offset_plus1 << 1251 channel->log2_element_size); 1252 1253 for (i = 0; 1254 i < channel->rd_host_buf_pos; 1255 i++) 1256 channel->rd_leftovers[i] = 1257 *tail++; 1258 } 1259 1260 if (bufidx == channel->rd_fpga_buf_idx) 1261 channel->rd_full = 1; 1262 1263 if (bufidx >= (channel->num_rd_buffers - 1)) 1264 channel->rd_host_buf_idx = 0; 1265 else 1266 channel->rd_host_buf_idx++; 1267 } 1268 } 1269 1270 /* 1271 * Marking our situation after the possible changes above, 1272 * for use after releasing the spinlock. 1273 * 1274 * full = full before change 1275 * exhasted = full after possible change 1276 */ 1277 1278 exhausted = channel->rd_full; 1279 1280 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1281 1282 if (!full) { /* Go on, now without the spinlock */ 1283 unsigned char *head = 1284 channel->rd_buffers[bufidx]->addr; 1285 int i; 1286 1287 if ((bufpos == 0) || /* Zero means it's virgin */ 1288 (channel->rd_leftovers[3] != 0)) { 1289 channel->endpoint->ephw->hw_sync_sgl_for_cpu( 1290 channel->endpoint, 1291 channel->rd_buffers[bufidx]->dma_addr, 1292 channel->rd_buf_size, 1293 DMA_TO_DEVICE); 1294 1295 /* Virgin, but leftovers are due */ 1296 for (i = 0; i < bufpos; i++) 1297 *head++ = channel->rd_leftovers[i]; 1298 1299 channel->rd_leftovers[3] = 0; /* Clear flag */ 1300 } 1301 1302 if (copy_from_user( 1303 channel->rd_buffers[bufidx]->addr + bufpos, 1304 userbuf, howmany)) 1305 rc = -EFAULT; 1306 1307 userbuf += howmany; 1308 bytes_done += howmany; 1309 1310 if (bufferdone) { 1311 channel->endpoint->ephw->hw_sync_sgl_for_device( 1312 channel->endpoint, 1313 channel->rd_buffers[bufidx]->dma_addr, 1314 channel->rd_buf_size, 1315 DMA_TO_DEVICE); 1316 1317 mutex_lock(&channel->endpoint->register_mutex); 1318 1319 iowrite32(end_offset_plus1 - 1, 1320 channel->endpoint->registers + 1321 fpga_buf_offset_reg); 1322 1323 iowrite32((channel->chan_num << 1) | 1324 (2 << 24) | /* 2 = submit buffer */ 1325 (bufidx << 12), 1326 channel->endpoint->registers + 1327 fpga_buf_ctrl_reg); 1328 1329 mutex_unlock(&channel->endpoint-> 1330 register_mutex); 1331 1332 channel->rd_leftovers[3] = 1333 (channel->rd_host_buf_pos != 0); 1334 } 1335 1336 if (rc) { 1337 mutex_unlock(&channel->rd_mutex); 1338 1339 if (channel->endpoint->fatal_error) 1340 return -EIO; 1341 1342 if (!channel->rd_synchronous) 1343 queue_delayed_work( 1344 xillybus_wq, 1345 &channel->rd_workitem, 1346 XILLY_RX_TIMEOUT); 1347 1348 return rc; 1349 } 1350 } 1351 1352 if (bytes_done >= count) 1353 break; 1354 1355 if (!exhausted) 1356 continue; /* If there's more space, just go on */ 1357 1358 if ((bytes_done > 0) && channel->rd_allow_partial) 1359 break; 1360 1361 /* 1362 * Indefinite sleep with mutex taken. With data waiting for 1363 * flushing, user should not be surprised if open() for write 1364 * sleeps. 1365 */ 1366 1367 if (filp->f_flags & O_NONBLOCK) { 1368 rc = -EAGAIN; 1369 break; 1370 } 1371 1372 if (wait_event_interruptible(channel->rd_wait, 1373 (!channel->rd_full))) { 1374 mutex_unlock(&channel->rd_mutex); 1375 1376 if (channel->endpoint->fatal_error) 1377 return -EIO; 1378 1379 if (bytes_done) 1380 return bytes_done; 1381 return -EINTR; 1382 } 1383 } 1384 1385 mutex_unlock(&channel->rd_mutex); 1386 1387 if (!channel->rd_synchronous) 1388 queue_delayed_work(xillybus_wq, 1389 &channel->rd_workitem, 1390 XILLY_RX_TIMEOUT); 1391 1392 if (channel->endpoint->fatal_error) 1393 return -EIO; 1394 1395 if (rc) 1396 return rc; 1397 1398 if ((channel->rd_synchronous) && (bytes_done > 0)) { 1399 rc = xillybus_myflush(filp->private_data, 0); /* No timeout */ 1400 1401 if (rc && (rc != -EINTR)) 1402 return rc; 1403 } 1404 1405 return bytes_done; 1406 } 1407 1408 static int xillybus_open(struct inode *inode, struct file *filp) 1409 { 1410 int rc = 0; 1411 unsigned long flags; 1412 int minor = iminor(inode); 1413 int major = imajor(inode); 1414 struct xilly_endpoint *ep_iter, *endpoint = NULL; 1415 struct xilly_channel *channel; 1416 1417 mutex_lock(&ep_list_lock); 1418 1419 list_for_each_entry(ep_iter, &list_of_endpoints, ep_list) { 1420 if ((ep_iter->major == major) && 1421 (minor >= ep_iter->lowest_minor) && 1422 (minor < (ep_iter->lowest_minor + 1423 ep_iter->num_channels))) { 1424 endpoint = ep_iter; 1425 break; 1426 } 1427 } 1428 mutex_unlock(&ep_list_lock); 1429 1430 if (!endpoint) { 1431 pr_err("xillybus: open() failed to find a device for major=%d and minor=%d\n", 1432 major, minor); 1433 return -ENODEV; 1434 } 1435 1436 if (endpoint->fatal_error) 1437 return -EIO; 1438 1439 channel = endpoint->channels[1 + minor - endpoint->lowest_minor]; 1440 filp->private_data = channel; 1441 1442 /* 1443 * It gets complicated because: 1444 * 1. We don't want to take a mutex we don't have to 1445 * 2. We don't want to open one direction if the other will fail. 1446 */ 1447 1448 if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers)) 1449 return -ENODEV; 1450 1451 if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers)) 1452 return -ENODEV; 1453 1454 if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) && 1455 (channel->wr_synchronous || !channel->wr_allow_partial || 1456 !channel->wr_supports_nonempty)) { 1457 dev_err(endpoint->dev, 1458 "open() failed: O_NONBLOCK not allowed for read on this device\n"); 1459 return -ENODEV; 1460 } 1461 1462 if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) && 1463 (channel->rd_synchronous || !channel->rd_allow_partial)) { 1464 dev_err(endpoint->dev, 1465 "open() failed: O_NONBLOCK not allowed for write on this device\n"); 1466 return -ENODEV; 1467 } 1468 1469 /* 1470 * Note: open() may block on getting mutexes despite O_NONBLOCK. 1471 * This shouldn't occur normally, since multiple open of the same 1472 * file descriptor is almost always prohibited anyhow 1473 * (*_exclusive_open is normally set in real-life systems). 1474 */ 1475 1476 if (filp->f_mode & FMODE_READ) { 1477 rc = mutex_lock_interruptible(&channel->wr_mutex); 1478 if (rc) 1479 return rc; 1480 } 1481 1482 if (filp->f_mode & FMODE_WRITE) { 1483 rc = mutex_lock_interruptible(&channel->rd_mutex); 1484 if (rc) 1485 goto unlock_wr; 1486 } 1487 1488 if ((filp->f_mode & FMODE_READ) && 1489 (channel->wr_ref_count != 0) && 1490 (channel->wr_exclusive_open)) { 1491 rc = -EBUSY; 1492 goto unlock; 1493 } 1494 1495 if ((filp->f_mode & FMODE_WRITE) && 1496 (channel->rd_ref_count != 0) && 1497 (channel->rd_exclusive_open)) { 1498 rc = -EBUSY; 1499 goto unlock; 1500 } 1501 1502 if (filp->f_mode & FMODE_READ) { 1503 if (channel->wr_ref_count == 0) { /* First open of file */ 1504 /* Move the host to first buffer */ 1505 spin_lock_irqsave(&channel->wr_spinlock, flags); 1506 channel->wr_host_buf_idx = 0; 1507 channel->wr_host_buf_pos = 0; 1508 channel->wr_fpga_buf_idx = -1; 1509 channel->wr_empty = 1; 1510 channel->wr_ready = 0; 1511 channel->wr_sleepy = 1; 1512 channel->wr_eof = -1; 1513 channel->wr_hangup = 0; 1514 1515 spin_unlock_irqrestore(&channel->wr_spinlock, flags); 1516 1517 iowrite32(1 | (channel->chan_num << 1) | 1518 (4 << 24) | /* Opcode 4, open channel */ 1519 ((channel->wr_synchronous & 1) << 23), 1520 channel->endpoint->registers + 1521 fpga_buf_ctrl_reg); 1522 } 1523 1524 channel->wr_ref_count++; 1525 } 1526 1527 if (filp->f_mode & FMODE_WRITE) { 1528 if (channel->rd_ref_count == 0) { /* First open of file */ 1529 /* Move the host to first buffer */ 1530 spin_lock_irqsave(&channel->rd_spinlock, flags); 1531 channel->rd_host_buf_idx = 0; 1532 channel->rd_host_buf_pos = 0; 1533 channel->rd_leftovers[3] = 0; /* No leftovers. */ 1534 channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1; 1535 channel->rd_full = 0; 1536 1537 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1538 1539 iowrite32((channel->chan_num << 1) | 1540 (4 << 24), /* Opcode 4, open channel */ 1541 channel->endpoint->registers + 1542 fpga_buf_ctrl_reg); 1543 } 1544 1545 channel->rd_ref_count++; 1546 } 1547 1548 unlock: 1549 if (filp->f_mode & FMODE_WRITE) 1550 mutex_unlock(&channel->rd_mutex); 1551 unlock_wr: 1552 if (filp->f_mode & FMODE_READ) 1553 mutex_unlock(&channel->wr_mutex); 1554 1555 if (!rc && (!channel->seekable)) 1556 return nonseekable_open(inode, filp); 1557 1558 return rc; 1559 } 1560 1561 static int xillybus_release(struct inode *inode, struct file *filp) 1562 { 1563 unsigned long flags; 1564 struct xilly_channel *channel = filp->private_data; 1565 1566 int buf_idx; 1567 int eof; 1568 1569 if (channel->endpoint->fatal_error) 1570 return -EIO; 1571 1572 if (filp->f_mode & FMODE_WRITE) { 1573 mutex_lock(&channel->rd_mutex); 1574 1575 channel->rd_ref_count--; 1576 1577 if (channel->rd_ref_count == 0) { 1578 /* 1579 * We rely on the kernel calling flush() 1580 * before we get here. 1581 */ 1582 1583 iowrite32((channel->chan_num << 1) | /* Channel ID */ 1584 (5 << 24), /* Opcode 5, close channel */ 1585 channel->endpoint->registers + 1586 fpga_buf_ctrl_reg); 1587 } 1588 mutex_unlock(&channel->rd_mutex); 1589 } 1590 1591 if (filp->f_mode & FMODE_READ) { 1592 mutex_lock(&channel->wr_mutex); 1593 1594 channel->wr_ref_count--; 1595 1596 if (channel->wr_ref_count == 0) { 1597 iowrite32(1 | (channel->chan_num << 1) | 1598 (5 << 24), /* Opcode 5, close channel */ 1599 channel->endpoint->registers + 1600 fpga_buf_ctrl_reg); 1601 1602 /* 1603 * This is crazily cautious: We make sure that not 1604 * only that we got an EOF (be it because we closed 1605 * the channel or because of a user's EOF), but verify 1606 * that it's one beyond the last buffer arrived, so 1607 * we have no leftover buffers pending before wrapping 1608 * up (which can only happen in asynchronous channels, 1609 * BTW) 1610 */ 1611 1612 while (1) { 1613 spin_lock_irqsave(&channel->wr_spinlock, 1614 flags); 1615 buf_idx = channel->wr_fpga_buf_idx; 1616 eof = channel->wr_eof; 1617 channel->wr_sleepy = 1; 1618 spin_unlock_irqrestore(&channel->wr_spinlock, 1619 flags); 1620 1621 /* 1622 * Check if eof points at the buffer after 1623 * the last one the FPGA submitted. Note that 1624 * no EOF is marked by negative eof. 1625 */ 1626 1627 buf_idx++; 1628 if (buf_idx == channel->num_wr_buffers) 1629 buf_idx = 0; 1630 1631 if (buf_idx == eof) 1632 break; 1633 1634 /* 1635 * Steal extra 100 ms if awaken by interrupt. 1636 * This is a simple workaround for an 1637 * interrupt pending when entering, which would 1638 * otherwise result in declaring the hardware 1639 * non-responsive. 1640 */ 1641 1642 if (wait_event_interruptible( 1643 channel->wr_wait, 1644 (!channel->wr_sleepy))) 1645 msleep(100); 1646 1647 if (channel->wr_sleepy) { 1648 mutex_unlock(&channel->wr_mutex); 1649 dev_warn(channel->endpoint->dev, 1650 "Hardware failed to respond to close command, therefore left in messy state.\n"); 1651 return -EINTR; 1652 } 1653 } 1654 } 1655 1656 mutex_unlock(&channel->wr_mutex); 1657 } 1658 1659 return 0; 1660 } 1661 1662 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence) 1663 { 1664 struct xilly_channel *channel = filp->private_data; 1665 loff_t pos = filp->f_pos; 1666 int rc = 0; 1667 1668 /* 1669 * Take both mutexes not allowing interrupts, since it seems like 1670 * common applications don't expect an -EINTR here. Besides, multiple 1671 * access to a single file descriptor on seekable devices is a mess 1672 * anyhow. 1673 */ 1674 1675 if (channel->endpoint->fatal_error) 1676 return -EIO; 1677 1678 mutex_lock(&channel->wr_mutex); 1679 mutex_lock(&channel->rd_mutex); 1680 1681 switch (whence) { 1682 case SEEK_SET: 1683 pos = offset; 1684 break; 1685 case SEEK_CUR: 1686 pos += offset; 1687 break; 1688 case SEEK_END: 1689 pos = offset; /* Going to the end => to the beginning */ 1690 break; 1691 default: 1692 rc = -EINVAL; 1693 goto end; 1694 } 1695 1696 /* In any case, we must finish on an element boundary */ 1697 if (pos & ((1 << channel->log2_element_size) - 1)) { 1698 rc = -EINVAL; 1699 goto end; 1700 } 1701 1702 mutex_lock(&channel->endpoint->register_mutex); 1703 1704 iowrite32(pos >> channel->log2_element_size, 1705 channel->endpoint->registers + fpga_buf_offset_reg); 1706 1707 iowrite32((channel->chan_num << 1) | 1708 (6 << 24), /* Opcode 6, set address */ 1709 channel->endpoint->registers + fpga_buf_ctrl_reg); 1710 1711 mutex_unlock(&channel->endpoint->register_mutex); 1712 1713 end: 1714 mutex_unlock(&channel->rd_mutex); 1715 mutex_unlock(&channel->wr_mutex); 1716 1717 if (rc) /* Return error after releasing mutexes */ 1718 return rc; 1719 1720 filp->f_pos = pos; 1721 1722 /* 1723 * Since seekable devices are allowed only when the channel is 1724 * synchronous, we assume that there is no data pending in either 1725 * direction (which holds true as long as no concurrent access on the 1726 * file descriptor takes place). 1727 * The only thing we may need to throw away is leftovers from partial 1728 * write() flush. 1729 */ 1730 1731 channel->rd_leftovers[3] = 0; 1732 1733 return pos; 1734 } 1735 1736 static __poll_t xillybus_poll(struct file *filp, poll_table *wait) 1737 { 1738 struct xilly_channel *channel = filp->private_data; 1739 __poll_t mask = 0; 1740 unsigned long flags; 1741 1742 poll_wait(filp, &channel->endpoint->ep_wait, wait); 1743 1744 /* 1745 * poll() won't play ball regarding read() channels which 1746 * aren't asynchronous and support the nonempty message. Allowing 1747 * that will create situations where data has been delivered at 1748 * the FPGA, and users expecting select() to wake up, which it may 1749 * not. 1750 */ 1751 1752 if (!channel->wr_synchronous && channel->wr_supports_nonempty) { 1753 poll_wait(filp, &channel->wr_wait, wait); 1754 poll_wait(filp, &channel->wr_ready_wait, wait); 1755 1756 spin_lock_irqsave(&channel->wr_spinlock, flags); 1757 if (!channel->wr_empty || channel->wr_ready) 1758 mask |= EPOLLIN | EPOLLRDNORM; 1759 1760 if (channel->wr_hangup) 1761 /* 1762 * Not EPOLLHUP, because its behavior is in the 1763 * mist, and EPOLLIN does what we want: Wake up 1764 * the read file descriptor so it sees EOF. 1765 */ 1766 mask |= EPOLLIN | EPOLLRDNORM; 1767 spin_unlock_irqrestore(&channel->wr_spinlock, flags); 1768 } 1769 1770 /* 1771 * If partial data write is disallowed on a write() channel, 1772 * it's pointless to ever signal OK to write, because is could 1773 * block despite some space being available. 1774 */ 1775 1776 if (channel->rd_allow_partial) { 1777 poll_wait(filp, &channel->rd_wait, wait); 1778 1779 spin_lock_irqsave(&channel->rd_spinlock, flags); 1780 if (!channel->rd_full) 1781 mask |= EPOLLOUT | EPOLLWRNORM; 1782 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1783 } 1784 1785 if (channel->endpoint->fatal_error) 1786 mask |= EPOLLERR; 1787 1788 return mask; 1789 } 1790 1791 static const struct file_operations xillybus_fops = { 1792 .owner = THIS_MODULE, 1793 .read = xillybus_read, 1794 .write = xillybus_write, 1795 .open = xillybus_open, 1796 .flush = xillybus_flush, 1797 .release = xillybus_release, 1798 .llseek = xillybus_llseek, 1799 .poll = xillybus_poll, 1800 }; 1801 1802 static int xillybus_init_chrdev(struct xilly_endpoint *endpoint, 1803 const unsigned char *idt) 1804 { 1805 int rc; 1806 dev_t dev; 1807 int devnum, i, minor, major; 1808 char devname[48]; 1809 struct device *device; 1810 1811 rc = alloc_chrdev_region(&dev, 0, /* minor start */ 1812 endpoint->num_channels, 1813 xillyname); 1814 if (rc) { 1815 dev_warn(endpoint->dev, "Failed to obtain major/minors"); 1816 return rc; 1817 } 1818 1819 endpoint->major = major = MAJOR(dev); 1820 endpoint->lowest_minor = minor = MINOR(dev); 1821 1822 cdev_init(&endpoint->cdev, &xillybus_fops); 1823 endpoint->cdev.owner = endpoint->ephw->owner; 1824 rc = cdev_add(&endpoint->cdev, MKDEV(major, minor), 1825 endpoint->num_channels); 1826 if (rc) { 1827 dev_warn(endpoint->dev, "Failed to add cdev. Aborting.\n"); 1828 goto unregister_chrdev; 1829 } 1830 1831 idt++; 1832 1833 for (i = minor, devnum = 0; 1834 devnum < endpoint->num_channels; 1835 devnum++, i++) { 1836 snprintf(devname, sizeof(devname)-1, "xillybus_%s", idt); 1837 1838 devname[sizeof(devname)-1] = 0; /* Should never matter */ 1839 1840 while (*idt++) 1841 /* Skip to next */; 1842 1843 device = device_create(xillybus_class, 1844 NULL, 1845 MKDEV(major, i), 1846 NULL, 1847 "%s", devname); 1848 1849 if (IS_ERR(device)) { 1850 dev_warn(endpoint->dev, 1851 "Failed to create %s device. Aborting.\n", 1852 devname); 1853 rc = -ENODEV; 1854 goto unroll_device_create; 1855 } 1856 } 1857 1858 dev_info(endpoint->dev, "Created %d device files.\n", 1859 endpoint->num_channels); 1860 return 0; /* succeed */ 1861 1862 unroll_device_create: 1863 devnum--; i--; 1864 for (; devnum >= 0; devnum--, i--) 1865 device_destroy(xillybus_class, MKDEV(major, i)); 1866 1867 cdev_del(&endpoint->cdev); 1868 unregister_chrdev: 1869 unregister_chrdev_region(MKDEV(major, minor), endpoint->num_channels); 1870 1871 return rc; 1872 } 1873 1874 static void xillybus_cleanup_chrdev(struct xilly_endpoint *endpoint) 1875 { 1876 int minor; 1877 1878 for (minor = endpoint->lowest_minor; 1879 minor < (endpoint->lowest_minor + endpoint->num_channels); 1880 minor++) 1881 device_destroy(xillybus_class, MKDEV(endpoint->major, minor)); 1882 cdev_del(&endpoint->cdev); 1883 unregister_chrdev_region(MKDEV(endpoint->major, 1884 endpoint->lowest_minor), 1885 endpoint->num_channels); 1886 1887 dev_info(endpoint->dev, "Removed %d device files.\n", 1888 endpoint->num_channels); 1889 } 1890 1891 struct xilly_endpoint *xillybus_init_endpoint(struct pci_dev *pdev, 1892 struct device *dev, 1893 struct xilly_endpoint_hardware 1894 *ephw) 1895 { 1896 struct xilly_endpoint *endpoint; 1897 1898 endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL); 1899 if (!endpoint) 1900 return NULL; 1901 1902 endpoint->pdev = pdev; 1903 endpoint->dev = dev; 1904 endpoint->ephw = ephw; 1905 endpoint->msg_counter = 0x0b; 1906 endpoint->failed_messages = 0; 1907 endpoint->fatal_error = 0; 1908 1909 init_waitqueue_head(&endpoint->ep_wait); 1910 mutex_init(&endpoint->register_mutex); 1911 1912 return endpoint; 1913 } 1914 EXPORT_SYMBOL(xillybus_init_endpoint); 1915 1916 static int xilly_quiesce(struct xilly_endpoint *endpoint) 1917 { 1918 long t; 1919 1920 endpoint->idtlen = -1; 1921 1922 iowrite32((u32) (endpoint->dma_using_dac & 0x0001), 1923 endpoint->registers + fpga_dma_control_reg); 1924 1925 t = wait_event_interruptible_timeout(endpoint->ep_wait, 1926 (endpoint->idtlen >= 0), 1927 XILLY_TIMEOUT); 1928 if (t <= 0) { 1929 dev_err(endpoint->dev, 1930 "Failed to quiesce the device on exit.\n"); 1931 return -ENODEV; 1932 } 1933 return 0; 1934 } 1935 1936 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint) 1937 { 1938 int rc; 1939 long t; 1940 1941 void *bootstrap_resources; 1942 int idtbuffersize = (1 << PAGE_SHIFT); 1943 struct device *dev = endpoint->dev; 1944 1945 /* 1946 * The bogus IDT is used during bootstrap for allocating the initial 1947 * message buffer, and then the message buffer and space for the IDT 1948 * itself. The initial message buffer is of a single page's size, but 1949 * it's soon replaced with a more modest one (and memory is freed). 1950 */ 1951 1952 unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0, 1953 3, 192, PAGE_SHIFT, 0 }; 1954 struct xilly_idt_handle idt_handle; 1955 1956 /* 1957 * Writing the value 0x00000001 to Endianness register signals which 1958 * endianness this processor is using, so the FPGA can swap words as 1959 * necessary. 1960 */ 1961 1962 iowrite32(1, endpoint->registers + fpga_endian_reg); 1963 1964 /* Bootstrap phase I: Allocate temporary message buffer */ 1965 1966 bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL); 1967 if (!bootstrap_resources) 1968 return -ENOMEM; 1969 1970 endpoint->num_channels = 0; 1971 1972 rc = xilly_setupchannels(endpoint, bogus_idt, 1); 1973 if (rc) 1974 return rc; 1975 1976 /* Clear the message subsystem (and counter in particular) */ 1977 iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg); 1978 1979 endpoint->idtlen = -1; 1980 1981 /* 1982 * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT 1983 * buffer size. 1984 */ 1985 iowrite32((u32) (endpoint->dma_using_dac & 0x0001), 1986 endpoint->registers + fpga_dma_control_reg); 1987 1988 t = wait_event_interruptible_timeout(endpoint->ep_wait, 1989 (endpoint->idtlen >= 0), 1990 XILLY_TIMEOUT); 1991 if (t <= 0) { 1992 dev_err(endpoint->dev, "No response from FPGA. Aborting.\n"); 1993 return -ENODEV; 1994 } 1995 1996 /* Enable DMA */ 1997 iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)), 1998 endpoint->registers + fpga_dma_control_reg); 1999 2000 /* Bootstrap phase II: Allocate buffer for IDT and obtain it */ 2001 while (endpoint->idtlen >= idtbuffersize) { 2002 idtbuffersize *= 2; 2003 bogus_idt[6]++; 2004 } 2005 2006 endpoint->num_channels = 1; 2007 2008 rc = xilly_setupchannels(endpoint, bogus_idt, 2); 2009 if (rc) 2010 goto failed_idt; 2011 2012 rc = xilly_obtain_idt(endpoint); 2013 if (rc) 2014 goto failed_idt; 2015 2016 rc = xilly_scan_idt(endpoint, &idt_handle); 2017 if (rc) 2018 goto failed_idt; 2019 2020 devres_close_group(dev, bootstrap_resources); 2021 2022 /* Bootstrap phase III: Allocate buffers according to IDT */ 2023 2024 rc = xilly_setupchannels(endpoint, 2025 idt_handle.chandesc, 2026 idt_handle.entries); 2027 if (rc) 2028 goto failed_idt; 2029 2030 /* 2031 * endpoint is now completely configured. We put it on the list 2032 * available to open() before registering the char device(s) 2033 */ 2034 2035 mutex_lock(&ep_list_lock); 2036 list_add_tail(&endpoint->ep_list, &list_of_endpoints); 2037 mutex_unlock(&ep_list_lock); 2038 2039 rc = xillybus_init_chrdev(endpoint, idt_handle.idt); 2040 if (rc) 2041 goto failed_chrdevs; 2042 2043 devres_release_group(dev, bootstrap_resources); 2044 2045 return 0; 2046 2047 failed_chrdevs: 2048 mutex_lock(&ep_list_lock); 2049 list_del(&endpoint->ep_list); 2050 mutex_unlock(&ep_list_lock); 2051 2052 failed_idt: 2053 xilly_quiesce(endpoint); 2054 flush_workqueue(xillybus_wq); 2055 2056 return rc; 2057 } 2058 EXPORT_SYMBOL(xillybus_endpoint_discovery); 2059 2060 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint) 2061 { 2062 xillybus_cleanup_chrdev(endpoint); 2063 2064 mutex_lock(&ep_list_lock); 2065 list_del(&endpoint->ep_list); 2066 mutex_unlock(&ep_list_lock); 2067 2068 xilly_quiesce(endpoint); 2069 2070 /* 2071 * Flushing is done upon endpoint release to prevent access to memory 2072 * just about to be released. This makes the quiesce complete. 2073 */ 2074 flush_workqueue(xillybus_wq); 2075 } 2076 EXPORT_SYMBOL(xillybus_endpoint_remove); 2077 2078 static int __init xillybus_init(void) 2079 { 2080 mutex_init(&ep_list_lock); 2081 2082 xillybus_class = class_create(THIS_MODULE, xillyname); 2083 if (IS_ERR(xillybus_class)) 2084 return PTR_ERR(xillybus_class); 2085 2086 xillybus_wq = alloc_workqueue(xillyname, 0, 0); 2087 if (!xillybus_wq) { 2088 class_destroy(xillybus_class); 2089 return -ENOMEM; 2090 } 2091 2092 return 0; 2093 } 2094 2095 static void __exit xillybus_exit(void) 2096 { 2097 /* flush_workqueue() was called for each endpoint released */ 2098 destroy_workqueue(xillybus_wq); 2099 2100 class_destroy(xillybus_class); 2101 } 2102 2103 module_init(xillybus_init); 2104 module_exit(xillybus_exit); 2105