1 /* 2 * linux/drivers/misc/xillybus_core.c 3 * 4 * Copyright 2011 Xillybus Ltd, http://xillybus.com 5 * 6 * Driver for the Xillybus FPGA/host framework. 7 * 8 * This driver interfaces with a special IP core in an FPGA, setting up 9 * a pipe between a hardware FIFO in the programmable logic and a device 10 * file in the host. The number of such pipes and their attributes are 11 * set up on the logic. This driver detects these automatically and 12 * creates the device files accordingly. 13 * 14 * This program is free software; you can redistribute it and/or modify 15 * it under the smems of the GNU General Public License as published by 16 * the Free Software Foundation; version 2 of the License. 17 */ 18 19 #include <linux/list.h> 20 #include <linux/device.h> 21 #include <linux/module.h> 22 #include <linux/io.h> 23 #include <linux/dma-mapping.h> 24 #include <linux/interrupt.h> 25 #include <linux/sched.h> 26 #include <linux/fs.h> 27 #include <linux/cdev.h> 28 #include <linux/spinlock.h> 29 #include <linux/mutex.h> 30 #include <linux/crc32.h> 31 #include <linux/poll.h> 32 #include <linux/delay.h> 33 #include <linux/slab.h> 34 #include <linux/workqueue.h> 35 #include "xillybus.h" 36 37 MODULE_DESCRIPTION("Xillybus core functions"); 38 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); 39 MODULE_VERSION("1.07"); 40 MODULE_ALIAS("xillybus_core"); 41 MODULE_LICENSE("GPL v2"); 42 43 /* General timeout is 100 ms, rx timeout is 10 ms */ 44 #define XILLY_RX_TIMEOUT (10*HZ/1000) 45 #define XILLY_TIMEOUT (100*HZ/1000) 46 47 #define fpga_msg_ctrl_reg 0x0008 48 #define fpga_dma_control_reg 0x0020 49 #define fpga_dma_bufno_reg 0x0024 50 #define fpga_dma_bufaddr_lowaddr_reg 0x0028 51 #define fpga_dma_bufaddr_highaddr_reg 0x002c 52 #define fpga_buf_ctrl_reg 0x0030 53 #define fpga_buf_offset_reg 0x0034 54 #define fpga_endian_reg 0x0040 55 56 #define XILLYMSG_OPCODE_RELEASEBUF 1 57 #define XILLYMSG_OPCODE_QUIESCEACK 2 58 #define XILLYMSG_OPCODE_FIFOEOF 3 59 #define XILLYMSG_OPCODE_FATAL_ERROR 4 60 #define XILLYMSG_OPCODE_NONEMPTY 5 61 62 static const char xillyname[] = "xillybus"; 63 64 static struct class *xillybus_class; 65 66 /* 67 * ep_list_lock is the last lock to be taken; No other lock requests are 68 * allowed while holding it. It merely protects list_of_endpoints, and not 69 * the endpoints listed in it. 70 */ 71 72 static LIST_HEAD(list_of_endpoints); 73 static struct mutex ep_list_lock; 74 static struct workqueue_struct *xillybus_wq; 75 76 /* 77 * Locking scheme: Mutexes protect invocations of character device methods. 78 * If both locks are taken, wr_mutex is taken first, rd_mutex second. 79 * 80 * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the 81 * buffers' end_offset fields against changes made by IRQ handler (and in 82 * theory, other file request handlers, but the mutex handles that). Nothing 83 * else. 84 * They are held for short direct memory manipulations. Needless to say, 85 * no mutex locking is allowed when a spinlock is held. 86 * 87 * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset. 88 * 89 * register_mutex is endpoint-specific, and is held when non-atomic 90 * register operations are performed. wr_mutex and rd_mutex may be 91 * held when register_mutex is taken, but none of the spinlocks. Note that 92 * register_mutex doesn't protect against sporadic buf_ctrl_reg writes 93 * which are unrelated to buf_offset_reg, since they are harmless. 94 * 95 * Blocking on the wait queues is allowed with mutexes held, but not with 96 * spinlocks. 97 * 98 * Only interruptible blocking is allowed on mutexes and wait queues. 99 * 100 * All in all, the locking order goes (with skips allowed, of course): 101 * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock 102 */ 103 104 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf) 105 { 106 int opcode; 107 int msg_channel, msg_bufno, msg_data, msg_dir; 108 109 opcode = (buf[0] >> 24) & 0xff; 110 msg_dir = buf[0] & 1; 111 msg_channel = (buf[0] >> 1) & 0x7ff; 112 msg_bufno = (buf[0] >> 12) & 0x3ff; 113 msg_data = buf[1] & 0xfffffff; 114 115 dev_warn(endpoint->dev, 116 "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n", 117 opcode, msg_channel, msg_dir, msg_bufno, msg_data); 118 } 119 120 /* 121 * xillybus_isr assumes the interrupt is allocated exclusively to it, 122 * which is the natural case MSI and several other hardware-oriented 123 * interrupts. Sharing is not allowed. 124 */ 125 126 irqreturn_t xillybus_isr(int irq, void *data) 127 { 128 struct xilly_endpoint *ep = data; 129 u32 *buf; 130 unsigned int buf_size; 131 int i; 132 int opcode; 133 unsigned int msg_channel, msg_bufno, msg_data, msg_dir; 134 struct xilly_channel *channel; 135 136 buf = ep->msgbuf_addr; 137 buf_size = ep->msg_buf_size/sizeof(u32); 138 139 ep->ephw->hw_sync_sgl_for_cpu(ep, 140 ep->msgbuf_dma_addr, 141 ep->msg_buf_size, 142 DMA_FROM_DEVICE); 143 144 for (i = 0; i < buf_size; i += 2) { 145 if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) { 146 malformed_message(ep, &buf[i]); 147 dev_warn(ep->dev, 148 "Sending a NACK on counter %x (instead of %x) on entry %d\n", 149 ((buf[i+1] >> 28) & 0xf), 150 ep->msg_counter, 151 i/2); 152 153 if (++ep->failed_messages > 10) { 154 dev_err(ep->dev, 155 "Lost sync with interrupt messages. Stopping.\n"); 156 } else { 157 ep->ephw->hw_sync_sgl_for_device( 158 ep, 159 ep->msgbuf_dma_addr, 160 ep->msg_buf_size, 161 DMA_FROM_DEVICE); 162 163 iowrite32(0x01, /* Message NACK */ 164 ep->registers + fpga_msg_ctrl_reg); 165 } 166 return IRQ_HANDLED; 167 } else if (buf[i] & (1 << 22)) /* Last message */ 168 break; 169 } 170 171 if (i >= buf_size) { 172 dev_err(ep->dev, "Bad interrupt message. Stopping.\n"); 173 return IRQ_HANDLED; 174 } 175 176 buf_size = i + 2; 177 178 for (i = 0; i < buf_size; i += 2) { /* Scan through messages */ 179 opcode = (buf[i] >> 24) & 0xff; 180 181 msg_dir = buf[i] & 1; 182 msg_channel = (buf[i] >> 1) & 0x7ff; 183 msg_bufno = (buf[i] >> 12) & 0x3ff; 184 msg_data = buf[i+1] & 0xfffffff; 185 186 switch (opcode) { 187 case XILLYMSG_OPCODE_RELEASEBUF: 188 if ((msg_channel > ep->num_channels) || 189 (msg_channel == 0)) { 190 malformed_message(ep, &buf[i]); 191 break; 192 } 193 194 channel = ep->channels[msg_channel]; 195 196 if (msg_dir) { /* Write channel */ 197 if (msg_bufno >= channel->num_wr_buffers) { 198 malformed_message(ep, &buf[i]); 199 break; 200 } 201 spin_lock(&channel->wr_spinlock); 202 channel->wr_buffers[msg_bufno]->end_offset = 203 msg_data; 204 channel->wr_fpga_buf_idx = msg_bufno; 205 channel->wr_empty = 0; 206 channel->wr_sleepy = 0; 207 spin_unlock(&channel->wr_spinlock); 208 209 wake_up_interruptible(&channel->wr_wait); 210 211 } else { 212 /* Read channel */ 213 214 if (msg_bufno >= channel->num_rd_buffers) { 215 malformed_message(ep, &buf[i]); 216 break; 217 } 218 219 spin_lock(&channel->rd_spinlock); 220 channel->rd_fpga_buf_idx = msg_bufno; 221 channel->rd_full = 0; 222 spin_unlock(&channel->rd_spinlock); 223 224 wake_up_interruptible(&channel->rd_wait); 225 if (!channel->rd_synchronous) 226 queue_delayed_work( 227 xillybus_wq, 228 &channel->rd_workitem, 229 XILLY_RX_TIMEOUT); 230 } 231 232 break; 233 case XILLYMSG_OPCODE_NONEMPTY: 234 if ((msg_channel > ep->num_channels) || 235 (msg_channel == 0) || (!msg_dir) || 236 !ep->channels[msg_channel]->wr_supports_nonempty) { 237 malformed_message(ep, &buf[i]); 238 break; 239 } 240 241 channel = ep->channels[msg_channel]; 242 243 if (msg_bufno >= channel->num_wr_buffers) { 244 malformed_message(ep, &buf[i]); 245 break; 246 } 247 spin_lock(&channel->wr_spinlock); 248 if (msg_bufno == channel->wr_host_buf_idx) 249 channel->wr_ready = 1; 250 spin_unlock(&channel->wr_spinlock); 251 252 wake_up_interruptible(&channel->wr_ready_wait); 253 254 break; 255 case XILLYMSG_OPCODE_QUIESCEACK: 256 ep->idtlen = msg_data; 257 wake_up_interruptible(&ep->ep_wait); 258 259 break; 260 case XILLYMSG_OPCODE_FIFOEOF: 261 if ((msg_channel > ep->num_channels) || 262 (msg_channel == 0) || (!msg_dir) || 263 !ep->channels[msg_channel]->num_wr_buffers) { 264 malformed_message(ep, &buf[i]); 265 break; 266 } 267 channel = ep->channels[msg_channel]; 268 spin_lock(&channel->wr_spinlock); 269 channel->wr_eof = msg_bufno; 270 channel->wr_sleepy = 0; 271 272 channel->wr_hangup = channel->wr_empty && 273 (channel->wr_host_buf_idx == msg_bufno); 274 275 spin_unlock(&channel->wr_spinlock); 276 277 wake_up_interruptible(&channel->wr_wait); 278 279 break; 280 case XILLYMSG_OPCODE_FATAL_ERROR: 281 ep->fatal_error = 1; 282 wake_up_interruptible(&ep->ep_wait); /* For select() */ 283 dev_err(ep->dev, 284 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n"); 285 break; 286 default: 287 malformed_message(ep, &buf[i]); 288 break; 289 } 290 } 291 292 ep->ephw->hw_sync_sgl_for_device(ep, 293 ep->msgbuf_dma_addr, 294 ep->msg_buf_size, 295 DMA_FROM_DEVICE); 296 297 ep->msg_counter = (ep->msg_counter + 1) & 0xf; 298 ep->failed_messages = 0; 299 iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */ 300 301 return IRQ_HANDLED; 302 } 303 EXPORT_SYMBOL(xillybus_isr); 304 305 /* 306 * A few trivial memory management functions. 307 * NOTE: These functions are used only on probe and remove, and therefore 308 * no locks are applied! 309 */ 310 311 static void xillybus_autoflush(struct work_struct *work); 312 313 struct xilly_alloc_state { 314 void *salami; 315 int left_of_salami; 316 int nbuffer; 317 enum dma_data_direction direction; 318 u32 regdirection; 319 }; 320 321 static int xilly_get_dma_buffers(struct xilly_endpoint *ep, 322 struct xilly_alloc_state *s, 323 struct xilly_buffer **buffers, 324 int bufnum, int bytebufsize) 325 { 326 int i, rc; 327 dma_addr_t dma_addr; 328 struct device *dev = ep->dev; 329 struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */ 330 331 if (buffers) { /* Not the message buffer */ 332 this_buffer = devm_kcalloc(dev, bufnum, 333 sizeof(struct xilly_buffer), 334 GFP_KERNEL); 335 if (!this_buffer) 336 return -ENOMEM; 337 } 338 339 for (i = 0; i < bufnum; i++) { 340 /* 341 * Buffers are expected in descending size order, so there 342 * is either enough space for this buffer or none at all. 343 */ 344 345 if ((s->left_of_salami < bytebufsize) && 346 (s->left_of_salami > 0)) { 347 dev_err(ep->dev, 348 "Corrupt buffer allocation in IDT. Aborting.\n"); 349 return -ENODEV; 350 } 351 352 if (s->left_of_salami == 0) { 353 int allocorder, allocsize; 354 355 allocsize = PAGE_SIZE; 356 allocorder = 0; 357 while (bytebufsize > allocsize) { 358 allocsize *= 2; 359 allocorder++; 360 } 361 362 s->salami = (void *) devm_get_free_pages( 363 dev, 364 GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO, 365 allocorder); 366 if (!s->salami) 367 return -ENOMEM; 368 369 s->left_of_salami = allocsize; 370 } 371 372 rc = ep->ephw->map_single(ep, s->salami, 373 bytebufsize, s->direction, 374 &dma_addr); 375 if (rc) 376 return rc; 377 378 iowrite32((u32) (dma_addr & 0xffffffff), 379 ep->registers + fpga_dma_bufaddr_lowaddr_reg); 380 iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)), 381 ep->registers + fpga_dma_bufaddr_highaddr_reg); 382 383 if (buffers) { /* Not the message buffer */ 384 this_buffer->addr = s->salami; 385 this_buffer->dma_addr = dma_addr; 386 buffers[i] = this_buffer++; 387 388 iowrite32(s->regdirection | s->nbuffer++, 389 ep->registers + fpga_dma_bufno_reg); 390 } else { 391 ep->msgbuf_addr = s->salami; 392 ep->msgbuf_dma_addr = dma_addr; 393 ep->msg_buf_size = bytebufsize; 394 395 iowrite32(s->regdirection, 396 ep->registers + fpga_dma_bufno_reg); 397 } 398 399 s->left_of_salami -= bytebufsize; 400 s->salami += bytebufsize; 401 } 402 return 0; 403 } 404 405 static int xilly_setupchannels(struct xilly_endpoint *ep, 406 unsigned char *chandesc, 407 int entries) 408 { 409 struct device *dev = ep->dev; 410 int i, entry, rc; 411 struct xilly_channel *channel; 412 int channelnum, bufnum, bufsize, format, is_writebuf; 413 int bytebufsize; 414 int synchronous, allowpartial, exclusive_open, seekable; 415 int supports_nonempty; 416 int msg_buf_done = 0; 417 418 struct xilly_alloc_state rd_alloc = { 419 .salami = NULL, 420 .left_of_salami = 0, 421 .nbuffer = 1, 422 .direction = DMA_TO_DEVICE, 423 .regdirection = 0, 424 }; 425 426 struct xilly_alloc_state wr_alloc = { 427 .salami = NULL, 428 .left_of_salami = 0, 429 .nbuffer = 1, 430 .direction = DMA_FROM_DEVICE, 431 .regdirection = 0x80000000, 432 }; 433 434 channel = devm_kcalloc(dev, ep->num_channels, 435 sizeof(struct xilly_channel), GFP_KERNEL); 436 if (!channel) 437 return -ENOMEM; 438 439 ep->channels = devm_kcalloc(dev, ep->num_channels + 1, 440 sizeof(struct xilly_channel *), 441 GFP_KERNEL); 442 if (!ep->channels) 443 return -ENOMEM; 444 445 ep->channels[0] = NULL; /* Channel 0 is message buf. */ 446 447 /* Initialize all channels with defaults */ 448 449 for (i = 1; i <= ep->num_channels; i++) { 450 channel->wr_buffers = NULL; 451 channel->rd_buffers = NULL; 452 channel->num_wr_buffers = 0; 453 channel->num_rd_buffers = 0; 454 channel->wr_fpga_buf_idx = -1; 455 channel->wr_host_buf_idx = 0; 456 channel->wr_host_buf_pos = 0; 457 channel->wr_empty = 1; 458 channel->wr_ready = 0; 459 channel->wr_sleepy = 1; 460 channel->rd_fpga_buf_idx = 0; 461 channel->rd_host_buf_idx = 0; 462 channel->rd_host_buf_pos = 0; 463 channel->rd_full = 0; 464 channel->wr_ref_count = 0; 465 channel->rd_ref_count = 0; 466 467 spin_lock_init(&channel->wr_spinlock); 468 spin_lock_init(&channel->rd_spinlock); 469 mutex_init(&channel->wr_mutex); 470 mutex_init(&channel->rd_mutex); 471 init_waitqueue_head(&channel->rd_wait); 472 init_waitqueue_head(&channel->wr_wait); 473 init_waitqueue_head(&channel->wr_ready_wait); 474 475 INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush); 476 477 channel->endpoint = ep; 478 channel->chan_num = i; 479 480 channel->log2_element_size = 0; 481 482 ep->channels[i] = channel++; 483 } 484 485 for (entry = 0; entry < entries; entry++, chandesc += 4) { 486 struct xilly_buffer **buffers = NULL; 487 488 is_writebuf = chandesc[0] & 0x01; 489 channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7); 490 format = (chandesc[1] >> 4) & 0x03; 491 allowpartial = (chandesc[1] >> 6) & 0x01; 492 synchronous = (chandesc[1] >> 7) & 0x01; 493 bufsize = 1 << (chandesc[2] & 0x1f); 494 bufnum = 1 << (chandesc[3] & 0x0f); 495 exclusive_open = (chandesc[2] >> 7) & 0x01; 496 seekable = (chandesc[2] >> 6) & 0x01; 497 supports_nonempty = (chandesc[2] >> 5) & 0x01; 498 499 if ((channelnum > ep->num_channels) || 500 ((channelnum == 0) && !is_writebuf)) { 501 dev_err(ep->dev, 502 "IDT requests channel out of range. Aborting.\n"); 503 return -ENODEV; 504 } 505 506 channel = ep->channels[channelnum]; /* NULL for msg channel */ 507 508 if (!is_writebuf || channelnum > 0) { 509 channel->log2_element_size = ((format > 2) ? 510 2 : format); 511 512 bytebufsize = channel->rd_buf_size = bufsize * 513 (1 << channel->log2_element_size); 514 515 buffers = devm_kcalloc(dev, bufnum, 516 sizeof(struct xilly_buffer *), 517 GFP_KERNEL); 518 if (!buffers) 519 return -ENOMEM; 520 } else { 521 bytebufsize = bufsize << 2; 522 } 523 524 if (!is_writebuf) { 525 channel->num_rd_buffers = bufnum; 526 channel->rd_allow_partial = allowpartial; 527 channel->rd_synchronous = synchronous; 528 channel->rd_exclusive_open = exclusive_open; 529 channel->seekable = seekable; 530 531 channel->rd_buffers = buffers; 532 rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers, 533 bufnum, bytebufsize); 534 } else if (channelnum > 0) { 535 channel->num_wr_buffers = bufnum; 536 537 channel->seekable = seekable; 538 channel->wr_supports_nonempty = supports_nonempty; 539 540 channel->wr_allow_partial = allowpartial; 541 channel->wr_synchronous = synchronous; 542 channel->wr_exclusive_open = exclusive_open; 543 544 channel->wr_buffers = buffers; 545 rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers, 546 bufnum, bytebufsize); 547 } else { 548 rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL, 549 bufnum, bytebufsize); 550 msg_buf_done++; 551 } 552 553 if (rc) 554 return -ENOMEM; 555 } 556 557 if (!msg_buf_done) { 558 dev_err(ep->dev, 559 "Corrupt IDT: No message buffer. Aborting.\n"); 560 return -ENODEV; 561 } 562 return 0; 563 } 564 565 static int xilly_scan_idt(struct xilly_endpoint *endpoint, 566 struct xilly_idt_handle *idt_handle) 567 { 568 int count = 0; 569 unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr; 570 unsigned char *end_of_idt = idt + endpoint->idtlen - 4; 571 unsigned char *scan; 572 int len; 573 574 scan = idt; 575 idt_handle->idt = idt; 576 577 scan++; /* Skip version number */ 578 579 while ((scan <= end_of_idt) && *scan) { 580 while ((scan <= end_of_idt) && *scan++) 581 /* Do nothing, just scan thru string */; 582 count++; 583 } 584 585 scan++; 586 587 if (scan > end_of_idt) { 588 dev_err(endpoint->dev, 589 "IDT device name list overflow. Aborting.\n"); 590 return -ENODEV; 591 } 592 idt_handle->chandesc = scan; 593 594 len = endpoint->idtlen - (3 + ((int) (scan - idt))); 595 596 if (len & 0x03) { 597 dev_err(endpoint->dev, 598 "Corrupt IDT device name list. Aborting.\n"); 599 return -ENODEV; 600 } 601 602 idt_handle->entries = len >> 2; 603 endpoint->num_channels = count; 604 605 return 0; 606 } 607 608 static int xilly_obtain_idt(struct xilly_endpoint *endpoint) 609 { 610 struct xilly_channel *channel; 611 unsigned char *version; 612 long t; 613 614 channel = endpoint->channels[1]; /* This should be generated ad-hoc */ 615 616 channel->wr_sleepy = 1; 617 618 iowrite32(1 | 619 (3 << 24), /* Opcode 3 for channel 0 = Send IDT */ 620 endpoint->registers + fpga_buf_ctrl_reg); 621 622 t = wait_event_interruptible_timeout(channel->wr_wait, 623 (!channel->wr_sleepy), 624 XILLY_TIMEOUT); 625 626 if (t <= 0) { 627 dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n"); 628 629 if (endpoint->fatal_error) 630 return -EIO; 631 632 return -ENODEV; 633 } 634 635 endpoint->ephw->hw_sync_sgl_for_cpu( 636 channel->endpoint, 637 channel->wr_buffers[0]->dma_addr, 638 channel->wr_buf_size, 639 DMA_FROM_DEVICE); 640 641 if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) { 642 dev_err(endpoint->dev, 643 "IDT length mismatch (%d != %d). Aborting.\n", 644 channel->wr_buffers[0]->end_offset, endpoint->idtlen); 645 return -ENODEV; 646 } 647 648 if (crc32_le(~0, channel->wr_buffers[0]->addr, 649 endpoint->idtlen+1) != 0) { 650 dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n"); 651 return -ENODEV; 652 } 653 654 version = channel->wr_buffers[0]->addr; 655 656 /* Check version number. Accept anything below 0x82 for now. */ 657 if (*version > 0x82) { 658 dev_err(endpoint->dev, 659 "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgarde. Aborting.\n", 660 *version); 661 return -ENODEV; 662 } 663 664 return 0; 665 } 666 667 static ssize_t xillybus_read(struct file *filp, char __user *userbuf, 668 size_t count, loff_t *f_pos) 669 { 670 ssize_t rc; 671 unsigned long flags; 672 int bytes_done = 0; 673 int no_time_left = 0; 674 long deadline, left_to_sleep; 675 struct xilly_channel *channel = filp->private_data; 676 677 int empty, reached_eof, exhausted, ready; 678 /* Initializations are there only to silence warnings */ 679 680 int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; 681 int waiting_bufidx; 682 683 if (channel->endpoint->fatal_error) 684 return -EIO; 685 686 deadline = jiffies + 1 + XILLY_RX_TIMEOUT; 687 688 rc = mutex_lock_interruptible(&channel->wr_mutex); 689 if (rc) 690 return rc; 691 692 while (1) { /* Note that we may drop mutex within this loop */ 693 int bytes_to_do = count - bytes_done; 694 695 spin_lock_irqsave(&channel->wr_spinlock, flags); 696 697 empty = channel->wr_empty; 698 ready = !empty || channel->wr_ready; 699 700 if (!empty) { 701 bufidx = channel->wr_host_buf_idx; 702 bufpos = channel->wr_host_buf_pos; 703 howmany = ((channel->wr_buffers[bufidx]->end_offset 704 + 1) << channel->log2_element_size) 705 - bufpos; 706 707 /* Update wr_host_* to its post-operation state */ 708 if (howmany > bytes_to_do) { 709 bufferdone = 0; 710 711 howmany = bytes_to_do; 712 channel->wr_host_buf_pos += howmany; 713 } else { 714 bufferdone = 1; 715 716 channel->wr_host_buf_pos = 0; 717 718 if (bufidx == channel->wr_fpga_buf_idx) { 719 channel->wr_empty = 1; 720 channel->wr_sleepy = 1; 721 channel->wr_ready = 0; 722 } 723 724 if (bufidx >= (channel->num_wr_buffers - 1)) 725 channel->wr_host_buf_idx = 0; 726 else 727 channel->wr_host_buf_idx++; 728 } 729 } 730 731 /* 732 * Marking our situation after the possible changes above, 733 * for use after releasing the spinlock. 734 * 735 * empty = empty before change 736 * exhasted = empty after possible change 737 */ 738 739 reached_eof = channel->wr_empty && 740 (channel->wr_host_buf_idx == channel->wr_eof); 741 channel->wr_hangup = reached_eof; 742 exhausted = channel->wr_empty; 743 waiting_bufidx = channel->wr_host_buf_idx; 744 745 spin_unlock_irqrestore(&channel->wr_spinlock, flags); 746 747 if (!empty) { /* Go on, now without the spinlock */ 748 749 if (bufpos == 0) /* Position zero means it's virgin */ 750 channel->endpoint->ephw->hw_sync_sgl_for_cpu( 751 channel->endpoint, 752 channel->wr_buffers[bufidx]->dma_addr, 753 channel->wr_buf_size, 754 DMA_FROM_DEVICE); 755 756 if (copy_to_user( 757 userbuf, 758 channel->wr_buffers[bufidx]->addr 759 + bufpos, howmany)) 760 rc = -EFAULT; 761 762 userbuf += howmany; 763 bytes_done += howmany; 764 765 if (bufferdone) { 766 channel->endpoint->ephw->hw_sync_sgl_for_device( 767 channel->endpoint, 768 channel->wr_buffers[bufidx]->dma_addr, 769 channel->wr_buf_size, 770 DMA_FROM_DEVICE); 771 772 /* 773 * Tell FPGA the buffer is done with. It's an 774 * atomic operation to the FPGA, so what 775 * happens with other channels doesn't matter, 776 * and the certain channel is protected with 777 * the channel-specific mutex. 778 */ 779 780 iowrite32(1 | (channel->chan_num << 1) | 781 (bufidx << 12), 782 channel->endpoint->registers + 783 fpga_buf_ctrl_reg); 784 } 785 786 if (rc) { 787 mutex_unlock(&channel->wr_mutex); 788 return rc; 789 } 790 } 791 792 /* This includes a zero-count return = EOF */ 793 if ((bytes_done >= count) || reached_eof) 794 break; 795 796 if (!exhausted) 797 continue; /* More in RAM buffer(s)? Just go on. */ 798 799 if ((bytes_done > 0) && 800 (no_time_left || 801 (channel->wr_synchronous && channel->wr_allow_partial))) 802 break; 803 804 /* 805 * Nonblocking read: The "ready" flag tells us that the FPGA 806 * has data to send. In non-blocking mode, if it isn't on, 807 * just return. But if there is, we jump directly to the point 808 * where we ask for the FPGA to send all it has, and wait 809 * until that data arrives. So in a sense, we *do* block in 810 * nonblocking mode, but only for a very short time. 811 */ 812 813 if (!no_time_left && (filp->f_flags & O_NONBLOCK)) { 814 if (bytes_done > 0) 815 break; 816 817 if (ready) 818 goto desperate; 819 820 rc = -EAGAIN; 821 break; 822 } 823 824 if (!no_time_left || (bytes_done > 0)) { 825 /* 826 * Note that in case of an element-misaligned read 827 * request, offsetlimit will include the last element, 828 * which will be partially read from. 829 */ 830 int offsetlimit = ((count - bytes_done) - 1) >> 831 channel->log2_element_size; 832 int buf_elements = channel->wr_buf_size >> 833 channel->log2_element_size; 834 835 /* 836 * In synchronous mode, always send an offset limit. 837 * Just don't send a value too big. 838 */ 839 840 if (channel->wr_synchronous) { 841 /* Don't request more than one buffer */ 842 if (channel->wr_allow_partial && 843 (offsetlimit >= buf_elements)) 844 offsetlimit = buf_elements - 1; 845 846 /* Don't request more than all buffers */ 847 if (!channel->wr_allow_partial && 848 (offsetlimit >= 849 (buf_elements * channel->num_wr_buffers))) 850 offsetlimit = buf_elements * 851 channel->num_wr_buffers - 1; 852 } 853 854 /* 855 * In asynchronous mode, force early flush of a buffer 856 * only if that will allow returning a full count. The 857 * "offsetlimit < ( ... )" rather than "<=" excludes 858 * requesting a full buffer, which would obviously 859 * cause a buffer transmission anyhow 860 */ 861 862 if (channel->wr_synchronous || 863 (offsetlimit < (buf_elements - 1))) { 864 mutex_lock(&channel->endpoint->register_mutex); 865 866 iowrite32(offsetlimit, 867 channel->endpoint->registers + 868 fpga_buf_offset_reg); 869 870 iowrite32(1 | (channel->chan_num << 1) | 871 (2 << 24) | /* 2 = offset limit */ 872 (waiting_bufidx << 12), 873 channel->endpoint->registers + 874 fpga_buf_ctrl_reg); 875 876 mutex_unlock(&channel->endpoint-> 877 register_mutex); 878 } 879 } 880 881 /* 882 * If partial completion is disallowed, there is no point in 883 * timeout sleeping. Neither if no_time_left is set and 884 * there's no data. 885 */ 886 887 if (!channel->wr_allow_partial || 888 (no_time_left && (bytes_done == 0))) { 889 /* 890 * This do-loop will run more than once if another 891 * thread reasserted wr_sleepy before we got the mutex 892 * back, so we try again. 893 */ 894 895 do { 896 mutex_unlock(&channel->wr_mutex); 897 898 if (wait_event_interruptible( 899 channel->wr_wait, 900 (!channel->wr_sleepy))) 901 goto interrupted; 902 903 if (mutex_lock_interruptible( 904 &channel->wr_mutex)) 905 goto interrupted; 906 } while (channel->wr_sleepy); 907 908 continue; 909 910 interrupted: /* Mutex is not held if got here */ 911 if (channel->endpoint->fatal_error) 912 return -EIO; 913 if (bytes_done) 914 return bytes_done; 915 if (filp->f_flags & O_NONBLOCK) 916 return -EAGAIN; /* Don't admit snoozing */ 917 return -EINTR; 918 } 919 920 left_to_sleep = deadline - ((long) jiffies); 921 922 /* 923 * If our time is out, skip the waiting. We may miss wr_sleepy 924 * being deasserted but hey, almost missing the train is like 925 * missing it. 926 */ 927 928 if (left_to_sleep > 0) { 929 left_to_sleep = 930 wait_event_interruptible_timeout( 931 channel->wr_wait, 932 (!channel->wr_sleepy), 933 left_to_sleep); 934 935 if (left_to_sleep > 0) /* wr_sleepy deasserted */ 936 continue; 937 938 if (left_to_sleep < 0) { /* Interrupt */ 939 mutex_unlock(&channel->wr_mutex); 940 if (channel->endpoint->fatal_error) 941 return -EIO; 942 if (bytes_done) 943 return bytes_done; 944 return -EINTR; 945 } 946 } 947 948 desperate: 949 no_time_left = 1; /* We're out of sleeping time. Desperate! */ 950 951 if (bytes_done == 0) { 952 /* 953 * Reaching here means that we allow partial return, 954 * that we've run out of time, and that we have 955 * nothing to return. 956 * So tell the FPGA to send anything it has or gets. 957 */ 958 959 iowrite32(1 | (channel->chan_num << 1) | 960 (3 << 24) | /* Opcode 3, flush it all! */ 961 (waiting_bufidx << 12), 962 channel->endpoint->registers + 963 fpga_buf_ctrl_reg); 964 } 965 966 /* 967 * Reaching here means that we *do* have data in the buffer, 968 * but the "partial" flag disallows returning less than 969 * required. And we don't have as much. So loop again, 970 * which is likely to end up blocking indefinitely until 971 * enough data has arrived. 972 */ 973 } 974 975 mutex_unlock(&channel->wr_mutex); 976 977 if (channel->endpoint->fatal_error) 978 return -EIO; 979 980 if (rc) 981 return rc; 982 983 return bytes_done; 984 } 985 986 /* 987 * The timeout argument takes values as follows: 988 * >0 : Flush with timeout 989 * ==0 : Flush, and wait idefinitely for the flush to complete 990 * <0 : Autoflush: Flush only if there's a single buffer occupied 991 */ 992 993 static int xillybus_myflush(struct xilly_channel *channel, long timeout) 994 { 995 int rc; 996 unsigned long flags; 997 998 int end_offset_plus1; 999 int bufidx, bufidx_minus1; 1000 int i; 1001 int empty; 1002 int new_rd_host_buf_pos; 1003 1004 if (channel->endpoint->fatal_error) 1005 return -EIO; 1006 rc = mutex_lock_interruptible(&channel->rd_mutex); 1007 if (rc) 1008 return rc; 1009 1010 /* 1011 * Don't flush a closed channel. This can happen when the work queued 1012 * autoflush thread fires off after the file has closed. This is not 1013 * an error, just something to dismiss. 1014 */ 1015 1016 if (!channel->rd_ref_count) 1017 goto done; 1018 1019 bufidx = channel->rd_host_buf_idx; 1020 1021 bufidx_minus1 = (bufidx == 0) ? 1022 channel->num_rd_buffers - 1 : 1023 bufidx - 1; 1024 1025 end_offset_plus1 = channel->rd_host_buf_pos >> 1026 channel->log2_element_size; 1027 1028 new_rd_host_buf_pos = channel->rd_host_buf_pos - 1029 (end_offset_plus1 << channel->log2_element_size); 1030 1031 /* Submit the current buffer if it's nonempty */ 1032 if (end_offset_plus1) { 1033 unsigned char *tail = channel->rd_buffers[bufidx]->addr + 1034 (end_offset_plus1 << channel->log2_element_size); 1035 1036 /* Copy unflushed data, so we can put it in next buffer */ 1037 for (i = 0; i < new_rd_host_buf_pos; i++) 1038 channel->rd_leftovers[i] = *tail++; 1039 1040 spin_lock_irqsave(&channel->rd_spinlock, flags); 1041 1042 /* Autoflush only if a single buffer is occupied */ 1043 1044 if ((timeout < 0) && 1045 (channel->rd_full || 1046 (bufidx_minus1 != channel->rd_fpga_buf_idx))) { 1047 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1048 /* 1049 * A new work item may be queued by the ISR exactly 1050 * now, since the execution of a work item allows the 1051 * queuing of a new one while it's running. 1052 */ 1053 goto done; 1054 } 1055 1056 /* The 4th element is never needed for data, so it's a flag */ 1057 channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0); 1058 1059 /* Set up rd_full to reflect a certain moment's state */ 1060 1061 if (bufidx == channel->rd_fpga_buf_idx) 1062 channel->rd_full = 1; 1063 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1064 1065 if (bufidx >= (channel->num_rd_buffers - 1)) 1066 channel->rd_host_buf_idx = 0; 1067 else 1068 channel->rd_host_buf_idx++; 1069 1070 channel->endpoint->ephw->hw_sync_sgl_for_device( 1071 channel->endpoint, 1072 channel->rd_buffers[bufidx]->dma_addr, 1073 channel->rd_buf_size, 1074 DMA_TO_DEVICE); 1075 1076 mutex_lock(&channel->endpoint->register_mutex); 1077 1078 iowrite32(end_offset_plus1 - 1, 1079 channel->endpoint->registers + fpga_buf_offset_reg); 1080 1081 iowrite32((channel->chan_num << 1) | /* Channel ID */ 1082 (2 << 24) | /* Opcode 2, submit buffer */ 1083 (bufidx << 12), 1084 channel->endpoint->registers + fpga_buf_ctrl_reg); 1085 1086 mutex_unlock(&channel->endpoint->register_mutex); 1087 } else if (bufidx == 0) { 1088 bufidx = channel->num_rd_buffers - 1; 1089 } else { 1090 bufidx--; 1091 } 1092 1093 channel->rd_host_buf_pos = new_rd_host_buf_pos; 1094 1095 if (timeout < 0) 1096 goto done; /* Autoflush */ 1097 1098 /* 1099 * bufidx is now the last buffer written to (or equal to 1100 * rd_fpga_buf_idx if buffer was never written to), and 1101 * channel->rd_host_buf_idx the one after it. 1102 * 1103 * If bufidx == channel->rd_fpga_buf_idx we're either empty or full. 1104 */ 1105 1106 while (1) { /* Loop waiting for draining of buffers */ 1107 spin_lock_irqsave(&channel->rd_spinlock, flags); 1108 1109 if (bufidx != channel->rd_fpga_buf_idx) 1110 channel->rd_full = 1; /* 1111 * Not really full, 1112 * but needs waiting. 1113 */ 1114 1115 empty = !channel->rd_full; 1116 1117 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1118 1119 if (empty) 1120 break; 1121 1122 /* 1123 * Indefinite sleep with mutex taken. With data waiting for 1124 * flushing user should not be surprised if open() for write 1125 * sleeps. 1126 */ 1127 if (timeout == 0) 1128 wait_event_interruptible(channel->rd_wait, 1129 (!channel->rd_full)); 1130 1131 else if (wait_event_interruptible_timeout( 1132 channel->rd_wait, 1133 (!channel->rd_full), 1134 timeout) == 0) { 1135 dev_warn(channel->endpoint->dev, 1136 "Timed out while flushing. Output data may be lost.\n"); 1137 1138 rc = -ETIMEDOUT; 1139 break; 1140 } 1141 1142 if (channel->rd_full) { 1143 rc = -EINTR; 1144 break; 1145 } 1146 } 1147 1148 done: 1149 mutex_unlock(&channel->rd_mutex); 1150 1151 if (channel->endpoint->fatal_error) 1152 return -EIO; 1153 1154 return rc; 1155 } 1156 1157 static int xillybus_flush(struct file *filp, fl_owner_t id) 1158 { 1159 if (!(filp->f_mode & FMODE_WRITE)) 1160 return 0; 1161 1162 return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */ 1163 } 1164 1165 static void xillybus_autoflush(struct work_struct *work) 1166 { 1167 struct delayed_work *workitem = container_of( 1168 work, struct delayed_work, work); 1169 struct xilly_channel *channel = container_of( 1170 workitem, struct xilly_channel, rd_workitem); 1171 int rc; 1172 1173 rc = xillybus_myflush(channel, -1); 1174 if (rc == -EINTR) 1175 dev_warn(channel->endpoint->dev, 1176 "Autoflush failed because work queue thread got a signal.\n"); 1177 else if (rc) 1178 dev_err(channel->endpoint->dev, 1179 "Autoflush failed under weird circumstances.\n"); 1180 } 1181 1182 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf, 1183 size_t count, loff_t *f_pos) 1184 { 1185 ssize_t rc; 1186 unsigned long flags; 1187 int bytes_done = 0; 1188 struct xilly_channel *channel = filp->private_data; 1189 1190 int full, exhausted; 1191 /* Initializations are there only to silence warnings */ 1192 1193 int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; 1194 int end_offset_plus1 = 0; 1195 1196 if (channel->endpoint->fatal_error) 1197 return -EIO; 1198 1199 rc = mutex_lock_interruptible(&channel->rd_mutex); 1200 if (rc) 1201 return rc; 1202 1203 while (1) { 1204 int bytes_to_do = count - bytes_done; 1205 1206 spin_lock_irqsave(&channel->rd_spinlock, flags); 1207 1208 full = channel->rd_full; 1209 1210 if (!full) { 1211 bufidx = channel->rd_host_buf_idx; 1212 bufpos = channel->rd_host_buf_pos; 1213 howmany = channel->rd_buf_size - bufpos; 1214 1215 /* 1216 * Update rd_host_* to its state after this operation. 1217 * count=0 means committing the buffer immediately, 1218 * which is like flushing, but not necessarily block. 1219 */ 1220 1221 if ((howmany > bytes_to_do) && 1222 (count || 1223 ((bufpos >> channel->log2_element_size) == 0))) { 1224 bufferdone = 0; 1225 1226 howmany = bytes_to_do; 1227 channel->rd_host_buf_pos += howmany; 1228 } else { 1229 bufferdone = 1; 1230 1231 if (count) { 1232 end_offset_plus1 = 1233 channel->rd_buf_size >> 1234 channel->log2_element_size; 1235 channel->rd_host_buf_pos = 0; 1236 } else { 1237 unsigned char *tail; 1238 int i; 1239 1240 howmany = 0; 1241 1242 end_offset_plus1 = bufpos >> 1243 channel->log2_element_size; 1244 1245 channel->rd_host_buf_pos -= 1246 end_offset_plus1 << 1247 channel->log2_element_size; 1248 1249 tail = channel-> 1250 rd_buffers[bufidx]->addr + 1251 (end_offset_plus1 << 1252 channel->log2_element_size); 1253 1254 for (i = 0; 1255 i < channel->rd_host_buf_pos; 1256 i++) 1257 channel->rd_leftovers[i] = 1258 *tail++; 1259 } 1260 1261 if (bufidx == channel->rd_fpga_buf_idx) 1262 channel->rd_full = 1; 1263 1264 if (bufidx >= (channel->num_rd_buffers - 1)) 1265 channel->rd_host_buf_idx = 0; 1266 else 1267 channel->rd_host_buf_idx++; 1268 } 1269 } 1270 1271 /* 1272 * Marking our situation after the possible changes above, 1273 * for use after releasing the spinlock. 1274 * 1275 * full = full before change 1276 * exhasted = full after possible change 1277 */ 1278 1279 exhausted = channel->rd_full; 1280 1281 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1282 1283 if (!full) { /* Go on, now without the spinlock */ 1284 unsigned char *head = 1285 channel->rd_buffers[bufidx]->addr; 1286 int i; 1287 1288 if ((bufpos == 0) || /* Zero means it's virgin */ 1289 (channel->rd_leftovers[3] != 0)) { 1290 channel->endpoint->ephw->hw_sync_sgl_for_cpu( 1291 channel->endpoint, 1292 channel->rd_buffers[bufidx]->dma_addr, 1293 channel->rd_buf_size, 1294 DMA_TO_DEVICE); 1295 1296 /* Virgin, but leftovers are due */ 1297 for (i = 0; i < bufpos; i++) 1298 *head++ = channel->rd_leftovers[i]; 1299 1300 channel->rd_leftovers[3] = 0; /* Clear flag */ 1301 } 1302 1303 if (copy_from_user( 1304 channel->rd_buffers[bufidx]->addr + bufpos, 1305 userbuf, howmany)) 1306 rc = -EFAULT; 1307 1308 userbuf += howmany; 1309 bytes_done += howmany; 1310 1311 if (bufferdone) { 1312 channel->endpoint->ephw->hw_sync_sgl_for_device( 1313 channel->endpoint, 1314 channel->rd_buffers[bufidx]->dma_addr, 1315 channel->rd_buf_size, 1316 DMA_TO_DEVICE); 1317 1318 mutex_lock(&channel->endpoint->register_mutex); 1319 1320 iowrite32(end_offset_plus1 - 1, 1321 channel->endpoint->registers + 1322 fpga_buf_offset_reg); 1323 1324 iowrite32((channel->chan_num << 1) | 1325 (2 << 24) | /* 2 = submit buffer */ 1326 (bufidx << 12), 1327 channel->endpoint->registers + 1328 fpga_buf_ctrl_reg); 1329 1330 mutex_unlock(&channel->endpoint-> 1331 register_mutex); 1332 1333 channel->rd_leftovers[3] = 1334 (channel->rd_host_buf_pos != 0); 1335 } 1336 1337 if (rc) { 1338 mutex_unlock(&channel->rd_mutex); 1339 1340 if (channel->endpoint->fatal_error) 1341 return -EIO; 1342 1343 if (!channel->rd_synchronous) 1344 queue_delayed_work( 1345 xillybus_wq, 1346 &channel->rd_workitem, 1347 XILLY_RX_TIMEOUT); 1348 1349 return rc; 1350 } 1351 } 1352 1353 if (bytes_done >= count) 1354 break; 1355 1356 if (!exhausted) 1357 continue; /* If there's more space, just go on */ 1358 1359 if ((bytes_done > 0) && channel->rd_allow_partial) 1360 break; 1361 1362 /* 1363 * Indefinite sleep with mutex taken. With data waiting for 1364 * flushing, user should not be surprised if open() for write 1365 * sleeps. 1366 */ 1367 1368 if (filp->f_flags & O_NONBLOCK) { 1369 rc = -EAGAIN; 1370 break; 1371 } 1372 1373 if (wait_event_interruptible(channel->rd_wait, 1374 (!channel->rd_full))) { 1375 mutex_unlock(&channel->rd_mutex); 1376 1377 if (channel->endpoint->fatal_error) 1378 return -EIO; 1379 1380 if (bytes_done) 1381 return bytes_done; 1382 return -EINTR; 1383 } 1384 } 1385 1386 mutex_unlock(&channel->rd_mutex); 1387 1388 if (!channel->rd_synchronous) 1389 queue_delayed_work(xillybus_wq, 1390 &channel->rd_workitem, 1391 XILLY_RX_TIMEOUT); 1392 1393 if (channel->endpoint->fatal_error) 1394 return -EIO; 1395 1396 if (rc) 1397 return rc; 1398 1399 if ((channel->rd_synchronous) && (bytes_done > 0)) { 1400 rc = xillybus_myflush(filp->private_data, 0); /* No timeout */ 1401 1402 if (rc && (rc != -EINTR)) 1403 return rc; 1404 } 1405 1406 return bytes_done; 1407 } 1408 1409 static int xillybus_open(struct inode *inode, struct file *filp) 1410 { 1411 int rc = 0; 1412 unsigned long flags; 1413 int minor = iminor(inode); 1414 int major = imajor(inode); 1415 struct xilly_endpoint *ep_iter, *endpoint = NULL; 1416 struct xilly_channel *channel; 1417 1418 mutex_lock(&ep_list_lock); 1419 1420 list_for_each_entry(ep_iter, &list_of_endpoints, ep_list) { 1421 if ((ep_iter->major == major) && 1422 (minor >= ep_iter->lowest_minor) && 1423 (minor < (ep_iter->lowest_minor + 1424 ep_iter->num_channels))) { 1425 endpoint = ep_iter; 1426 break; 1427 } 1428 } 1429 mutex_unlock(&ep_list_lock); 1430 1431 if (!endpoint) { 1432 pr_err("xillybus: open() failed to find a device for major=%d and minor=%d\n", 1433 major, minor); 1434 return -ENODEV; 1435 } 1436 1437 if (endpoint->fatal_error) 1438 return -EIO; 1439 1440 channel = endpoint->channels[1 + minor - endpoint->lowest_minor]; 1441 filp->private_data = channel; 1442 1443 /* 1444 * It gets complicated because: 1445 * 1. We don't want to take a mutex we don't have to 1446 * 2. We don't want to open one direction if the other will fail. 1447 */ 1448 1449 if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers)) 1450 return -ENODEV; 1451 1452 if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers)) 1453 return -ENODEV; 1454 1455 if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) && 1456 (channel->wr_synchronous || !channel->wr_allow_partial || 1457 !channel->wr_supports_nonempty)) { 1458 dev_err(endpoint->dev, 1459 "open() failed: O_NONBLOCK not allowed for read on this device\n"); 1460 return -ENODEV; 1461 } 1462 1463 if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) && 1464 (channel->rd_synchronous || !channel->rd_allow_partial)) { 1465 dev_err(endpoint->dev, 1466 "open() failed: O_NONBLOCK not allowed for write on this device\n"); 1467 return -ENODEV; 1468 } 1469 1470 /* 1471 * Note: open() may block on getting mutexes despite O_NONBLOCK. 1472 * This shouldn't occur normally, since multiple open of the same 1473 * file descriptor is almost always prohibited anyhow 1474 * (*_exclusive_open is normally set in real-life systems). 1475 */ 1476 1477 if (filp->f_mode & FMODE_READ) { 1478 rc = mutex_lock_interruptible(&channel->wr_mutex); 1479 if (rc) 1480 return rc; 1481 } 1482 1483 if (filp->f_mode & FMODE_WRITE) { 1484 rc = mutex_lock_interruptible(&channel->rd_mutex); 1485 if (rc) 1486 goto unlock_wr; 1487 } 1488 1489 if ((filp->f_mode & FMODE_READ) && 1490 (channel->wr_ref_count != 0) && 1491 (channel->wr_exclusive_open)) { 1492 rc = -EBUSY; 1493 goto unlock; 1494 } 1495 1496 if ((filp->f_mode & FMODE_WRITE) && 1497 (channel->rd_ref_count != 0) && 1498 (channel->rd_exclusive_open)) { 1499 rc = -EBUSY; 1500 goto unlock; 1501 } 1502 1503 if (filp->f_mode & FMODE_READ) { 1504 if (channel->wr_ref_count == 0) { /* First open of file */ 1505 /* Move the host to first buffer */ 1506 spin_lock_irqsave(&channel->wr_spinlock, flags); 1507 channel->wr_host_buf_idx = 0; 1508 channel->wr_host_buf_pos = 0; 1509 channel->wr_fpga_buf_idx = -1; 1510 channel->wr_empty = 1; 1511 channel->wr_ready = 0; 1512 channel->wr_sleepy = 1; 1513 channel->wr_eof = -1; 1514 channel->wr_hangup = 0; 1515 1516 spin_unlock_irqrestore(&channel->wr_spinlock, flags); 1517 1518 iowrite32(1 | (channel->chan_num << 1) | 1519 (4 << 24) | /* Opcode 4, open channel */ 1520 ((channel->wr_synchronous & 1) << 23), 1521 channel->endpoint->registers + 1522 fpga_buf_ctrl_reg); 1523 } 1524 1525 channel->wr_ref_count++; 1526 } 1527 1528 if (filp->f_mode & FMODE_WRITE) { 1529 if (channel->rd_ref_count == 0) { /* First open of file */ 1530 /* Move the host to first buffer */ 1531 spin_lock_irqsave(&channel->rd_spinlock, flags); 1532 channel->rd_host_buf_idx = 0; 1533 channel->rd_host_buf_pos = 0; 1534 channel->rd_leftovers[3] = 0; /* No leftovers. */ 1535 channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1; 1536 channel->rd_full = 0; 1537 1538 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1539 1540 iowrite32((channel->chan_num << 1) | 1541 (4 << 24), /* Opcode 4, open channel */ 1542 channel->endpoint->registers + 1543 fpga_buf_ctrl_reg); 1544 } 1545 1546 channel->rd_ref_count++; 1547 } 1548 1549 unlock: 1550 if (filp->f_mode & FMODE_WRITE) 1551 mutex_unlock(&channel->rd_mutex); 1552 unlock_wr: 1553 if (filp->f_mode & FMODE_READ) 1554 mutex_unlock(&channel->wr_mutex); 1555 1556 if (!rc && (!channel->seekable)) 1557 return nonseekable_open(inode, filp); 1558 1559 return rc; 1560 } 1561 1562 static int xillybus_release(struct inode *inode, struct file *filp) 1563 { 1564 unsigned long flags; 1565 struct xilly_channel *channel = filp->private_data; 1566 1567 int buf_idx; 1568 int eof; 1569 1570 if (channel->endpoint->fatal_error) 1571 return -EIO; 1572 1573 if (filp->f_mode & FMODE_WRITE) { 1574 mutex_lock(&channel->rd_mutex); 1575 1576 channel->rd_ref_count--; 1577 1578 if (channel->rd_ref_count == 0) { 1579 /* 1580 * We rely on the kernel calling flush() 1581 * before we get here. 1582 */ 1583 1584 iowrite32((channel->chan_num << 1) | /* Channel ID */ 1585 (5 << 24), /* Opcode 5, close channel */ 1586 channel->endpoint->registers + 1587 fpga_buf_ctrl_reg); 1588 } 1589 mutex_unlock(&channel->rd_mutex); 1590 } 1591 1592 if (filp->f_mode & FMODE_READ) { 1593 mutex_lock(&channel->wr_mutex); 1594 1595 channel->wr_ref_count--; 1596 1597 if (channel->wr_ref_count == 0) { 1598 iowrite32(1 | (channel->chan_num << 1) | 1599 (5 << 24), /* Opcode 5, close channel */ 1600 channel->endpoint->registers + 1601 fpga_buf_ctrl_reg); 1602 1603 /* 1604 * This is crazily cautious: We make sure that not 1605 * only that we got an EOF (be it because we closed 1606 * the channel or because of a user's EOF), but verify 1607 * that it's one beyond the last buffer arrived, so 1608 * we have no leftover buffers pending before wrapping 1609 * up (which can only happen in asynchronous channels, 1610 * BTW) 1611 */ 1612 1613 while (1) { 1614 spin_lock_irqsave(&channel->wr_spinlock, 1615 flags); 1616 buf_idx = channel->wr_fpga_buf_idx; 1617 eof = channel->wr_eof; 1618 channel->wr_sleepy = 1; 1619 spin_unlock_irqrestore(&channel->wr_spinlock, 1620 flags); 1621 1622 /* 1623 * Check if eof points at the buffer after 1624 * the last one the FPGA submitted. Note that 1625 * no EOF is marked by negative eof. 1626 */ 1627 1628 buf_idx++; 1629 if (buf_idx == channel->num_wr_buffers) 1630 buf_idx = 0; 1631 1632 if (buf_idx == eof) 1633 break; 1634 1635 /* 1636 * Steal extra 100 ms if awaken by interrupt. 1637 * This is a simple workaround for an 1638 * interrupt pending when entering, which would 1639 * otherwise result in declaring the hardware 1640 * non-responsive. 1641 */ 1642 1643 if (wait_event_interruptible( 1644 channel->wr_wait, 1645 (!channel->wr_sleepy))) 1646 msleep(100); 1647 1648 if (channel->wr_sleepy) { 1649 mutex_unlock(&channel->wr_mutex); 1650 dev_warn(channel->endpoint->dev, 1651 "Hardware failed to respond to close command, therefore left in messy state.\n"); 1652 return -EINTR; 1653 } 1654 } 1655 } 1656 1657 mutex_unlock(&channel->wr_mutex); 1658 } 1659 1660 return 0; 1661 } 1662 1663 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence) 1664 { 1665 struct xilly_channel *channel = filp->private_data; 1666 loff_t pos = filp->f_pos; 1667 int rc = 0; 1668 1669 /* 1670 * Take both mutexes not allowing interrupts, since it seems like 1671 * common applications don't expect an -EINTR here. Besides, multiple 1672 * access to a single file descriptor on seekable devices is a mess 1673 * anyhow. 1674 */ 1675 1676 if (channel->endpoint->fatal_error) 1677 return -EIO; 1678 1679 mutex_lock(&channel->wr_mutex); 1680 mutex_lock(&channel->rd_mutex); 1681 1682 switch (whence) { 1683 case SEEK_SET: 1684 pos = offset; 1685 break; 1686 case SEEK_CUR: 1687 pos += offset; 1688 break; 1689 case SEEK_END: 1690 pos = offset; /* Going to the end => to the beginning */ 1691 break; 1692 default: 1693 rc = -EINVAL; 1694 goto end; 1695 } 1696 1697 /* In any case, we must finish on an element boundary */ 1698 if (pos & ((1 << channel->log2_element_size) - 1)) { 1699 rc = -EINVAL; 1700 goto end; 1701 } 1702 1703 mutex_lock(&channel->endpoint->register_mutex); 1704 1705 iowrite32(pos >> channel->log2_element_size, 1706 channel->endpoint->registers + fpga_buf_offset_reg); 1707 1708 iowrite32((channel->chan_num << 1) | 1709 (6 << 24), /* Opcode 6, set address */ 1710 channel->endpoint->registers + fpga_buf_ctrl_reg); 1711 1712 mutex_unlock(&channel->endpoint->register_mutex); 1713 1714 end: 1715 mutex_unlock(&channel->rd_mutex); 1716 mutex_unlock(&channel->wr_mutex); 1717 1718 if (rc) /* Return error after releasing mutexes */ 1719 return rc; 1720 1721 filp->f_pos = pos; 1722 1723 /* 1724 * Since seekable devices are allowed only when the channel is 1725 * synchronous, we assume that there is no data pending in either 1726 * direction (which holds true as long as no concurrent access on the 1727 * file descriptor takes place). 1728 * The only thing we may need to throw away is leftovers from partial 1729 * write() flush. 1730 */ 1731 1732 channel->rd_leftovers[3] = 0; 1733 1734 return pos; 1735 } 1736 1737 static unsigned int xillybus_poll(struct file *filp, poll_table *wait) 1738 { 1739 struct xilly_channel *channel = filp->private_data; 1740 unsigned int mask = 0; 1741 unsigned long flags; 1742 1743 poll_wait(filp, &channel->endpoint->ep_wait, wait); 1744 1745 /* 1746 * poll() won't play ball regarding read() channels which 1747 * aren't asynchronous and support the nonempty message. Allowing 1748 * that will create situations where data has been delivered at 1749 * the FPGA, and users expecting select() to wake up, which it may 1750 * not. 1751 */ 1752 1753 if (!channel->wr_synchronous && channel->wr_supports_nonempty) { 1754 poll_wait(filp, &channel->wr_wait, wait); 1755 poll_wait(filp, &channel->wr_ready_wait, wait); 1756 1757 spin_lock_irqsave(&channel->wr_spinlock, flags); 1758 if (!channel->wr_empty || channel->wr_ready) 1759 mask |= POLLIN | POLLRDNORM; 1760 1761 if (channel->wr_hangup) 1762 /* 1763 * Not POLLHUP, because its behavior is in the 1764 * mist, and POLLIN does what we want: Wake up 1765 * the read file descriptor so it sees EOF. 1766 */ 1767 mask |= POLLIN | POLLRDNORM; 1768 spin_unlock_irqrestore(&channel->wr_spinlock, flags); 1769 } 1770 1771 /* 1772 * If partial data write is disallowed on a write() channel, 1773 * it's pointless to ever signal OK to write, because is could 1774 * block despite some space being available. 1775 */ 1776 1777 if (channel->rd_allow_partial) { 1778 poll_wait(filp, &channel->rd_wait, wait); 1779 1780 spin_lock_irqsave(&channel->rd_spinlock, flags); 1781 if (!channel->rd_full) 1782 mask |= POLLOUT | POLLWRNORM; 1783 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1784 } 1785 1786 if (channel->endpoint->fatal_error) 1787 mask |= POLLERR; 1788 1789 return mask; 1790 } 1791 1792 static const struct file_operations xillybus_fops = { 1793 .owner = THIS_MODULE, 1794 .read = xillybus_read, 1795 .write = xillybus_write, 1796 .open = xillybus_open, 1797 .flush = xillybus_flush, 1798 .release = xillybus_release, 1799 .llseek = xillybus_llseek, 1800 .poll = xillybus_poll, 1801 }; 1802 1803 static int xillybus_init_chrdev(struct xilly_endpoint *endpoint, 1804 const unsigned char *idt) 1805 { 1806 int rc; 1807 dev_t dev; 1808 int devnum, i, minor, major; 1809 char devname[48]; 1810 struct device *device; 1811 1812 rc = alloc_chrdev_region(&dev, 0, /* minor start */ 1813 endpoint->num_channels, 1814 xillyname); 1815 if (rc) { 1816 dev_warn(endpoint->dev, "Failed to obtain major/minors"); 1817 return rc; 1818 } 1819 1820 endpoint->major = major = MAJOR(dev); 1821 endpoint->lowest_minor = minor = MINOR(dev); 1822 1823 cdev_init(&endpoint->cdev, &xillybus_fops); 1824 endpoint->cdev.owner = endpoint->ephw->owner; 1825 rc = cdev_add(&endpoint->cdev, MKDEV(major, minor), 1826 endpoint->num_channels); 1827 if (rc) { 1828 dev_warn(endpoint->dev, "Failed to add cdev. Aborting.\n"); 1829 goto unregister_chrdev; 1830 } 1831 1832 idt++; 1833 1834 for (i = minor, devnum = 0; 1835 devnum < endpoint->num_channels; 1836 devnum++, i++) { 1837 snprintf(devname, sizeof(devname)-1, "xillybus_%s", idt); 1838 1839 devname[sizeof(devname)-1] = 0; /* Should never matter */ 1840 1841 while (*idt++) 1842 /* Skip to next */; 1843 1844 device = device_create(xillybus_class, 1845 NULL, 1846 MKDEV(major, i), 1847 NULL, 1848 "%s", devname); 1849 1850 if (IS_ERR(device)) { 1851 dev_warn(endpoint->dev, 1852 "Failed to create %s device. Aborting.\n", 1853 devname); 1854 rc = -ENODEV; 1855 goto unroll_device_create; 1856 } 1857 } 1858 1859 dev_info(endpoint->dev, "Created %d device files.\n", 1860 endpoint->num_channels); 1861 return 0; /* succeed */ 1862 1863 unroll_device_create: 1864 devnum--; i--; 1865 for (; devnum >= 0; devnum--, i--) 1866 device_destroy(xillybus_class, MKDEV(major, i)); 1867 1868 cdev_del(&endpoint->cdev); 1869 unregister_chrdev: 1870 unregister_chrdev_region(MKDEV(major, minor), endpoint->num_channels); 1871 1872 return rc; 1873 } 1874 1875 static void xillybus_cleanup_chrdev(struct xilly_endpoint *endpoint) 1876 { 1877 int minor; 1878 1879 for (minor = endpoint->lowest_minor; 1880 minor < (endpoint->lowest_minor + endpoint->num_channels); 1881 minor++) 1882 device_destroy(xillybus_class, MKDEV(endpoint->major, minor)); 1883 cdev_del(&endpoint->cdev); 1884 unregister_chrdev_region(MKDEV(endpoint->major, 1885 endpoint->lowest_minor), 1886 endpoint->num_channels); 1887 1888 dev_info(endpoint->dev, "Removed %d device files.\n", 1889 endpoint->num_channels); 1890 } 1891 1892 struct xilly_endpoint *xillybus_init_endpoint(struct pci_dev *pdev, 1893 struct device *dev, 1894 struct xilly_endpoint_hardware 1895 *ephw) 1896 { 1897 struct xilly_endpoint *endpoint; 1898 1899 endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL); 1900 if (!endpoint) 1901 return NULL; 1902 1903 endpoint->pdev = pdev; 1904 endpoint->dev = dev; 1905 endpoint->ephw = ephw; 1906 endpoint->msg_counter = 0x0b; 1907 endpoint->failed_messages = 0; 1908 endpoint->fatal_error = 0; 1909 1910 init_waitqueue_head(&endpoint->ep_wait); 1911 mutex_init(&endpoint->register_mutex); 1912 1913 return endpoint; 1914 } 1915 EXPORT_SYMBOL(xillybus_init_endpoint); 1916 1917 static int xilly_quiesce(struct xilly_endpoint *endpoint) 1918 { 1919 long t; 1920 1921 endpoint->idtlen = -1; 1922 1923 iowrite32((u32) (endpoint->dma_using_dac & 0x0001), 1924 endpoint->registers + fpga_dma_control_reg); 1925 1926 t = wait_event_interruptible_timeout(endpoint->ep_wait, 1927 (endpoint->idtlen >= 0), 1928 XILLY_TIMEOUT); 1929 if (t <= 0) { 1930 dev_err(endpoint->dev, 1931 "Failed to quiesce the device on exit.\n"); 1932 return -ENODEV; 1933 } 1934 return 0; 1935 } 1936 1937 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint) 1938 { 1939 int rc; 1940 long t; 1941 1942 void *bootstrap_resources; 1943 int idtbuffersize = (1 << PAGE_SHIFT); 1944 struct device *dev = endpoint->dev; 1945 1946 /* 1947 * The bogus IDT is used during bootstrap for allocating the initial 1948 * message buffer, and then the message buffer and space for the IDT 1949 * itself. The initial message buffer is of a single page's size, but 1950 * it's soon replaced with a more modest one (and memory is freed). 1951 */ 1952 1953 unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0, 1954 3, 192, PAGE_SHIFT, 0 }; 1955 struct xilly_idt_handle idt_handle; 1956 1957 /* 1958 * Writing the value 0x00000001 to Endianness register signals which 1959 * endianness this processor is using, so the FPGA can swap words as 1960 * necessary. 1961 */ 1962 1963 iowrite32(1, endpoint->registers + fpga_endian_reg); 1964 1965 /* Bootstrap phase I: Allocate temporary message buffer */ 1966 1967 bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL); 1968 if (!bootstrap_resources) 1969 return -ENOMEM; 1970 1971 endpoint->num_channels = 0; 1972 1973 rc = xilly_setupchannels(endpoint, bogus_idt, 1); 1974 if (rc) 1975 return rc; 1976 1977 /* Clear the message subsystem (and counter in particular) */ 1978 iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg); 1979 1980 endpoint->idtlen = -1; 1981 1982 /* 1983 * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT 1984 * buffer size. 1985 */ 1986 iowrite32((u32) (endpoint->dma_using_dac & 0x0001), 1987 endpoint->registers + fpga_dma_control_reg); 1988 1989 t = wait_event_interruptible_timeout(endpoint->ep_wait, 1990 (endpoint->idtlen >= 0), 1991 XILLY_TIMEOUT); 1992 if (t <= 0) { 1993 dev_err(endpoint->dev, "No response from FPGA. Aborting.\n"); 1994 return -ENODEV; 1995 } 1996 1997 /* Enable DMA */ 1998 iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)), 1999 endpoint->registers + fpga_dma_control_reg); 2000 2001 /* Bootstrap phase II: Allocate buffer for IDT and obtain it */ 2002 while (endpoint->idtlen >= idtbuffersize) { 2003 idtbuffersize *= 2; 2004 bogus_idt[6]++; 2005 } 2006 2007 endpoint->num_channels = 1; 2008 2009 rc = xilly_setupchannels(endpoint, bogus_idt, 2); 2010 if (rc) 2011 goto failed_idt; 2012 2013 rc = xilly_obtain_idt(endpoint); 2014 if (rc) 2015 goto failed_idt; 2016 2017 rc = xilly_scan_idt(endpoint, &idt_handle); 2018 if (rc) 2019 goto failed_idt; 2020 2021 devres_close_group(dev, bootstrap_resources); 2022 2023 /* Bootstrap phase III: Allocate buffers according to IDT */ 2024 2025 rc = xilly_setupchannels(endpoint, 2026 idt_handle.chandesc, 2027 idt_handle.entries); 2028 if (rc) 2029 goto failed_idt; 2030 2031 /* 2032 * endpoint is now completely configured. We put it on the list 2033 * available to open() before registering the char device(s) 2034 */ 2035 2036 mutex_lock(&ep_list_lock); 2037 list_add_tail(&endpoint->ep_list, &list_of_endpoints); 2038 mutex_unlock(&ep_list_lock); 2039 2040 rc = xillybus_init_chrdev(endpoint, idt_handle.idt); 2041 if (rc) 2042 goto failed_chrdevs; 2043 2044 devres_release_group(dev, bootstrap_resources); 2045 2046 return 0; 2047 2048 failed_chrdevs: 2049 mutex_lock(&ep_list_lock); 2050 list_del(&endpoint->ep_list); 2051 mutex_unlock(&ep_list_lock); 2052 2053 failed_idt: 2054 xilly_quiesce(endpoint); 2055 flush_workqueue(xillybus_wq); 2056 2057 return rc; 2058 } 2059 EXPORT_SYMBOL(xillybus_endpoint_discovery); 2060 2061 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint) 2062 { 2063 xillybus_cleanup_chrdev(endpoint); 2064 2065 mutex_lock(&ep_list_lock); 2066 list_del(&endpoint->ep_list); 2067 mutex_unlock(&ep_list_lock); 2068 2069 xilly_quiesce(endpoint); 2070 2071 /* 2072 * Flushing is done upon endpoint release to prevent access to memory 2073 * just about to be released. This makes the quiesce complete. 2074 */ 2075 flush_workqueue(xillybus_wq); 2076 } 2077 EXPORT_SYMBOL(xillybus_endpoint_remove); 2078 2079 static int __init xillybus_init(void) 2080 { 2081 mutex_init(&ep_list_lock); 2082 2083 xillybus_class = class_create(THIS_MODULE, xillyname); 2084 if (IS_ERR(xillybus_class)) 2085 return PTR_ERR(xillybus_class); 2086 2087 xillybus_wq = alloc_workqueue(xillyname, 0, 0); 2088 if (!xillybus_wq) { 2089 class_destroy(xillybus_class); 2090 return -ENOMEM; 2091 } 2092 2093 return 0; 2094 } 2095 2096 static void __exit xillybus_exit(void) 2097 { 2098 /* flush_workqueue() was called for each endpoint released */ 2099 destroy_workqueue(xillybus_wq); 2100 2101 class_destroy(xillybus_class); 2102 } 2103 2104 module_init(xillybus_init); 2105 module_exit(xillybus_exit); 2106