1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * BSD LICENSE 14 * 15 * Copyright(c) 2012 Intel Corporation. All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 21 * * Redistributions of source code must retain the above copyright 22 * notice, this list of conditions and the following disclaimer. 23 * * Redistributions in binary form must reproduce the above copy 24 * notice, this list of conditions and the following disclaimer in 25 * the documentation and/or other materials provided with the 26 * distribution. 27 * * Neither the name of Intel Corporation nor the names of its 28 * contributors may be used to endorse or promote products derived 29 * from this software without specific prior written permission. 30 * 31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * Intel PCIe NTB Linux driver 44 * 45 * Contact Information: 46 * Jon Mason <jon.mason@intel.com> 47 */ 48 #include <linux/debugfs.h> 49 #include <linux/delay.h> 50 #include <linux/dmaengine.h> 51 #include <linux/dma-mapping.h> 52 #include <linux/errno.h> 53 #include <linux/export.h> 54 #include <linux/interrupt.h> 55 #include <linux/module.h> 56 #include <linux/pci.h> 57 #include <linux/slab.h> 58 #include <linux/types.h> 59 #include <linux/ntb.h> 60 #include "ntb_hw.h" 61 62 #define NTB_TRANSPORT_VERSION 3 63 64 static unsigned int transport_mtu = 0x401E; 65 module_param(transport_mtu, uint, 0644); 66 MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); 67 68 static unsigned char max_num_clients; 69 module_param(max_num_clients, byte, 0644); 70 MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients"); 71 72 static unsigned int copy_bytes = 1024; 73 module_param(copy_bytes, uint, 0644); 74 MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); 75 76 struct ntb_queue_entry { 77 /* ntb_queue list reference */ 78 struct list_head entry; 79 /* pointers to data to be transfered */ 80 void *cb_data; 81 void *buf; 82 unsigned int len; 83 unsigned int flags; 84 85 struct ntb_transport_qp *qp; 86 union { 87 struct ntb_payload_header __iomem *tx_hdr; 88 struct ntb_payload_header *rx_hdr; 89 }; 90 unsigned int index; 91 }; 92 93 struct ntb_rx_info { 94 unsigned int entry; 95 }; 96 97 struct ntb_transport_qp { 98 struct ntb_transport *transport; 99 struct ntb_device *ndev; 100 void *cb_data; 101 struct dma_chan *dma_chan; 102 103 bool client_ready; 104 bool qp_link; 105 u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ 106 107 struct ntb_rx_info __iomem *rx_info; 108 struct ntb_rx_info *remote_rx_info; 109 110 void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data, 111 void *data, int len); 112 struct list_head tx_free_q; 113 spinlock_t ntb_tx_free_q_lock; 114 void __iomem *tx_mw; 115 dma_addr_t tx_mw_phys; 116 unsigned int tx_index; 117 unsigned int tx_max_entry; 118 unsigned int tx_max_frame; 119 120 void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data, 121 void *data, int len); 122 struct tasklet_struct rx_work; 123 struct list_head rx_pend_q; 124 struct list_head rx_free_q; 125 spinlock_t ntb_rx_pend_q_lock; 126 spinlock_t ntb_rx_free_q_lock; 127 void *rx_buff; 128 unsigned int rx_index; 129 unsigned int rx_max_entry; 130 unsigned int rx_max_frame; 131 dma_cookie_t last_cookie; 132 133 void (*event_handler) (void *data, int status); 134 struct delayed_work link_work; 135 struct work_struct link_cleanup; 136 137 struct dentry *debugfs_dir; 138 struct dentry *debugfs_stats; 139 140 /* Stats */ 141 u64 rx_bytes; 142 u64 rx_pkts; 143 u64 rx_ring_empty; 144 u64 rx_err_no_buf; 145 u64 rx_err_oflow; 146 u64 rx_err_ver; 147 u64 rx_memcpy; 148 u64 rx_async; 149 u64 tx_bytes; 150 u64 tx_pkts; 151 u64 tx_ring_full; 152 u64 tx_err_no_buf; 153 u64 tx_memcpy; 154 u64 tx_async; 155 }; 156 157 struct ntb_transport_mw { 158 size_t size; 159 void *virt_addr; 160 dma_addr_t dma_addr; 161 }; 162 163 struct ntb_transport_client_dev { 164 struct list_head entry; 165 struct device dev; 166 }; 167 168 struct ntb_transport { 169 struct list_head entry; 170 struct list_head client_devs; 171 172 struct ntb_device *ndev; 173 struct ntb_transport_mw *mw; 174 struct ntb_transport_qp *qps; 175 unsigned int max_qps; 176 unsigned long qp_bitmap; 177 bool transport_link; 178 struct delayed_work link_work; 179 struct work_struct link_cleanup; 180 }; 181 182 enum { 183 DESC_DONE_FLAG = 1 << 0, 184 LINK_DOWN_FLAG = 1 << 1, 185 }; 186 187 struct ntb_payload_header { 188 unsigned int ver; 189 unsigned int len; 190 unsigned int flags; 191 }; 192 193 enum { 194 VERSION = 0, 195 QP_LINKS, 196 NUM_QPS, 197 NUM_MWS, 198 MW0_SZ_HIGH, 199 MW0_SZ_LOW, 200 MW1_SZ_HIGH, 201 MW1_SZ_LOW, 202 MAX_SPAD, 203 }; 204 205 #define QP_TO_MW(ndev, qp) ((qp) % ntb_max_mw(ndev)) 206 #define NTB_QP_DEF_NUM_ENTRIES 100 207 #define NTB_LINK_DOWN_TIMEOUT 10 208 209 static int ntb_match_bus(struct device *dev, struct device_driver *drv) 210 { 211 return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); 212 } 213 214 static int ntb_client_probe(struct device *dev) 215 { 216 const struct ntb_client *drv = container_of(dev->driver, 217 struct ntb_client, driver); 218 struct pci_dev *pdev = container_of(dev->parent, struct pci_dev, dev); 219 int rc = -EINVAL; 220 221 get_device(dev); 222 if (drv && drv->probe) 223 rc = drv->probe(pdev); 224 if (rc) 225 put_device(dev); 226 227 return rc; 228 } 229 230 static int ntb_client_remove(struct device *dev) 231 { 232 const struct ntb_client *drv = container_of(dev->driver, 233 struct ntb_client, driver); 234 struct pci_dev *pdev = container_of(dev->parent, struct pci_dev, dev); 235 236 if (drv && drv->remove) 237 drv->remove(pdev); 238 239 put_device(dev); 240 241 return 0; 242 } 243 244 static struct bus_type ntb_bus_type = { 245 .name = "ntb_bus", 246 .match = ntb_match_bus, 247 .probe = ntb_client_probe, 248 .remove = ntb_client_remove, 249 }; 250 251 static LIST_HEAD(ntb_transport_list); 252 253 static int ntb_bus_init(struct ntb_transport *nt) 254 { 255 if (list_empty(&ntb_transport_list)) { 256 int rc = bus_register(&ntb_bus_type); 257 if (rc) 258 return rc; 259 } 260 261 list_add(&nt->entry, &ntb_transport_list); 262 263 return 0; 264 } 265 266 static void ntb_bus_remove(struct ntb_transport *nt) 267 { 268 struct ntb_transport_client_dev *client_dev, *cd; 269 270 list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) { 271 dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n", 272 dev_name(&client_dev->dev)); 273 list_del(&client_dev->entry); 274 device_unregister(&client_dev->dev); 275 } 276 277 list_del(&nt->entry); 278 279 if (list_empty(&ntb_transport_list)) 280 bus_unregister(&ntb_bus_type); 281 } 282 283 static void ntb_client_release(struct device *dev) 284 { 285 struct ntb_transport_client_dev *client_dev; 286 client_dev = container_of(dev, struct ntb_transport_client_dev, dev); 287 288 kfree(client_dev); 289 } 290 291 /** 292 * ntb_unregister_client_dev - Unregister NTB client device 293 * @device_name: Name of NTB client device 294 * 295 * Unregister an NTB client device with the NTB transport layer 296 */ 297 void ntb_unregister_client_dev(char *device_name) 298 { 299 struct ntb_transport_client_dev *client, *cd; 300 struct ntb_transport *nt; 301 302 list_for_each_entry(nt, &ntb_transport_list, entry) 303 list_for_each_entry_safe(client, cd, &nt->client_devs, entry) 304 if (!strncmp(dev_name(&client->dev), device_name, 305 strlen(device_name))) { 306 list_del(&client->entry); 307 device_unregister(&client->dev); 308 } 309 } 310 EXPORT_SYMBOL_GPL(ntb_unregister_client_dev); 311 312 /** 313 * ntb_register_client_dev - Register NTB client device 314 * @device_name: Name of NTB client device 315 * 316 * Register an NTB client device with the NTB transport layer 317 */ 318 int ntb_register_client_dev(char *device_name) 319 { 320 struct ntb_transport_client_dev *client_dev; 321 struct ntb_transport *nt; 322 int rc, i = 0; 323 324 if (list_empty(&ntb_transport_list)) 325 return -ENODEV; 326 327 list_for_each_entry(nt, &ntb_transport_list, entry) { 328 struct device *dev; 329 330 client_dev = kzalloc(sizeof(struct ntb_transport_client_dev), 331 GFP_KERNEL); 332 if (!client_dev) { 333 rc = -ENOMEM; 334 goto err; 335 } 336 337 dev = &client_dev->dev; 338 339 /* setup and register client devices */ 340 dev_set_name(dev, "%s%d", device_name, i); 341 dev->bus = &ntb_bus_type; 342 dev->release = ntb_client_release; 343 dev->parent = &ntb_query_pdev(nt->ndev)->dev; 344 345 rc = device_register(dev); 346 if (rc) { 347 kfree(client_dev); 348 goto err; 349 } 350 351 list_add_tail(&client_dev->entry, &nt->client_devs); 352 i++; 353 } 354 355 return 0; 356 357 err: 358 ntb_unregister_client_dev(device_name); 359 360 return rc; 361 } 362 EXPORT_SYMBOL_GPL(ntb_register_client_dev); 363 364 /** 365 * ntb_register_client - Register NTB client driver 366 * @drv: NTB client driver to be registered 367 * 368 * Register an NTB client driver with the NTB transport layer 369 * 370 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 371 */ 372 int ntb_register_client(struct ntb_client *drv) 373 { 374 drv->driver.bus = &ntb_bus_type; 375 376 if (list_empty(&ntb_transport_list)) 377 return -ENODEV; 378 379 return driver_register(&drv->driver); 380 } 381 EXPORT_SYMBOL_GPL(ntb_register_client); 382 383 /** 384 * ntb_unregister_client - Unregister NTB client driver 385 * @drv: NTB client driver to be unregistered 386 * 387 * Unregister an NTB client driver with the NTB transport layer 388 * 389 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 390 */ 391 void ntb_unregister_client(struct ntb_client *drv) 392 { 393 driver_unregister(&drv->driver); 394 } 395 EXPORT_SYMBOL_GPL(ntb_unregister_client); 396 397 static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, 398 loff_t *offp) 399 { 400 struct ntb_transport_qp *qp; 401 char *buf; 402 ssize_t ret, out_offset, out_count; 403 404 out_count = 1000; 405 406 buf = kmalloc(out_count, GFP_KERNEL); 407 if (!buf) 408 return -ENOMEM; 409 410 qp = filp->private_data; 411 out_offset = 0; 412 out_offset += snprintf(buf + out_offset, out_count - out_offset, 413 "NTB QP stats\n"); 414 out_offset += snprintf(buf + out_offset, out_count - out_offset, 415 "rx_bytes - \t%llu\n", qp->rx_bytes); 416 out_offset += snprintf(buf + out_offset, out_count - out_offset, 417 "rx_pkts - \t%llu\n", qp->rx_pkts); 418 out_offset += snprintf(buf + out_offset, out_count - out_offset, 419 "rx_memcpy - \t%llu\n", qp->rx_memcpy); 420 out_offset += snprintf(buf + out_offset, out_count - out_offset, 421 "rx_async - \t%llu\n", qp->rx_async); 422 out_offset += snprintf(buf + out_offset, out_count - out_offset, 423 "rx_ring_empty - %llu\n", qp->rx_ring_empty); 424 out_offset += snprintf(buf + out_offset, out_count - out_offset, 425 "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); 426 out_offset += snprintf(buf + out_offset, out_count - out_offset, 427 "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); 428 out_offset += snprintf(buf + out_offset, out_count - out_offset, 429 "rx_err_ver - \t%llu\n", qp->rx_err_ver); 430 out_offset += snprintf(buf + out_offset, out_count - out_offset, 431 "rx_buff - \t%p\n", qp->rx_buff); 432 out_offset += snprintf(buf + out_offset, out_count - out_offset, 433 "rx_index - \t%u\n", qp->rx_index); 434 out_offset += snprintf(buf + out_offset, out_count - out_offset, 435 "rx_max_entry - \t%u\n", qp->rx_max_entry); 436 437 out_offset += snprintf(buf + out_offset, out_count - out_offset, 438 "tx_bytes - \t%llu\n", qp->tx_bytes); 439 out_offset += snprintf(buf + out_offset, out_count - out_offset, 440 "tx_pkts - \t%llu\n", qp->tx_pkts); 441 out_offset += snprintf(buf + out_offset, out_count - out_offset, 442 "tx_memcpy - \t%llu\n", qp->tx_memcpy); 443 out_offset += snprintf(buf + out_offset, out_count - out_offset, 444 "tx_async - \t%llu\n", qp->tx_async); 445 out_offset += snprintf(buf + out_offset, out_count - out_offset, 446 "tx_ring_full - \t%llu\n", qp->tx_ring_full); 447 out_offset += snprintf(buf + out_offset, out_count - out_offset, 448 "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); 449 out_offset += snprintf(buf + out_offset, out_count - out_offset, 450 "tx_mw - \t%p\n", qp->tx_mw); 451 out_offset += snprintf(buf + out_offset, out_count - out_offset, 452 "tx_index - \t%u\n", qp->tx_index); 453 out_offset += snprintf(buf + out_offset, out_count - out_offset, 454 "tx_max_entry - \t%u\n", qp->tx_max_entry); 455 456 out_offset += snprintf(buf + out_offset, out_count - out_offset, 457 "\nQP Link %s\n", (qp->qp_link == NTB_LINK_UP) ? 458 "Up" : "Down"); 459 if (out_offset > out_count) 460 out_offset = out_count; 461 462 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); 463 kfree(buf); 464 return ret; 465 } 466 467 static const struct file_operations ntb_qp_debugfs_stats = { 468 .owner = THIS_MODULE, 469 .open = simple_open, 470 .read = debugfs_read, 471 }; 472 473 static void ntb_list_add(spinlock_t *lock, struct list_head *entry, 474 struct list_head *list) 475 { 476 unsigned long flags; 477 478 spin_lock_irqsave(lock, flags); 479 list_add_tail(entry, list); 480 spin_unlock_irqrestore(lock, flags); 481 } 482 483 static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, 484 struct list_head *list) 485 { 486 struct ntb_queue_entry *entry; 487 unsigned long flags; 488 489 spin_lock_irqsave(lock, flags); 490 if (list_empty(list)) { 491 entry = NULL; 492 goto out; 493 } 494 entry = list_first_entry(list, struct ntb_queue_entry, entry); 495 list_del(&entry->entry); 496 out: 497 spin_unlock_irqrestore(lock, flags); 498 499 return entry; 500 } 501 502 static void ntb_transport_setup_qp_mw(struct ntb_transport *nt, 503 unsigned int qp_num) 504 { 505 struct ntb_transport_qp *qp = &nt->qps[qp_num]; 506 unsigned int rx_size, num_qps_mw; 507 u8 mw_num, mw_max; 508 unsigned int i; 509 510 mw_max = ntb_max_mw(nt->ndev); 511 mw_num = QP_TO_MW(nt->ndev, qp_num); 512 513 WARN_ON(nt->mw[mw_num].virt_addr == NULL); 514 515 if (nt->max_qps % mw_max && mw_num < nt->max_qps % mw_max) 516 num_qps_mw = nt->max_qps / mw_max + 1; 517 else 518 num_qps_mw = nt->max_qps / mw_max; 519 520 rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw; 521 qp->rx_buff = nt->mw[mw_num].virt_addr + qp_num / mw_max * rx_size; 522 rx_size -= sizeof(struct ntb_rx_info); 523 524 qp->remote_rx_info = qp->rx_buff + rx_size; 525 526 /* Due to housekeeping, there must be atleast 2 buffs */ 527 qp->rx_max_frame = min(transport_mtu, rx_size / 2); 528 qp->rx_max_entry = rx_size / qp->rx_max_frame; 529 qp->rx_index = 0; 530 531 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 532 533 /* setup the hdr offsets with 0's */ 534 for (i = 0; i < qp->rx_max_entry; i++) { 535 void *offset = qp->rx_buff + qp->rx_max_frame * (i + 1) - 536 sizeof(struct ntb_payload_header); 537 memset(offset, 0, sizeof(struct ntb_payload_header)); 538 } 539 540 qp->rx_pkts = 0; 541 qp->tx_pkts = 0; 542 qp->tx_index = 0; 543 } 544 545 static void ntb_free_mw(struct ntb_transport *nt, int num_mw) 546 { 547 struct ntb_transport_mw *mw = &nt->mw[num_mw]; 548 struct pci_dev *pdev = ntb_query_pdev(nt->ndev); 549 550 if (!mw->virt_addr) 551 return; 552 553 dma_free_coherent(&pdev->dev, mw->size, mw->virt_addr, mw->dma_addr); 554 mw->virt_addr = NULL; 555 } 556 557 static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size) 558 { 559 struct ntb_transport_mw *mw = &nt->mw[num_mw]; 560 struct pci_dev *pdev = ntb_query_pdev(nt->ndev); 561 562 /* No need to re-setup */ 563 if (mw->size == ALIGN(size, 4096)) 564 return 0; 565 566 if (mw->size != 0) 567 ntb_free_mw(nt, num_mw); 568 569 /* Alloc memory for receiving data. Must be 4k aligned */ 570 mw->size = ALIGN(size, 4096); 571 572 mw->virt_addr = dma_alloc_coherent(&pdev->dev, mw->size, &mw->dma_addr, 573 GFP_KERNEL); 574 if (!mw->virt_addr) { 575 mw->size = 0; 576 dev_err(&pdev->dev, "Unable to allocate MW buffer of size %d\n", 577 (int) mw->size); 578 return -ENOMEM; 579 } 580 581 /* Notify HW the memory location of the receive buffer */ 582 ntb_set_mw_addr(nt->ndev, num_mw, mw->dma_addr); 583 584 return 0; 585 } 586 587 static void ntb_qp_link_cleanup(struct work_struct *work) 588 { 589 struct ntb_transport_qp *qp = container_of(work, 590 struct ntb_transport_qp, 591 link_cleanup); 592 struct ntb_transport *nt = qp->transport; 593 struct pci_dev *pdev = ntb_query_pdev(nt->ndev); 594 595 if (qp->qp_link == NTB_LINK_DOWN) { 596 cancel_delayed_work_sync(&qp->link_work); 597 return; 598 } 599 600 if (qp->event_handler) 601 qp->event_handler(qp->cb_data, NTB_LINK_DOWN); 602 603 dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); 604 qp->qp_link = NTB_LINK_DOWN; 605 606 if (nt->transport_link == NTB_LINK_UP) 607 schedule_delayed_work(&qp->link_work, 608 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 609 } 610 611 static void ntb_qp_link_down(struct ntb_transport_qp *qp) 612 { 613 schedule_work(&qp->link_cleanup); 614 } 615 616 static void ntb_transport_link_cleanup(struct work_struct *work) 617 { 618 struct ntb_transport *nt = container_of(work, struct ntb_transport, 619 link_cleanup); 620 int i; 621 622 if (nt->transport_link == NTB_LINK_DOWN) 623 cancel_delayed_work_sync(&nt->link_work); 624 else 625 nt->transport_link = NTB_LINK_DOWN; 626 627 /* Pass along the info to any clients */ 628 for (i = 0; i < nt->max_qps; i++) 629 if (!test_bit(i, &nt->qp_bitmap)) 630 ntb_qp_link_down(&nt->qps[i]); 631 632 /* The scratchpad registers keep the values if the remote side 633 * goes down, blast them now to give them a sane value the next 634 * time they are accessed 635 */ 636 for (i = 0; i < MAX_SPAD; i++) 637 ntb_write_local_spad(nt->ndev, i, 0); 638 } 639 640 static void ntb_transport_event_callback(void *data, enum ntb_hw_event event) 641 { 642 struct ntb_transport *nt = data; 643 644 switch (event) { 645 case NTB_EVENT_HW_LINK_UP: 646 schedule_delayed_work(&nt->link_work, 0); 647 break; 648 case NTB_EVENT_HW_LINK_DOWN: 649 schedule_work(&nt->link_cleanup); 650 break; 651 default: 652 BUG(); 653 } 654 } 655 656 static void ntb_transport_link_work(struct work_struct *work) 657 { 658 struct ntb_transport *nt = container_of(work, struct ntb_transport, 659 link_work.work); 660 struct ntb_device *ndev = nt->ndev; 661 struct pci_dev *pdev = ntb_query_pdev(ndev); 662 u32 val; 663 int rc, i; 664 665 /* send the local info, in the opposite order of the way we read it */ 666 for (i = 0; i < ntb_max_mw(ndev); i++) { 667 rc = ntb_write_remote_spad(ndev, MW0_SZ_HIGH + (i * 2), 668 ntb_get_mw_size(ndev, i) >> 32); 669 if (rc) { 670 dev_err(&pdev->dev, "Error writing %u to remote spad %d\n", 671 (u32)(ntb_get_mw_size(ndev, i) >> 32), 672 MW0_SZ_HIGH + (i * 2)); 673 goto out; 674 } 675 676 rc = ntb_write_remote_spad(ndev, MW0_SZ_LOW + (i * 2), 677 (u32) ntb_get_mw_size(ndev, i)); 678 if (rc) { 679 dev_err(&pdev->dev, "Error writing %u to remote spad %d\n", 680 (u32) ntb_get_mw_size(ndev, i), 681 MW0_SZ_LOW + (i * 2)); 682 goto out; 683 } 684 } 685 686 rc = ntb_write_remote_spad(ndev, NUM_MWS, ntb_max_mw(ndev)); 687 if (rc) { 688 dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", 689 ntb_max_mw(ndev), NUM_MWS); 690 goto out; 691 } 692 693 rc = ntb_write_remote_spad(ndev, NUM_QPS, nt->max_qps); 694 if (rc) { 695 dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", 696 nt->max_qps, NUM_QPS); 697 goto out; 698 } 699 700 rc = ntb_write_remote_spad(ndev, VERSION, NTB_TRANSPORT_VERSION); 701 if (rc) { 702 dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", 703 NTB_TRANSPORT_VERSION, VERSION); 704 goto out; 705 } 706 707 /* Query the remote side for its info */ 708 rc = ntb_read_remote_spad(ndev, VERSION, &val); 709 if (rc) { 710 dev_err(&pdev->dev, "Error reading remote spad %d\n", VERSION); 711 goto out; 712 } 713 714 if (val != NTB_TRANSPORT_VERSION) 715 goto out; 716 dev_dbg(&pdev->dev, "Remote version = %d\n", val); 717 718 rc = ntb_read_remote_spad(ndev, NUM_QPS, &val); 719 if (rc) { 720 dev_err(&pdev->dev, "Error reading remote spad %d\n", NUM_QPS); 721 goto out; 722 } 723 724 if (val != nt->max_qps) 725 goto out; 726 dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); 727 728 rc = ntb_read_remote_spad(ndev, NUM_MWS, &val); 729 if (rc) { 730 dev_err(&pdev->dev, "Error reading remote spad %d\n", NUM_MWS); 731 goto out; 732 } 733 734 if (val != ntb_max_mw(ndev)) 735 goto out; 736 dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); 737 738 for (i = 0; i < ntb_max_mw(ndev); i++) { 739 u64 val64; 740 741 rc = ntb_read_remote_spad(ndev, MW0_SZ_HIGH + (i * 2), &val); 742 if (rc) { 743 dev_err(&pdev->dev, "Error reading remote spad %d\n", 744 MW0_SZ_HIGH + (i * 2)); 745 goto out1; 746 } 747 748 val64 = (u64) val << 32; 749 750 rc = ntb_read_remote_spad(ndev, MW0_SZ_LOW + (i * 2), &val); 751 if (rc) { 752 dev_err(&pdev->dev, "Error reading remote spad %d\n", 753 MW0_SZ_LOW + (i * 2)); 754 goto out1; 755 } 756 757 val64 |= val; 758 759 dev_dbg(&pdev->dev, "Remote MW%d size = %llu\n", i, val64); 760 761 rc = ntb_set_mw(nt, i, val64); 762 if (rc) 763 goto out1; 764 } 765 766 nt->transport_link = NTB_LINK_UP; 767 768 for (i = 0; i < nt->max_qps; i++) { 769 struct ntb_transport_qp *qp = &nt->qps[i]; 770 771 ntb_transport_setup_qp_mw(nt, i); 772 773 if (qp->client_ready == NTB_LINK_UP) 774 schedule_delayed_work(&qp->link_work, 0); 775 } 776 777 return; 778 779 out1: 780 for (i = 0; i < ntb_max_mw(ndev); i++) 781 ntb_free_mw(nt, i); 782 out: 783 if (ntb_hw_link_status(ndev)) 784 schedule_delayed_work(&nt->link_work, 785 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 786 } 787 788 static void ntb_qp_link_work(struct work_struct *work) 789 { 790 struct ntb_transport_qp *qp = container_of(work, 791 struct ntb_transport_qp, 792 link_work.work); 793 struct pci_dev *pdev = ntb_query_pdev(qp->ndev); 794 struct ntb_transport *nt = qp->transport; 795 int rc, val; 796 797 WARN_ON(nt->transport_link != NTB_LINK_UP); 798 799 rc = ntb_read_local_spad(nt->ndev, QP_LINKS, &val); 800 if (rc) { 801 dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS); 802 return; 803 } 804 805 rc = ntb_write_remote_spad(nt->ndev, QP_LINKS, val | 1 << qp->qp_num); 806 if (rc) 807 dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", 808 val | 1 << qp->qp_num, QP_LINKS); 809 810 /* query remote spad for qp ready bits */ 811 rc = ntb_read_remote_spad(nt->ndev, QP_LINKS, &val); 812 if (rc) 813 dev_err(&pdev->dev, "Error reading remote spad %d\n", QP_LINKS); 814 815 dev_dbg(&pdev->dev, "Remote QP link status = %x\n", val); 816 817 /* See if the remote side is up */ 818 if (1 << qp->qp_num & val) { 819 qp->qp_link = NTB_LINK_UP; 820 821 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); 822 if (qp->event_handler) 823 qp->event_handler(qp->cb_data, NTB_LINK_UP); 824 } else if (nt->transport_link == NTB_LINK_UP) 825 schedule_delayed_work(&qp->link_work, 826 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 827 } 828 829 static int ntb_transport_init_queue(struct ntb_transport *nt, 830 unsigned int qp_num) 831 { 832 struct ntb_transport_qp *qp; 833 unsigned int num_qps_mw, tx_size; 834 u8 mw_num, mw_max; 835 u64 qp_offset; 836 837 mw_max = ntb_max_mw(nt->ndev); 838 mw_num = QP_TO_MW(nt->ndev, qp_num); 839 840 qp = &nt->qps[qp_num]; 841 qp->qp_num = qp_num; 842 qp->transport = nt; 843 qp->ndev = nt->ndev; 844 qp->qp_link = NTB_LINK_DOWN; 845 qp->client_ready = NTB_LINK_DOWN; 846 qp->event_handler = NULL; 847 848 if (nt->max_qps % mw_max && mw_num < nt->max_qps % mw_max) 849 num_qps_mw = nt->max_qps / mw_max + 1; 850 else 851 num_qps_mw = nt->max_qps / mw_max; 852 853 tx_size = (unsigned int) ntb_get_mw_size(qp->ndev, mw_num) / num_qps_mw; 854 qp_offset = qp_num / mw_max * tx_size; 855 qp->tx_mw = ntb_get_mw_vbase(nt->ndev, mw_num) + qp_offset; 856 if (!qp->tx_mw) 857 return -EINVAL; 858 859 qp->tx_mw_phys = ntb_get_mw_base(qp->ndev, mw_num) + qp_offset; 860 if (!qp->tx_mw_phys) 861 return -EINVAL; 862 863 tx_size -= sizeof(struct ntb_rx_info); 864 qp->rx_info = qp->tx_mw + tx_size; 865 866 /* Due to housekeeping, there must be atleast 2 buffs */ 867 qp->tx_max_frame = min(transport_mtu, tx_size / 2); 868 qp->tx_max_entry = tx_size / qp->tx_max_frame; 869 870 if (ntb_query_debugfs(nt->ndev)) { 871 char debugfs_name[4]; 872 873 snprintf(debugfs_name, 4, "qp%d", qp_num); 874 qp->debugfs_dir = debugfs_create_dir(debugfs_name, 875 ntb_query_debugfs(nt->ndev)); 876 877 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, 878 qp->debugfs_dir, qp, 879 &ntb_qp_debugfs_stats); 880 } 881 882 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); 883 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup); 884 885 spin_lock_init(&qp->ntb_rx_pend_q_lock); 886 spin_lock_init(&qp->ntb_rx_free_q_lock); 887 spin_lock_init(&qp->ntb_tx_free_q_lock); 888 889 INIT_LIST_HEAD(&qp->rx_pend_q); 890 INIT_LIST_HEAD(&qp->rx_free_q); 891 INIT_LIST_HEAD(&qp->tx_free_q); 892 893 return 0; 894 } 895 896 int ntb_transport_init(struct pci_dev *pdev) 897 { 898 struct ntb_transport *nt; 899 int rc, i; 900 901 nt = kzalloc(sizeof(struct ntb_transport), GFP_KERNEL); 902 if (!nt) 903 return -ENOMEM; 904 905 nt->ndev = ntb_register_transport(pdev, nt); 906 if (!nt->ndev) { 907 rc = -EIO; 908 goto err; 909 } 910 911 nt->mw = kcalloc(ntb_max_mw(nt->ndev), sizeof(struct ntb_transport_mw), 912 GFP_KERNEL); 913 if (!nt->mw) { 914 rc = -ENOMEM; 915 goto err1; 916 } 917 918 if (max_num_clients) 919 nt->max_qps = min(ntb_max_cbs(nt->ndev), max_num_clients); 920 else 921 nt->max_qps = min(ntb_max_cbs(nt->ndev), ntb_max_mw(nt->ndev)); 922 923 nt->qps = kcalloc(nt->max_qps, sizeof(struct ntb_transport_qp), 924 GFP_KERNEL); 925 if (!nt->qps) { 926 rc = -ENOMEM; 927 goto err2; 928 } 929 930 nt->qp_bitmap = ((u64) 1 << nt->max_qps) - 1; 931 932 for (i = 0; i < nt->max_qps; i++) { 933 rc = ntb_transport_init_queue(nt, i); 934 if (rc) 935 goto err3; 936 } 937 938 INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); 939 INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup); 940 941 rc = ntb_register_event_callback(nt->ndev, 942 ntb_transport_event_callback); 943 if (rc) 944 goto err3; 945 946 INIT_LIST_HEAD(&nt->client_devs); 947 rc = ntb_bus_init(nt); 948 if (rc) 949 goto err4; 950 951 if (ntb_hw_link_status(nt->ndev)) 952 schedule_delayed_work(&nt->link_work, 0); 953 954 return 0; 955 956 err4: 957 ntb_unregister_event_callback(nt->ndev); 958 err3: 959 kfree(nt->qps); 960 err2: 961 kfree(nt->mw); 962 err1: 963 ntb_unregister_transport(nt->ndev); 964 err: 965 kfree(nt); 966 return rc; 967 } 968 969 void ntb_transport_free(void *transport) 970 { 971 struct ntb_transport *nt = transport; 972 struct ntb_device *ndev = nt->ndev; 973 int i; 974 975 nt->transport_link = NTB_LINK_DOWN; 976 977 /* verify that all the qp's are freed */ 978 for (i = 0; i < nt->max_qps; i++) { 979 if (!test_bit(i, &nt->qp_bitmap)) 980 ntb_transport_free_queue(&nt->qps[i]); 981 debugfs_remove_recursive(nt->qps[i].debugfs_dir); 982 } 983 984 ntb_bus_remove(nt); 985 986 cancel_delayed_work_sync(&nt->link_work); 987 988 ntb_unregister_event_callback(ndev); 989 990 for (i = 0; i < ntb_max_mw(ndev); i++) 991 ntb_free_mw(nt, i); 992 993 kfree(nt->qps); 994 kfree(nt->mw); 995 ntb_unregister_transport(ndev); 996 kfree(nt); 997 } 998 999 static void ntb_rx_copy_callback(void *data) 1000 { 1001 struct ntb_queue_entry *entry = data; 1002 struct ntb_transport_qp *qp = entry->qp; 1003 void *cb_data = entry->cb_data; 1004 unsigned int len = entry->len; 1005 struct ntb_payload_header *hdr = entry->rx_hdr; 1006 1007 /* Ensure that the data is fully copied out before clearing the flag */ 1008 wmb(); 1009 hdr->flags = 0; 1010 1011 iowrite32(entry->index, &qp->rx_info->entry); 1012 1013 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q); 1014 1015 if (qp->rx_handler && qp->client_ready == NTB_LINK_UP) 1016 qp->rx_handler(qp, qp->cb_data, cb_data, len); 1017 } 1018 1019 static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) 1020 { 1021 void *buf = entry->buf; 1022 size_t len = entry->len; 1023 1024 memcpy(buf, offset, len); 1025 1026 ntb_rx_copy_callback(entry); 1027 } 1028 1029 static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset, 1030 size_t len) 1031 { 1032 struct dma_async_tx_descriptor *txd; 1033 struct ntb_transport_qp *qp = entry->qp; 1034 struct dma_chan *chan = qp->dma_chan; 1035 struct dma_device *device; 1036 size_t pay_off, buff_off; 1037 dma_addr_t src, dest; 1038 dma_cookie_t cookie; 1039 void *buf = entry->buf; 1040 unsigned long flags; 1041 1042 entry->len = len; 1043 1044 if (!chan) 1045 goto err; 1046 1047 if (len < copy_bytes) 1048 goto err1; 1049 1050 device = chan->device; 1051 pay_off = (size_t) offset & ~PAGE_MASK; 1052 buff_off = (size_t) buf & ~PAGE_MASK; 1053 1054 if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) 1055 goto err1; 1056 1057 dest = dma_map_single(device->dev, buf, len, DMA_FROM_DEVICE); 1058 if (dma_mapping_error(device->dev, dest)) 1059 goto err1; 1060 1061 src = dma_map_single(device->dev, offset, len, DMA_TO_DEVICE); 1062 if (dma_mapping_error(device->dev, src)) 1063 goto err2; 1064 1065 flags = DMA_COMPL_DEST_UNMAP_SINGLE | DMA_COMPL_SRC_UNMAP_SINGLE | 1066 DMA_PREP_INTERRUPT; 1067 txd = device->device_prep_dma_memcpy(chan, dest, src, len, flags); 1068 if (!txd) 1069 goto err3; 1070 1071 txd->callback = ntb_rx_copy_callback; 1072 txd->callback_param = entry; 1073 1074 cookie = dmaengine_submit(txd); 1075 if (dma_submit_error(cookie)) 1076 goto err3; 1077 1078 qp->last_cookie = cookie; 1079 1080 qp->rx_async++; 1081 1082 return; 1083 1084 err3: 1085 dma_unmap_single(device->dev, src, len, DMA_TO_DEVICE); 1086 err2: 1087 dma_unmap_single(device->dev, dest, len, DMA_FROM_DEVICE); 1088 err1: 1089 /* If the callbacks come out of order, the writing of the index to the 1090 * last completed will be out of order. This may result in the 1091 * receive stalling forever. 1092 */ 1093 dma_sync_wait(chan, qp->last_cookie); 1094 err: 1095 ntb_memcpy_rx(entry, offset); 1096 qp->rx_memcpy++; 1097 } 1098 1099 static int ntb_process_rxc(struct ntb_transport_qp *qp) 1100 { 1101 struct ntb_payload_header *hdr; 1102 struct ntb_queue_entry *entry; 1103 void *offset; 1104 1105 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; 1106 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); 1107 1108 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); 1109 if (!entry) { 1110 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, 1111 "no buffer - HDR ver %u, len %d, flags %x\n", 1112 hdr->ver, hdr->len, hdr->flags); 1113 qp->rx_err_no_buf++; 1114 return -ENOMEM; 1115 } 1116 1117 if (!(hdr->flags & DESC_DONE_FLAG)) { 1118 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, 1119 &qp->rx_pend_q); 1120 qp->rx_ring_empty++; 1121 return -EAGAIN; 1122 } 1123 1124 if (hdr->ver != (u32) qp->rx_pkts) { 1125 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, 1126 "qp %d: version mismatch, expected %llu - got %u\n", 1127 qp->qp_num, qp->rx_pkts, hdr->ver); 1128 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, 1129 &qp->rx_pend_q); 1130 qp->rx_err_ver++; 1131 return -EIO; 1132 } 1133 1134 if (hdr->flags & LINK_DOWN_FLAG) { 1135 ntb_qp_link_down(qp); 1136 1137 goto err; 1138 } 1139 1140 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, 1141 "rx offset %u, ver %u - %d payload received, buf size %d\n", 1142 qp->rx_index, hdr->ver, hdr->len, entry->len); 1143 1144 qp->rx_bytes += hdr->len; 1145 qp->rx_pkts++; 1146 1147 if (hdr->len > entry->len) { 1148 qp->rx_err_oflow++; 1149 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, 1150 "RX overflow! Wanted %d got %d\n", 1151 hdr->len, entry->len); 1152 1153 goto err; 1154 } 1155 1156 entry->index = qp->rx_index; 1157 entry->rx_hdr = hdr; 1158 1159 ntb_async_rx(entry, offset, hdr->len); 1160 1161 out: 1162 qp->rx_index++; 1163 qp->rx_index %= qp->rx_max_entry; 1164 1165 return 0; 1166 1167 err: 1168 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, 1169 &qp->rx_pend_q); 1170 /* Ensure that the data is fully copied out before clearing the flag */ 1171 wmb(); 1172 hdr->flags = 0; 1173 iowrite32(qp->rx_index, &qp->rx_info->entry); 1174 1175 goto out; 1176 } 1177 1178 static void ntb_transport_rx(unsigned long data) 1179 { 1180 struct ntb_transport_qp *qp = (struct ntb_transport_qp *)data; 1181 int rc, i; 1182 1183 /* Limit the number of packets processed in a single interrupt to 1184 * provide fairness to others 1185 */ 1186 for (i = 0; i < qp->rx_max_entry; i++) { 1187 rc = ntb_process_rxc(qp); 1188 if (rc) 1189 break; 1190 } 1191 1192 if (qp->dma_chan) 1193 dma_async_issue_pending(qp->dma_chan); 1194 } 1195 1196 static void ntb_transport_rxc_db(void *data, int db_num) 1197 { 1198 struct ntb_transport_qp *qp = data; 1199 1200 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n", 1201 __func__, db_num); 1202 1203 tasklet_schedule(&qp->rx_work); 1204 } 1205 1206 static void ntb_tx_copy_callback(void *data) 1207 { 1208 struct ntb_queue_entry *entry = data; 1209 struct ntb_transport_qp *qp = entry->qp; 1210 struct ntb_payload_header __iomem *hdr = entry->tx_hdr; 1211 1212 /* Ensure that the data is fully copied out before setting the flags */ 1213 wmb(); 1214 iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); 1215 1216 ntb_ring_doorbell(qp->ndev, qp->qp_num); 1217 1218 /* The entry length can only be zero if the packet is intended to be a 1219 * "link down" or similar. Since no payload is being sent in these 1220 * cases, there is nothing to add to the completion queue. 1221 */ 1222 if (entry->len > 0) { 1223 qp->tx_bytes += entry->len; 1224 1225 if (qp->tx_handler) 1226 qp->tx_handler(qp, qp->cb_data, entry->cb_data, 1227 entry->len); 1228 } 1229 1230 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); 1231 } 1232 1233 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) 1234 { 1235 memcpy_toio(offset, entry->buf, entry->len); 1236 1237 ntb_tx_copy_callback(entry); 1238 } 1239 1240 static void ntb_async_tx(struct ntb_transport_qp *qp, 1241 struct ntb_queue_entry *entry) 1242 { 1243 struct ntb_payload_header __iomem *hdr; 1244 struct dma_async_tx_descriptor *txd; 1245 struct dma_chan *chan = qp->dma_chan; 1246 struct dma_device *device; 1247 size_t dest_off, buff_off; 1248 dma_addr_t src, dest; 1249 dma_cookie_t cookie; 1250 void __iomem *offset; 1251 size_t len = entry->len; 1252 void *buf = entry->buf; 1253 unsigned long flags; 1254 1255 offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; 1256 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); 1257 entry->tx_hdr = hdr; 1258 1259 iowrite32(entry->len, &hdr->len); 1260 iowrite32((u32) qp->tx_pkts, &hdr->ver); 1261 1262 if (!chan) 1263 goto err; 1264 1265 if (len < copy_bytes) 1266 goto err; 1267 1268 device = chan->device; 1269 dest = qp->tx_mw_phys + qp->tx_max_frame * qp->tx_index; 1270 buff_off = (size_t) buf & ~PAGE_MASK; 1271 dest_off = (size_t) dest & ~PAGE_MASK; 1272 1273 if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) 1274 goto err; 1275 1276 src = dma_map_single(device->dev, buf, len, DMA_TO_DEVICE); 1277 if (dma_mapping_error(device->dev, src)) 1278 goto err; 1279 1280 flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_PREP_INTERRUPT; 1281 txd = device->device_prep_dma_memcpy(chan, dest, src, len, flags); 1282 if (!txd) 1283 goto err1; 1284 1285 txd->callback = ntb_tx_copy_callback; 1286 txd->callback_param = entry; 1287 1288 cookie = dmaengine_submit(txd); 1289 if (dma_submit_error(cookie)) 1290 goto err1; 1291 1292 dma_async_issue_pending(chan); 1293 qp->tx_async++; 1294 1295 return; 1296 err1: 1297 dma_unmap_single(device->dev, src, len, DMA_TO_DEVICE); 1298 err: 1299 ntb_memcpy_tx(entry, offset); 1300 qp->tx_memcpy++; 1301 } 1302 1303 static int ntb_process_tx(struct ntb_transport_qp *qp, 1304 struct ntb_queue_entry *entry) 1305 { 1306 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%lld - tx %u, entry len %d flags %x buff %p\n", 1307 qp->tx_pkts, qp->tx_index, entry->len, entry->flags, 1308 entry->buf); 1309 if (qp->tx_index == qp->remote_rx_info->entry) { 1310 qp->tx_ring_full++; 1311 return -EAGAIN; 1312 } 1313 1314 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 1315 if (qp->tx_handler) 1316 qp->tx_handler(qp->cb_data, qp, NULL, -EIO); 1317 1318 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1319 &qp->tx_free_q); 1320 return 0; 1321 } 1322 1323 ntb_async_tx(qp, entry); 1324 1325 qp->tx_index++; 1326 qp->tx_index %= qp->tx_max_entry; 1327 1328 qp->tx_pkts++; 1329 1330 return 0; 1331 } 1332 1333 static void ntb_send_link_down(struct ntb_transport_qp *qp) 1334 { 1335 struct pci_dev *pdev = ntb_query_pdev(qp->ndev); 1336 struct ntb_queue_entry *entry; 1337 int i, rc; 1338 1339 if (qp->qp_link == NTB_LINK_DOWN) 1340 return; 1341 1342 qp->qp_link = NTB_LINK_DOWN; 1343 dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); 1344 1345 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1346 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1347 if (entry) 1348 break; 1349 msleep(100); 1350 } 1351 1352 if (!entry) 1353 return; 1354 1355 entry->cb_data = NULL; 1356 entry->buf = NULL; 1357 entry->len = 0; 1358 entry->flags = LINK_DOWN_FLAG; 1359 1360 rc = ntb_process_tx(qp, entry); 1361 if (rc) 1362 dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", 1363 qp->qp_num); 1364 } 1365 1366 /** 1367 * ntb_transport_create_queue - Create a new NTB transport layer queue 1368 * @rx_handler: receive callback function 1369 * @tx_handler: transmit callback function 1370 * @event_handler: event callback function 1371 * 1372 * Create a new NTB transport layer queue and provide the queue with a callback 1373 * routine for both transmit and receive. The receive callback routine will be 1374 * used to pass up data when the transport has received it on the queue. The 1375 * transmit callback routine will be called when the transport has completed the 1376 * transmission of the data on the queue and the data is ready to be freed. 1377 * 1378 * RETURNS: pointer to newly created ntb_queue, NULL on error. 1379 */ 1380 struct ntb_transport_qp * 1381 ntb_transport_create_queue(void *data, struct pci_dev *pdev, 1382 const struct ntb_queue_handlers *handlers) 1383 { 1384 struct ntb_queue_entry *entry; 1385 struct ntb_transport_qp *qp; 1386 struct ntb_transport *nt; 1387 unsigned int free_queue; 1388 int rc, i; 1389 1390 nt = ntb_find_transport(pdev); 1391 if (!nt) 1392 goto err; 1393 1394 free_queue = ffs(nt->qp_bitmap); 1395 if (!free_queue) 1396 goto err; 1397 1398 /* decrement free_queue to make it zero based */ 1399 free_queue--; 1400 1401 clear_bit(free_queue, &nt->qp_bitmap); 1402 1403 qp = &nt->qps[free_queue]; 1404 qp->cb_data = data; 1405 qp->rx_handler = handlers->rx_handler; 1406 qp->tx_handler = handlers->tx_handler; 1407 qp->event_handler = handlers->event_handler; 1408 1409 qp->dma_chan = dma_find_channel(DMA_MEMCPY); 1410 if (!qp->dma_chan) 1411 dev_info(&pdev->dev, "Unable to allocate DMA channel, using CPU instead\n"); 1412 else 1413 dmaengine_get(); 1414 1415 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 1416 entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC); 1417 if (!entry) 1418 goto err1; 1419 1420 entry->qp = qp; 1421 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, 1422 &qp->rx_free_q); 1423 } 1424 1425 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 1426 entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC); 1427 if (!entry) 1428 goto err2; 1429 1430 entry->qp = qp; 1431 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1432 &qp->tx_free_q); 1433 } 1434 1435 tasklet_init(&qp->rx_work, ntb_transport_rx, (unsigned long) qp); 1436 1437 rc = ntb_register_db_callback(qp->ndev, free_queue, qp, 1438 ntb_transport_rxc_db); 1439 if (rc) 1440 goto err3; 1441 1442 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); 1443 1444 return qp; 1445 1446 err3: 1447 tasklet_disable(&qp->rx_work); 1448 err2: 1449 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 1450 kfree(entry); 1451 err1: 1452 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 1453 kfree(entry); 1454 set_bit(free_queue, &nt->qp_bitmap); 1455 err: 1456 return NULL; 1457 } 1458 EXPORT_SYMBOL_GPL(ntb_transport_create_queue); 1459 1460 /** 1461 * ntb_transport_free_queue - Frees NTB transport queue 1462 * @qp: NTB queue to be freed 1463 * 1464 * Frees NTB transport queue 1465 */ 1466 void ntb_transport_free_queue(struct ntb_transport_qp *qp) 1467 { 1468 struct pci_dev *pdev; 1469 struct ntb_queue_entry *entry; 1470 1471 if (!qp) 1472 return; 1473 1474 pdev = ntb_query_pdev(qp->ndev); 1475 1476 if (qp->dma_chan) { 1477 struct dma_chan *chan = qp->dma_chan; 1478 /* Putting the dma_chan to NULL will force any new traffic to be 1479 * processed by the CPU instead of the DAM engine 1480 */ 1481 qp->dma_chan = NULL; 1482 1483 /* Try to be nice and wait for any queued DMA engine 1484 * transactions to process before smashing it with a rock 1485 */ 1486 dma_sync_wait(chan, qp->last_cookie); 1487 dmaengine_terminate_all(chan); 1488 dmaengine_put(); 1489 } 1490 1491 ntb_unregister_db_callback(qp->ndev, qp->qp_num); 1492 tasklet_disable(&qp->rx_work); 1493 1494 cancel_delayed_work_sync(&qp->link_work); 1495 1496 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 1497 kfree(entry); 1498 1499 while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q))) { 1500 dev_warn(&pdev->dev, "Freeing item from a non-empty queue\n"); 1501 kfree(entry); 1502 } 1503 1504 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 1505 kfree(entry); 1506 1507 set_bit(qp->qp_num, &qp->transport->qp_bitmap); 1508 1509 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); 1510 } 1511 EXPORT_SYMBOL_GPL(ntb_transport_free_queue); 1512 1513 /** 1514 * ntb_transport_rx_remove - Dequeues enqueued rx packet 1515 * @qp: NTB queue to be freed 1516 * @len: pointer to variable to write enqueued buffers length 1517 * 1518 * Dequeues unused buffers from receive queue. Should only be used during 1519 * shutdown of qp. 1520 * 1521 * RETURNS: NULL error value on error, or void* for success. 1522 */ 1523 void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) 1524 { 1525 struct ntb_queue_entry *entry; 1526 void *buf; 1527 1528 if (!qp || qp->client_ready == NTB_LINK_UP) 1529 return NULL; 1530 1531 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); 1532 if (!entry) 1533 return NULL; 1534 1535 buf = entry->cb_data; 1536 *len = entry->len; 1537 1538 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q); 1539 1540 return buf; 1541 } 1542 EXPORT_SYMBOL_GPL(ntb_transport_rx_remove); 1543 1544 /** 1545 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry 1546 * @qp: NTB transport layer queue the entry is to be enqueued on 1547 * @cb: per buffer pointer for callback function to use 1548 * @data: pointer to data buffer that incoming packets will be copied into 1549 * @len: length of the data buffer 1550 * 1551 * Enqueue a new receive buffer onto the transport queue into which a NTB 1552 * payload can be received into. 1553 * 1554 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1555 */ 1556 int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 1557 unsigned int len) 1558 { 1559 struct ntb_queue_entry *entry; 1560 1561 if (!qp) 1562 return -EINVAL; 1563 1564 entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q); 1565 if (!entry) 1566 return -ENOMEM; 1567 1568 entry->cb_data = cb; 1569 entry->buf = data; 1570 entry->len = len; 1571 1572 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, &qp->rx_pend_q); 1573 1574 return 0; 1575 } 1576 EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue); 1577 1578 /** 1579 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 1580 * @qp: NTB transport layer queue the entry is to be enqueued on 1581 * @cb: per buffer pointer for callback function to use 1582 * @data: pointer to data buffer that will be sent 1583 * @len: length of the data buffer 1584 * 1585 * Enqueue a new transmit buffer onto the transport queue from which a NTB 1586 * payload will be transmitted. This assumes that a lock is being held to 1587 * serialize access to the qp. 1588 * 1589 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1590 */ 1591 int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 1592 unsigned int len) 1593 { 1594 struct ntb_queue_entry *entry; 1595 int rc; 1596 1597 if (!qp || qp->qp_link != NTB_LINK_UP || !len) 1598 return -EINVAL; 1599 1600 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1601 if (!entry) { 1602 qp->tx_err_no_buf++; 1603 return -ENOMEM; 1604 } 1605 1606 entry->cb_data = cb; 1607 entry->buf = data; 1608 entry->len = len; 1609 entry->flags = 0; 1610 1611 rc = ntb_process_tx(qp, entry); 1612 if (rc) 1613 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1614 &qp->tx_free_q); 1615 1616 return rc; 1617 } 1618 EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue); 1619 1620 /** 1621 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 1622 * @qp: NTB transport layer queue to be enabled 1623 * 1624 * Notify NTB transport layer of client readiness to use queue 1625 */ 1626 void ntb_transport_link_up(struct ntb_transport_qp *qp) 1627 { 1628 if (!qp) 1629 return; 1630 1631 qp->client_ready = NTB_LINK_UP; 1632 1633 if (qp->transport->transport_link == NTB_LINK_UP) 1634 schedule_delayed_work(&qp->link_work, 0); 1635 } 1636 EXPORT_SYMBOL_GPL(ntb_transport_link_up); 1637 1638 /** 1639 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 1640 * @qp: NTB transport layer queue to be disabled 1641 * 1642 * Notify NTB transport layer of client's desire to no longer receive data on 1643 * transport queue specified. It is the client's responsibility to ensure all 1644 * entries on queue are purged or otherwise handled appropriately. 1645 */ 1646 void ntb_transport_link_down(struct ntb_transport_qp *qp) 1647 { 1648 struct pci_dev *pdev; 1649 int rc, val; 1650 1651 if (!qp) 1652 return; 1653 1654 pdev = ntb_query_pdev(qp->ndev); 1655 qp->client_ready = NTB_LINK_DOWN; 1656 1657 rc = ntb_read_local_spad(qp->ndev, QP_LINKS, &val); 1658 if (rc) { 1659 dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS); 1660 return; 1661 } 1662 1663 rc = ntb_write_remote_spad(qp->ndev, QP_LINKS, 1664 val & ~(1 << qp->qp_num)); 1665 if (rc) 1666 dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", 1667 val & ~(1 << qp->qp_num), QP_LINKS); 1668 1669 if (qp->qp_link == NTB_LINK_UP) 1670 ntb_send_link_down(qp); 1671 else 1672 cancel_delayed_work_sync(&qp->link_work); 1673 } 1674 EXPORT_SYMBOL_GPL(ntb_transport_link_down); 1675 1676 /** 1677 * ntb_transport_link_query - Query transport link state 1678 * @qp: NTB transport layer queue to be queried 1679 * 1680 * Query connectivity to the remote system of the NTB transport queue 1681 * 1682 * RETURNS: true for link up or false for link down 1683 */ 1684 bool ntb_transport_link_query(struct ntb_transport_qp *qp) 1685 { 1686 if (!qp) 1687 return false; 1688 1689 return qp->qp_link == NTB_LINK_UP; 1690 } 1691 EXPORT_SYMBOL_GPL(ntb_transport_link_query); 1692 1693 /** 1694 * ntb_transport_qp_num - Query the qp number 1695 * @qp: NTB transport layer queue to be queried 1696 * 1697 * Query qp number of the NTB transport queue 1698 * 1699 * RETURNS: a zero based number specifying the qp number 1700 */ 1701 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) 1702 { 1703 if (!qp) 1704 return 0; 1705 1706 return qp->qp_num; 1707 } 1708 EXPORT_SYMBOL_GPL(ntb_transport_qp_num); 1709 1710 /** 1711 * ntb_transport_max_size - Query the max payload size of a qp 1712 * @qp: NTB transport layer queue to be queried 1713 * 1714 * Query the maximum payload size permissible on the given qp 1715 * 1716 * RETURNS: the max payload size of a qp 1717 */ 1718 unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) 1719 { 1720 unsigned int max; 1721 1722 if (!qp) 1723 return 0; 1724 1725 if (!qp->dma_chan) 1726 return qp->tx_max_frame - sizeof(struct ntb_payload_header); 1727 1728 /* If DMA engine usage is possible, try to find the max size for that */ 1729 max = qp->tx_max_frame - sizeof(struct ntb_payload_header); 1730 max -= max % (1 << qp->dma_chan->device->copy_align); 1731 1732 return max; 1733 } 1734 EXPORT_SYMBOL_GPL(ntb_transport_max_size); 1735