1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * BSD LICENSE 15 * 16 * Copyright(c) 2012 Intel Corporation. All rights reserved. 17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 23 * * Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * * Redistributions in binary form must reproduce the above copy 26 * notice, this list of conditions and the following disclaimer in 27 * the documentation and/or other materials provided with the 28 * distribution. 29 * * Neither the name of Intel Corporation nor the names of its 30 * contributors may be used to endorse or promote products derived 31 * from this software without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 44 * 45 * PCIe NTB Transport Linux driver 46 * 47 * Contact Information: 48 * Jon Mason <jon.mason@intel.com> 49 */ 50 #include <linux/debugfs.h> 51 #include <linux/delay.h> 52 #include <linux/dmaengine.h> 53 #include <linux/dma-mapping.h> 54 #include <linux/errno.h> 55 #include <linux/export.h> 56 #include <linux/interrupt.h> 57 #include <linux/module.h> 58 #include <linux/pci.h> 59 #include <linux/slab.h> 60 #include <linux/types.h> 61 #include <linux/uaccess.h> 62 #include "linux/ntb.h" 63 #include "linux/ntb_transport.h" 64 65 #define NTB_TRANSPORT_VERSION 4 66 #define NTB_TRANSPORT_VER "4" 67 #define NTB_TRANSPORT_NAME "ntb_transport" 68 #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB" 69 #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2) 70 71 MODULE_DESCRIPTION(NTB_TRANSPORT_DESC); 72 MODULE_VERSION(NTB_TRANSPORT_VER); 73 MODULE_LICENSE("Dual BSD/GPL"); 74 MODULE_AUTHOR("Intel Corporation"); 75 76 static unsigned long max_mw_size; 77 module_param(max_mw_size, ulong, 0644); 78 MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); 79 80 static unsigned int transport_mtu = 0x10000; 81 module_param(transport_mtu, uint, 0644); 82 MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); 83 84 static unsigned char max_num_clients; 85 module_param(max_num_clients, byte, 0644); 86 MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients"); 87 88 static unsigned int copy_bytes = 1024; 89 module_param(copy_bytes, uint, 0644); 90 MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); 91 92 static bool use_dma; 93 module_param(use_dma, bool, 0644); 94 MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy"); 95 96 static struct dentry *nt_debugfs_dir; 97 98 /* Only two-ports NTB devices are supported */ 99 #define PIDX NTB_DEF_PEER_IDX 100 101 struct ntb_queue_entry { 102 /* ntb_queue list reference */ 103 struct list_head entry; 104 /* pointers to data to be transferred */ 105 void *cb_data; 106 void *buf; 107 unsigned int len; 108 unsigned int flags; 109 int retries; 110 int errors; 111 unsigned int tx_index; 112 unsigned int rx_index; 113 114 struct ntb_transport_qp *qp; 115 union { 116 struct ntb_payload_header __iomem *tx_hdr; 117 struct ntb_payload_header *rx_hdr; 118 }; 119 }; 120 121 struct ntb_rx_info { 122 unsigned int entry; 123 }; 124 125 struct ntb_transport_qp { 126 struct ntb_transport_ctx *transport; 127 struct ntb_dev *ndev; 128 void *cb_data; 129 struct dma_chan *tx_dma_chan; 130 struct dma_chan *rx_dma_chan; 131 132 bool client_ready; 133 bool link_is_up; 134 bool active; 135 136 u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ 137 u64 qp_bit; 138 139 struct ntb_rx_info __iomem *rx_info; 140 struct ntb_rx_info *remote_rx_info; 141 142 void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, 143 void *data, int len); 144 struct list_head tx_free_q; 145 spinlock_t ntb_tx_free_q_lock; 146 void __iomem *tx_mw; 147 phys_addr_t tx_mw_phys; 148 size_t tx_mw_size; 149 dma_addr_t tx_mw_dma_addr; 150 unsigned int tx_index; 151 unsigned int tx_max_entry; 152 unsigned int tx_max_frame; 153 154 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 155 void *data, int len); 156 struct list_head rx_post_q; 157 struct list_head rx_pend_q; 158 struct list_head rx_free_q; 159 /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */ 160 spinlock_t ntb_rx_q_lock; 161 void *rx_buff; 162 unsigned int rx_index; 163 unsigned int rx_max_entry; 164 unsigned int rx_max_frame; 165 unsigned int rx_alloc_entry; 166 dma_cookie_t last_cookie; 167 struct tasklet_struct rxc_db_work; 168 169 void (*event_handler)(void *data, int status); 170 struct delayed_work link_work; 171 struct work_struct link_cleanup; 172 173 struct dentry *debugfs_dir; 174 struct dentry *debugfs_stats; 175 176 /* Stats */ 177 u64 rx_bytes; 178 u64 rx_pkts; 179 u64 rx_ring_empty; 180 u64 rx_err_no_buf; 181 u64 rx_err_oflow; 182 u64 rx_err_ver; 183 u64 rx_memcpy; 184 u64 rx_async; 185 u64 tx_bytes; 186 u64 tx_pkts; 187 u64 tx_ring_full; 188 u64 tx_err_no_buf; 189 u64 tx_memcpy; 190 u64 tx_async; 191 }; 192 193 struct ntb_transport_mw { 194 phys_addr_t phys_addr; 195 resource_size_t phys_size; 196 void __iomem *vbase; 197 size_t xlat_size; 198 size_t buff_size; 199 size_t alloc_size; 200 void *alloc_addr; 201 void *virt_addr; 202 dma_addr_t dma_addr; 203 }; 204 205 struct ntb_transport_client_dev { 206 struct list_head entry; 207 struct ntb_transport_ctx *nt; 208 struct device dev; 209 }; 210 211 struct ntb_transport_ctx { 212 struct list_head entry; 213 struct list_head client_devs; 214 215 struct ntb_dev *ndev; 216 217 struct ntb_transport_mw *mw_vec; 218 struct ntb_transport_qp *qp_vec; 219 unsigned int mw_count; 220 unsigned int qp_count; 221 u64 qp_bitmap; 222 u64 qp_bitmap_free; 223 224 bool link_is_up; 225 struct delayed_work link_work; 226 struct work_struct link_cleanup; 227 228 struct dentry *debugfs_node_dir; 229 }; 230 231 enum { 232 DESC_DONE_FLAG = BIT(0), 233 LINK_DOWN_FLAG = BIT(1), 234 }; 235 236 struct ntb_payload_header { 237 unsigned int ver; 238 unsigned int len; 239 unsigned int flags; 240 }; 241 242 enum { 243 VERSION = 0, 244 QP_LINKS, 245 NUM_QPS, 246 NUM_MWS, 247 MW0_SZ_HIGH, 248 MW0_SZ_LOW, 249 }; 250 251 #define dev_client_dev(__dev) \ 252 container_of((__dev), struct ntb_transport_client_dev, dev) 253 254 #define drv_client(__drv) \ 255 container_of((__drv), struct ntb_transport_client, driver) 256 257 #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) 258 #define NTB_QP_DEF_NUM_ENTRIES 100 259 #define NTB_LINK_DOWN_TIMEOUT 10 260 261 static void ntb_transport_rxc_db(unsigned long data); 262 static const struct ntb_ctx_ops ntb_transport_ops; 263 static struct ntb_client ntb_transport_client; 264 static int ntb_async_tx_submit(struct ntb_transport_qp *qp, 265 struct ntb_queue_entry *entry); 266 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset); 267 static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset); 268 static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset); 269 270 271 static int ntb_transport_bus_match(struct device *dev, 272 struct device_driver *drv) 273 { 274 return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); 275 } 276 277 static int ntb_transport_bus_probe(struct device *dev) 278 { 279 const struct ntb_transport_client *client; 280 int rc = -EINVAL; 281 282 get_device(dev); 283 284 client = drv_client(dev->driver); 285 rc = client->probe(dev); 286 if (rc) 287 put_device(dev); 288 289 return rc; 290 } 291 292 static int ntb_transport_bus_remove(struct device *dev) 293 { 294 const struct ntb_transport_client *client; 295 296 client = drv_client(dev->driver); 297 client->remove(dev); 298 299 put_device(dev); 300 301 return 0; 302 } 303 304 static struct bus_type ntb_transport_bus = { 305 .name = "ntb_transport", 306 .match = ntb_transport_bus_match, 307 .probe = ntb_transport_bus_probe, 308 .remove = ntb_transport_bus_remove, 309 }; 310 311 static LIST_HEAD(ntb_transport_list); 312 313 static int ntb_bus_init(struct ntb_transport_ctx *nt) 314 { 315 list_add_tail(&nt->entry, &ntb_transport_list); 316 return 0; 317 } 318 319 static void ntb_bus_remove(struct ntb_transport_ctx *nt) 320 { 321 struct ntb_transport_client_dev *client_dev, *cd; 322 323 list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) { 324 dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n", 325 dev_name(&client_dev->dev)); 326 list_del(&client_dev->entry); 327 device_unregister(&client_dev->dev); 328 } 329 330 list_del(&nt->entry); 331 } 332 333 static void ntb_transport_client_release(struct device *dev) 334 { 335 struct ntb_transport_client_dev *client_dev; 336 337 client_dev = dev_client_dev(dev); 338 kfree(client_dev); 339 } 340 341 /** 342 * ntb_transport_unregister_client_dev - Unregister NTB client device 343 * @device_name: Name of NTB client device 344 * 345 * Unregister an NTB client device with the NTB transport layer 346 */ 347 void ntb_transport_unregister_client_dev(char *device_name) 348 { 349 struct ntb_transport_client_dev *client, *cd; 350 struct ntb_transport_ctx *nt; 351 352 list_for_each_entry(nt, &ntb_transport_list, entry) 353 list_for_each_entry_safe(client, cd, &nt->client_devs, entry) 354 if (!strncmp(dev_name(&client->dev), device_name, 355 strlen(device_name))) { 356 list_del(&client->entry); 357 device_unregister(&client->dev); 358 } 359 } 360 EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev); 361 362 /** 363 * ntb_transport_register_client_dev - Register NTB client device 364 * @device_name: Name of NTB client device 365 * 366 * Register an NTB client device with the NTB transport layer 367 */ 368 int ntb_transport_register_client_dev(char *device_name) 369 { 370 struct ntb_transport_client_dev *client_dev; 371 struct ntb_transport_ctx *nt; 372 int node; 373 int rc, i = 0; 374 375 if (list_empty(&ntb_transport_list)) 376 return -ENODEV; 377 378 list_for_each_entry(nt, &ntb_transport_list, entry) { 379 struct device *dev; 380 381 node = dev_to_node(&nt->ndev->dev); 382 383 client_dev = kzalloc_node(sizeof(*client_dev), 384 GFP_KERNEL, node); 385 if (!client_dev) { 386 rc = -ENOMEM; 387 goto err; 388 } 389 390 dev = &client_dev->dev; 391 392 /* setup and register client devices */ 393 dev_set_name(dev, "%s%d", device_name, i); 394 dev->bus = &ntb_transport_bus; 395 dev->release = ntb_transport_client_release; 396 dev->parent = &nt->ndev->dev; 397 398 rc = device_register(dev); 399 if (rc) { 400 kfree(client_dev); 401 goto err; 402 } 403 404 list_add_tail(&client_dev->entry, &nt->client_devs); 405 i++; 406 } 407 408 return 0; 409 410 err: 411 ntb_transport_unregister_client_dev(device_name); 412 413 return rc; 414 } 415 EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev); 416 417 /** 418 * ntb_transport_register_client - Register NTB client driver 419 * @drv: NTB client driver to be registered 420 * 421 * Register an NTB client driver with the NTB transport layer 422 * 423 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 424 */ 425 int ntb_transport_register_client(struct ntb_transport_client *drv) 426 { 427 drv->driver.bus = &ntb_transport_bus; 428 429 if (list_empty(&ntb_transport_list)) 430 return -ENODEV; 431 432 return driver_register(&drv->driver); 433 } 434 EXPORT_SYMBOL_GPL(ntb_transport_register_client); 435 436 /** 437 * ntb_transport_unregister_client - Unregister NTB client driver 438 * @drv: NTB client driver to be unregistered 439 * 440 * Unregister an NTB client driver with the NTB transport layer 441 * 442 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 443 */ 444 void ntb_transport_unregister_client(struct ntb_transport_client *drv) 445 { 446 driver_unregister(&drv->driver); 447 } 448 EXPORT_SYMBOL_GPL(ntb_transport_unregister_client); 449 450 static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, 451 loff_t *offp) 452 { 453 struct ntb_transport_qp *qp; 454 char *buf; 455 ssize_t ret, out_offset, out_count; 456 457 qp = filp->private_data; 458 459 if (!qp || !qp->link_is_up) 460 return 0; 461 462 out_count = 1000; 463 464 buf = kmalloc(out_count, GFP_KERNEL); 465 if (!buf) 466 return -ENOMEM; 467 468 out_offset = 0; 469 out_offset += snprintf(buf + out_offset, out_count - out_offset, 470 "\nNTB QP stats:\n\n"); 471 out_offset += snprintf(buf + out_offset, out_count - out_offset, 472 "rx_bytes - \t%llu\n", qp->rx_bytes); 473 out_offset += snprintf(buf + out_offset, out_count - out_offset, 474 "rx_pkts - \t%llu\n", qp->rx_pkts); 475 out_offset += snprintf(buf + out_offset, out_count - out_offset, 476 "rx_memcpy - \t%llu\n", qp->rx_memcpy); 477 out_offset += snprintf(buf + out_offset, out_count - out_offset, 478 "rx_async - \t%llu\n", qp->rx_async); 479 out_offset += snprintf(buf + out_offset, out_count - out_offset, 480 "rx_ring_empty - %llu\n", qp->rx_ring_empty); 481 out_offset += snprintf(buf + out_offset, out_count - out_offset, 482 "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); 483 out_offset += snprintf(buf + out_offset, out_count - out_offset, 484 "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); 485 out_offset += snprintf(buf + out_offset, out_count - out_offset, 486 "rx_err_ver - \t%llu\n", qp->rx_err_ver); 487 out_offset += snprintf(buf + out_offset, out_count - out_offset, 488 "rx_buff - \t0x%p\n", qp->rx_buff); 489 out_offset += snprintf(buf + out_offset, out_count - out_offset, 490 "rx_index - \t%u\n", qp->rx_index); 491 out_offset += snprintf(buf + out_offset, out_count - out_offset, 492 "rx_max_entry - \t%u\n", qp->rx_max_entry); 493 out_offset += snprintf(buf + out_offset, out_count - out_offset, 494 "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry); 495 496 out_offset += snprintf(buf + out_offset, out_count - out_offset, 497 "tx_bytes - \t%llu\n", qp->tx_bytes); 498 out_offset += snprintf(buf + out_offset, out_count - out_offset, 499 "tx_pkts - \t%llu\n", qp->tx_pkts); 500 out_offset += snprintf(buf + out_offset, out_count - out_offset, 501 "tx_memcpy - \t%llu\n", qp->tx_memcpy); 502 out_offset += snprintf(buf + out_offset, out_count - out_offset, 503 "tx_async - \t%llu\n", qp->tx_async); 504 out_offset += snprintf(buf + out_offset, out_count - out_offset, 505 "tx_ring_full - \t%llu\n", qp->tx_ring_full); 506 out_offset += snprintf(buf + out_offset, out_count - out_offset, 507 "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); 508 out_offset += snprintf(buf + out_offset, out_count - out_offset, 509 "tx_mw - \t0x%p\n", qp->tx_mw); 510 out_offset += snprintf(buf + out_offset, out_count - out_offset, 511 "tx_index (H) - \t%u\n", qp->tx_index); 512 out_offset += snprintf(buf + out_offset, out_count - out_offset, 513 "RRI (T) - \t%u\n", 514 qp->remote_rx_info->entry); 515 out_offset += snprintf(buf + out_offset, out_count - out_offset, 516 "tx_max_entry - \t%u\n", qp->tx_max_entry); 517 out_offset += snprintf(buf + out_offset, out_count - out_offset, 518 "free tx - \t%u\n", 519 ntb_transport_tx_free_entry(qp)); 520 521 out_offset += snprintf(buf + out_offset, out_count - out_offset, 522 "\n"); 523 out_offset += snprintf(buf + out_offset, out_count - out_offset, 524 "Using TX DMA - \t%s\n", 525 qp->tx_dma_chan ? "Yes" : "No"); 526 out_offset += snprintf(buf + out_offset, out_count - out_offset, 527 "Using RX DMA - \t%s\n", 528 qp->rx_dma_chan ? "Yes" : "No"); 529 out_offset += snprintf(buf + out_offset, out_count - out_offset, 530 "QP Link - \t%s\n", 531 qp->link_is_up ? "Up" : "Down"); 532 out_offset += snprintf(buf + out_offset, out_count - out_offset, 533 "\n"); 534 535 if (out_offset > out_count) 536 out_offset = out_count; 537 538 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); 539 kfree(buf); 540 return ret; 541 } 542 543 static const struct file_operations ntb_qp_debugfs_stats = { 544 .owner = THIS_MODULE, 545 .open = simple_open, 546 .read = debugfs_read, 547 }; 548 549 static void ntb_list_add(spinlock_t *lock, struct list_head *entry, 550 struct list_head *list) 551 { 552 unsigned long flags; 553 554 spin_lock_irqsave(lock, flags); 555 list_add_tail(entry, list); 556 spin_unlock_irqrestore(lock, flags); 557 } 558 559 static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, 560 struct list_head *list) 561 { 562 struct ntb_queue_entry *entry; 563 unsigned long flags; 564 565 spin_lock_irqsave(lock, flags); 566 if (list_empty(list)) { 567 entry = NULL; 568 goto out; 569 } 570 entry = list_first_entry(list, struct ntb_queue_entry, entry); 571 list_del(&entry->entry); 572 573 out: 574 spin_unlock_irqrestore(lock, flags); 575 576 return entry; 577 } 578 579 static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock, 580 struct list_head *list, 581 struct list_head *to_list) 582 { 583 struct ntb_queue_entry *entry; 584 unsigned long flags; 585 586 spin_lock_irqsave(lock, flags); 587 588 if (list_empty(list)) { 589 entry = NULL; 590 } else { 591 entry = list_first_entry(list, struct ntb_queue_entry, entry); 592 list_move_tail(&entry->entry, to_list); 593 } 594 595 spin_unlock_irqrestore(lock, flags); 596 597 return entry; 598 } 599 600 static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, 601 unsigned int qp_num) 602 { 603 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 604 struct ntb_transport_mw *mw; 605 struct ntb_dev *ndev = nt->ndev; 606 struct ntb_queue_entry *entry; 607 unsigned int rx_size, num_qps_mw; 608 unsigned int mw_num, mw_count, qp_count; 609 unsigned int i; 610 int node; 611 612 mw_count = nt->mw_count; 613 qp_count = nt->qp_count; 614 615 mw_num = QP_TO_MW(nt, qp_num); 616 mw = &nt->mw_vec[mw_num]; 617 618 if (!mw->virt_addr) 619 return -ENOMEM; 620 621 if (mw_num < qp_count % mw_count) 622 num_qps_mw = qp_count / mw_count + 1; 623 else 624 num_qps_mw = qp_count / mw_count; 625 626 rx_size = (unsigned int)mw->xlat_size / num_qps_mw; 627 qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); 628 rx_size -= sizeof(struct ntb_rx_info); 629 630 qp->remote_rx_info = qp->rx_buff + rx_size; 631 632 /* Due to housekeeping, there must be atleast 2 buffs */ 633 qp->rx_max_frame = min(transport_mtu, rx_size / 2); 634 qp->rx_max_entry = rx_size / qp->rx_max_frame; 635 qp->rx_index = 0; 636 637 /* 638 * Checking to see if we have more entries than the default. 639 * We should add additional entries if that is the case so we 640 * can be in sync with the transport frames. 641 */ 642 node = dev_to_node(&ndev->dev); 643 for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) { 644 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 645 if (!entry) 646 return -ENOMEM; 647 648 entry->qp = qp; 649 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, 650 &qp->rx_free_q); 651 qp->rx_alloc_entry++; 652 } 653 654 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 655 656 /* setup the hdr offsets with 0's */ 657 for (i = 0; i < qp->rx_max_entry; i++) { 658 void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) - 659 sizeof(struct ntb_payload_header)); 660 memset(offset, 0, sizeof(struct ntb_payload_header)); 661 } 662 663 qp->rx_pkts = 0; 664 qp->tx_pkts = 0; 665 qp->tx_index = 0; 666 667 return 0; 668 } 669 670 static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) 671 { 672 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 673 struct pci_dev *pdev = nt->ndev->pdev; 674 675 if (!mw->virt_addr) 676 return; 677 678 ntb_mw_clear_trans(nt->ndev, PIDX, num_mw); 679 dma_free_coherent(&pdev->dev, mw->alloc_size, 680 mw->alloc_addr, mw->dma_addr); 681 mw->xlat_size = 0; 682 mw->buff_size = 0; 683 mw->alloc_size = 0; 684 mw->alloc_addr = NULL; 685 mw->virt_addr = NULL; 686 } 687 688 static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw, 689 struct device *dma_dev, size_t align) 690 { 691 dma_addr_t dma_addr; 692 void *alloc_addr, *virt_addr; 693 int rc; 694 695 alloc_addr = dma_alloc_coherent(dma_dev, mw->alloc_size, 696 &dma_addr, GFP_KERNEL); 697 if (!alloc_addr) { 698 dev_err(dma_dev, "Unable to alloc MW buff of size %zu\n", 699 mw->alloc_size); 700 return -ENOMEM; 701 } 702 virt_addr = alloc_addr; 703 704 /* 705 * we must ensure that the memory address allocated is BAR size 706 * aligned in order for the XLAT register to take the value. This 707 * is a requirement of the hardware. It is recommended to setup CMA 708 * for BAR sizes equal or greater than 4MB. 709 */ 710 if (!IS_ALIGNED(dma_addr, align)) { 711 if (mw->alloc_size > mw->buff_size) { 712 virt_addr = PTR_ALIGN(alloc_addr, align); 713 dma_addr = ALIGN(dma_addr, align); 714 } else { 715 rc = -ENOMEM; 716 goto err; 717 } 718 } 719 720 mw->alloc_addr = alloc_addr; 721 mw->virt_addr = virt_addr; 722 mw->dma_addr = dma_addr; 723 724 return 0; 725 726 err: 727 dma_free_coherent(dma_dev, mw->alloc_size, alloc_addr, dma_addr); 728 729 return rc; 730 } 731 732 static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, 733 resource_size_t size) 734 { 735 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 736 struct pci_dev *pdev = nt->ndev->pdev; 737 size_t xlat_size, buff_size; 738 resource_size_t xlat_align; 739 resource_size_t xlat_align_size; 740 int rc; 741 742 if (!size) 743 return -EINVAL; 744 745 rc = ntb_mw_get_align(nt->ndev, PIDX, num_mw, &xlat_align, 746 &xlat_align_size, NULL); 747 if (rc) 748 return rc; 749 750 xlat_size = round_up(size, xlat_align_size); 751 buff_size = round_up(size, xlat_align); 752 753 /* No need to re-setup */ 754 if (mw->xlat_size == xlat_size) 755 return 0; 756 757 if (mw->buff_size) 758 ntb_free_mw(nt, num_mw); 759 760 /* Alloc memory for receiving data. Must be aligned */ 761 mw->xlat_size = xlat_size; 762 mw->buff_size = buff_size; 763 mw->alloc_size = buff_size; 764 765 rc = ntb_alloc_mw_buffer(mw, &pdev->dev, xlat_align); 766 if (rc) { 767 mw->alloc_size *= 2; 768 rc = ntb_alloc_mw_buffer(mw, &pdev->dev, xlat_align); 769 if (rc) { 770 dev_err(&pdev->dev, 771 "Unable to alloc aligned MW buff\n"); 772 mw->xlat_size = 0; 773 mw->buff_size = 0; 774 mw->alloc_size = 0; 775 return rc; 776 } 777 } 778 779 /* Notify HW the memory location of the receive buffer */ 780 rc = ntb_mw_set_trans(nt->ndev, PIDX, num_mw, mw->dma_addr, 781 mw->xlat_size); 782 if (rc) { 783 dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw); 784 ntb_free_mw(nt, num_mw); 785 return -EIO; 786 } 787 788 return 0; 789 } 790 791 static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp) 792 { 793 qp->link_is_up = false; 794 qp->active = false; 795 796 qp->tx_index = 0; 797 qp->rx_index = 0; 798 qp->rx_bytes = 0; 799 qp->rx_pkts = 0; 800 qp->rx_ring_empty = 0; 801 qp->rx_err_no_buf = 0; 802 qp->rx_err_oflow = 0; 803 qp->rx_err_ver = 0; 804 qp->rx_memcpy = 0; 805 qp->rx_async = 0; 806 qp->tx_bytes = 0; 807 qp->tx_pkts = 0; 808 qp->tx_ring_full = 0; 809 qp->tx_err_no_buf = 0; 810 qp->tx_memcpy = 0; 811 qp->tx_async = 0; 812 } 813 814 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 815 { 816 struct ntb_transport_ctx *nt = qp->transport; 817 struct pci_dev *pdev = nt->ndev->pdev; 818 819 dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num); 820 821 cancel_delayed_work_sync(&qp->link_work); 822 ntb_qp_link_down_reset(qp); 823 824 if (qp->event_handler) 825 qp->event_handler(qp->cb_data, qp->link_is_up); 826 } 827 828 static void ntb_qp_link_cleanup_work(struct work_struct *work) 829 { 830 struct ntb_transport_qp *qp = container_of(work, 831 struct ntb_transport_qp, 832 link_cleanup); 833 struct ntb_transport_ctx *nt = qp->transport; 834 835 ntb_qp_link_cleanup(qp); 836 837 if (nt->link_is_up) 838 schedule_delayed_work(&qp->link_work, 839 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 840 } 841 842 static void ntb_qp_link_down(struct ntb_transport_qp *qp) 843 { 844 schedule_work(&qp->link_cleanup); 845 } 846 847 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) 848 { 849 struct ntb_transport_qp *qp; 850 u64 qp_bitmap_alloc; 851 unsigned int i, count; 852 853 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 854 855 /* Pass along the info to any clients */ 856 for (i = 0; i < nt->qp_count; i++) 857 if (qp_bitmap_alloc & BIT_ULL(i)) { 858 qp = &nt->qp_vec[i]; 859 ntb_qp_link_cleanup(qp); 860 cancel_work_sync(&qp->link_cleanup); 861 cancel_delayed_work_sync(&qp->link_work); 862 } 863 864 if (!nt->link_is_up) 865 cancel_delayed_work_sync(&nt->link_work); 866 867 for (i = 0; i < nt->mw_count; i++) 868 ntb_free_mw(nt, i); 869 870 /* The scratchpad registers keep the values if the remote side 871 * goes down, blast them now to give them a sane value the next 872 * time they are accessed 873 */ 874 count = ntb_spad_count(nt->ndev); 875 for (i = 0; i < count; i++) 876 ntb_spad_write(nt->ndev, i, 0); 877 } 878 879 static void ntb_transport_link_cleanup_work(struct work_struct *work) 880 { 881 struct ntb_transport_ctx *nt = 882 container_of(work, struct ntb_transport_ctx, link_cleanup); 883 884 ntb_transport_link_cleanup(nt); 885 } 886 887 static void ntb_transport_event_callback(void *data) 888 { 889 struct ntb_transport_ctx *nt = data; 890 891 if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1) 892 schedule_delayed_work(&nt->link_work, 0); 893 else 894 schedule_work(&nt->link_cleanup); 895 } 896 897 static void ntb_transport_link_work(struct work_struct *work) 898 { 899 struct ntb_transport_ctx *nt = 900 container_of(work, struct ntb_transport_ctx, link_work.work); 901 struct ntb_dev *ndev = nt->ndev; 902 struct pci_dev *pdev = ndev->pdev; 903 resource_size_t size; 904 u32 val; 905 int rc = 0, i, spad; 906 907 /* send the local info, in the opposite order of the way we read it */ 908 for (i = 0; i < nt->mw_count; i++) { 909 size = nt->mw_vec[i].phys_size; 910 911 if (max_mw_size && size > max_mw_size) 912 size = max_mw_size; 913 914 spad = MW0_SZ_HIGH + (i * 2); 915 ntb_peer_spad_write(ndev, PIDX, spad, upper_32_bits(size)); 916 917 spad = MW0_SZ_LOW + (i * 2); 918 ntb_peer_spad_write(ndev, PIDX, spad, lower_32_bits(size)); 919 } 920 921 ntb_peer_spad_write(ndev, PIDX, NUM_MWS, nt->mw_count); 922 923 ntb_peer_spad_write(ndev, PIDX, NUM_QPS, nt->qp_count); 924 925 ntb_peer_spad_write(ndev, PIDX, VERSION, NTB_TRANSPORT_VERSION); 926 927 /* Query the remote side for its info */ 928 val = ntb_spad_read(ndev, VERSION); 929 dev_dbg(&pdev->dev, "Remote version = %d\n", val); 930 if (val != NTB_TRANSPORT_VERSION) 931 goto out; 932 933 val = ntb_spad_read(ndev, NUM_QPS); 934 dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); 935 if (val != nt->qp_count) 936 goto out; 937 938 val = ntb_spad_read(ndev, NUM_MWS); 939 dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); 940 if (val != nt->mw_count) 941 goto out; 942 943 for (i = 0; i < nt->mw_count; i++) { 944 u64 val64; 945 946 val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2)); 947 val64 = (u64)val << 32; 948 949 val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2)); 950 val64 |= val; 951 952 dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64); 953 954 rc = ntb_set_mw(nt, i, val64); 955 if (rc) 956 goto out1; 957 } 958 959 nt->link_is_up = true; 960 961 for (i = 0; i < nt->qp_count; i++) { 962 struct ntb_transport_qp *qp = &nt->qp_vec[i]; 963 964 ntb_transport_setup_qp_mw(nt, i); 965 966 if (qp->client_ready) 967 schedule_delayed_work(&qp->link_work, 0); 968 } 969 970 return; 971 972 out1: 973 for (i = 0; i < nt->mw_count; i++) 974 ntb_free_mw(nt, i); 975 976 /* if there's an actual failure, we should just bail */ 977 if (rc < 0) 978 return; 979 980 out: 981 if (ntb_link_is_up(ndev, NULL, NULL) == 1) 982 schedule_delayed_work(&nt->link_work, 983 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 984 } 985 986 static void ntb_qp_link_work(struct work_struct *work) 987 { 988 struct ntb_transport_qp *qp = container_of(work, 989 struct ntb_transport_qp, 990 link_work.work); 991 struct pci_dev *pdev = qp->ndev->pdev; 992 struct ntb_transport_ctx *nt = qp->transport; 993 int val; 994 995 WARN_ON(!nt->link_is_up); 996 997 val = ntb_spad_read(nt->ndev, QP_LINKS); 998 999 ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num)); 1000 1001 /* query remote spad for qp ready bits */ 1002 dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val); 1003 1004 /* See if the remote side is up */ 1005 if (val & BIT(qp->qp_num)) { 1006 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); 1007 qp->link_is_up = true; 1008 qp->active = true; 1009 1010 if (qp->event_handler) 1011 qp->event_handler(qp->cb_data, qp->link_is_up); 1012 1013 if (qp->active) 1014 tasklet_schedule(&qp->rxc_db_work); 1015 } else if (nt->link_is_up) 1016 schedule_delayed_work(&qp->link_work, 1017 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 1018 } 1019 1020 static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, 1021 unsigned int qp_num) 1022 { 1023 struct ntb_transport_qp *qp; 1024 phys_addr_t mw_base; 1025 resource_size_t mw_size; 1026 unsigned int num_qps_mw, tx_size; 1027 unsigned int mw_num, mw_count, qp_count; 1028 u64 qp_offset; 1029 1030 mw_count = nt->mw_count; 1031 qp_count = nt->qp_count; 1032 1033 mw_num = QP_TO_MW(nt, qp_num); 1034 1035 qp = &nt->qp_vec[qp_num]; 1036 qp->qp_num = qp_num; 1037 qp->transport = nt; 1038 qp->ndev = nt->ndev; 1039 qp->client_ready = false; 1040 qp->event_handler = NULL; 1041 ntb_qp_link_down_reset(qp); 1042 1043 if (mw_num < qp_count % mw_count) 1044 num_qps_mw = qp_count / mw_count + 1; 1045 else 1046 num_qps_mw = qp_count / mw_count; 1047 1048 mw_base = nt->mw_vec[mw_num].phys_addr; 1049 mw_size = nt->mw_vec[mw_num].phys_size; 1050 1051 if (max_mw_size && mw_size > max_mw_size) 1052 mw_size = max_mw_size; 1053 1054 tx_size = (unsigned int)mw_size / num_qps_mw; 1055 qp_offset = tx_size * (qp_num / mw_count); 1056 1057 qp->tx_mw_size = tx_size; 1058 qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; 1059 if (!qp->tx_mw) 1060 return -EINVAL; 1061 1062 qp->tx_mw_phys = mw_base + qp_offset; 1063 if (!qp->tx_mw_phys) 1064 return -EINVAL; 1065 1066 tx_size -= sizeof(struct ntb_rx_info); 1067 qp->rx_info = qp->tx_mw + tx_size; 1068 1069 /* Due to housekeeping, there must be atleast 2 buffs */ 1070 qp->tx_max_frame = min(transport_mtu, tx_size / 2); 1071 qp->tx_max_entry = tx_size / qp->tx_max_frame; 1072 1073 if (nt->debugfs_node_dir) { 1074 char debugfs_name[4]; 1075 1076 snprintf(debugfs_name, 4, "qp%d", qp_num); 1077 qp->debugfs_dir = debugfs_create_dir(debugfs_name, 1078 nt->debugfs_node_dir); 1079 1080 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, 1081 qp->debugfs_dir, qp, 1082 &ntb_qp_debugfs_stats); 1083 } else { 1084 qp->debugfs_dir = NULL; 1085 qp->debugfs_stats = NULL; 1086 } 1087 1088 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); 1089 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); 1090 1091 spin_lock_init(&qp->ntb_rx_q_lock); 1092 spin_lock_init(&qp->ntb_tx_free_q_lock); 1093 1094 INIT_LIST_HEAD(&qp->rx_post_q); 1095 INIT_LIST_HEAD(&qp->rx_pend_q); 1096 INIT_LIST_HEAD(&qp->rx_free_q); 1097 INIT_LIST_HEAD(&qp->tx_free_q); 1098 1099 tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db, 1100 (unsigned long)qp); 1101 1102 return 0; 1103 } 1104 1105 static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) 1106 { 1107 struct ntb_transport_ctx *nt; 1108 struct ntb_transport_mw *mw; 1109 unsigned int mw_count, qp_count, spad_count, max_mw_count_for_spads; 1110 u64 qp_bitmap; 1111 int node; 1112 int rc, i; 1113 1114 mw_count = ntb_peer_mw_count(ndev); 1115 1116 if (!ndev->ops->mw_set_trans) { 1117 dev_err(&ndev->dev, "Inbound MW based NTB API is required\n"); 1118 return -EINVAL; 1119 } 1120 1121 if (ntb_db_is_unsafe(ndev)) 1122 dev_dbg(&ndev->dev, 1123 "doorbell is unsafe, proceed anyway...\n"); 1124 if (ntb_spad_is_unsafe(ndev)) 1125 dev_dbg(&ndev->dev, 1126 "scratchpad is unsafe, proceed anyway...\n"); 1127 1128 if (ntb_peer_port_count(ndev) != NTB_DEF_PEER_CNT) 1129 dev_warn(&ndev->dev, "Multi-port NTB devices unsupported\n"); 1130 1131 node = dev_to_node(&ndev->dev); 1132 1133 nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node); 1134 if (!nt) 1135 return -ENOMEM; 1136 1137 nt->ndev = ndev; 1138 spad_count = ntb_spad_count(ndev); 1139 1140 /* Limit the MW's based on the availability of scratchpads */ 1141 1142 if (spad_count < NTB_TRANSPORT_MIN_SPADS) { 1143 nt->mw_count = 0; 1144 rc = -EINVAL; 1145 goto err; 1146 } 1147 1148 max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2; 1149 nt->mw_count = min(mw_count, max_mw_count_for_spads); 1150 1151 nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec), 1152 GFP_KERNEL, node); 1153 if (!nt->mw_vec) { 1154 rc = -ENOMEM; 1155 goto err; 1156 } 1157 1158 for (i = 0; i < mw_count; i++) { 1159 mw = &nt->mw_vec[i]; 1160 1161 rc = ntb_peer_mw_get_addr(ndev, i, &mw->phys_addr, 1162 &mw->phys_size); 1163 if (rc) 1164 goto err1; 1165 1166 mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size); 1167 if (!mw->vbase) { 1168 rc = -ENOMEM; 1169 goto err1; 1170 } 1171 1172 mw->buff_size = 0; 1173 mw->xlat_size = 0; 1174 mw->virt_addr = NULL; 1175 mw->dma_addr = 0; 1176 } 1177 1178 qp_bitmap = ntb_db_valid_mask(ndev); 1179 1180 qp_count = ilog2(qp_bitmap); 1181 if (max_num_clients && max_num_clients < qp_count) 1182 qp_count = max_num_clients; 1183 else if (nt->mw_count < qp_count) 1184 qp_count = nt->mw_count; 1185 1186 qp_bitmap &= BIT_ULL(qp_count) - 1; 1187 1188 nt->qp_count = qp_count; 1189 nt->qp_bitmap = qp_bitmap; 1190 nt->qp_bitmap_free = qp_bitmap; 1191 1192 nt->qp_vec = kcalloc_node(qp_count, sizeof(*nt->qp_vec), 1193 GFP_KERNEL, node); 1194 if (!nt->qp_vec) { 1195 rc = -ENOMEM; 1196 goto err1; 1197 } 1198 1199 if (nt_debugfs_dir) { 1200 nt->debugfs_node_dir = 1201 debugfs_create_dir(pci_name(ndev->pdev), 1202 nt_debugfs_dir); 1203 } 1204 1205 for (i = 0; i < qp_count; i++) { 1206 rc = ntb_transport_init_queue(nt, i); 1207 if (rc) 1208 goto err2; 1209 } 1210 1211 INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); 1212 INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); 1213 1214 rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops); 1215 if (rc) 1216 goto err2; 1217 1218 INIT_LIST_HEAD(&nt->client_devs); 1219 rc = ntb_bus_init(nt); 1220 if (rc) 1221 goto err3; 1222 1223 nt->link_is_up = false; 1224 ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1225 ntb_link_event(ndev); 1226 1227 return 0; 1228 1229 err3: 1230 ntb_clear_ctx(ndev); 1231 err2: 1232 kfree(nt->qp_vec); 1233 err1: 1234 while (i--) { 1235 mw = &nt->mw_vec[i]; 1236 iounmap(mw->vbase); 1237 } 1238 kfree(nt->mw_vec); 1239 err: 1240 kfree(nt); 1241 return rc; 1242 } 1243 1244 static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev) 1245 { 1246 struct ntb_transport_ctx *nt = ndev->ctx; 1247 struct ntb_transport_qp *qp; 1248 u64 qp_bitmap_alloc; 1249 int i; 1250 1251 ntb_transport_link_cleanup(nt); 1252 cancel_work_sync(&nt->link_cleanup); 1253 cancel_delayed_work_sync(&nt->link_work); 1254 1255 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 1256 1257 /* verify that all the qp's are freed */ 1258 for (i = 0; i < nt->qp_count; i++) { 1259 qp = &nt->qp_vec[i]; 1260 if (qp_bitmap_alloc & BIT_ULL(i)) 1261 ntb_transport_free_queue(qp); 1262 debugfs_remove_recursive(qp->debugfs_dir); 1263 } 1264 1265 ntb_link_disable(ndev); 1266 ntb_clear_ctx(ndev); 1267 1268 ntb_bus_remove(nt); 1269 1270 for (i = nt->mw_count; i--; ) { 1271 ntb_free_mw(nt, i); 1272 iounmap(nt->mw_vec[i].vbase); 1273 } 1274 1275 kfree(nt->qp_vec); 1276 kfree(nt->mw_vec); 1277 kfree(nt); 1278 } 1279 1280 static void ntb_complete_rxc(struct ntb_transport_qp *qp) 1281 { 1282 struct ntb_queue_entry *entry; 1283 void *cb_data; 1284 unsigned int len; 1285 unsigned long irqflags; 1286 1287 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); 1288 1289 while (!list_empty(&qp->rx_post_q)) { 1290 entry = list_first_entry(&qp->rx_post_q, 1291 struct ntb_queue_entry, entry); 1292 if (!(entry->flags & DESC_DONE_FLAG)) 1293 break; 1294 1295 entry->rx_hdr->flags = 0; 1296 iowrite32(entry->rx_index, &qp->rx_info->entry); 1297 1298 cb_data = entry->cb_data; 1299 len = entry->len; 1300 1301 list_move_tail(&entry->entry, &qp->rx_free_q); 1302 1303 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); 1304 1305 if (qp->rx_handler && qp->client_ready) 1306 qp->rx_handler(qp, qp->cb_data, cb_data, len); 1307 1308 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); 1309 } 1310 1311 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); 1312 } 1313 1314 static void ntb_rx_copy_callback(void *data, 1315 const struct dmaengine_result *res) 1316 { 1317 struct ntb_queue_entry *entry = data; 1318 1319 /* we need to check DMA results if we are using DMA */ 1320 if (res) { 1321 enum dmaengine_tx_result dma_err = res->result; 1322 1323 switch (dma_err) { 1324 case DMA_TRANS_READ_FAILED: 1325 case DMA_TRANS_WRITE_FAILED: 1326 entry->errors++; 1327 /* fall through */ 1328 case DMA_TRANS_ABORTED: 1329 { 1330 struct ntb_transport_qp *qp = entry->qp; 1331 void *offset = qp->rx_buff + qp->rx_max_frame * 1332 qp->rx_index; 1333 1334 ntb_memcpy_rx(entry, offset); 1335 qp->rx_memcpy++; 1336 return; 1337 } 1338 1339 case DMA_TRANS_NOERROR: 1340 default: 1341 break; 1342 } 1343 } 1344 1345 entry->flags |= DESC_DONE_FLAG; 1346 1347 ntb_complete_rxc(entry->qp); 1348 } 1349 1350 static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) 1351 { 1352 void *buf = entry->buf; 1353 size_t len = entry->len; 1354 1355 memcpy(buf, offset, len); 1356 1357 /* Ensure that the data is fully copied out before clearing the flag */ 1358 wmb(); 1359 1360 ntb_rx_copy_callback(entry, NULL); 1361 } 1362 1363 static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset) 1364 { 1365 struct dma_async_tx_descriptor *txd; 1366 struct ntb_transport_qp *qp = entry->qp; 1367 struct dma_chan *chan = qp->rx_dma_chan; 1368 struct dma_device *device; 1369 size_t pay_off, buff_off, len; 1370 struct dmaengine_unmap_data *unmap; 1371 dma_cookie_t cookie; 1372 void *buf = entry->buf; 1373 1374 len = entry->len; 1375 device = chan->device; 1376 pay_off = (size_t)offset & ~PAGE_MASK; 1377 buff_off = (size_t)buf & ~PAGE_MASK; 1378 1379 if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) 1380 goto err; 1381 1382 unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); 1383 if (!unmap) 1384 goto err; 1385 1386 unmap->len = len; 1387 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset), 1388 pay_off, len, DMA_TO_DEVICE); 1389 if (dma_mapping_error(device->dev, unmap->addr[0])) 1390 goto err_get_unmap; 1391 1392 unmap->to_cnt = 1; 1393 1394 unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf), 1395 buff_off, len, DMA_FROM_DEVICE); 1396 if (dma_mapping_error(device->dev, unmap->addr[1])) 1397 goto err_get_unmap; 1398 1399 unmap->from_cnt = 1; 1400 1401 txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], 1402 unmap->addr[0], len, 1403 DMA_PREP_INTERRUPT); 1404 if (!txd) 1405 goto err_get_unmap; 1406 1407 txd->callback_result = ntb_rx_copy_callback; 1408 txd->callback_param = entry; 1409 dma_set_unmap(txd, unmap); 1410 1411 cookie = dmaengine_submit(txd); 1412 if (dma_submit_error(cookie)) 1413 goto err_set_unmap; 1414 1415 dmaengine_unmap_put(unmap); 1416 1417 qp->last_cookie = cookie; 1418 1419 qp->rx_async++; 1420 1421 return 0; 1422 1423 err_set_unmap: 1424 dmaengine_unmap_put(unmap); 1425 err_get_unmap: 1426 dmaengine_unmap_put(unmap); 1427 err: 1428 return -ENXIO; 1429 } 1430 1431 static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) 1432 { 1433 struct ntb_transport_qp *qp = entry->qp; 1434 struct dma_chan *chan = qp->rx_dma_chan; 1435 int res; 1436 1437 if (!chan) 1438 goto err; 1439 1440 if (entry->len < copy_bytes) 1441 goto err; 1442 1443 res = ntb_async_rx_submit(entry, offset); 1444 if (res < 0) 1445 goto err; 1446 1447 if (!entry->retries) 1448 qp->rx_async++; 1449 1450 return; 1451 1452 err: 1453 ntb_memcpy_rx(entry, offset); 1454 qp->rx_memcpy++; 1455 } 1456 1457 static int ntb_process_rxc(struct ntb_transport_qp *qp) 1458 { 1459 struct ntb_payload_header *hdr; 1460 struct ntb_queue_entry *entry; 1461 void *offset; 1462 1463 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; 1464 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); 1465 1466 dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n", 1467 qp->qp_num, hdr->ver, hdr->len, hdr->flags); 1468 1469 if (!(hdr->flags & DESC_DONE_FLAG)) { 1470 dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n"); 1471 qp->rx_ring_empty++; 1472 return -EAGAIN; 1473 } 1474 1475 if (hdr->flags & LINK_DOWN_FLAG) { 1476 dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n"); 1477 ntb_qp_link_down(qp); 1478 hdr->flags = 0; 1479 return -EAGAIN; 1480 } 1481 1482 if (hdr->ver != (u32)qp->rx_pkts) { 1483 dev_dbg(&qp->ndev->pdev->dev, 1484 "version mismatch, expected %llu - got %u\n", 1485 qp->rx_pkts, hdr->ver); 1486 qp->rx_err_ver++; 1487 return -EIO; 1488 } 1489 1490 entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q); 1491 if (!entry) { 1492 dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n"); 1493 qp->rx_err_no_buf++; 1494 return -EAGAIN; 1495 } 1496 1497 entry->rx_hdr = hdr; 1498 entry->rx_index = qp->rx_index; 1499 1500 if (hdr->len > entry->len) { 1501 dev_dbg(&qp->ndev->pdev->dev, 1502 "receive buffer overflow! Wanted %d got %d\n", 1503 hdr->len, entry->len); 1504 qp->rx_err_oflow++; 1505 1506 entry->len = -EIO; 1507 entry->flags |= DESC_DONE_FLAG; 1508 1509 ntb_complete_rxc(qp); 1510 } else { 1511 dev_dbg(&qp->ndev->pdev->dev, 1512 "RX OK index %u ver %u size %d into buf size %d\n", 1513 qp->rx_index, hdr->ver, hdr->len, entry->len); 1514 1515 qp->rx_bytes += hdr->len; 1516 qp->rx_pkts++; 1517 1518 entry->len = hdr->len; 1519 1520 ntb_async_rx(entry, offset); 1521 } 1522 1523 qp->rx_index++; 1524 qp->rx_index %= qp->rx_max_entry; 1525 1526 return 0; 1527 } 1528 1529 static void ntb_transport_rxc_db(unsigned long data) 1530 { 1531 struct ntb_transport_qp *qp = (void *)data; 1532 int rc, i; 1533 1534 dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n", 1535 __func__, qp->qp_num); 1536 1537 /* Limit the number of packets processed in a single interrupt to 1538 * provide fairness to others 1539 */ 1540 for (i = 0; i < qp->rx_max_entry; i++) { 1541 rc = ntb_process_rxc(qp); 1542 if (rc) 1543 break; 1544 } 1545 1546 if (i && qp->rx_dma_chan) 1547 dma_async_issue_pending(qp->rx_dma_chan); 1548 1549 if (i == qp->rx_max_entry) { 1550 /* there is more work to do */ 1551 if (qp->active) 1552 tasklet_schedule(&qp->rxc_db_work); 1553 } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) { 1554 /* the doorbell bit is set: clear it */ 1555 ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num)); 1556 /* ntb_db_read ensures ntb_db_clear write is committed */ 1557 ntb_db_read(qp->ndev); 1558 1559 /* an interrupt may have arrived between finishing 1560 * ntb_process_rxc and clearing the doorbell bit: 1561 * there might be some more work to do. 1562 */ 1563 if (qp->active) 1564 tasklet_schedule(&qp->rxc_db_work); 1565 } 1566 } 1567 1568 static void ntb_tx_copy_callback(void *data, 1569 const struct dmaengine_result *res) 1570 { 1571 struct ntb_queue_entry *entry = data; 1572 struct ntb_transport_qp *qp = entry->qp; 1573 struct ntb_payload_header __iomem *hdr = entry->tx_hdr; 1574 1575 /* we need to check DMA results if we are using DMA */ 1576 if (res) { 1577 enum dmaengine_tx_result dma_err = res->result; 1578 1579 switch (dma_err) { 1580 case DMA_TRANS_READ_FAILED: 1581 case DMA_TRANS_WRITE_FAILED: 1582 entry->errors++; 1583 /* fall through */ 1584 case DMA_TRANS_ABORTED: 1585 { 1586 void __iomem *offset = 1587 qp->tx_mw + qp->tx_max_frame * 1588 entry->tx_index; 1589 1590 /* resubmit via CPU */ 1591 ntb_memcpy_tx(entry, offset); 1592 qp->tx_memcpy++; 1593 return; 1594 } 1595 1596 case DMA_TRANS_NOERROR: 1597 default: 1598 break; 1599 } 1600 } 1601 1602 iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); 1603 1604 ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); 1605 1606 /* The entry length can only be zero if the packet is intended to be a 1607 * "link down" or similar. Since no payload is being sent in these 1608 * cases, there is nothing to add to the completion queue. 1609 */ 1610 if (entry->len > 0) { 1611 qp->tx_bytes += entry->len; 1612 1613 if (qp->tx_handler) 1614 qp->tx_handler(qp, qp->cb_data, entry->cb_data, 1615 entry->len); 1616 } 1617 1618 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); 1619 } 1620 1621 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) 1622 { 1623 #ifdef ARCH_HAS_NOCACHE_UACCESS 1624 /* 1625 * Using non-temporal mov to improve performance on non-cached 1626 * writes, even though we aren't actually copying from user space. 1627 */ 1628 __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); 1629 #else 1630 memcpy_toio(offset, entry->buf, entry->len); 1631 #endif 1632 1633 /* Ensure that the data is fully copied out before setting the flags */ 1634 wmb(); 1635 1636 ntb_tx_copy_callback(entry, NULL); 1637 } 1638 1639 static int ntb_async_tx_submit(struct ntb_transport_qp *qp, 1640 struct ntb_queue_entry *entry) 1641 { 1642 struct dma_async_tx_descriptor *txd; 1643 struct dma_chan *chan = qp->tx_dma_chan; 1644 struct dma_device *device; 1645 size_t len = entry->len; 1646 void *buf = entry->buf; 1647 size_t dest_off, buff_off; 1648 struct dmaengine_unmap_data *unmap; 1649 dma_addr_t dest; 1650 dma_cookie_t cookie; 1651 1652 device = chan->device; 1653 dest = qp->tx_mw_dma_addr + qp->tx_max_frame * entry->tx_index; 1654 buff_off = (size_t)buf & ~PAGE_MASK; 1655 dest_off = (size_t)dest & ~PAGE_MASK; 1656 1657 if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) 1658 goto err; 1659 1660 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); 1661 if (!unmap) 1662 goto err; 1663 1664 unmap->len = len; 1665 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf), 1666 buff_off, len, DMA_TO_DEVICE); 1667 if (dma_mapping_error(device->dev, unmap->addr[0])) 1668 goto err_get_unmap; 1669 1670 unmap->to_cnt = 1; 1671 1672 txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len, 1673 DMA_PREP_INTERRUPT); 1674 if (!txd) 1675 goto err_get_unmap; 1676 1677 txd->callback_result = ntb_tx_copy_callback; 1678 txd->callback_param = entry; 1679 dma_set_unmap(txd, unmap); 1680 1681 cookie = dmaengine_submit(txd); 1682 if (dma_submit_error(cookie)) 1683 goto err_set_unmap; 1684 1685 dmaengine_unmap_put(unmap); 1686 1687 dma_async_issue_pending(chan); 1688 1689 return 0; 1690 err_set_unmap: 1691 dmaengine_unmap_put(unmap); 1692 err_get_unmap: 1693 dmaengine_unmap_put(unmap); 1694 err: 1695 return -ENXIO; 1696 } 1697 1698 static void ntb_async_tx(struct ntb_transport_qp *qp, 1699 struct ntb_queue_entry *entry) 1700 { 1701 struct ntb_payload_header __iomem *hdr; 1702 struct dma_chan *chan = qp->tx_dma_chan; 1703 void __iomem *offset; 1704 int res; 1705 1706 entry->tx_index = qp->tx_index; 1707 offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index; 1708 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); 1709 entry->tx_hdr = hdr; 1710 1711 iowrite32(entry->len, &hdr->len); 1712 iowrite32((u32)qp->tx_pkts, &hdr->ver); 1713 1714 if (!chan) 1715 goto err; 1716 1717 if (entry->len < copy_bytes) 1718 goto err; 1719 1720 res = ntb_async_tx_submit(qp, entry); 1721 if (res < 0) 1722 goto err; 1723 1724 if (!entry->retries) 1725 qp->tx_async++; 1726 1727 return; 1728 1729 err: 1730 ntb_memcpy_tx(entry, offset); 1731 qp->tx_memcpy++; 1732 } 1733 1734 static int ntb_process_tx(struct ntb_transport_qp *qp, 1735 struct ntb_queue_entry *entry) 1736 { 1737 if (qp->tx_index == qp->remote_rx_info->entry) { 1738 qp->tx_ring_full++; 1739 return -EAGAIN; 1740 } 1741 1742 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 1743 if (qp->tx_handler) 1744 qp->tx_handler(qp, qp->cb_data, NULL, -EIO); 1745 1746 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1747 &qp->tx_free_q); 1748 return 0; 1749 } 1750 1751 ntb_async_tx(qp, entry); 1752 1753 qp->tx_index++; 1754 qp->tx_index %= qp->tx_max_entry; 1755 1756 qp->tx_pkts++; 1757 1758 return 0; 1759 } 1760 1761 static void ntb_send_link_down(struct ntb_transport_qp *qp) 1762 { 1763 struct pci_dev *pdev = qp->ndev->pdev; 1764 struct ntb_queue_entry *entry; 1765 int i, rc; 1766 1767 if (!qp->link_is_up) 1768 return; 1769 1770 dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num); 1771 1772 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1773 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1774 if (entry) 1775 break; 1776 msleep(100); 1777 } 1778 1779 if (!entry) 1780 return; 1781 1782 entry->cb_data = NULL; 1783 entry->buf = NULL; 1784 entry->len = 0; 1785 entry->flags = LINK_DOWN_FLAG; 1786 1787 rc = ntb_process_tx(qp, entry); 1788 if (rc) 1789 dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", 1790 qp->qp_num); 1791 1792 ntb_qp_link_down_reset(qp); 1793 } 1794 1795 static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node) 1796 { 1797 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; 1798 } 1799 1800 /** 1801 * ntb_transport_create_queue - Create a new NTB transport layer queue 1802 * @rx_handler: receive callback function 1803 * @tx_handler: transmit callback function 1804 * @event_handler: event callback function 1805 * 1806 * Create a new NTB transport layer queue and provide the queue with a callback 1807 * routine for both transmit and receive. The receive callback routine will be 1808 * used to pass up data when the transport has received it on the queue. The 1809 * transmit callback routine will be called when the transport has completed the 1810 * transmission of the data on the queue and the data is ready to be freed. 1811 * 1812 * RETURNS: pointer to newly created ntb_queue, NULL on error. 1813 */ 1814 struct ntb_transport_qp * 1815 ntb_transport_create_queue(void *data, struct device *client_dev, 1816 const struct ntb_queue_handlers *handlers) 1817 { 1818 struct ntb_dev *ndev; 1819 struct pci_dev *pdev; 1820 struct ntb_transport_ctx *nt; 1821 struct ntb_queue_entry *entry; 1822 struct ntb_transport_qp *qp; 1823 u64 qp_bit; 1824 unsigned int free_queue; 1825 dma_cap_mask_t dma_mask; 1826 int node; 1827 int i; 1828 1829 ndev = dev_ntb(client_dev->parent); 1830 pdev = ndev->pdev; 1831 nt = ndev->ctx; 1832 1833 node = dev_to_node(&ndev->dev); 1834 1835 free_queue = ffs(nt->qp_bitmap_free); 1836 if (!free_queue) 1837 goto err; 1838 1839 /* decrement free_queue to make it zero based */ 1840 free_queue--; 1841 1842 qp = &nt->qp_vec[free_queue]; 1843 qp_bit = BIT_ULL(qp->qp_num); 1844 1845 nt->qp_bitmap_free &= ~qp_bit; 1846 1847 qp->cb_data = data; 1848 qp->rx_handler = handlers->rx_handler; 1849 qp->tx_handler = handlers->tx_handler; 1850 qp->event_handler = handlers->event_handler; 1851 1852 dma_cap_zero(dma_mask); 1853 dma_cap_set(DMA_MEMCPY, dma_mask); 1854 1855 if (use_dma) { 1856 qp->tx_dma_chan = 1857 dma_request_channel(dma_mask, ntb_dma_filter_fn, 1858 (void *)(unsigned long)node); 1859 if (!qp->tx_dma_chan) 1860 dev_info(&pdev->dev, "Unable to allocate TX DMA channel\n"); 1861 1862 qp->rx_dma_chan = 1863 dma_request_channel(dma_mask, ntb_dma_filter_fn, 1864 (void *)(unsigned long)node); 1865 if (!qp->rx_dma_chan) 1866 dev_info(&pdev->dev, "Unable to allocate RX DMA channel\n"); 1867 } else { 1868 qp->tx_dma_chan = NULL; 1869 qp->rx_dma_chan = NULL; 1870 } 1871 1872 qp->tx_mw_dma_addr = 0; 1873 if (qp->tx_dma_chan) { 1874 qp->tx_mw_dma_addr = 1875 dma_map_resource(qp->tx_dma_chan->device->dev, 1876 qp->tx_mw_phys, qp->tx_mw_size, 1877 DMA_FROM_DEVICE, 0); 1878 if (dma_mapping_error(qp->tx_dma_chan->device->dev, 1879 qp->tx_mw_dma_addr)) { 1880 qp->tx_mw_dma_addr = 0; 1881 goto err1; 1882 } 1883 } 1884 1885 dev_dbg(&pdev->dev, "Using %s memcpy for TX\n", 1886 qp->tx_dma_chan ? "DMA" : "CPU"); 1887 1888 dev_dbg(&pdev->dev, "Using %s memcpy for RX\n", 1889 qp->rx_dma_chan ? "DMA" : "CPU"); 1890 1891 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 1892 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 1893 if (!entry) 1894 goto err1; 1895 1896 entry->qp = qp; 1897 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, 1898 &qp->rx_free_q); 1899 } 1900 qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES; 1901 1902 for (i = 0; i < qp->tx_max_entry; i++) { 1903 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 1904 if (!entry) 1905 goto err2; 1906 1907 entry->qp = qp; 1908 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1909 &qp->tx_free_q); 1910 } 1911 1912 ntb_db_clear(qp->ndev, qp_bit); 1913 ntb_db_clear_mask(qp->ndev, qp_bit); 1914 1915 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); 1916 1917 return qp; 1918 1919 err2: 1920 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 1921 kfree(entry); 1922 err1: 1923 qp->rx_alloc_entry = 0; 1924 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 1925 kfree(entry); 1926 if (qp->tx_mw_dma_addr) 1927 dma_unmap_resource(qp->tx_dma_chan->device->dev, 1928 qp->tx_mw_dma_addr, qp->tx_mw_size, 1929 DMA_FROM_DEVICE, 0); 1930 if (qp->tx_dma_chan) 1931 dma_release_channel(qp->tx_dma_chan); 1932 if (qp->rx_dma_chan) 1933 dma_release_channel(qp->rx_dma_chan); 1934 nt->qp_bitmap_free |= qp_bit; 1935 err: 1936 return NULL; 1937 } 1938 EXPORT_SYMBOL_GPL(ntb_transport_create_queue); 1939 1940 /** 1941 * ntb_transport_free_queue - Frees NTB transport queue 1942 * @qp: NTB queue to be freed 1943 * 1944 * Frees NTB transport queue 1945 */ 1946 void ntb_transport_free_queue(struct ntb_transport_qp *qp) 1947 { 1948 struct pci_dev *pdev; 1949 struct ntb_queue_entry *entry; 1950 u64 qp_bit; 1951 1952 if (!qp) 1953 return; 1954 1955 pdev = qp->ndev->pdev; 1956 1957 qp->active = false; 1958 1959 if (qp->tx_dma_chan) { 1960 struct dma_chan *chan = qp->tx_dma_chan; 1961 /* Putting the dma_chan to NULL will force any new traffic to be 1962 * processed by the CPU instead of the DAM engine 1963 */ 1964 qp->tx_dma_chan = NULL; 1965 1966 /* Try to be nice and wait for any queued DMA engine 1967 * transactions to process before smashing it with a rock 1968 */ 1969 dma_sync_wait(chan, qp->last_cookie); 1970 dmaengine_terminate_all(chan); 1971 1972 dma_unmap_resource(chan->device->dev, 1973 qp->tx_mw_dma_addr, qp->tx_mw_size, 1974 DMA_FROM_DEVICE, 0); 1975 1976 dma_release_channel(chan); 1977 } 1978 1979 if (qp->rx_dma_chan) { 1980 struct dma_chan *chan = qp->rx_dma_chan; 1981 /* Putting the dma_chan to NULL will force any new traffic to be 1982 * processed by the CPU instead of the DAM engine 1983 */ 1984 qp->rx_dma_chan = NULL; 1985 1986 /* Try to be nice and wait for any queued DMA engine 1987 * transactions to process before smashing it with a rock 1988 */ 1989 dma_sync_wait(chan, qp->last_cookie); 1990 dmaengine_terminate_all(chan); 1991 dma_release_channel(chan); 1992 } 1993 1994 qp_bit = BIT_ULL(qp->qp_num); 1995 1996 ntb_db_set_mask(qp->ndev, qp_bit); 1997 tasklet_kill(&qp->rxc_db_work); 1998 1999 cancel_delayed_work_sync(&qp->link_work); 2000 2001 qp->cb_data = NULL; 2002 qp->rx_handler = NULL; 2003 qp->tx_handler = NULL; 2004 qp->event_handler = NULL; 2005 2006 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 2007 kfree(entry); 2008 2009 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) { 2010 dev_warn(&pdev->dev, "Freeing item from non-empty rx_pend_q\n"); 2011 kfree(entry); 2012 } 2013 2014 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) { 2015 dev_warn(&pdev->dev, "Freeing item from non-empty rx_post_q\n"); 2016 kfree(entry); 2017 } 2018 2019 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 2020 kfree(entry); 2021 2022 qp->transport->qp_bitmap_free |= qp_bit; 2023 2024 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); 2025 } 2026 EXPORT_SYMBOL_GPL(ntb_transport_free_queue); 2027 2028 /** 2029 * ntb_transport_rx_remove - Dequeues enqueued rx packet 2030 * @qp: NTB queue to be freed 2031 * @len: pointer to variable to write enqueued buffers length 2032 * 2033 * Dequeues unused buffers from receive queue. Should only be used during 2034 * shutdown of qp. 2035 * 2036 * RETURNS: NULL error value on error, or void* for success. 2037 */ 2038 void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) 2039 { 2040 struct ntb_queue_entry *entry; 2041 void *buf; 2042 2043 if (!qp || qp->client_ready) 2044 return NULL; 2045 2046 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q); 2047 if (!entry) 2048 return NULL; 2049 2050 buf = entry->cb_data; 2051 *len = entry->len; 2052 2053 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q); 2054 2055 return buf; 2056 } 2057 EXPORT_SYMBOL_GPL(ntb_transport_rx_remove); 2058 2059 /** 2060 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry 2061 * @qp: NTB transport layer queue the entry is to be enqueued on 2062 * @cb: per buffer pointer for callback function to use 2063 * @data: pointer to data buffer that incoming packets will be copied into 2064 * @len: length of the data buffer 2065 * 2066 * Enqueue a new receive buffer onto the transport queue into which a NTB 2067 * payload can be received into. 2068 * 2069 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 2070 */ 2071 int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 2072 unsigned int len) 2073 { 2074 struct ntb_queue_entry *entry; 2075 2076 if (!qp) 2077 return -EINVAL; 2078 2079 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q); 2080 if (!entry) 2081 return -ENOMEM; 2082 2083 entry->cb_data = cb; 2084 entry->buf = data; 2085 entry->len = len; 2086 entry->flags = 0; 2087 entry->retries = 0; 2088 entry->errors = 0; 2089 entry->rx_index = 0; 2090 2091 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q); 2092 2093 if (qp->active) 2094 tasklet_schedule(&qp->rxc_db_work); 2095 2096 return 0; 2097 } 2098 EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue); 2099 2100 /** 2101 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 2102 * @qp: NTB transport layer queue the entry is to be enqueued on 2103 * @cb: per buffer pointer for callback function to use 2104 * @data: pointer to data buffer that will be sent 2105 * @len: length of the data buffer 2106 * 2107 * Enqueue a new transmit buffer onto the transport queue from which a NTB 2108 * payload will be transmitted. This assumes that a lock is being held to 2109 * serialize access to the qp. 2110 * 2111 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 2112 */ 2113 int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 2114 unsigned int len) 2115 { 2116 struct ntb_queue_entry *entry; 2117 int rc; 2118 2119 if (!qp || !qp->link_is_up || !len) 2120 return -EINVAL; 2121 2122 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 2123 if (!entry) { 2124 qp->tx_err_no_buf++; 2125 return -EBUSY; 2126 } 2127 2128 entry->cb_data = cb; 2129 entry->buf = data; 2130 entry->len = len; 2131 entry->flags = 0; 2132 entry->errors = 0; 2133 entry->retries = 0; 2134 entry->tx_index = 0; 2135 2136 rc = ntb_process_tx(qp, entry); 2137 if (rc) 2138 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 2139 &qp->tx_free_q); 2140 2141 return rc; 2142 } 2143 EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue); 2144 2145 /** 2146 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 2147 * @qp: NTB transport layer queue to be enabled 2148 * 2149 * Notify NTB transport layer of client readiness to use queue 2150 */ 2151 void ntb_transport_link_up(struct ntb_transport_qp *qp) 2152 { 2153 if (!qp) 2154 return; 2155 2156 qp->client_ready = true; 2157 2158 if (qp->transport->link_is_up) 2159 schedule_delayed_work(&qp->link_work, 0); 2160 } 2161 EXPORT_SYMBOL_GPL(ntb_transport_link_up); 2162 2163 /** 2164 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 2165 * @qp: NTB transport layer queue to be disabled 2166 * 2167 * Notify NTB transport layer of client's desire to no longer receive data on 2168 * transport queue specified. It is the client's responsibility to ensure all 2169 * entries on queue are purged or otherwise handled appropriately. 2170 */ 2171 void ntb_transport_link_down(struct ntb_transport_qp *qp) 2172 { 2173 int val; 2174 2175 if (!qp) 2176 return; 2177 2178 qp->client_ready = false; 2179 2180 val = ntb_spad_read(qp->ndev, QP_LINKS); 2181 2182 ntb_peer_spad_write(qp->ndev, PIDX, QP_LINKS, val & ~BIT(qp->qp_num)); 2183 2184 if (qp->link_is_up) 2185 ntb_send_link_down(qp); 2186 else 2187 cancel_delayed_work_sync(&qp->link_work); 2188 } 2189 EXPORT_SYMBOL_GPL(ntb_transport_link_down); 2190 2191 /** 2192 * ntb_transport_link_query - Query transport link state 2193 * @qp: NTB transport layer queue to be queried 2194 * 2195 * Query connectivity to the remote system of the NTB transport queue 2196 * 2197 * RETURNS: true for link up or false for link down 2198 */ 2199 bool ntb_transport_link_query(struct ntb_transport_qp *qp) 2200 { 2201 if (!qp) 2202 return false; 2203 2204 return qp->link_is_up; 2205 } 2206 EXPORT_SYMBOL_GPL(ntb_transport_link_query); 2207 2208 /** 2209 * ntb_transport_qp_num - Query the qp number 2210 * @qp: NTB transport layer queue to be queried 2211 * 2212 * Query qp number of the NTB transport queue 2213 * 2214 * RETURNS: a zero based number specifying the qp number 2215 */ 2216 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) 2217 { 2218 if (!qp) 2219 return 0; 2220 2221 return qp->qp_num; 2222 } 2223 EXPORT_SYMBOL_GPL(ntb_transport_qp_num); 2224 2225 /** 2226 * ntb_transport_max_size - Query the max payload size of a qp 2227 * @qp: NTB transport layer queue to be queried 2228 * 2229 * Query the maximum payload size permissible on the given qp 2230 * 2231 * RETURNS: the max payload size of a qp 2232 */ 2233 unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) 2234 { 2235 unsigned int max_size; 2236 unsigned int copy_align; 2237 struct dma_chan *rx_chan, *tx_chan; 2238 2239 if (!qp) 2240 return 0; 2241 2242 rx_chan = qp->rx_dma_chan; 2243 tx_chan = qp->tx_dma_chan; 2244 2245 copy_align = max(rx_chan ? rx_chan->device->copy_align : 0, 2246 tx_chan ? tx_chan->device->copy_align : 0); 2247 2248 /* If DMA engine usage is possible, try to find the max size for that */ 2249 max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header); 2250 max_size = round_down(max_size, 1 << copy_align); 2251 2252 return max_size; 2253 } 2254 EXPORT_SYMBOL_GPL(ntb_transport_max_size); 2255 2256 unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp) 2257 { 2258 unsigned int head = qp->tx_index; 2259 unsigned int tail = qp->remote_rx_info->entry; 2260 2261 return tail > head ? tail - head : qp->tx_max_entry + tail - head; 2262 } 2263 EXPORT_SYMBOL_GPL(ntb_transport_tx_free_entry); 2264 2265 static void ntb_transport_doorbell_callback(void *data, int vector) 2266 { 2267 struct ntb_transport_ctx *nt = data; 2268 struct ntb_transport_qp *qp; 2269 u64 db_bits; 2270 unsigned int qp_num; 2271 2272 db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free & 2273 ntb_db_vector_mask(nt->ndev, vector)); 2274 2275 while (db_bits) { 2276 qp_num = __ffs(db_bits); 2277 qp = &nt->qp_vec[qp_num]; 2278 2279 if (qp->active) 2280 tasklet_schedule(&qp->rxc_db_work); 2281 2282 db_bits &= ~BIT_ULL(qp_num); 2283 } 2284 } 2285 2286 static const struct ntb_ctx_ops ntb_transport_ops = { 2287 .link_event = ntb_transport_event_callback, 2288 .db_event = ntb_transport_doorbell_callback, 2289 }; 2290 2291 static struct ntb_client ntb_transport_client = { 2292 .ops = { 2293 .probe = ntb_transport_probe, 2294 .remove = ntb_transport_free, 2295 }, 2296 }; 2297 2298 static int __init ntb_transport_init(void) 2299 { 2300 int rc; 2301 2302 pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER); 2303 2304 if (debugfs_initialized()) 2305 nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 2306 2307 rc = bus_register(&ntb_transport_bus); 2308 if (rc) 2309 goto err_bus; 2310 2311 rc = ntb_register_client(&ntb_transport_client); 2312 if (rc) 2313 goto err_client; 2314 2315 return 0; 2316 2317 err_client: 2318 bus_unregister(&ntb_transport_bus); 2319 err_bus: 2320 debugfs_remove_recursive(nt_debugfs_dir); 2321 return rc; 2322 } 2323 module_init(ntb_transport_init); 2324 2325 static void __exit ntb_transport_exit(void) 2326 { 2327 ntb_unregister_client(&ntb_transport_client); 2328 bus_unregister(&ntb_transport_bus); 2329 debugfs_remove_recursive(nt_debugfs_dir); 2330 } 2331 module_exit(ntb_transport_exit); 2332