1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * BSD LICENSE 15 * 16 * Copyright(c) 2012 Intel Corporation. All rights reserved. 17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 23 * * Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * * Redistributions in binary form must reproduce the above copy 26 * notice, this list of conditions and the following disclaimer in 27 * the documentation and/or other materials provided with the 28 * distribution. 29 * * Neither the name of Intel Corporation nor the names of its 30 * contributors may be used to endorse or promote products derived 31 * from this software without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 44 * 45 * PCIe NTB Transport Linux driver 46 * 47 * Contact Information: 48 * Jon Mason <jon.mason@intel.com> 49 */ 50 #include <linux/debugfs.h> 51 #include <linux/delay.h> 52 #include <linux/dmaengine.h> 53 #include <linux/dma-mapping.h> 54 #include <linux/errno.h> 55 #include <linux/export.h> 56 #include <linux/interrupt.h> 57 #include <linux/module.h> 58 #include <linux/pci.h> 59 #include <linux/slab.h> 60 #include <linux/types.h> 61 #include <linux/uaccess.h> 62 #include "linux/ntb.h" 63 #include "linux/ntb_transport.h" 64 65 #define NTB_TRANSPORT_VERSION 4 66 #define NTB_TRANSPORT_VER "4" 67 #define NTB_TRANSPORT_NAME "ntb_transport" 68 #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB" 69 #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2) 70 71 MODULE_DESCRIPTION(NTB_TRANSPORT_DESC); 72 MODULE_VERSION(NTB_TRANSPORT_VER); 73 MODULE_LICENSE("Dual BSD/GPL"); 74 MODULE_AUTHOR("Intel Corporation"); 75 76 static unsigned long max_mw_size; 77 module_param(max_mw_size, ulong, 0644); 78 MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); 79 80 static unsigned int transport_mtu = 0x10000; 81 module_param(transport_mtu, uint, 0644); 82 MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); 83 84 static unsigned char max_num_clients; 85 module_param(max_num_clients, byte, 0644); 86 MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients"); 87 88 static unsigned int copy_bytes = 1024; 89 module_param(copy_bytes, uint, 0644); 90 MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); 91 92 static bool use_dma; 93 module_param(use_dma, bool, 0644); 94 MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy"); 95 96 static struct dentry *nt_debugfs_dir; 97 98 /* Only two-ports NTB devices are supported */ 99 #define PIDX NTB_DEF_PEER_IDX 100 101 struct ntb_queue_entry { 102 /* ntb_queue list reference */ 103 struct list_head entry; 104 /* pointers to data to be transferred */ 105 void *cb_data; 106 void *buf; 107 unsigned int len; 108 unsigned int flags; 109 int retries; 110 int errors; 111 unsigned int tx_index; 112 unsigned int rx_index; 113 114 struct ntb_transport_qp *qp; 115 union { 116 struct ntb_payload_header __iomem *tx_hdr; 117 struct ntb_payload_header *rx_hdr; 118 }; 119 }; 120 121 struct ntb_rx_info { 122 unsigned int entry; 123 }; 124 125 struct ntb_transport_qp { 126 struct ntb_transport_ctx *transport; 127 struct ntb_dev *ndev; 128 void *cb_data; 129 struct dma_chan *tx_dma_chan; 130 struct dma_chan *rx_dma_chan; 131 132 bool client_ready; 133 bool link_is_up; 134 bool active; 135 136 u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ 137 u64 qp_bit; 138 139 struct ntb_rx_info __iomem *rx_info; 140 struct ntb_rx_info *remote_rx_info; 141 142 void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, 143 void *data, int len); 144 struct list_head tx_free_q; 145 spinlock_t ntb_tx_free_q_lock; 146 void __iomem *tx_mw; 147 dma_addr_t tx_mw_phys; 148 unsigned int tx_index; 149 unsigned int tx_max_entry; 150 unsigned int tx_max_frame; 151 152 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 153 void *data, int len); 154 struct list_head rx_post_q; 155 struct list_head rx_pend_q; 156 struct list_head rx_free_q; 157 /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */ 158 spinlock_t ntb_rx_q_lock; 159 void *rx_buff; 160 unsigned int rx_index; 161 unsigned int rx_max_entry; 162 unsigned int rx_max_frame; 163 unsigned int rx_alloc_entry; 164 dma_cookie_t last_cookie; 165 struct tasklet_struct rxc_db_work; 166 167 void (*event_handler)(void *data, int status); 168 struct delayed_work link_work; 169 struct work_struct link_cleanup; 170 171 struct dentry *debugfs_dir; 172 struct dentry *debugfs_stats; 173 174 /* Stats */ 175 u64 rx_bytes; 176 u64 rx_pkts; 177 u64 rx_ring_empty; 178 u64 rx_err_no_buf; 179 u64 rx_err_oflow; 180 u64 rx_err_ver; 181 u64 rx_memcpy; 182 u64 rx_async; 183 u64 tx_bytes; 184 u64 tx_pkts; 185 u64 tx_ring_full; 186 u64 tx_err_no_buf; 187 u64 tx_memcpy; 188 u64 tx_async; 189 }; 190 191 struct ntb_transport_mw { 192 phys_addr_t phys_addr; 193 resource_size_t phys_size; 194 void __iomem *vbase; 195 size_t xlat_size; 196 size_t buff_size; 197 size_t alloc_size; 198 void *alloc_addr; 199 void *virt_addr; 200 dma_addr_t dma_addr; 201 }; 202 203 struct ntb_transport_client_dev { 204 struct list_head entry; 205 struct ntb_transport_ctx *nt; 206 struct device dev; 207 }; 208 209 struct ntb_transport_ctx { 210 struct list_head entry; 211 struct list_head client_devs; 212 213 struct ntb_dev *ndev; 214 215 struct ntb_transport_mw *mw_vec; 216 struct ntb_transport_qp *qp_vec; 217 unsigned int mw_count; 218 unsigned int qp_count; 219 u64 qp_bitmap; 220 u64 qp_bitmap_free; 221 222 bool link_is_up; 223 struct delayed_work link_work; 224 struct work_struct link_cleanup; 225 226 struct dentry *debugfs_node_dir; 227 }; 228 229 enum { 230 DESC_DONE_FLAG = BIT(0), 231 LINK_DOWN_FLAG = BIT(1), 232 }; 233 234 struct ntb_payload_header { 235 unsigned int ver; 236 unsigned int len; 237 unsigned int flags; 238 }; 239 240 enum { 241 VERSION = 0, 242 QP_LINKS, 243 NUM_QPS, 244 NUM_MWS, 245 MW0_SZ_HIGH, 246 MW0_SZ_LOW, 247 }; 248 249 #define dev_client_dev(__dev) \ 250 container_of((__dev), struct ntb_transport_client_dev, dev) 251 252 #define drv_client(__drv) \ 253 container_of((__drv), struct ntb_transport_client, driver) 254 255 #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) 256 #define NTB_QP_DEF_NUM_ENTRIES 100 257 #define NTB_LINK_DOWN_TIMEOUT 10 258 259 static void ntb_transport_rxc_db(unsigned long data); 260 static const struct ntb_ctx_ops ntb_transport_ops; 261 static struct ntb_client ntb_transport_client; 262 static int ntb_async_tx_submit(struct ntb_transport_qp *qp, 263 struct ntb_queue_entry *entry); 264 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset); 265 static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset); 266 static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset); 267 268 269 static int ntb_transport_bus_match(struct device *dev, 270 struct device_driver *drv) 271 { 272 return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); 273 } 274 275 static int ntb_transport_bus_probe(struct device *dev) 276 { 277 const struct ntb_transport_client *client; 278 int rc = -EINVAL; 279 280 get_device(dev); 281 282 client = drv_client(dev->driver); 283 rc = client->probe(dev); 284 if (rc) 285 put_device(dev); 286 287 return rc; 288 } 289 290 static int ntb_transport_bus_remove(struct device *dev) 291 { 292 const struct ntb_transport_client *client; 293 294 client = drv_client(dev->driver); 295 client->remove(dev); 296 297 put_device(dev); 298 299 return 0; 300 } 301 302 static struct bus_type ntb_transport_bus = { 303 .name = "ntb_transport", 304 .match = ntb_transport_bus_match, 305 .probe = ntb_transport_bus_probe, 306 .remove = ntb_transport_bus_remove, 307 }; 308 309 static LIST_HEAD(ntb_transport_list); 310 311 static int ntb_bus_init(struct ntb_transport_ctx *nt) 312 { 313 list_add_tail(&nt->entry, &ntb_transport_list); 314 return 0; 315 } 316 317 static void ntb_bus_remove(struct ntb_transport_ctx *nt) 318 { 319 struct ntb_transport_client_dev *client_dev, *cd; 320 321 list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) { 322 dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n", 323 dev_name(&client_dev->dev)); 324 list_del(&client_dev->entry); 325 device_unregister(&client_dev->dev); 326 } 327 328 list_del(&nt->entry); 329 } 330 331 static void ntb_transport_client_release(struct device *dev) 332 { 333 struct ntb_transport_client_dev *client_dev; 334 335 client_dev = dev_client_dev(dev); 336 kfree(client_dev); 337 } 338 339 /** 340 * ntb_transport_unregister_client_dev - Unregister NTB client device 341 * @device_name: Name of NTB client device 342 * 343 * Unregister an NTB client device with the NTB transport layer 344 */ 345 void ntb_transport_unregister_client_dev(char *device_name) 346 { 347 struct ntb_transport_client_dev *client, *cd; 348 struct ntb_transport_ctx *nt; 349 350 list_for_each_entry(nt, &ntb_transport_list, entry) 351 list_for_each_entry_safe(client, cd, &nt->client_devs, entry) 352 if (!strncmp(dev_name(&client->dev), device_name, 353 strlen(device_name))) { 354 list_del(&client->entry); 355 device_unregister(&client->dev); 356 } 357 } 358 EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev); 359 360 /** 361 * ntb_transport_register_client_dev - Register NTB client device 362 * @device_name: Name of NTB client device 363 * 364 * Register an NTB client device with the NTB transport layer 365 */ 366 int ntb_transport_register_client_dev(char *device_name) 367 { 368 struct ntb_transport_client_dev *client_dev; 369 struct ntb_transport_ctx *nt; 370 int node; 371 int rc, i = 0; 372 373 if (list_empty(&ntb_transport_list)) 374 return -ENODEV; 375 376 list_for_each_entry(nt, &ntb_transport_list, entry) { 377 struct device *dev; 378 379 node = dev_to_node(&nt->ndev->dev); 380 381 client_dev = kzalloc_node(sizeof(*client_dev), 382 GFP_KERNEL, node); 383 if (!client_dev) { 384 rc = -ENOMEM; 385 goto err; 386 } 387 388 dev = &client_dev->dev; 389 390 /* setup and register client devices */ 391 dev_set_name(dev, "%s%d", device_name, i); 392 dev->bus = &ntb_transport_bus; 393 dev->release = ntb_transport_client_release; 394 dev->parent = &nt->ndev->dev; 395 396 rc = device_register(dev); 397 if (rc) { 398 kfree(client_dev); 399 goto err; 400 } 401 402 list_add_tail(&client_dev->entry, &nt->client_devs); 403 i++; 404 } 405 406 return 0; 407 408 err: 409 ntb_transport_unregister_client_dev(device_name); 410 411 return rc; 412 } 413 EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev); 414 415 /** 416 * ntb_transport_register_client - Register NTB client driver 417 * @drv: NTB client driver to be registered 418 * 419 * Register an NTB client driver with the NTB transport layer 420 * 421 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 422 */ 423 int ntb_transport_register_client(struct ntb_transport_client *drv) 424 { 425 drv->driver.bus = &ntb_transport_bus; 426 427 if (list_empty(&ntb_transport_list)) 428 return -ENODEV; 429 430 return driver_register(&drv->driver); 431 } 432 EXPORT_SYMBOL_GPL(ntb_transport_register_client); 433 434 /** 435 * ntb_transport_unregister_client - Unregister NTB client driver 436 * @drv: NTB client driver to be unregistered 437 * 438 * Unregister an NTB client driver with the NTB transport layer 439 * 440 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 441 */ 442 void ntb_transport_unregister_client(struct ntb_transport_client *drv) 443 { 444 driver_unregister(&drv->driver); 445 } 446 EXPORT_SYMBOL_GPL(ntb_transport_unregister_client); 447 448 static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, 449 loff_t *offp) 450 { 451 struct ntb_transport_qp *qp; 452 char *buf; 453 ssize_t ret, out_offset, out_count; 454 455 qp = filp->private_data; 456 457 if (!qp || !qp->link_is_up) 458 return 0; 459 460 out_count = 1000; 461 462 buf = kmalloc(out_count, GFP_KERNEL); 463 if (!buf) 464 return -ENOMEM; 465 466 out_offset = 0; 467 out_offset += snprintf(buf + out_offset, out_count - out_offset, 468 "\nNTB QP stats:\n\n"); 469 out_offset += snprintf(buf + out_offset, out_count - out_offset, 470 "rx_bytes - \t%llu\n", qp->rx_bytes); 471 out_offset += snprintf(buf + out_offset, out_count - out_offset, 472 "rx_pkts - \t%llu\n", qp->rx_pkts); 473 out_offset += snprintf(buf + out_offset, out_count - out_offset, 474 "rx_memcpy - \t%llu\n", qp->rx_memcpy); 475 out_offset += snprintf(buf + out_offset, out_count - out_offset, 476 "rx_async - \t%llu\n", qp->rx_async); 477 out_offset += snprintf(buf + out_offset, out_count - out_offset, 478 "rx_ring_empty - %llu\n", qp->rx_ring_empty); 479 out_offset += snprintf(buf + out_offset, out_count - out_offset, 480 "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); 481 out_offset += snprintf(buf + out_offset, out_count - out_offset, 482 "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); 483 out_offset += snprintf(buf + out_offset, out_count - out_offset, 484 "rx_err_ver - \t%llu\n", qp->rx_err_ver); 485 out_offset += snprintf(buf + out_offset, out_count - out_offset, 486 "rx_buff - \t0x%p\n", qp->rx_buff); 487 out_offset += snprintf(buf + out_offset, out_count - out_offset, 488 "rx_index - \t%u\n", qp->rx_index); 489 out_offset += snprintf(buf + out_offset, out_count - out_offset, 490 "rx_max_entry - \t%u\n", qp->rx_max_entry); 491 out_offset += snprintf(buf + out_offset, out_count - out_offset, 492 "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry); 493 494 out_offset += snprintf(buf + out_offset, out_count - out_offset, 495 "tx_bytes - \t%llu\n", qp->tx_bytes); 496 out_offset += snprintf(buf + out_offset, out_count - out_offset, 497 "tx_pkts - \t%llu\n", qp->tx_pkts); 498 out_offset += snprintf(buf + out_offset, out_count - out_offset, 499 "tx_memcpy - \t%llu\n", qp->tx_memcpy); 500 out_offset += snprintf(buf + out_offset, out_count - out_offset, 501 "tx_async - \t%llu\n", qp->tx_async); 502 out_offset += snprintf(buf + out_offset, out_count - out_offset, 503 "tx_ring_full - \t%llu\n", qp->tx_ring_full); 504 out_offset += snprintf(buf + out_offset, out_count - out_offset, 505 "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); 506 out_offset += snprintf(buf + out_offset, out_count - out_offset, 507 "tx_mw - \t0x%p\n", qp->tx_mw); 508 out_offset += snprintf(buf + out_offset, out_count - out_offset, 509 "tx_index (H) - \t%u\n", qp->tx_index); 510 out_offset += snprintf(buf + out_offset, out_count - out_offset, 511 "RRI (T) - \t%u\n", 512 qp->remote_rx_info->entry); 513 out_offset += snprintf(buf + out_offset, out_count - out_offset, 514 "tx_max_entry - \t%u\n", qp->tx_max_entry); 515 out_offset += snprintf(buf + out_offset, out_count - out_offset, 516 "free tx - \t%u\n", 517 ntb_transport_tx_free_entry(qp)); 518 519 out_offset += snprintf(buf + out_offset, out_count - out_offset, 520 "\n"); 521 out_offset += snprintf(buf + out_offset, out_count - out_offset, 522 "Using TX DMA - \t%s\n", 523 qp->tx_dma_chan ? "Yes" : "No"); 524 out_offset += snprintf(buf + out_offset, out_count - out_offset, 525 "Using RX DMA - \t%s\n", 526 qp->rx_dma_chan ? "Yes" : "No"); 527 out_offset += snprintf(buf + out_offset, out_count - out_offset, 528 "QP Link - \t%s\n", 529 qp->link_is_up ? "Up" : "Down"); 530 out_offset += snprintf(buf + out_offset, out_count - out_offset, 531 "\n"); 532 533 if (out_offset > out_count) 534 out_offset = out_count; 535 536 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); 537 kfree(buf); 538 return ret; 539 } 540 541 static const struct file_operations ntb_qp_debugfs_stats = { 542 .owner = THIS_MODULE, 543 .open = simple_open, 544 .read = debugfs_read, 545 }; 546 547 static void ntb_list_add(spinlock_t *lock, struct list_head *entry, 548 struct list_head *list) 549 { 550 unsigned long flags; 551 552 spin_lock_irqsave(lock, flags); 553 list_add_tail(entry, list); 554 spin_unlock_irqrestore(lock, flags); 555 } 556 557 static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, 558 struct list_head *list) 559 { 560 struct ntb_queue_entry *entry; 561 unsigned long flags; 562 563 spin_lock_irqsave(lock, flags); 564 if (list_empty(list)) { 565 entry = NULL; 566 goto out; 567 } 568 entry = list_first_entry(list, struct ntb_queue_entry, entry); 569 list_del(&entry->entry); 570 571 out: 572 spin_unlock_irqrestore(lock, flags); 573 574 return entry; 575 } 576 577 static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock, 578 struct list_head *list, 579 struct list_head *to_list) 580 { 581 struct ntb_queue_entry *entry; 582 unsigned long flags; 583 584 spin_lock_irqsave(lock, flags); 585 586 if (list_empty(list)) { 587 entry = NULL; 588 } else { 589 entry = list_first_entry(list, struct ntb_queue_entry, entry); 590 list_move_tail(&entry->entry, to_list); 591 } 592 593 spin_unlock_irqrestore(lock, flags); 594 595 return entry; 596 } 597 598 static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, 599 unsigned int qp_num) 600 { 601 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 602 struct ntb_transport_mw *mw; 603 struct ntb_dev *ndev = nt->ndev; 604 struct ntb_queue_entry *entry; 605 unsigned int rx_size, num_qps_mw; 606 unsigned int mw_num, mw_count, qp_count; 607 unsigned int i; 608 int node; 609 610 mw_count = nt->mw_count; 611 qp_count = nt->qp_count; 612 613 mw_num = QP_TO_MW(nt, qp_num); 614 mw = &nt->mw_vec[mw_num]; 615 616 if (!mw->virt_addr) 617 return -ENOMEM; 618 619 if (mw_num < qp_count % mw_count) 620 num_qps_mw = qp_count / mw_count + 1; 621 else 622 num_qps_mw = qp_count / mw_count; 623 624 rx_size = (unsigned int)mw->xlat_size / num_qps_mw; 625 qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); 626 rx_size -= sizeof(struct ntb_rx_info); 627 628 qp->remote_rx_info = qp->rx_buff + rx_size; 629 630 /* Due to housekeeping, there must be atleast 2 buffs */ 631 qp->rx_max_frame = min(transport_mtu, rx_size / 2); 632 qp->rx_max_entry = rx_size / qp->rx_max_frame; 633 qp->rx_index = 0; 634 635 /* 636 * Checking to see if we have more entries than the default. 637 * We should add additional entries if that is the case so we 638 * can be in sync with the transport frames. 639 */ 640 node = dev_to_node(&ndev->dev); 641 for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) { 642 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 643 if (!entry) 644 return -ENOMEM; 645 646 entry->qp = qp; 647 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, 648 &qp->rx_free_q); 649 qp->rx_alloc_entry++; 650 } 651 652 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 653 654 /* setup the hdr offsets with 0's */ 655 for (i = 0; i < qp->rx_max_entry; i++) { 656 void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) - 657 sizeof(struct ntb_payload_header)); 658 memset(offset, 0, sizeof(struct ntb_payload_header)); 659 } 660 661 qp->rx_pkts = 0; 662 qp->tx_pkts = 0; 663 qp->tx_index = 0; 664 665 return 0; 666 } 667 668 static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) 669 { 670 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 671 struct pci_dev *pdev = nt->ndev->pdev; 672 673 if (!mw->virt_addr) 674 return; 675 676 ntb_mw_clear_trans(nt->ndev, PIDX, num_mw); 677 dma_free_coherent(&pdev->dev, mw->alloc_size, 678 mw->alloc_addr, mw->dma_addr); 679 mw->xlat_size = 0; 680 mw->buff_size = 0; 681 mw->alloc_size = 0; 682 mw->alloc_addr = NULL; 683 mw->virt_addr = NULL; 684 } 685 686 static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw, 687 struct device *dma_dev, size_t align) 688 { 689 dma_addr_t dma_addr; 690 void *alloc_addr, *virt_addr; 691 int rc; 692 693 alloc_addr = dma_alloc_coherent(dma_dev, mw->alloc_size, 694 &dma_addr, GFP_KERNEL); 695 if (!alloc_addr) { 696 dev_err(dma_dev, "Unable to alloc MW buff of size %zu\n", 697 mw->alloc_size); 698 return -ENOMEM; 699 } 700 virt_addr = alloc_addr; 701 702 /* 703 * we must ensure that the memory address allocated is BAR size 704 * aligned in order for the XLAT register to take the value. This 705 * is a requirement of the hardware. It is recommended to setup CMA 706 * for BAR sizes equal or greater than 4MB. 707 */ 708 if (!IS_ALIGNED(dma_addr, align)) { 709 if (mw->alloc_size > mw->buff_size) { 710 virt_addr = PTR_ALIGN(alloc_addr, align); 711 dma_addr = ALIGN(dma_addr, align); 712 } else { 713 rc = -ENOMEM; 714 goto err; 715 } 716 } 717 718 mw->alloc_addr = alloc_addr; 719 mw->virt_addr = virt_addr; 720 mw->dma_addr = dma_addr; 721 722 return 0; 723 724 err: 725 dma_free_coherent(dma_dev, mw->alloc_size, alloc_addr, dma_addr); 726 727 return rc; 728 } 729 730 static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, 731 resource_size_t size) 732 { 733 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 734 struct pci_dev *pdev = nt->ndev->pdev; 735 size_t xlat_size, buff_size; 736 resource_size_t xlat_align; 737 resource_size_t xlat_align_size; 738 int rc; 739 740 if (!size) 741 return -EINVAL; 742 743 rc = ntb_mw_get_align(nt->ndev, PIDX, num_mw, &xlat_align, 744 &xlat_align_size, NULL); 745 if (rc) 746 return rc; 747 748 xlat_size = round_up(size, xlat_align_size); 749 buff_size = round_up(size, xlat_align); 750 751 /* No need to re-setup */ 752 if (mw->xlat_size == xlat_size) 753 return 0; 754 755 if (mw->buff_size) 756 ntb_free_mw(nt, num_mw); 757 758 /* Alloc memory for receiving data. Must be aligned */ 759 mw->xlat_size = xlat_size; 760 mw->buff_size = buff_size; 761 mw->alloc_size = buff_size; 762 763 rc = ntb_alloc_mw_buffer(mw, &pdev->dev, xlat_align); 764 if (rc) { 765 mw->alloc_size *= 2; 766 rc = ntb_alloc_mw_buffer(mw, &pdev->dev, xlat_align); 767 if (rc) { 768 dev_err(&pdev->dev, 769 "Unable to alloc aligned MW buff\n"); 770 mw->xlat_size = 0; 771 mw->buff_size = 0; 772 mw->alloc_size = 0; 773 return rc; 774 } 775 } 776 777 /* Notify HW the memory location of the receive buffer */ 778 rc = ntb_mw_set_trans(nt->ndev, PIDX, num_mw, mw->dma_addr, 779 mw->xlat_size); 780 if (rc) { 781 dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw); 782 ntb_free_mw(nt, num_mw); 783 return -EIO; 784 } 785 786 return 0; 787 } 788 789 static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp) 790 { 791 qp->link_is_up = false; 792 qp->active = false; 793 794 qp->tx_index = 0; 795 qp->rx_index = 0; 796 qp->rx_bytes = 0; 797 qp->rx_pkts = 0; 798 qp->rx_ring_empty = 0; 799 qp->rx_err_no_buf = 0; 800 qp->rx_err_oflow = 0; 801 qp->rx_err_ver = 0; 802 qp->rx_memcpy = 0; 803 qp->rx_async = 0; 804 qp->tx_bytes = 0; 805 qp->tx_pkts = 0; 806 qp->tx_ring_full = 0; 807 qp->tx_err_no_buf = 0; 808 qp->tx_memcpy = 0; 809 qp->tx_async = 0; 810 } 811 812 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 813 { 814 struct ntb_transport_ctx *nt = qp->transport; 815 struct pci_dev *pdev = nt->ndev->pdev; 816 817 dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num); 818 819 cancel_delayed_work_sync(&qp->link_work); 820 ntb_qp_link_down_reset(qp); 821 822 if (qp->event_handler) 823 qp->event_handler(qp->cb_data, qp->link_is_up); 824 } 825 826 static void ntb_qp_link_cleanup_work(struct work_struct *work) 827 { 828 struct ntb_transport_qp *qp = container_of(work, 829 struct ntb_transport_qp, 830 link_cleanup); 831 struct ntb_transport_ctx *nt = qp->transport; 832 833 ntb_qp_link_cleanup(qp); 834 835 if (nt->link_is_up) 836 schedule_delayed_work(&qp->link_work, 837 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 838 } 839 840 static void ntb_qp_link_down(struct ntb_transport_qp *qp) 841 { 842 schedule_work(&qp->link_cleanup); 843 } 844 845 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) 846 { 847 struct ntb_transport_qp *qp; 848 u64 qp_bitmap_alloc; 849 unsigned int i, count; 850 851 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 852 853 /* Pass along the info to any clients */ 854 for (i = 0; i < nt->qp_count; i++) 855 if (qp_bitmap_alloc & BIT_ULL(i)) { 856 qp = &nt->qp_vec[i]; 857 ntb_qp_link_cleanup(qp); 858 cancel_work_sync(&qp->link_cleanup); 859 cancel_delayed_work_sync(&qp->link_work); 860 } 861 862 if (!nt->link_is_up) 863 cancel_delayed_work_sync(&nt->link_work); 864 865 /* The scratchpad registers keep the values if the remote side 866 * goes down, blast them now to give them a sane value the next 867 * time they are accessed 868 */ 869 count = ntb_spad_count(nt->ndev); 870 for (i = 0; i < count; i++) 871 ntb_spad_write(nt->ndev, i, 0); 872 } 873 874 static void ntb_transport_link_cleanup_work(struct work_struct *work) 875 { 876 struct ntb_transport_ctx *nt = 877 container_of(work, struct ntb_transport_ctx, link_cleanup); 878 879 ntb_transport_link_cleanup(nt); 880 } 881 882 static void ntb_transport_event_callback(void *data) 883 { 884 struct ntb_transport_ctx *nt = data; 885 886 if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1) 887 schedule_delayed_work(&nt->link_work, 0); 888 else 889 schedule_work(&nt->link_cleanup); 890 } 891 892 static void ntb_transport_link_work(struct work_struct *work) 893 { 894 struct ntb_transport_ctx *nt = 895 container_of(work, struct ntb_transport_ctx, link_work.work); 896 struct ntb_dev *ndev = nt->ndev; 897 struct pci_dev *pdev = ndev->pdev; 898 resource_size_t size; 899 u32 val; 900 int rc = 0, i, spad; 901 902 /* send the local info, in the opposite order of the way we read it */ 903 for (i = 0; i < nt->mw_count; i++) { 904 size = nt->mw_vec[i].phys_size; 905 906 if (max_mw_size && size > max_mw_size) 907 size = max_mw_size; 908 909 spad = MW0_SZ_HIGH + (i * 2); 910 ntb_peer_spad_write(ndev, PIDX, spad, upper_32_bits(size)); 911 912 spad = MW0_SZ_LOW + (i * 2); 913 ntb_peer_spad_write(ndev, PIDX, spad, lower_32_bits(size)); 914 } 915 916 ntb_peer_spad_write(ndev, PIDX, NUM_MWS, nt->mw_count); 917 918 ntb_peer_spad_write(ndev, PIDX, NUM_QPS, nt->qp_count); 919 920 ntb_peer_spad_write(ndev, PIDX, VERSION, NTB_TRANSPORT_VERSION); 921 922 /* Query the remote side for its info */ 923 val = ntb_spad_read(ndev, VERSION); 924 dev_dbg(&pdev->dev, "Remote version = %d\n", val); 925 if (val != NTB_TRANSPORT_VERSION) 926 goto out; 927 928 val = ntb_spad_read(ndev, NUM_QPS); 929 dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); 930 if (val != nt->qp_count) 931 goto out; 932 933 val = ntb_spad_read(ndev, NUM_MWS); 934 dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); 935 if (val != nt->mw_count) 936 goto out; 937 938 for (i = 0; i < nt->mw_count; i++) { 939 u64 val64; 940 941 val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2)); 942 val64 = (u64)val << 32; 943 944 val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2)); 945 val64 |= val; 946 947 dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64); 948 949 rc = ntb_set_mw(nt, i, val64); 950 if (rc) 951 goto out1; 952 } 953 954 nt->link_is_up = true; 955 956 for (i = 0; i < nt->qp_count; i++) { 957 struct ntb_transport_qp *qp = &nt->qp_vec[i]; 958 959 ntb_transport_setup_qp_mw(nt, i); 960 961 if (qp->client_ready) 962 schedule_delayed_work(&qp->link_work, 0); 963 } 964 965 return; 966 967 out1: 968 for (i = 0; i < nt->mw_count; i++) 969 ntb_free_mw(nt, i); 970 971 /* if there's an actual failure, we should just bail */ 972 if (rc < 0) 973 return; 974 975 out: 976 if (ntb_link_is_up(ndev, NULL, NULL) == 1) 977 schedule_delayed_work(&nt->link_work, 978 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 979 } 980 981 static void ntb_qp_link_work(struct work_struct *work) 982 { 983 struct ntb_transport_qp *qp = container_of(work, 984 struct ntb_transport_qp, 985 link_work.work); 986 struct pci_dev *pdev = qp->ndev->pdev; 987 struct ntb_transport_ctx *nt = qp->transport; 988 int val; 989 990 WARN_ON(!nt->link_is_up); 991 992 val = ntb_spad_read(nt->ndev, QP_LINKS); 993 994 ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num)); 995 996 /* query remote spad for qp ready bits */ 997 dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val); 998 999 /* See if the remote side is up */ 1000 if (val & BIT(qp->qp_num)) { 1001 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); 1002 qp->link_is_up = true; 1003 qp->active = true; 1004 1005 if (qp->event_handler) 1006 qp->event_handler(qp->cb_data, qp->link_is_up); 1007 1008 if (qp->active) 1009 tasklet_schedule(&qp->rxc_db_work); 1010 } else if (nt->link_is_up) 1011 schedule_delayed_work(&qp->link_work, 1012 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 1013 } 1014 1015 static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, 1016 unsigned int qp_num) 1017 { 1018 struct ntb_transport_qp *qp; 1019 phys_addr_t mw_base; 1020 resource_size_t mw_size; 1021 unsigned int num_qps_mw, tx_size; 1022 unsigned int mw_num, mw_count, qp_count; 1023 u64 qp_offset; 1024 1025 mw_count = nt->mw_count; 1026 qp_count = nt->qp_count; 1027 1028 mw_num = QP_TO_MW(nt, qp_num); 1029 1030 qp = &nt->qp_vec[qp_num]; 1031 qp->qp_num = qp_num; 1032 qp->transport = nt; 1033 qp->ndev = nt->ndev; 1034 qp->client_ready = false; 1035 qp->event_handler = NULL; 1036 ntb_qp_link_down_reset(qp); 1037 1038 if (mw_num < qp_count % mw_count) 1039 num_qps_mw = qp_count / mw_count + 1; 1040 else 1041 num_qps_mw = qp_count / mw_count; 1042 1043 mw_base = nt->mw_vec[mw_num].phys_addr; 1044 mw_size = nt->mw_vec[mw_num].phys_size; 1045 1046 if (max_mw_size && mw_size > max_mw_size) 1047 mw_size = max_mw_size; 1048 1049 tx_size = (unsigned int)mw_size / num_qps_mw; 1050 qp_offset = tx_size * (qp_num / mw_count); 1051 1052 qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; 1053 if (!qp->tx_mw) 1054 return -EINVAL; 1055 1056 qp->tx_mw_phys = mw_base + qp_offset; 1057 if (!qp->tx_mw_phys) 1058 return -EINVAL; 1059 1060 tx_size -= sizeof(struct ntb_rx_info); 1061 qp->rx_info = qp->tx_mw + tx_size; 1062 1063 /* Due to housekeeping, there must be atleast 2 buffs */ 1064 qp->tx_max_frame = min(transport_mtu, tx_size / 2); 1065 qp->tx_max_entry = tx_size / qp->tx_max_frame; 1066 1067 if (nt->debugfs_node_dir) { 1068 char debugfs_name[4]; 1069 1070 snprintf(debugfs_name, 4, "qp%d", qp_num); 1071 qp->debugfs_dir = debugfs_create_dir(debugfs_name, 1072 nt->debugfs_node_dir); 1073 1074 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, 1075 qp->debugfs_dir, qp, 1076 &ntb_qp_debugfs_stats); 1077 } else { 1078 qp->debugfs_dir = NULL; 1079 qp->debugfs_stats = NULL; 1080 } 1081 1082 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); 1083 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); 1084 1085 spin_lock_init(&qp->ntb_rx_q_lock); 1086 spin_lock_init(&qp->ntb_tx_free_q_lock); 1087 1088 INIT_LIST_HEAD(&qp->rx_post_q); 1089 INIT_LIST_HEAD(&qp->rx_pend_q); 1090 INIT_LIST_HEAD(&qp->rx_free_q); 1091 INIT_LIST_HEAD(&qp->tx_free_q); 1092 1093 tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db, 1094 (unsigned long)qp); 1095 1096 return 0; 1097 } 1098 1099 static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) 1100 { 1101 struct ntb_transport_ctx *nt; 1102 struct ntb_transport_mw *mw; 1103 unsigned int mw_count, qp_count, spad_count, max_mw_count_for_spads; 1104 u64 qp_bitmap; 1105 int node; 1106 int rc, i; 1107 1108 mw_count = ntb_peer_mw_count(ndev); 1109 1110 if (!ndev->ops->mw_set_trans) { 1111 dev_err(&ndev->dev, "Inbound MW based NTB API is required\n"); 1112 return -EINVAL; 1113 } 1114 1115 if (ntb_db_is_unsafe(ndev)) 1116 dev_dbg(&ndev->dev, 1117 "doorbell is unsafe, proceed anyway...\n"); 1118 if (ntb_spad_is_unsafe(ndev)) 1119 dev_dbg(&ndev->dev, 1120 "scratchpad is unsafe, proceed anyway...\n"); 1121 1122 if (ntb_peer_port_count(ndev) != NTB_DEF_PEER_CNT) 1123 dev_warn(&ndev->dev, "Multi-port NTB devices unsupported\n"); 1124 1125 node = dev_to_node(&ndev->dev); 1126 1127 nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node); 1128 if (!nt) 1129 return -ENOMEM; 1130 1131 nt->ndev = ndev; 1132 spad_count = ntb_spad_count(ndev); 1133 1134 /* Limit the MW's based on the availability of scratchpads */ 1135 1136 if (spad_count < NTB_TRANSPORT_MIN_SPADS) { 1137 nt->mw_count = 0; 1138 rc = -EINVAL; 1139 goto err; 1140 } 1141 1142 max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2; 1143 nt->mw_count = min(mw_count, max_mw_count_for_spads); 1144 1145 nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec), 1146 GFP_KERNEL, node); 1147 if (!nt->mw_vec) { 1148 rc = -ENOMEM; 1149 goto err; 1150 } 1151 1152 for (i = 0; i < mw_count; i++) { 1153 mw = &nt->mw_vec[i]; 1154 1155 rc = ntb_peer_mw_get_addr(ndev, i, &mw->phys_addr, 1156 &mw->phys_size); 1157 if (rc) 1158 goto err1; 1159 1160 mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size); 1161 if (!mw->vbase) { 1162 rc = -ENOMEM; 1163 goto err1; 1164 } 1165 1166 mw->buff_size = 0; 1167 mw->xlat_size = 0; 1168 mw->virt_addr = NULL; 1169 mw->dma_addr = 0; 1170 } 1171 1172 qp_bitmap = ntb_db_valid_mask(ndev); 1173 1174 qp_count = ilog2(qp_bitmap); 1175 if (max_num_clients && max_num_clients < qp_count) 1176 qp_count = max_num_clients; 1177 else if (nt->mw_count < qp_count) 1178 qp_count = nt->mw_count; 1179 1180 qp_bitmap &= BIT_ULL(qp_count) - 1; 1181 1182 nt->qp_count = qp_count; 1183 nt->qp_bitmap = qp_bitmap; 1184 nt->qp_bitmap_free = qp_bitmap; 1185 1186 nt->qp_vec = kcalloc_node(qp_count, sizeof(*nt->qp_vec), 1187 GFP_KERNEL, node); 1188 if (!nt->qp_vec) { 1189 rc = -ENOMEM; 1190 goto err1; 1191 } 1192 1193 if (nt_debugfs_dir) { 1194 nt->debugfs_node_dir = 1195 debugfs_create_dir(pci_name(ndev->pdev), 1196 nt_debugfs_dir); 1197 } 1198 1199 for (i = 0; i < qp_count; i++) { 1200 rc = ntb_transport_init_queue(nt, i); 1201 if (rc) 1202 goto err2; 1203 } 1204 1205 INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); 1206 INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); 1207 1208 rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops); 1209 if (rc) 1210 goto err2; 1211 1212 INIT_LIST_HEAD(&nt->client_devs); 1213 rc = ntb_bus_init(nt); 1214 if (rc) 1215 goto err3; 1216 1217 nt->link_is_up = false; 1218 ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1219 ntb_link_event(ndev); 1220 1221 return 0; 1222 1223 err3: 1224 ntb_clear_ctx(ndev); 1225 err2: 1226 kfree(nt->qp_vec); 1227 err1: 1228 while (i--) { 1229 mw = &nt->mw_vec[i]; 1230 iounmap(mw->vbase); 1231 } 1232 kfree(nt->mw_vec); 1233 err: 1234 kfree(nt); 1235 return rc; 1236 } 1237 1238 static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev) 1239 { 1240 struct ntb_transport_ctx *nt = ndev->ctx; 1241 struct ntb_transport_qp *qp; 1242 u64 qp_bitmap_alloc; 1243 int i; 1244 1245 ntb_transport_link_cleanup(nt); 1246 cancel_work_sync(&nt->link_cleanup); 1247 cancel_delayed_work_sync(&nt->link_work); 1248 1249 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 1250 1251 /* verify that all the qp's are freed */ 1252 for (i = 0; i < nt->qp_count; i++) { 1253 qp = &nt->qp_vec[i]; 1254 if (qp_bitmap_alloc & BIT_ULL(i)) 1255 ntb_transport_free_queue(qp); 1256 debugfs_remove_recursive(qp->debugfs_dir); 1257 } 1258 1259 ntb_link_disable(ndev); 1260 ntb_clear_ctx(ndev); 1261 1262 ntb_bus_remove(nt); 1263 1264 for (i = nt->mw_count; i--; ) { 1265 ntb_free_mw(nt, i); 1266 iounmap(nt->mw_vec[i].vbase); 1267 } 1268 1269 kfree(nt->qp_vec); 1270 kfree(nt->mw_vec); 1271 kfree(nt); 1272 } 1273 1274 static void ntb_complete_rxc(struct ntb_transport_qp *qp) 1275 { 1276 struct ntb_queue_entry *entry; 1277 void *cb_data; 1278 unsigned int len; 1279 unsigned long irqflags; 1280 1281 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); 1282 1283 while (!list_empty(&qp->rx_post_q)) { 1284 entry = list_first_entry(&qp->rx_post_q, 1285 struct ntb_queue_entry, entry); 1286 if (!(entry->flags & DESC_DONE_FLAG)) 1287 break; 1288 1289 entry->rx_hdr->flags = 0; 1290 iowrite32(entry->rx_index, &qp->rx_info->entry); 1291 1292 cb_data = entry->cb_data; 1293 len = entry->len; 1294 1295 list_move_tail(&entry->entry, &qp->rx_free_q); 1296 1297 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); 1298 1299 if (qp->rx_handler && qp->client_ready) 1300 qp->rx_handler(qp, qp->cb_data, cb_data, len); 1301 1302 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); 1303 } 1304 1305 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); 1306 } 1307 1308 static void ntb_rx_copy_callback(void *data, 1309 const struct dmaengine_result *res) 1310 { 1311 struct ntb_queue_entry *entry = data; 1312 1313 /* we need to check DMA results if we are using DMA */ 1314 if (res) { 1315 enum dmaengine_tx_result dma_err = res->result; 1316 1317 switch (dma_err) { 1318 case DMA_TRANS_READ_FAILED: 1319 case DMA_TRANS_WRITE_FAILED: 1320 entry->errors++; 1321 /* fall through */ 1322 case DMA_TRANS_ABORTED: 1323 { 1324 struct ntb_transport_qp *qp = entry->qp; 1325 void *offset = qp->rx_buff + qp->rx_max_frame * 1326 qp->rx_index; 1327 1328 ntb_memcpy_rx(entry, offset); 1329 qp->rx_memcpy++; 1330 return; 1331 } 1332 1333 case DMA_TRANS_NOERROR: 1334 default: 1335 break; 1336 } 1337 } 1338 1339 entry->flags |= DESC_DONE_FLAG; 1340 1341 ntb_complete_rxc(entry->qp); 1342 } 1343 1344 static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) 1345 { 1346 void *buf = entry->buf; 1347 size_t len = entry->len; 1348 1349 memcpy(buf, offset, len); 1350 1351 /* Ensure that the data is fully copied out before clearing the flag */ 1352 wmb(); 1353 1354 ntb_rx_copy_callback(entry, NULL); 1355 } 1356 1357 static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset) 1358 { 1359 struct dma_async_tx_descriptor *txd; 1360 struct ntb_transport_qp *qp = entry->qp; 1361 struct dma_chan *chan = qp->rx_dma_chan; 1362 struct dma_device *device; 1363 size_t pay_off, buff_off, len; 1364 struct dmaengine_unmap_data *unmap; 1365 dma_cookie_t cookie; 1366 void *buf = entry->buf; 1367 1368 len = entry->len; 1369 device = chan->device; 1370 pay_off = (size_t)offset & ~PAGE_MASK; 1371 buff_off = (size_t)buf & ~PAGE_MASK; 1372 1373 if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) 1374 goto err; 1375 1376 unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); 1377 if (!unmap) 1378 goto err; 1379 1380 unmap->len = len; 1381 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset), 1382 pay_off, len, DMA_TO_DEVICE); 1383 if (dma_mapping_error(device->dev, unmap->addr[0])) 1384 goto err_get_unmap; 1385 1386 unmap->to_cnt = 1; 1387 1388 unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf), 1389 buff_off, len, DMA_FROM_DEVICE); 1390 if (dma_mapping_error(device->dev, unmap->addr[1])) 1391 goto err_get_unmap; 1392 1393 unmap->from_cnt = 1; 1394 1395 txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], 1396 unmap->addr[0], len, 1397 DMA_PREP_INTERRUPT); 1398 if (!txd) 1399 goto err_get_unmap; 1400 1401 txd->callback_result = ntb_rx_copy_callback; 1402 txd->callback_param = entry; 1403 dma_set_unmap(txd, unmap); 1404 1405 cookie = dmaengine_submit(txd); 1406 if (dma_submit_error(cookie)) 1407 goto err_set_unmap; 1408 1409 dmaengine_unmap_put(unmap); 1410 1411 qp->last_cookie = cookie; 1412 1413 qp->rx_async++; 1414 1415 return 0; 1416 1417 err_set_unmap: 1418 dmaengine_unmap_put(unmap); 1419 err_get_unmap: 1420 dmaengine_unmap_put(unmap); 1421 err: 1422 return -ENXIO; 1423 } 1424 1425 static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) 1426 { 1427 struct ntb_transport_qp *qp = entry->qp; 1428 struct dma_chan *chan = qp->rx_dma_chan; 1429 int res; 1430 1431 if (!chan) 1432 goto err; 1433 1434 if (entry->len < copy_bytes) 1435 goto err; 1436 1437 res = ntb_async_rx_submit(entry, offset); 1438 if (res < 0) 1439 goto err; 1440 1441 if (!entry->retries) 1442 qp->rx_async++; 1443 1444 return; 1445 1446 err: 1447 ntb_memcpy_rx(entry, offset); 1448 qp->rx_memcpy++; 1449 } 1450 1451 static int ntb_process_rxc(struct ntb_transport_qp *qp) 1452 { 1453 struct ntb_payload_header *hdr; 1454 struct ntb_queue_entry *entry; 1455 void *offset; 1456 1457 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; 1458 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); 1459 1460 dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n", 1461 qp->qp_num, hdr->ver, hdr->len, hdr->flags); 1462 1463 if (!(hdr->flags & DESC_DONE_FLAG)) { 1464 dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n"); 1465 qp->rx_ring_empty++; 1466 return -EAGAIN; 1467 } 1468 1469 if (hdr->flags & LINK_DOWN_FLAG) { 1470 dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n"); 1471 ntb_qp_link_down(qp); 1472 hdr->flags = 0; 1473 return -EAGAIN; 1474 } 1475 1476 if (hdr->ver != (u32)qp->rx_pkts) { 1477 dev_dbg(&qp->ndev->pdev->dev, 1478 "version mismatch, expected %llu - got %u\n", 1479 qp->rx_pkts, hdr->ver); 1480 qp->rx_err_ver++; 1481 return -EIO; 1482 } 1483 1484 entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q); 1485 if (!entry) { 1486 dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n"); 1487 qp->rx_err_no_buf++; 1488 return -EAGAIN; 1489 } 1490 1491 entry->rx_hdr = hdr; 1492 entry->rx_index = qp->rx_index; 1493 1494 if (hdr->len > entry->len) { 1495 dev_dbg(&qp->ndev->pdev->dev, 1496 "receive buffer overflow! Wanted %d got %d\n", 1497 hdr->len, entry->len); 1498 qp->rx_err_oflow++; 1499 1500 entry->len = -EIO; 1501 entry->flags |= DESC_DONE_FLAG; 1502 1503 ntb_complete_rxc(qp); 1504 } else { 1505 dev_dbg(&qp->ndev->pdev->dev, 1506 "RX OK index %u ver %u size %d into buf size %d\n", 1507 qp->rx_index, hdr->ver, hdr->len, entry->len); 1508 1509 qp->rx_bytes += hdr->len; 1510 qp->rx_pkts++; 1511 1512 entry->len = hdr->len; 1513 1514 ntb_async_rx(entry, offset); 1515 } 1516 1517 qp->rx_index++; 1518 qp->rx_index %= qp->rx_max_entry; 1519 1520 return 0; 1521 } 1522 1523 static void ntb_transport_rxc_db(unsigned long data) 1524 { 1525 struct ntb_transport_qp *qp = (void *)data; 1526 int rc, i; 1527 1528 dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n", 1529 __func__, qp->qp_num); 1530 1531 /* Limit the number of packets processed in a single interrupt to 1532 * provide fairness to others 1533 */ 1534 for (i = 0; i < qp->rx_max_entry; i++) { 1535 rc = ntb_process_rxc(qp); 1536 if (rc) 1537 break; 1538 } 1539 1540 if (i && qp->rx_dma_chan) 1541 dma_async_issue_pending(qp->rx_dma_chan); 1542 1543 if (i == qp->rx_max_entry) { 1544 /* there is more work to do */ 1545 if (qp->active) 1546 tasklet_schedule(&qp->rxc_db_work); 1547 } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) { 1548 /* the doorbell bit is set: clear it */ 1549 ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num)); 1550 /* ntb_db_read ensures ntb_db_clear write is committed */ 1551 ntb_db_read(qp->ndev); 1552 1553 /* an interrupt may have arrived between finishing 1554 * ntb_process_rxc and clearing the doorbell bit: 1555 * there might be some more work to do. 1556 */ 1557 if (qp->active) 1558 tasklet_schedule(&qp->rxc_db_work); 1559 } 1560 } 1561 1562 static void ntb_tx_copy_callback(void *data, 1563 const struct dmaengine_result *res) 1564 { 1565 struct ntb_queue_entry *entry = data; 1566 struct ntb_transport_qp *qp = entry->qp; 1567 struct ntb_payload_header __iomem *hdr = entry->tx_hdr; 1568 1569 /* we need to check DMA results if we are using DMA */ 1570 if (res) { 1571 enum dmaengine_tx_result dma_err = res->result; 1572 1573 switch (dma_err) { 1574 case DMA_TRANS_READ_FAILED: 1575 case DMA_TRANS_WRITE_FAILED: 1576 entry->errors++; 1577 /* fall through */ 1578 case DMA_TRANS_ABORTED: 1579 { 1580 void __iomem *offset = 1581 qp->tx_mw + qp->tx_max_frame * 1582 entry->tx_index; 1583 1584 /* resubmit via CPU */ 1585 ntb_memcpy_tx(entry, offset); 1586 qp->tx_memcpy++; 1587 return; 1588 } 1589 1590 case DMA_TRANS_NOERROR: 1591 default: 1592 break; 1593 } 1594 } 1595 1596 iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); 1597 1598 ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); 1599 1600 /* The entry length can only be zero if the packet is intended to be a 1601 * "link down" or similar. Since no payload is being sent in these 1602 * cases, there is nothing to add to the completion queue. 1603 */ 1604 if (entry->len > 0) { 1605 qp->tx_bytes += entry->len; 1606 1607 if (qp->tx_handler) 1608 qp->tx_handler(qp, qp->cb_data, entry->cb_data, 1609 entry->len); 1610 } 1611 1612 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); 1613 } 1614 1615 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) 1616 { 1617 #ifdef ARCH_HAS_NOCACHE_UACCESS 1618 /* 1619 * Using non-temporal mov to improve performance on non-cached 1620 * writes, even though we aren't actually copying from user space. 1621 */ 1622 __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); 1623 #else 1624 memcpy_toio(offset, entry->buf, entry->len); 1625 #endif 1626 1627 /* Ensure that the data is fully copied out before setting the flags */ 1628 wmb(); 1629 1630 ntb_tx_copy_callback(entry, NULL); 1631 } 1632 1633 static int ntb_async_tx_submit(struct ntb_transport_qp *qp, 1634 struct ntb_queue_entry *entry) 1635 { 1636 struct dma_async_tx_descriptor *txd; 1637 struct dma_chan *chan = qp->tx_dma_chan; 1638 struct dma_device *device; 1639 size_t len = entry->len; 1640 void *buf = entry->buf; 1641 size_t dest_off, buff_off; 1642 struct dmaengine_unmap_data *unmap; 1643 dma_addr_t dest; 1644 dma_cookie_t cookie; 1645 1646 device = chan->device; 1647 dest = qp->tx_mw_phys + qp->tx_max_frame * entry->tx_index; 1648 buff_off = (size_t)buf & ~PAGE_MASK; 1649 dest_off = (size_t)dest & ~PAGE_MASK; 1650 1651 if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) 1652 goto err; 1653 1654 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); 1655 if (!unmap) 1656 goto err; 1657 1658 unmap->len = len; 1659 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf), 1660 buff_off, len, DMA_TO_DEVICE); 1661 if (dma_mapping_error(device->dev, unmap->addr[0])) 1662 goto err_get_unmap; 1663 1664 unmap->to_cnt = 1; 1665 1666 txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len, 1667 DMA_PREP_INTERRUPT); 1668 if (!txd) 1669 goto err_get_unmap; 1670 1671 txd->callback_result = ntb_tx_copy_callback; 1672 txd->callback_param = entry; 1673 dma_set_unmap(txd, unmap); 1674 1675 cookie = dmaengine_submit(txd); 1676 if (dma_submit_error(cookie)) 1677 goto err_set_unmap; 1678 1679 dmaengine_unmap_put(unmap); 1680 1681 dma_async_issue_pending(chan); 1682 1683 return 0; 1684 err_set_unmap: 1685 dmaengine_unmap_put(unmap); 1686 err_get_unmap: 1687 dmaengine_unmap_put(unmap); 1688 err: 1689 return -ENXIO; 1690 } 1691 1692 static void ntb_async_tx(struct ntb_transport_qp *qp, 1693 struct ntb_queue_entry *entry) 1694 { 1695 struct ntb_payload_header __iomem *hdr; 1696 struct dma_chan *chan = qp->tx_dma_chan; 1697 void __iomem *offset; 1698 int res; 1699 1700 entry->tx_index = qp->tx_index; 1701 offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index; 1702 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); 1703 entry->tx_hdr = hdr; 1704 1705 iowrite32(entry->len, &hdr->len); 1706 iowrite32((u32)qp->tx_pkts, &hdr->ver); 1707 1708 if (!chan) 1709 goto err; 1710 1711 if (entry->len < copy_bytes) 1712 goto err; 1713 1714 res = ntb_async_tx_submit(qp, entry); 1715 if (res < 0) 1716 goto err; 1717 1718 if (!entry->retries) 1719 qp->tx_async++; 1720 1721 return; 1722 1723 err: 1724 ntb_memcpy_tx(entry, offset); 1725 qp->tx_memcpy++; 1726 } 1727 1728 static int ntb_process_tx(struct ntb_transport_qp *qp, 1729 struct ntb_queue_entry *entry) 1730 { 1731 if (qp->tx_index == qp->remote_rx_info->entry) { 1732 qp->tx_ring_full++; 1733 return -EAGAIN; 1734 } 1735 1736 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 1737 if (qp->tx_handler) 1738 qp->tx_handler(qp, qp->cb_data, NULL, -EIO); 1739 1740 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1741 &qp->tx_free_q); 1742 return 0; 1743 } 1744 1745 ntb_async_tx(qp, entry); 1746 1747 qp->tx_index++; 1748 qp->tx_index %= qp->tx_max_entry; 1749 1750 qp->tx_pkts++; 1751 1752 return 0; 1753 } 1754 1755 static void ntb_send_link_down(struct ntb_transport_qp *qp) 1756 { 1757 struct pci_dev *pdev = qp->ndev->pdev; 1758 struct ntb_queue_entry *entry; 1759 int i, rc; 1760 1761 if (!qp->link_is_up) 1762 return; 1763 1764 dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num); 1765 1766 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1767 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1768 if (entry) 1769 break; 1770 msleep(100); 1771 } 1772 1773 if (!entry) 1774 return; 1775 1776 entry->cb_data = NULL; 1777 entry->buf = NULL; 1778 entry->len = 0; 1779 entry->flags = LINK_DOWN_FLAG; 1780 1781 rc = ntb_process_tx(qp, entry); 1782 if (rc) 1783 dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", 1784 qp->qp_num); 1785 1786 ntb_qp_link_down_reset(qp); 1787 } 1788 1789 static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node) 1790 { 1791 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; 1792 } 1793 1794 /** 1795 * ntb_transport_create_queue - Create a new NTB transport layer queue 1796 * @rx_handler: receive callback function 1797 * @tx_handler: transmit callback function 1798 * @event_handler: event callback function 1799 * 1800 * Create a new NTB transport layer queue and provide the queue with a callback 1801 * routine for both transmit and receive. The receive callback routine will be 1802 * used to pass up data when the transport has received it on the queue. The 1803 * transmit callback routine will be called when the transport has completed the 1804 * transmission of the data on the queue and the data is ready to be freed. 1805 * 1806 * RETURNS: pointer to newly created ntb_queue, NULL on error. 1807 */ 1808 struct ntb_transport_qp * 1809 ntb_transport_create_queue(void *data, struct device *client_dev, 1810 const struct ntb_queue_handlers *handlers) 1811 { 1812 struct ntb_dev *ndev; 1813 struct pci_dev *pdev; 1814 struct ntb_transport_ctx *nt; 1815 struct ntb_queue_entry *entry; 1816 struct ntb_transport_qp *qp; 1817 u64 qp_bit; 1818 unsigned int free_queue; 1819 dma_cap_mask_t dma_mask; 1820 int node; 1821 int i; 1822 1823 ndev = dev_ntb(client_dev->parent); 1824 pdev = ndev->pdev; 1825 nt = ndev->ctx; 1826 1827 node = dev_to_node(&ndev->dev); 1828 1829 free_queue = ffs(nt->qp_bitmap_free); 1830 if (!free_queue) 1831 goto err; 1832 1833 /* decrement free_queue to make it zero based */ 1834 free_queue--; 1835 1836 qp = &nt->qp_vec[free_queue]; 1837 qp_bit = BIT_ULL(qp->qp_num); 1838 1839 nt->qp_bitmap_free &= ~qp_bit; 1840 1841 qp->cb_data = data; 1842 qp->rx_handler = handlers->rx_handler; 1843 qp->tx_handler = handlers->tx_handler; 1844 qp->event_handler = handlers->event_handler; 1845 1846 dma_cap_zero(dma_mask); 1847 dma_cap_set(DMA_MEMCPY, dma_mask); 1848 1849 if (use_dma) { 1850 qp->tx_dma_chan = 1851 dma_request_channel(dma_mask, ntb_dma_filter_fn, 1852 (void *)(unsigned long)node); 1853 if (!qp->tx_dma_chan) 1854 dev_info(&pdev->dev, "Unable to allocate TX DMA channel\n"); 1855 1856 qp->rx_dma_chan = 1857 dma_request_channel(dma_mask, ntb_dma_filter_fn, 1858 (void *)(unsigned long)node); 1859 if (!qp->rx_dma_chan) 1860 dev_info(&pdev->dev, "Unable to allocate RX DMA channel\n"); 1861 } else { 1862 qp->tx_dma_chan = NULL; 1863 qp->rx_dma_chan = NULL; 1864 } 1865 1866 dev_dbg(&pdev->dev, "Using %s memcpy for TX\n", 1867 qp->tx_dma_chan ? "DMA" : "CPU"); 1868 1869 dev_dbg(&pdev->dev, "Using %s memcpy for RX\n", 1870 qp->rx_dma_chan ? "DMA" : "CPU"); 1871 1872 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 1873 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 1874 if (!entry) 1875 goto err1; 1876 1877 entry->qp = qp; 1878 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, 1879 &qp->rx_free_q); 1880 } 1881 qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES; 1882 1883 for (i = 0; i < qp->tx_max_entry; i++) { 1884 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 1885 if (!entry) 1886 goto err2; 1887 1888 entry->qp = qp; 1889 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1890 &qp->tx_free_q); 1891 } 1892 1893 ntb_db_clear(qp->ndev, qp_bit); 1894 ntb_db_clear_mask(qp->ndev, qp_bit); 1895 1896 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); 1897 1898 return qp; 1899 1900 err2: 1901 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 1902 kfree(entry); 1903 err1: 1904 qp->rx_alloc_entry = 0; 1905 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 1906 kfree(entry); 1907 if (qp->tx_dma_chan) 1908 dma_release_channel(qp->tx_dma_chan); 1909 if (qp->rx_dma_chan) 1910 dma_release_channel(qp->rx_dma_chan); 1911 nt->qp_bitmap_free |= qp_bit; 1912 err: 1913 return NULL; 1914 } 1915 EXPORT_SYMBOL_GPL(ntb_transport_create_queue); 1916 1917 /** 1918 * ntb_transport_free_queue - Frees NTB transport queue 1919 * @qp: NTB queue to be freed 1920 * 1921 * Frees NTB transport queue 1922 */ 1923 void ntb_transport_free_queue(struct ntb_transport_qp *qp) 1924 { 1925 struct pci_dev *pdev; 1926 struct ntb_queue_entry *entry; 1927 u64 qp_bit; 1928 1929 if (!qp) 1930 return; 1931 1932 pdev = qp->ndev->pdev; 1933 1934 qp->active = false; 1935 1936 if (qp->tx_dma_chan) { 1937 struct dma_chan *chan = qp->tx_dma_chan; 1938 /* Putting the dma_chan to NULL will force any new traffic to be 1939 * processed by the CPU instead of the DAM engine 1940 */ 1941 qp->tx_dma_chan = NULL; 1942 1943 /* Try to be nice and wait for any queued DMA engine 1944 * transactions to process before smashing it with a rock 1945 */ 1946 dma_sync_wait(chan, qp->last_cookie); 1947 dmaengine_terminate_all(chan); 1948 dma_release_channel(chan); 1949 } 1950 1951 if (qp->rx_dma_chan) { 1952 struct dma_chan *chan = qp->rx_dma_chan; 1953 /* Putting the dma_chan to NULL will force any new traffic to be 1954 * processed by the CPU instead of the DAM engine 1955 */ 1956 qp->rx_dma_chan = NULL; 1957 1958 /* Try to be nice and wait for any queued DMA engine 1959 * transactions to process before smashing it with a rock 1960 */ 1961 dma_sync_wait(chan, qp->last_cookie); 1962 dmaengine_terminate_all(chan); 1963 dma_release_channel(chan); 1964 } 1965 1966 qp_bit = BIT_ULL(qp->qp_num); 1967 1968 ntb_db_set_mask(qp->ndev, qp_bit); 1969 tasklet_kill(&qp->rxc_db_work); 1970 1971 cancel_delayed_work_sync(&qp->link_work); 1972 1973 qp->cb_data = NULL; 1974 qp->rx_handler = NULL; 1975 qp->tx_handler = NULL; 1976 qp->event_handler = NULL; 1977 1978 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 1979 kfree(entry); 1980 1981 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) { 1982 dev_warn(&pdev->dev, "Freeing item from non-empty rx_pend_q\n"); 1983 kfree(entry); 1984 } 1985 1986 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) { 1987 dev_warn(&pdev->dev, "Freeing item from non-empty rx_post_q\n"); 1988 kfree(entry); 1989 } 1990 1991 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 1992 kfree(entry); 1993 1994 qp->transport->qp_bitmap_free |= qp_bit; 1995 1996 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); 1997 } 1998 EXPORT_SYMBOL_GPL(ntb_transport_free_queue); 1999 2000 /** 2001 * ntb_transport_rx_remove - Dequeues enqueued rx packet 2002 * @qp: NTB queue to be freed 2003 * @len: pointer to variable to write enqueued buffers length 2004 * 2005 * Dequeues unused buffers from receive queue. Should only be used during 2006 * shutdown of qp. 2007 * 2008 * RETURNS: NULL error value on error, or void* for success. 2009 */ 2010 void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) 2011 { 2012 struct ntb_queue_entry *entry; 2013 void *buf; 2014 2015 if (!qp || qp->client_ready) 2016 return NULL; 2017 2018 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q); 2019 if (!entry) 2020 return NULL; 2021 2022 buf = entry->cb_data; 2023 *len = entry->len; 2024 2025 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q); 2026 2027 return buf; 2028 } 2029 EXPORT_SYMBOL_GPL(ntb_transport_rx_remove); 2030 2031 /** 2032 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry 2033 * @qp: NTB transport layer queue the entry is to be enqueued on 2034 * @cb: per buffer pointer for callback function to use 2035 * @data: pointer to data buffer that incoming packets will be copied into 2036 * @len: length of the data buffer 2037 * 2038 * Enqueue a new receive buffer onto the transport queue into which a NTB 2039 * payload can be received into. 2040 * 2041 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 2042 */ 2043 int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 2044 unsigned int len) 2045 { 2046 struct ntb_queue_entry *entry; 2047 2048 if (!qp) 2049 return -EINVAL; 2050 2051 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q); 2052 if (!entry) 2053 return -ENOMEM; 2054 2055 entry->cb_data = cb; 2056 entry->buf = data; 2057 entry->len = len; 2058 entry->flags = 0; 2059 entry->retries = 0; 2060 entry->errors = 0; 2061 entry->rx_index = 0; 2062 2063 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q); 2064 2065 if (qp->active) 2066 tasklet_schedule(&qp->rxc_db_work); 2067 2068 return 0; 2069 } 2070 EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue); 2071 2072 /** 2073 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 2074 * @qp: NTB transport layer queue the entry is to be enqueued on 2075 * @cb: per buffer pointer for callback function to use 2076 * @data: pointer to data buffer that will be sent 2077 * @len: length of the data buffer 2078 * 2079 * Enqueue a new transmit buffer onto the transport queue from which a NTB 2080 * payload will be transmitted. This assumes that a lock is being held to 2081 * serialize access to the qp. 2082 * 2083 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 2084 */ 2085 int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 2086 unsigned int len) 2087 { 2088 struct ntb_queue_entry *entry; 2089 int rc; 2090 2091 if (!qp || !qp->link_is_up || !len) 2092 return -EINVAL; 2093 2094 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 2095 if (!entry) { 2096 qp->tx_err_no_buf++; 2097 return -EBUSY; 2098 } 2099 2100 entry->cb_data = cb; 2101 entry->buf = data; 2102 entry->len = len; 2103 entry->flags = 0; 2104 entry->errors = 0; 2105 entry->retries = 0; 2106 entry->tx_index = 0; 2107 2108 rc = ntb_process_tx(qp, entry); 2109 if (rc) 2110 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 2111 &qp->tx_free_q); 2112 2113 return rc; 2114 } 2115 EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue); 2116 2117 /** 2118 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 2119 * @qp: NTB transport layer queue to be enabled 2120 * 2121 * Notify NTB transport layer of client readiness to use queue 2122 */ 2123 void ntb_transport_link_up(struct ntb_transport_qp *qp) 2124 { 2125 if (!qp) 2126 return; 2127 2128 qp->client_ready = true; 2129 2130 if (qp->transport->link_is_up) 2131 schedule_delayed_work(&qp->link_work, 0); 2132 } 2133 EXPORT_SYMBOL_GPL(ntb_transport_link_up); 2134 2135 /** 2136 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 2137 * @qp: NTB transport layer queue to be disabled 2138 * 2139 * Notify NTB transport layer of client's desire to no longer receive data on 2140 * transport queue specified. It is the client's responsibility to ensure all 2141 * entries on queue are purged or otherwise handled appropriately. 2142 */ 2143 void ntb_transport_link_down(struct ntb_transport_qp *qp) 2144 { 2145 int val; 2146 2147 if (!qp) 2148 return; 2149 2150 qp->client_ready = false; 2151 2152 val = ntb_spad_read(qp->ndev, QP_LINKS); 2153 2154 ntb_peer_spad_write(qp->ndev, PIDX, QP_LINKS, val & ~BIT(qp->qp_num)); 2155 2156 if (qp->link_is_up) 2157 ntb_send_link_down(qp); 2158 else 2159 cancel_delayed_work_sync(&qp->link_work); 2160 } 2161 EXPORT_SYMBOL_GPL(ntb_transport_link_down); 2162 2163 /** 2164 * ntb_transport_link_query - Query transport link state 2165 * @qp: NTB transport layer queue to be queried 2166 * 2167 * Query connectivity to the remote system of the NTB transport queue 2168 * 2169 * RETURNS: true for link up or false for link down 2170 */ 2171 bool ntb_transport_link_query(struct ntb_transport_qp *qp) 2172 { 2173 if (!qp) 2174 return false; 2175 2176 return qp->link_is_up; 2177 } 2178 EXPORT_SYMBOL_GPL(ntb_transport_link_query); 2179 2180 /** 2181 * ntb_transport_qp_num - Query the qp number 2182 * @qp: NTB transport layer queue to be queried 2183 * 2184 * Query qp number of the NTB transport queue 2185 * 2186 * RETURNS: a zero based number specifying the qp number 2187 */ 2188 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) 2189 { 2190 if (!qp) 2191 return 0; 2192 2193 return qp->qp_num; 2194 } 2195 EXPORT_SYMBOL_GPL(ntb_transport_qp_num); 2196 2197 /** 2198 * ntb_transport_max_size - Query the max payload size of a qp 2199 * @qp: NTB transport layer queue to be queried 2200 * 2201 * Query the maximum payload size permissible on the given qp 2202 * 2203 * RETURNS: the max payload size of a qp 2204 */ 2205 unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) 2206 { 2207 unsigned int max_size; 2208 unsigned int copy_align; 2209 struct dma_chan *rx_chan, *tx_chan; 2210 2211 if (!qp) 2212 return 0; 2213 2214 rx_chan = qp->rx_dma_chan; 2215 tx_chan = qp->tx_dma_chan; 2216 2217 copy_align = max(rx_chan ? rx_chan->device->copy_align : 0, 2218 tx_chan ? tx_chan->device->copy_align : 0); 2219 2220 /* If DMA engine usage is possible, try to find the max size for that */ 2221 max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header); 2222 max_size = round_down(max_size, 1 << copy_align); 2223 2224 return max_size; 2225 } 2226 EXPORT_SYMBOL_GPL(ntb_transport_max_size); 2227 2228 unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp) 2229 { 2230 unsigned int head = qp->tx_index; 2231 unsigned int tail = qp->remote_rx_info->entry; 2232 2233 return tail > head ? tail - head : qp->tx_max_entry + tail - head; 2234 } 2235 EXPORT_SYMBOL_GPL(ntb_transport_tx_free_entry); 2236 2237 static void ntb_transport_doorbell_callback(void *data, int vector) 2238 { 2239 struct ntb_transport_ctx *nt = data; 2240 struct ntb_transport_qp *qp; 2241 u64 db_bits; 2242 unsigned int qp_num; 2243 2244 db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free & 2245 ntb_db_vector_mask(nt->ndev, vector)); 2246 2247 while (db_bits) { 2248 qp_num = __ffs(db_bits); 2249 qp = &nt->qp_vec[qp_num]; 2250 2251 if (qp->active) 2252 tasklet_schedule(&qp->rxc_db_work); 2253 2254 db_bits &= ~BIT_ULL(qp_num); 2255 } 2256 } 2257 2258 static const struct ntb_ctx_ops ntb_transport_ops = { 2259 .link_event = ntb_transport_event_callback, 2260 .db_event = ntb_transport_doorbell_callback, 2261 }; 2262 2263 static struct ntb_client ntb_transport_client = { 2264 .ops = { 2265 .probe = ntb_transport_probe, 2266 .remove = ntb_transport_free, 2267 }, 2268 }; 2269 2270 static int __init ntb_transport_init(void) 2271 { 2272 int rc; 2273 2274 pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER); 2275 2276 if (debugfs_initialized()) 2277 nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 2278 2279 rc = bus_register(&ntb_transport_bus); 2280 if (rc) 2281 goto err_bus; 2282 2283 rc = ntb_register_client(&ntb_transport_client); 2284 if (rc) 2285 goto err_client; 2286 2287 return 0; 2288 2289 err_client: 2290 bus_unregister(&ntb_transport_bus); 2291 err_bus: 2292 debugfs_remove_recursive(nt_debugfs_dir); 2293 return rc; 2294 } 2295 module_init(ntb_transport_init); 2296 2297 static void __exit ntb_transport_exit(void) 2298 { 2299 ntb_unregister_client(&ntb_transport_client); 2300 bus_unregister(&ntb_transport_bus); 2301 debugfs_remove_recursive(nt_debugfs_dir); 2302 } 2303 module_exit(ntb_transport_exit); 2304