1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * BSD LICENSE 15 * 16 * Copyright(c) 2012 Intel Corporation. All rights reserved. 17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 23 * * Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * * Redistributions in binary form must reproduce the above copy 26 * notice, this list of conditions and the following disclaimer in 27 * the documentation and/or other materials provided with the 28 * distribution. 29 * * Neither the name of Intel Corporation nor the names of its 30 * contributors may be used to endorse or promote products derived 31 * from this software without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 44 * 45 * PCIe NTB Transport Linux driver 46 * 47 * Contact Information: 48 * Jon Mason <jon.mason@intel.com> 49 */ 50 #include <linux/debugfs.h> 51 #include <linux/delay.h> 52 #include <linux/dmaengine.h> 53 #include <linux/dma-mapping.h> 54 #include <linux/errno.h> 55 #include <linux/export.h> 56 #include <linux/interrupt.h> 57 #include <linux/module.h> 58 #include <linux/pci.h> 59 #include <linux/slab.h> 60 #include <linux/types.h> 61 #include <linux/uaccess.h> 62 #include "linux/ntb.h" 63 #include "linux/ntb_transport.h" 64 65 #define NTB_TRANSPORT_VERSION 4 66 #define NTB_TRANSPORT_VER "4" 67 #define NTB_TRANSPORT_NAME "ntb_transport" 68 #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB" 69 #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2) 70 71 MODULE_DESCRIPTION(NTB_TRANSPORT_DESC); 72 MODULE_VERSION(NTB_TRANSPORT_VER); 73 MODULE_LICENSE("Dual BSD/GPL"); 74 MODULE_AUTHOR("Intel Corporation"); 75 76 static unsigned long max_mw_size; 77 module_param(max_mw_size, ulong, 0644); 78 MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); 79 80 static unsigned int transport_mtu = 0x10000; 81 module_param(transport_mtu, uint, 0644); 82 MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); 83 84 static unsigned char max_num_clients; 85 module_param(max_num_clients, byte, 0644); 86 MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients"); 87 88 static unsigned int copy_bytes = 1024; 89 module_param(copy_bytes, uint, 0644); 90 MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); 91 92 static bool use_dma; 93 module_param(use_dma, bool, 0644); 94 MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy"); 95 96 static struct dentry *nt_debugfs_dir; 97 98 struct ntb_queue_entry { 99 /* ntb_queue list reference */ 100 struct list_head entry; 101 /* pointers to data to be transferred */ 102 void *cb_data; 103 void *buf; 104 unsigned int len; 105 unsigned int flags; 106 int retries; 107 int errors; 108 unsigned int tx_index; 109 unsigned int rx_index; 110 111 struct ntb_transport_qp *qp; 112 union { 113 struct ntb_payload_header __iomem *tx_hdr; 114 struct ntb_payload_header *rx_hdr; 115 }; 116 }; 117 118 struct ntb_rx_info { 119 unsigned int entry; 120 }; 121 122 struct ntb_transport_qp { 123 struct ntb_transport_ctx *transport; 124 struct ntb_dev *ndev; 125 void *cb_data; 126 struct dma_chan *tx_dma_chan; 127 struct dma_chan *rx_dma_chan; 128 129 bool client_ready; 130 bool link_is_up; 131 bool active; 132 133 u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ 134 u64 qp_bit; 135 136 struct ntb_rx_info __iomem *rx_info; 137 struct ntb_rx_info *remote_rx_info; 138 139 void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, 140 void *data, int len); 141 struct list_head tx_free_q; 142 spinlock_t ntb_tx_free_q_lock; 143 void __iomem *tx_mw; 144 dma_addr_t tx_mw_phys; 145 unsigned int tx_index; 146 unsigned int tx_max_entry; 147 unsigned int tx_max_frame; 148 149 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 150 void *data, int len); 151 struct list_head rx_post_q; 152 struct list_head rx_pend_q; 153 struct list_head rx_free_q; 154 /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */ 155 spinlock_t ntb_rx_q_lock; 156 void *rx_buff; 157 unsigned int rx_index; 158 unsigned int rx_max_entry; 159 unsigned int rx_max_frame; 160 unsigned int rx_alloc_entry; 161 dma_cookie_t last_cookie; 162 struct tasklet_struct rxc_db_work; 163 164 void (*event_handler)(void *data, int status); 165 struct delayed_work link_work; 166 struct work_struct link_cleanup; 167 168 struct dentry *debugfs_dir; 169 struct dentry *debugfs_stats; 170 171 /* Stats */ 172 u64 rx_bytes; 173 u64 rx_pkts; 174 u64 rx_ring_empty; 175 u64 rx_err_no_buf; 176 u64 rx_err_oflow; 177 u64 rx_err_ver; 178 u64 rx_memcpy; 179 u64 rx_async; 180 u64 dma_rx_prep_err; 181 u64 tx_bytes; 182 u64 tx_pkts; 183 u64 tx_ring_full; 184 u64 tx_err_no_buf; 185 u64 tx_memcpy; 186 u64 tx_async; 187 u64 dma_tx_prep_err; 188 }; 189 190 struct ntb_transport_mw { 191 phys_addr_t phys_addr; 192 resource_size_t phys_size; 193 resource_size_t xlat_align; 194 resource_size_t xlat_align_size; 195 void __iomem *vbase; 196 size_t xlat_size; 197 size_t buff_size; 198 void *virt_addr; 199 dma_addr_t dma_addr; 200 }; 201 202 struct ntb_transport_client_dev { 203 struct list_head entry; 204 struct ntb_transport_ctx *nt; 205 struct device dev; 206 }; 207 208 struct ntb_transport_ctx { 209 struct list_head entry; 210 struct list_head client_devs; 211 212 struct ntb_dev *ndev; 213 214 struct ntb_transport_mw *mw_vec; 215 struct ntb_transport_qp *qp_vec; 216 unsigned int mw_count; 217 unsigned int qp_count; 218 u64 qp_bitmap; 219 u64 qp_bitmap_free; 220 221 bool link_is_up; 222 struct delayed_work link_work; 223 struct work_struct link_cleanup; 224 225 struct dentry *debugfs_node_dir; 226 }; 227 228 enum { 229 DESC_DONE_FLAG = BIT(0), 230 LINK_DOWN_FLAG = BIT(1), 231 }; 232 233 struct ntb_payload_header { 234 unsigned int ver; 235 unsigned int len; 236 unsigned int flags; 237 }; 238 239 enum { 240 VERSION = 0, 241 QP_LINKS, 242 NUM_QPS, 243 NUM_MWS, 244 MW0_SZ_HIGH, 245 MW0_SZ_LOW, 246 }; 247 248 #define dev_client_dev(__dev) \ 249 container_of((__dev), struct ntb_transport_client_dev, dev) 250 251 #define drv_client(__drv) \ 252 container_of((__drv), struct ntb_transport_client, driver) 253 254 #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) 255 #define NTB_QP_DEF_NUM_ENTRIES 100 256 #define NTB_LINK_DOWN_TIMEOUT 10 257 #define DMA_RETRIES 20 258 #define DMA_OUT_RESOURCE_TO msecs_to_jiffies(50) 259 260 static void ntb_transport_rxc_db(unsigned long data); 261 static const struct ntb_ctx_ops ntb_transport_ops; 262 static struct ntb_client ntb_transport_client; 263 static int ntb_async_tx_submit(struct ntb_transport_qp *qp, 264 struct ntb_queue_entry *entry); 265 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset); 266 static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset); 267 static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset); 268 269 270 static int ntb_transport_bus_match(struct device *dev, 271 struct device_driver *drv) 272 { 273 return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); 274 } 275 276 static int ntb_transport_bus_probe(struct device *dev) 277 { 278 const struct ntb_transport_client *client; 279 int rc = -EINVAL; 280 281 get_device(dev); 282 283 client = drv_client(dev->driver); 284 rc = client->probe(dev); 285 if (rc) 286 put_device(dev); 287 288 return rc; 289 } 290 291 static int ntb_transport_bus_remove(struct device *dev) 292 { 293 const struct ntb_transport_client *client; 294 295 client = drv_client(dev->driver); 296 client->remove(dev); 297 298 put_device(dev); 299 300 return 0; 301 } 302 303 static struct bus_type ntb_transport_bus = { 304 .name = "ntb_transport", 305 .match = ntb_transport_bus_match, 306 .probe = ntb_transport_bus_probe, 307 .remove = ntb_transport_bus_remove, 308 }; 309 310 static LIST_HEAD(ntb_transport_list); 311 312 static int ntb_bus_init(struct ntb_transport_ctx *nt) 313 { 314 list_add_tail(&nt->entry, &ntb_transport_list); 315 return 0; 316 } 317 318 static void ntb_bus_remove(struct ntb_transport_ctx *nt) 319 { 320 struct ntb_transport_client_dev *client_dev, *cd; 321 322 list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) { 323 dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n", 324 dev_name(&client_dev->dev)); 325 list_del(&client_dev->entry); 326 device_unregister(&client_dev->dev); 327 } 328 329 list_del(&nt->entry); 330 } 331 332 static void ntb_transport_client_release(struct device *dev) 333 { 334 struct ntb_transport_client_dev *client_dev; 335 336 client_dev = dev_client_dev(dev); 337 kfree(client_dev); 338 } 339 340 /** 341 * ntb_transport_unregister_client_dev - Unregister NTB client device 342 * @device_name: Name of NTB client device 343 * 344 * Unregister an NTB client device with the NTB transport layer 345 */ 346 void ntb_transport_unregister_client_dev(char *device_name) 347 { 348 struct ntb_transport_client_dev *client, *cd; 349 struct ntb_transport_ctx *nt; 350 351 list_for_each_entry(nt, &ntb_transport_list, entry) 352 list_for_each_entry_safe(client, cd, &nt->client_devs, entry) 353 if (!strncmp(dev_name(&client->dev), device_name, 354 strlen(device_name))) { 355 list_del(&client->entry); 356 device_unregister(&client->dev); 357 } 358 } 359 EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev); 360 361 /** 362 * ntb_transport_register_client_dev - Register NTB client device 363 * @device_name: Name of NTB client device 364 * 365 * Register an NTB client device with the NTB transport layer 366 */ 367 int ntb_transport_register_client_dev(char *device_name) 368 { 369 struct ntb_transport_client_dev *client_dev; 370 struct ntb_transport_ctx *nt; 371 int node; 372 int rc, i = 0; 373 374 if (list_empty(&ntb_transport_list)) 375 return -ENODEV; 376 377 list_for_each_entry(nt, &ntb_transport_list, entry) { 378 struct device *dev; 379 380 node = dev_to_node(&nt->ndev->dev); 381 382 client_dev = kzalloc_node(sizeof(*client_dev), 383 GFP_KERNEL, node); 384 if (!client_dev) { 385 rc = -ENOMEM; 386 goto err; 387 } 388 389 dev = &client_dev->dev; 390 391 /* setup and register client devices */ 392 dev_set_name(dev, "%s%d", device_name, i); 393 dev->bus = &ntb_transport_bus; 394 dev->release = ntb_transport_client_release; 395 dev->parent = &nt->ndev->dev; 396 397 rc = device_register(dev); 398 if (rc) { 399 kfree(client_dev); 400 goto err; 401 } 402 403 list_add_tail(&client_dev->entry, &nt->client_devs); 404 i++; 405 } 406 407 return 0; 408 409 err: 410 ntb_transport_unregister_client_dev(device_name); 411 412 return rc; 413 } 414 EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev); 415 416 /** 417 * ntb_transport_register_client - Register NTB client driver 418 * @drv: NTB client driver to be registered 419 * 420 * Register an NTB client driver with the NTB transport layer 421 * 422 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 423 */ 424 int ntb_transport_register_client(struct ntb_transport_client *drv) 425 { 426 drv->driver.bus = &ntb_transport_bus; 427 428 if (list_empty(&ntb_transport_list)) 429 return -ENODEV; 430 431 return driver_register(&drv->driver); 432 } 433 EXPORT_SYMBOL_GPL(ntb_transport_register_client); 434 435 /** 436 * ntb_transport_unregister_client - Unregister NTB client driver 437 * @drv: NTB client driver to be unregistered 438 * 439 * Unregister an NTB client driver with the NTB transport layer 440 * 441 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 442 */ 443 void ntb_transport_unregister_client(struct ntb_transport_client *drv) 444 { 445 driver_unregister(&drv->driver); 446 } 447 EXPORT_SYMBOL_GPL(ntb_transport_unregister_client); 448 449 static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, 450 loff_t *offp) 451 { 452 struct ntb_transport_qp *qp; 453 char *buf; 454 ssize_t ret, out_offset, out_count; 455 456 qp = filp->private_data; 457 458 if (!qp || !qp->link_is_up) 459 return 0; 460 461 out_count = 1000; 462 463 buf = kmalloc(out_count, GFP_KERNEL); 464 if (!buf) 465 return -ENOMEM; 466 467 out_offset = 0; 468 out_offset += snprintf(buf + out_offset, out_count - out_offset, 469 "\nNTB QP stats:\n\n"); 470 out_offset += snprintf(buf + out_offset, out_count - out_offset, 471 "rx_bytes - \t%llu\n", qp->rx_bytes); 472 out_offset += snprintf(buf + out_offset, out_count - out_offset, 473 "rx_pkts - \t%llu\n", qp->rx_pkts); 474 out_offset += snprintf(buf + out_offset, out_count - out_offset, 475 "rx_memcpy - \t%llu\n", qp->rx_memcpy); 476 out_offset += snprintf(buf + out_offset, out_count - out_offset, 477 "rx_async - \t%llu\n", qp->rx_async); 478 out_offset += snprintf(buf + out_offset, out_count - out_offset, 479 "rx_ring_empty - %llu\n", qp->rx_ring_empty); 480 out_offset += snprintf(buf + out_offset, out_count - out_offset, 481 "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); 482 out_offset += snprintf(buf + out_offset, out_count - out_offset, 483 "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); 484 out_offset += snprintf(buf + out_offset, out_count - out_offset, 485 "rx_err_ver - \t%llu\n", qp->rx_err_ver); 486 out_offset += snprintf(buf + out_offset, out_count - out_offset, 487 "rx_buff - \t0x%p\n", qp->rx_buff); 488 out_offset += snprintf(buf + out_offset, out_count - out_offset, 489 "rx_index - \t%u\n", qp->rx_index); 490 out_offset += snprintf(buf + out_offset, out_count - out_offset, 491 "rx_max_entry - \t%u\n", qp->rx_max_entry); 492 out_offset += snprintf(buf + out_offset, out_count - out_offset, 493 "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry); 494 495 out_offset += snprintf(buf + out_offset, out_count - out_offset, 496 "tx_bytes - \t%llu\n", qp->tx_bytes); 497 out_offset += snprintf(buf + out_offset, out_count - out_offset, 498 "tx_pkts - \t%llu\n", qp->tx_pkts); 499 out_offset += snprintf(buf + out_offset, out_count - out_offset, 500 "tx_memcpy - \t%llu\n", qp->tx_memcpy); 501 out_offset += snprintf(buf + out_offset, out_count - out_offset, 502 "tx_async - \t%llu\n", qp->tx_async); 503 out_offset += snprintf(buf + out_offset, out_count - out_offset, 504 "tx_ring_full - \t%llu\n", qp->tx_ring_full); 505 out_offset += snprintf(buf + out_offset, out_count - out_offset, 506 "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); 507 out_offset += snprintf(buf + out_offset, out_count - out_offset, 508 "tx_mw - \t0x%p\n", qp->tx_mw); 509 out_offset += snprintf(buf + out_offset, out_count - out_offset, 510 "tx_index (H) - \t%u\n", qp->tx_index); 511 out_offset += snprintf(buf + out_offset, out_count - out_offset, 512 "RRI (T) - \t%u\n", 513 qp->remote_rx_info->entry); 514 out_offset += snprintf(buf + out_offset, out_count - out_offset, 515 "tx_max_entry - \t%u\n", qp->tx_max_entry); 516 out_offset += snprintf(buf + out_offset, out_count - out_offset, 517 "free tx - \t%u\n", 518 ntb_transport_tx_free_entry(qp)); 519 out_offset += snprintf(buf + out_offset, out_count - out_offset, 520 "DMA tx prep err - \t%llu\n", 521 qp->dma_tx_prep_err); 522 out_offset += snprintf(buf + out_offset, out_count - out_offset, 523 "DMA rx prep err - \t%llu\n", 524 qp->dma_rx_prep_err); 525 526 out_offset += snprintf(buf + out_offset, out_count - out_offset, 527 "\n"); 528 out_offset += snprintf(buf + out_offset, out_count - out_offset, 529 "Using TX DMA - \t%s\n", 530 qp->tx_dma_chan ? "Yes" : "No"); 531 out_offset += snprintf(buf + out_offset, out_count - out_offset, 532 "Using RX DMA - \t%s\n", 533 qp->rx_dma_chan ? "Yes" : "No"); 534 out_offset += snprintf(buf + out_offset, out_count - out_offset, 535 "QP Link - \t%s\n", 536 qp->link_is_up ? "Up" : "Down"); 537 out_offset += snprintf(buf + out_offset, out_count - out_offset, 538 "\n"); 539 540 if (out_offset > out_count) 541 out_offset = out_count; 542 543 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); 544 kfree(buf); 545 return ret; 546 } 547 548 static const struct file_operations ntb_qp_debugfs_stats = { 549 .owner = THIS_MODULE, 550 .open = simple_open, 551 .read = debugfs_read, 552 }; 553 554 static void ntb_list_add(spinlock_t *lock, struct list_head *entry, 555 struct list_head *list) 556 { 557 unsigned long flags; 558 559 spin_lock_irqsave(lock, flags); 560 list_add_tail(entry, list); 561 spin_unlock_irqrestore(lock, flags); 562 } 563 564 static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, 565 struct list_head *list) 566 { 567 struct ntb_queue_entry *entry; 568 unsigned long flags; 569 570 spin_lock_irqsave(lock, flags); 571 if (list_empty(list)) { 572 entry = NULL; 573 goto out; 574 } 575 entry = list_first_entry(list, struct ntb_queue_entry, entry); 576 list_del(&entry->entry); 577 578 out: 579 spin_unlock_irqrestore(lock, flags); 580 581 return entry; 582 } 583 584 static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock, 585 struct list_head *list, 586 struct list_head *to_list) 587 { 588 struct ntb_queue_entry *entry; 589 unsigned long flags; 590 591 spin_lock_irqsave(lock, flags); 592 593 if (list_empty(list)) { 594 entry = NULL; 595 } else { 596 entry = list_first_entry(list, struct ntb_queue_entry, entry); 597 list_move_tail(&entry->entry, to_list); 598 } 599 600 spin_unlock_irqrestore(lock, flags); 601 602 return entry; 603 } 604 605 static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, 606 unsigned int qp_num) 607 { 608 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 609 struct ntb_transport_mw *mw; 610 struct ntb_dev *ndev = nt->ndev; 611 struct ntb_queue_entry *entry; 612 unsigned int rx_size, num_qps_mw; 613 unsigned int mw_num, mw_count, qp_count; 614 unsigned int i; 615 int node; 616 617 mw_count = nt->mw_count; 618 qp_count = nt->qp_count; 619 620 mw_num = QP_TO_MW(nt, qp_num); 621 mw = &nt->mw_vec[mw_num]; 622 623 if (!mw->virt_addr) 624 return -ENOMEM; 625 626 if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count) 627 num_qps_mw = qp_count / mw_count + 1; 628 else 629 num_qps_mw = qp_count / mw_count; 630 631 rx_size = (unsigned int)mw->xlat_size / num_qps_mw; 632 qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); 633 rx_size -= sizeof(struct ntb_rx_info); 634 635 qp->remote_rx_info = qp->rx_buff + rx_size; 636 637 /* Due to housekeeping, there must be atleast 2 buffs */ 638 qp->rx_max_frame = min(transport_mtu, rx_size / 2); 639 qp->rx_max_entry = rx_size / qp->rx_max_frame; 640 qp->rx_index = 0; 641 642 /* 643 * Checking to see if we have more entries than the default. 644 * We should add additional entries if that is the case so we 645 * can be in sync with the transport frames. 646 */ 647 node = dev_to_node(&ndev->dev); 648 for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) { 649 entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); 650 if (!entry) 651 return -ENOMEM; 652 653 entry->qp = qp; 654 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, 655 &qp->rx_free_q); 656 qp->rx_alloc_entry++; 657 } 658 659 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 660 661 /* setup the hdr offsets with 0's */ 662 for (i = 0; i < qp->rx_max_entry; i++) { 663 void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) - 664 sizeof(struct ntb_payload_header)); 665 memset(offset, 0, sizeof(struct ntb_payload_header)); 666 } 667 668 qp->rx_pkts = 0; 669 qp->tx_pkts = 0; 670 qp->tx_index = 0; 671 672 return 0; 673 } 674 675 static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) 676 { 677 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 678 struct pci_dev *pdev = nt->ndev->pdev; 679 680 if (!mw->virt_addr) 681 return; 682 683 ntb_mw_clear_trans(nt->ndev, num_mw); 684 dma_free_coherent(&pdev->dev, mw->buff_size, 685 mw->virt_addr, mw->dma_addr); 686 mw->xlat_size = 0; 687 mw->buff_size = 0; 688 mw->virt_addr = NULL; 689 } 690 691 static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, 692 resource_size_t size) 693 { 694 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 695 struct pci_dev *pdev = nt->ndev->pdev; 696 size_t xlat_size, buff_size; 697 int rc; 698 699 if (!size) 700 return -EINVAL; 701 702 xlat_size = round_up(size, mw->xlat_align_size); 703 buff_size = round_up(size, mw->xlat_align); 704 705 /* No need to re-setup */ 706 if (mw->xlat_size == xlat_size) 707 return 0; 708 709 if (mw->buff_size) 710 ntb_free_mw(nt, num_mw); 711 712 /* Alloc memory for receiving data. Must be aligned */ 713 mw->xlat_size = xlat_size; 714 mw->buff_size = buff_size; 715 716 mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size, 717 &mw->dma_addr, GFP_KERNEL); 718 if (!mw->virt_addr) { 719 mw->xlat_size = 0; 720 mw->buff_size = 0; 721 dev_err(&pdev->dev, "Unable to alloc MW buff of size %zu\n", 722 buff_size); 723 return -ENOMEM; 724 } 725 726 /* 727 * we must ensure that the memory address allocated is BAR size 728 * aligned in order for the XLAT register to take the value. This 729 * is a requirement of the hardware. It is recommended to setup CMA 730 * for BAR sizes equal or greater than 4MB. 731 */ 732 if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) { 733 dev_err(&pdev->dev, "DMA memory %pad is not aligned\n", 734 &mw->dma_addr); 735 ntb_free_mw(nt, num_mw); 736 return -ENOMEM; 737 } 738 739 /* Notify HW the memory location of the receive buffer */ 740 rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size); 741 if (rc) { 742 dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw); 743 ntb_free_mw(nt, num_mw); 744 return -EIO; 745 } 746 747 return 0; 748 } 749 750 static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp) 751 { 752 qp->link_is_up = false; 753 qp->active = false; 754 755 qp->tx_index = 0; 756 qp->rx_index = 0; 757 qp->rx_bytes = 0; 758 qp->rx_pkts = 0; 759 qp->rx_ring_empty = 0; 760 qp->rx_err_no_buf = 0; 761 qp->rx_err_oflow = 0; 762 qp->rx_err_ver = 0; 763 qp->rx_memcpy = 0; 764 qp->rx_async = 0; 765 qp->tx_bytes = 0; 766 qp->tx_pkts = 0; 767 qp->tx_ring_full = 0; 768 qp->tx_err_no_buf = 0; 769 qp->tx_memcpy = 0; 770 qp->tx_async = 0; 771 qp->dma_tx_prep_err = 0; 772 qp->dma_rx_prep_err = 0; 773 } 774 775 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 776 { 777 struct ntb_transport_ctx *nt = qp->transport; 778 struct pci_dev *pdev = nt->ndev->pdev; 779 780 dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num); 781 782 cancel_delayed_work_sync(&qp->link_work); 783 ntb_qp_link_down_reset(qp); 784 785 if (qp->event_handler) 786 qp->event_handler(qp->cb_data, qp->link_is_up); 787 } 788 789 static void ntb_qp_link_cleanup_work(struct work_struct *work) 790 { 791 struct ntb_transport_qp *qp = container_of(work, 792 struct ntb_transport_qp, 793 link_cleanup); 794 struct ntb_transport_ctx *nt = qp->transport; 795 796 ntb_qp_link_cleanup(qp); 797 798 if (nt->link_is_up) 799 schedule_delayed_work(&qp->link_work, 800 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 801 } 802 803 static void ntb_qp_link_down(struct ntb_transport_qp *qp) 804 { 805 schedule_work(&qp->link_cleanup); 806 } 807 808 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) 809 { 810 struct ntb_transport_qp *qp; 811 u64 qp_bitmap_alloc; 812 unsigned int i, count; 813 814 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 815 816 /* Pass along the info to any clients */ 817 for (i = 0; i < nt->qp_count; i++) 818 if (qp_bitmap_alloc & BIT_ULL(i)) { 819 qp = &nt->qp_vec[i]; 820 ntb_qp_link_cleanup(qp); 821 cancel_work_sync(&qp->link_cleanup); 822 cancel_delayed_work_sync(&qp->link_work); 823 } 824 825 if (!nt->link_is_up) 826 cancel_delayed_work_sync(&nt->link_work); 827 828 /* The scratchpad registers keep the values if the remote side 829 * goes down, blast them now to give them a sane value the next 830 * time they are accessed 831 */ 832 count = ntb_spad_count(nt->ndev); 833 for (i = 0; i < count; i++) 834 ntb_spad_write(nt->ndev, i, 0); 835 } 836 837 static void ntb_transport_link_cleanup_work(struct work_struct *work) 838 { 839 struct ntb_transport_ctx *nt = 840 container_of(work, struct ntb_transport_ctx, link_cleanup); 841 842 ntb_transport_link_cleanup(nt); 843 } 844 845 static void ntb_transport_event_callback(void *data) 846 { 847 struct ntb_transport_ctx *nt = data; 848 849 if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1) 850 schedule_delayed_work(&nt->link_work, 0); 851 else 852 schedule_work(&nt->link_cleanup); 853 } 854 855 static void ntb_transport_link_work(struct work_struct *work) 856 { 857 struct ntb_transport_ctx *nt = 858 container_of(work, struct ntb_transport_ctx, link_work.work); 859 struct ntb_dev *ndev = nt->ndev; 860 struct pci_dev *pdev = ndev->pdev; 861 resource_size_t size; 862 u32 val; 863 int rc = 0, i, spad; 864 865 /* send the local info, in the opposite order of the way we read it */ 866 for (i = 0; i < nt->mw_count; i++) { 867 size = nt->mw_vec[i].phys_size; 868 869 if (max_mw_size && size > max_mw_size) 870 size = max_mw_size; 871 872 spad = MW0_SZ_HIGH + (i * 2); 873 ntb_peer_spad_write(ndev, spad, upper_32_bits(size)); 874 875 spad = MW0_SZ_LOW + (i * 2); 876 ntb_peer_spad_write(ndev, spad, lower_32_bits(size)); 877 } 878 879 ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count); 880 881 ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count); 882 883 ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION); 884 885 /* Query the remote side for its info */ 886 val = ntb_spad_read(ndev, VERSION); 887 dev_dbg(&pdev->dev, "Remote version = %d\n", val); 888 if (val != NTB_TRANSPORT_VERSION) 889 goto out; 890 891 val = ntb_spad_read(ndev, NUM_QPS); 892 dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); 893 if (val != nt->qp_count) 894 goto out; 895 896 val = ntb_spad_read(ndev, NUM_MWS); 897 dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); 898 if (val != nt->mw_count) 899 goto out; 900 901 for (i = 0; i < nt->mw_count; i++) { 902 u64 val64; 903 904 val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2)); 905 val64 = (u64)val << 32; 906 907 val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2)); 908 val64 |= val; 909 910 dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64); 911 912 rc = ntb_set_mw(nt, i, val64); 913 if (rc) 914 goto out1; 915 } 916 917 nt->link_is_up = true; 918 919 for (i = 0; i < nt->qp_count; i++) { 920 struct ntb_transport_qp *qp = &nt->qp_vec[i]; 921 922 ntb_transport_setup_qp_mw(nt, i); 923 924 if (qp->client_ready) 925 schedule_delayed_work(&qp->link_work, 0); 926 } 927 928 return; 929 930 out1: 931 for (i = 0; i < nt->mw_count; i++) 932 ntb_free_mw(nt, i); 933 934 /* if there's an actual failure, we should just bail */ 935 if (rc < 0) { 936 ntb_link_disable(ndev); 937 return; 938 } 939 940 out: 941 if (ntb_link_is_up(ndev, NULL, NULL) == 1) 942 schedule_delayed_work(&nt->link_work, 943 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 944 } 945 946 static void ntb_qp_link_work(struct work_struct *work) 947 { 948 struct ntb_transport_qp *qp = container_of(work, 949 struct ntb_transport_qp, 950 link_work.work); 951 struct pci_dev *pdev = qp->ndev->pdev; 952 struct ntb_transport_ctx *nt = qp->transport; 953 int val; 954 955 WARN_ON(!nt->link_is_up); 956 957 val = ntb_spad_read(nt->ndev, QP_LINKS); 958 959 ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num)); 960 961 /* query remote spad for qp ready bits */ 962 dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val); 963 964 /* See if the remote side is up */ 965 if (val & BIT(qp->qp_num)) { 966 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); 967 qp->link_is_up = true; 968 qp->active = true; 969 970 if (qp->event_handler) 971 qp->event_handler(qp->cb_data, qp->link_is_up); 972 973 if (qp->active) 974 tasklet_schedule(&qp->rxc_db_work); 975 } else if (nt->link_is_up) 976 schedule_delayed_work(&qp->link_work, 977 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 978 } 979 980 static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, 981 unsigned int qp_num) 982 { 983 struct ntb_transport_qp *qp; 984 phys_addr_t mw_base; 985 resource_size_t mw_size; 986 unsigned int num_qps_mw, tx_size; 987 unsigned int mw_num, mw_count, qp_count; 988 u64 qp_offset; 989 990 mw_count = nt->mw_count; 991 qp_count = nt->qp_count; 992 993 mw_num = QP_TO_MW(nt, qp_num); 994 995 qp = &nt->qp_vec[qp_num]; 996 qp->qp_num = qp_num; 997 qp->transport = nt; 998 qp->ndev = nt->ndev; 999 qp->client_ready = false; 1000 qp->event_handler = NULL; 1001 ntb_qp_link_down_reset(qp); 1002 1003 if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count) 1004 num_qps_mw = qp_count / mw_count + 1; 1005 else 1006 num_qps_mw = qp_count / mw_count; 1007 1008 mw_base = nt->mw_vec[mw_num].phys_addr; 1009 mw_size = nt->mw_vec[mw_num].phys_size; 1010 1011 tx_size = (unsigned int)mw_size / num_qps_mw; 1012 qp_offset = tx_size * (qp_num / mw_count); 1013 1014 qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; 1015 if (!qp->tx_mw) 1016 return -EINVAL; 1017 1018 qp->tx_mw_phys = mw_base + qp_offset; 1019 if (!qp->tx_mw_phys) 1020 return -EINVAL; 1021 1022 tx_size -= sizeof(struct ntb_rx_info); 1023 qp->rx_info = qp->tx_mw + tx_size; 1024 1025 /* Due to housekeeping, there must be atleast 2 buffs */ 1026 qp->tx_max_frame = min(transport_mtu, tx_size / 2); 1027 qp->tx_max_entry = tx_size / qp->tx_max_frame; 1028 1029 if (nt->debugfs_node_dir) { 1030 char debugfs_name[4]; 1031 1032 snprintf(debugfs_name, 4, "qp%d", qp_num); 1033 qp->debugfs_dir = debugfs_create_dir(debugfs_name, 1034 nt->debugfs_node_dir); 1035 1036 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, 1037 qp->debugfs_dir, qp, 1038 &ntb_qp_debugfs_stats); 1039 } else { 1040 qp->debugfs_dir = NULL; 1041 qp->debugfs_stats = NULL; 1042 } 1043 1044 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); 1045 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); 1046 1047 spin_lock_init(&qp->ntb_rx_q_lock); 1048 spin_lock_init(&qp->ntb_tx_free_q_lock); 1049 1050 INIT_LIST_HEAD(&qp->rx_post_q); 1051 INIT_LIST_HEAD(&qp->rx_pend_q); 1052 INIT_LIST_HEAD(&qp->rx_free_q); 1053 INIT_LIST_HEAD(&qp->tx_free_q); 1054 1055 tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db, 1056 (unsigned long)qp); 1057 1058 return 0; 1059 } 1060 1061 static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) 1062 { 1063 struct ntb_transport_ctx *nt; 1064 struct ntb_transport_mw *mw; 1065 unsigned int mw_count, qp_count, spad_count, max_mw_count_for_spads; 1066 u64 qp_bitmap; 1067 int node; 1068 int rc, i; 1069 1070 mw_count = ntb_mw_count(ndev); 1071 1072 if (ntb_db_is_unsafe(ndev)) 1073 dev_dbg(&ndev->dev, 1074 "doorbell is unsafe, proceed anyway...\n"); 1075 if (ntb_spad_is_unsafe(ndev)) 1076 dev_dbg(&ndev->dev, 1077 "scratchpad is unsafe, proceed anyway...\n"); 1078 1079 node = dev_to_node(&ndev->dev); 1080 1081 nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node); 1082 if (!nt) 1083 return -ENOMEM; 1084 1085 nt->ndev = ndev; 1086 spad_count = ntb_spad_count(ndev); 1087 1088 /* Limit the MW's based on the availability of scratchpads */ 1089 1090 if (spad_count < NTB_TRANSPORT_MIN_SPADS) { 1091 nt->mw_count = 0; 1092 rc = -EINVAL; 1093 goto err; 1094 } 1095 1096 max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2; 1097 nt->mw_count = min(mw_count, max_mw_count_for_spads); 1098 1099 nt->mw_vec = kzalloc_node(mw_count * sizeof(*nt->mw_vec), 1100 GFP_KERNEL, node); 1101 if (!nt->mw_vec) { 1102 rc = -ENOMEM; 1103 goto err; 1104 } 1105 1106 for (i = 0; i < mw_count; i++) { 1107 mw = &nt->mw_vec[i]; 1108 1109 rc = ntb_mw_get_range(ndev, i, &mw->phys_addr, &mw->phys_size, 1110 &mw->xlat_align, &mw->xlat_align_size); 1111 if (rc) 1112 goto err1; 1113 1114 mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size); 1115 if (!mw->vbase) { 1116 rc = -ENOMEM; 1117 goto err1; 1118 } 1119 1120 mw->buff_size = 0; 1121 mw->xlat_size = 0; 1122 mw->virt_addr = NULL; 1123 mw->dma_addr = 0; 1124 } 1125 1126 qp_bitmap = ntb_db_valid_mask(ndev); 1127 1128 qp_count = ilog2(qp_bitmap); 1129 if (max_num_clients && max_num_clients < qp_count) 1130 qp_count = max_num_clients; 1131 else if (nt->mw_count < qp_count) 1132 qp_count = nt->mw_count; 1133 1134 qp_bitmap &= BIT_ULL(qp_count) - 1; 1135 1136 nt->qp_count = qp_count; 1137 nt->qp_bitmap = qp_bitmap; 1138 nt->qp_bitmap_free = qp_bitmap; 1139 1140 nt->qp_vec = kzalloc_node(qp_count * sizeof(*nt->qp_vec), 1141 GFP_KERNEL, node); 1142 if (!nt->qp_vec) { 1143 rc = -ENOMEM; 1144 goto err1; 1145 } 1146 1147 if (nt_debugfs_dir) { 1148 nt->debugfs_node_dir = 1149 debugfs_create_dir(pci_name(ndev->pdev), 1150 nt_debugfs_dir); 1151 } 1152 1153 for (i = 0; i < qp_count; i++) { 1154 rc = ntb_transport_init_queue(nt, i); 1155 if (rc) 1156 goto err2; 1157 } 1158 1159 INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); 1160 INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); 1161 1162 rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops); 1163 if (rc) 1164 goto err2; 1165 1166 INIT_LIST_HEAD(&nt->client_devs); 1167 rc = ntb_bus_init(nt); 1168 if (rc) 1169 goto err3; 1170 1171 nt->link_is_up = false; 1172 ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1173 ntb_link_event(ndev); 1174 1175 return 0; 1176 1177 err3: 1178 ntb_clear_ctx(ndev); 1179 err2: 1180 kfree(nt->qp_vec); 1181 err1: 1182 while (i--) { 1183 mw = &nt->mw_vec[i]; 1184 iounmap(mw->vbase); 1185 } 1186 kfree(nt->mw_vec); 1187 err: 1188 kfree(nt); 1189 return rc; 1190 } 1191 1192 static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev) 1193 { 1194 struct ntb_transport_ctx *nt = ndev->ctx; 1195 struct ntb_transport_qp *qp; 1196 u64 qp_bitmap_alloc; 1197 int i; 1198 1199 ntb_transport_link_cleanup(nt); 1200 cancel_work_sync(&nt->link_cleanup); 1201 cancel_delayed_work_sync(&nt->link_work); 1202 1203 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 1204 1205 /* verify that all the qp's are freed */ 1206 for (i = 0; i < nt->qp_count; i++) { 1207 qp = &nt->qp_vec[i]; 1208 if (qp_bitmap_alloc & BIT_ULL(i)) 1209 ntb_transport_free_queue(qp); 1210 debugfs_remove_recursive(qp->debugfs_dir); 1211 } 1212 1213 ntb_link_disable(ndev); 1214 ntb_clear_ctx(ndev); 1215 1216 ntb_bus_remove(nt); 1217 1218 for (i = nt->mw_count; i--; ) { 1219 ntb_free_mw(nt, i); 1220 iounmap(nt->mw_vec[i].vbase); 1221 } 1222 1223 kfree(nt->qp_vec); 1224 kfree(nt->mw_vec); 1225 kfree(nt); 1226 } 1227 1228 static void ntb_complete_rxc(struct ntb_transport_qp *qp) 1229 { 1230 struct ntb_queue_entry *entry; 1231 void *cb_data; 1232 unsigned int len; 1233 unsigned long irqflags; 1234 1235 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); 1236 1237 while (!list_empty(&qp->rx_post_q)) { 1238 entry = list_first_entry(&qp->rx_post_q, 1239 struct ntb_queue_entry, entry); 1240 if (!(entry->flags & DESC_DONE_FLAG)) 1241 break; 1242 1243 entry->rx_hdr->flags = 0; 1244 iowrite32(entry->rx_index, &qp->rx_info->entry); 1245 1246 cb_data = entry->cb_data; 1247 len = entry->len; 1248 1249 list_move_tail(&entry->entry, &qp->rx_free_q); 1250 1251 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); 1252 1253 if (qp->rx_handler && qp->client_ready) 1254 qp->rx_handler(qp, qp->cb_data, cb_data, len); 1255 1256 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); 1257 } 1258 1259 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); 1260 } 1261 1262 static void ntb_rx_copy_callback(void *data, 1263 const struct dmaengine_result *res) 1264 { 1265 struct ntb_queue_entry *entry = data; 1266 1267 /* we need to check DMA results if we are using DMA */ 1268 if (res) { 1269 enum dmaengine_tx_result dma_err = res->result; 1270 1271 switch (dma_err) { 1272 case DMA_TRANS_READ_FAILED: 1273 case DMA_TRANS_WRITE_FAILED: 1274 entry->errors++; 1275 case DMA_TRANS_ABORTED: 1276 { 1277 struct ntb_transport_qp *qp = entry->qp; 1278 void *offset = qp->rx_buff + qp->rx_max_frame * 1279 qp->rx_index; 1280 1281 ntb_memcpy_rx(entry, offset); 1282 qp->rx_memcpy++; 1283 return; 1284 } 1285 1286 case DMA_TRANS_NOERROR: 1287 default: 1288 break; 1289 } 1290 } 1291 1292 entry->flags |= DESC_DONE_FLAG; 1293 1294 ntb_complete_rxc(entry->qp); 1295 } 1296 1297 static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) 1298 { 1299 void *buf = entry->buf; 1300 size_t len = entry->len; 1301 1302 memcpy(buf, offset, len); 1303 1304 /* Ensure that the data is fully copied out before clearing the flag */ 1305 wmb(); 1306 1307 ntb_rx_copy_callback(entry, NULL); 1308 } 1309 1310 static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset) 1311 { 1312 struct dma_async_tx_descriptor *txd; 1313 struct ntb_transport_qp *qp = entry->qp; 1314 struct dma_chan *chan = qp->rx_dma_chan; 1315 struct dma_device *device; 1316 size_t pay_off, buff_off, len; 1317 struct dmaengine_unmap_data *unmap; 1318 dma_cookie_t cookie; 1319 void *buf = entry->buf; 1320 int retries = 0; 1321 1322 len = entry->len; 1323 device = chan->device; 1324 pay_off = (size_t)offset & ~PAGE_MASK; 1325 buff_off = (size_t)buf & ~PAGE_MASK; 1326 1327 if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) 1328 goto err; 1329 1330 unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); 1331 if (!unmap) 1332 goto err; 1333 1334 unmap->len = len; 1335 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset), 1336 pay_off, len, DMA_TO_DEVICE); 1337 if (dma_mapping_error(device->dev, unmap->addr[0])) 1338 goto err_get_unmap; 1339 1340 unmap->to_cnt = 1; 1341 1342 unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf), 1343 buff_off, len, DMA_FROM_DEVICE); 1344 if (dma_mapping_error(device->dev, unmap->addr[1])) 1345 goto err_get_unmap; 1346 1347 unmap->from_cnt = 1; 1348 1349 for (retries = 0; retries < DMA_RETRIES; retries++) { 1350 txd = device->device_prep_dma_memcpy(chan, 1351 unmap->addr[1], 1352 unmap->addr[0], len, 1353 DMA_PREP_INTERRUPT); 1354 if (txd) 1355 break; 1356 1357 set_current_state(TASK_INTERRUPTIBLE); 1358 schedule_timeout(DMA_OUT_RESOURCE_TO); 1359 } 1360 1361 if (!txd) { 1362 qp->dma_rx_prep_err++; 1363 goto err_get_unmap; 1364 } 1365 1366 txd->callback_result = ntb_rx_copy_callback; 1367 txd->callback_param = entry; 1368 dma_set_unmap(txd, unmap); 1369 1370 cookie = dmaengine_submit(txd); 1371 if (dma_submit_error(cookie)) 1372 goto err_set_unmap; 1373 1374 dmaengine_unmap_put(unmap); 1375 1376 qp->last_cookie = cookie; 1377 1378 qp->rx_async++; 1379 1380 return 0; 1381 1382 err_set_unmap: 1383 dmaengine_unmap_put(unmap); 1384 err_get_unmap: 1385 dmaengine_unmap_put(unmap); 1386 err: 1387 return -ENXIO; 1388 } 1389 1390 static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) 1391 { 1392 struct ntb_transport_qp *qp = entry->qp; 1393 struct dma_chan *chan = qp->rx_dma_chan; 1394 int res; 1395 1396 if (!chan) 1397 goto err; 1398 1399 if (entry->len < copy_bytes) 1400 goto err; 1401 1402 res = ntb_async_rx_submit(entry, offset); 1403 if (res < 0) 1404 goto err; 1405 1406 if (!entry->retries) 1407 qp->rx_async++; 1408 1409 return; 1410 1411 err: 1412 ntb_memcpy_rx(entry, offset); 1413 qp->rx_memcpy++; 1414 } 1415 1416 static int ntb_process_rxc(struct ntb_transport_qp *qp) 1417 { 1418 struct ntb_payload_header *hdr; 1419 struct ntb_queue_entry *entry; 1420 void *offset; 1421 1422 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; 1423 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); 1424 1425 dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n", 1426 qp->qp_num, hdr->ver, hdr->len, hdr->flags); 1427 1428 if (!(hdr->flags & DESC_DONE_FLAG)) { 1429 dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n"); 1430 qp->rx_ring_empty++; 1431 return -EAGAIN; 1432 } 1433 1434 if (hdr->flags & LINK_DOWN_FLAG) { 1435 dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n"); 1436 ntb_qp_link_down(qp); 1437 hdr->flags = 0; 1438 return -EAGAIN; 1439 } 1440 1441 if (hdr->ver != (u32)qp->rx_pkts) { 1442 dev_dbg(&qp->ndev->pdev->dev, 1443 "version mismatch, expected %llu - got %u\n", 1444 qp->rx_pkts, hdr->ver); 1445 qp->rx_err_ver++; 1446 return -EIO; 1447 } 1448 1449 entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q); 1450 if (!entry) { 1451 dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n"); 1452 qp->rx_err_no_buf++; 1453 return -EAGAIN; 1454 } 1455 1456 entry->rx_hdr = hdr; 1457 entry->rx_index = qp->rx_index; 1458 1459 if (hdr->len > entry->len) { 1460 dev_dbg(&qp->ndev->pdev->dev, 1461 "receive buffer overflow! Wanted %d got %d\n", 1462 hdr->len, entry->len); 1463 qp->rx_err_oflow++; 1464 1465 entry->len = -EIO; 1466 entry->flags |= DESC_DONE_FLAG; 1467 1468 ntb_complete_rxc(qp); 1469 } else { 1470 dev_dbg(&qp->ndev->pdev->dev, 1471 "RX OK index %u ver %u size %d into buf size %d\n", 1472 qp->rx_index, hdr->ver, hdr->len, entry->len); 1473 1474 qp->rx_bytes += hdr->len; 1475 qp->rx_pkts++; 1476 1477 entry->len = hdr->len; 1478 1479 ntb_async_rx(entry, offset); 1480 } 1481 1482 qp->rx_index++; 1483 qp->rx_index %= qp->rx_max_entry; 1484 1485 return 0; 1486 } 1487 1488 static void ntb_transport_rxc_db(unsigned long data) 1489 { 1490 struct ntb_transport_qp *qp = (void *)data; 1491 int rc, i; 1492 1493 dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n", 1494 __func__, qp->qp_num); 1495 1496 /* Limit the number of packets processed in a single interrupt to 1497 * provide fairness to others 1498 */ 1499 for (i = 0; i < qp->rx_max_entry; i++) { 1500 rc = ntb_process_rxc(qp); 1501 if (rc) 1502 break; 1503 } 1504 1505 if (i && qp->rx_dma_chan) 1506 dma_async_issue_pending(qp->rx_dma_chan); 1507 1508 if (i == qp->rx_max_entry) { 1509 /* there is more work to do */ 1510 if (qp->active) 1511 tasklet_schedule(&qp->rxc_db_work); 1512 } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) { 1513 /* the doorbell bit is set: clear it */ 1514 ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num)); 1515 /* ntb_db_read ensures ntb_db_clear write is committed */ 1516 ntb_db_read(qp->ndev); 1517 1518 /* an interrupt may have arrived between finishing 1519 * ntb_process_rxc and clearing the doorbell bit: 1520 * there might be some more work to do. 1521 */ 1522 if (qp->active) 1523 tasklet_schedule(&qp->rxc_db_work); 1524 } 1525 } 1526 1527 static void ntb_tx_copy_callback(void *data, 1528 const struct dmaengine_result *res) 1529 { 1530 struct ntb_queue_entry *entry = data; 1531 struct ntb_transport_qp *qp = entry->qp; 1532 struct ntb_payload_header __iomem *hdr = entry->tx_hdr; 1533 1534 /* we need to check DMA results if we are using DMA */ 1535 if (res) { 1536 enum dmaengine_tx_result dma_err = res->result; 1537 1538 switch (dma_err) { 1539 case DMA_TRANS_READ_FAILED: 1540 case DMA_TRANS_WRITE_FAILED: 1541 entry->errors++; 1542 case DMA_TRANS_ABORTED: 1543 { 1544 void __iomem *offset = 1545 qp->tx_mw + qp->tx_max_frame * 1546 entry->tx_index; 1547 1548 /* resubmit via CPU */ 1549 ntb_memcpy_tx(entry, offset); 1550 qp->tx_memcpy++; 1551 return; 1552 } 1553 1554 case DMA_TRANS_NOERROR: 1555 default: 1556 break; 1557 } 1558 } 1559 1560 iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); 1561 1562 ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); 1563 1564 /* The entry length can only be zero if the packet is intended to be a 1565 * "link down" or similar. Since no payload is being sent in these 1566 * cases, there is nothing to add to the completion queue. 1567 */ 1568 if (entry->len > 0) { 1569 qp->tx_bytes += entry->len; 1570 1571 if (qp->tx_handler) 1572 qp->tx_handler(qp, qp->cb_data, entry->cb_data, 1573 entry->len); 1574 } 1575 1576 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); 1577 } 1578 1579 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) 1580 { 1581 #ifdef ARCH_HAS_NOCACHE_UACCESS 1582 /* 1583 * Using non-temporal mov to improve performance on non-cached 1584 * writes, even though we aren't actually copying from user space. 1585 */ 1586 __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); 1587 #else 1588 memcpy_toio(offset, entry->buf, entry->len); 1589 #endif 1590 1591 /* Ensure that the data is fully copied out before setting the flags */ 1592 wmb(); 1593 1594 ntb_tx_copy_callback(entry, NULL); 1595 } 1596 1597 static int ntb_async_tx_submit(struct ntb_transport_qp *qp, 1598 struct ntb_queue_entry *entry) 1599 { 1600 struct dma_async_tx_descriptor *txd; 1601 struct dma_chan *chan = qp->tx_dma_chan; 1602 struct dma_device *device; 1603 size_t len = entry->len; 1604 void *buf = entry->buf; 1605 size_t dest_off, buff_off; 1606 struct dmaengine_unmap_data *unmap; 1607 dma_addr_t dest; 1608 dma_cookie_t cookie; 1609 int retries = 0; 1610 1611 device = chan->device; 1612 dest = qp->tx_mw_phys + qp->tx_max_frame * entry->tx_index; 1613 buff_off = (size_t)buf & ~PAGE_MASK; 1614 dest_off = (size_t)dest & ~PAGE_MASK; 1615 1616 if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) 1617 goto err; 1618 1619 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); 1620 if (!unmap) 1621 goto err; 1622 1623 unmap->len = len; 1624 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf), 1625 buff_off, len, DMA_TO_DEVICE); 1626 if (dma_mapping_error(device->dev, unmap->addr[0])) 1627 goto err_get_unmap; 1628 1629 unmap->to_cnt = 1; 1630 1631 for (retries = 0; retries < DMA_RETRIES; retries++) { 1632 txd = device->device_prep_dma_memcpy(chan, dest, 1633 unmap->addr[0], len, 1634 DMA_PREP_INTERRUPT); 1635 if (txd) 1636 break; 1637 1638 set_current_state(TASK_INTERRUPTIBLE); 1639 schedule_timeout(DMA_OUT_RESOURCE_TO); 1640 } 1641 1642 if (!txd) { 1643 qp->dma_tx_prep_err++; 1644 goto err_get_unmap; 1645 } 1646 1647 txd->callback_result = ntb_tx_copy_callback; 1648 txd->callback_param = entry; 1649 dma_set_unmap(txd, unmap); 1650 1651 cookie = dmaengine_submit(txd); 1652 if (dma_submit_error(cookie)) 1653 goto err_set_unmap; 1654 1655 dmaengine_unmap_put(unmap); 1656 1657 dma_async_issue_pending(chan); 1658 1659 return 0; 1660 err_set_unmap: 1661 dmaengine_unmap_put(unmap); 1662 err_get_unmap: 1663 dmaengine_unmap_put(unmap); 1664 err: 1665 return -ENXIO; 1666 } 1667 1668 static void ntb_async_tx(struct ntb_transport_qp *qp, 1669 struct ntb_queue_entry *entry) 1670 { 1671 struct ntb_payload_header __iomem *hdr; 1672 struct dma_chan *chan = qp->tx_dma_chan; 1673 void __iomem *offset; 1674 int res; 1675 1676 entry->tx_index = qp->tx_index; 1677 offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index; 1678 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); 1679 entry->tx_hdr = hdr; 1680 1681 iowrite32(entry->len, &hdr->len); 1682 iowrite32((u32)qp->tx_pkts, &hdr->ver); 1683 1684 if (!chan) 1685 goto err; 1686 1687 if (entry->len < copy_bytes) 1688 goto err; 1689 1690 res = ntb_async_tx_submit(qp, entry); 1691 if (res < 0) 1692 goto err; 1693 1694 if (!entry->retries) 1695 qp->tx_async++; 1696 1697 return; 1698 1699 err: 1700 ntb_memcpy_tx(entry, offset); 1701 qp->tx_memcpy++; 1702 } 1703 1704 static int ntb_process_tx(struct ntb_transport_qp *qp, 1705 struct ntb_queue_entry *entry) 1706 { 1707 if (qp->tx_index == qp->remote_rx_info->entry) { 1708 qp->tx_ring_full++; 1709 return -EAGAIN; 1710 } 1711 1712 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 1713 if (qp->tx_handler) 1714 qp->tx_handler(qp, qp->cb_data, NULL, -EIO); 1715 1716 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1717 &qp->tx_free_q); 1718 return 0; 1719 } 1720 1721 ntb_async_tx(qp, entry); 1722 1723 qp->tx_index++; 1724 qp->tx_index %= qp->tx_max_entry; 1725 1726 qp->tx_pkts++; 1727 1728 return 0; 1729 } 1730 1731 static void ntb_send_link_down(struct ntb_transport_qp *qp) 1732 { 1733 struct pci_dev *pdev = qp->ndev->pdev; 1734 struct ntb_queue_entry *entry; 1735 int i, rc; 1736 1737 if (!qp->link_is_up) 1738 return; 1739 1740 dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num); 1741 1742 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1743 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1744 if (entry) 1745 break; 1746 msleep(100); 1747 } 1748 1749 if (!entry) 1750 return; 1751 1752 entry->cb_data = NULL; 1753 entry->buf = NULL; 1754 entry->len = 0; 1755 entry->flags = LINK_DOWN_FLAG; 1756 1757 rc = ntb_process_tx(qp, entry); 1758 if (rc) 1759 dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", 1760 qp->qp_num); 1761 1762 ntb_qp_link_down_reset(qp); 1763 } 1764 1765 static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node) 1766 { 1767 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; 1768 } 1769 1770 /** 1771 * ntb_transport_create_queue - Create a new NTB transport layer queue 1772 * @rx_handler: receive callback function 1773 * @tx_handler: transmit callback function 1774 * @event_handler: event callback function 1775 * 1776 * Create a new NTB transport layer queue and provide the queue with a callback 1777 * routine for both transmit and receive. The receive callback routine will be 1778 * used to pass up data when the transport has received it on the queue. The 1779 * transmit callback routine will be called when the transport has completed the 1780 * transmission of the data on the queue and the data is ready to be freed. 1781 * 1782 * RETURNS: pointer to newly created ntb_queue, NULL on error. 1783 */ 1784 struct ntb_transport_qp * 1785 ntb_transport_create_queue(void *data, struct device *client_dev, 1786 const struct ntb_queue_handlers *handlers) 1787 { 1788 struct ntb_dev *ndev; 1789 struct pci_dev *pdev; 1790 struct ntb_transport_ctx *nt; 1791 struct ntb_queue_entry *entry; 1792 struct ntb_transport_qp *qp; 1793 u64 qp_bit; 1794 unsigned int free_queue; 1795 dma_cap_mask_t dma_mask; 1796 int node; 1797 int i; 1798 1799 ndev = dev_ntb(client_dev->parent); 1800 pdev = ndev->pdev; 1801 nt = ndev->ctx; 1802 1803 node = dev_to_node(&ndev->dev); 1804 1805 free_queue = ffs(nt->qp_bitmap_free); 1806 if (!free_queue) 1807 goto err; 1808 1809 /* decrement free_queue to make it zero based */ 1810 free_queue--; 1811 1812 qp = &nt->qp_vec[free_queue]; 1813 qp_bit = BIT_ULL(qp->qp_num); 1814 1815 nt->qp_bitmap_free &= ~qp_bit; 1816 1817 qp->cb_data = data; 1818 qp->rx_handler = handlers->rx_handler; 1819 qp->tx_handler = handlers->tx_handler; 1820 qp->event_handler = handlers->event_handler; 1821 1822 dma_cap_zero(dma_mask); 1823 dma_cap_set(DMA_MEMCPY, dma_mask); 1824 1825 if (use_dma) { 1826 qp->tx_dma_chan = 1827 dma_request_channel(dma_mask, ntb_dma_filter_fn, 1828 (void *)(unsigned long)node); 1829 if (!qp->tx_dma_chan) 1830 dev_info(&pdev->dev, "Unable to allocate TX DMA channel\n"); 1831 1832 qp->rx_dma_chan = 1833 dma_request_channel(dma_mask, ntb_dma_filter_fn, 1834 (void *)(unsigned long)node); 1835 if (!qp->rx_dma_chan) 1836 dev_info(&pdev->dev, "Unable to allocate RX DMA channel\n"); 1837 } else { 1838 qp->tx_dma_chan = NULL; 1839 qp->rx_dma_chan = NULL; 1840 } 1841 1842 dev_dbg(&pdev->dev, "Using %s memcpy for TX\n", 1843 qp->tx_dma_chan ? "DMA" : "CPU"); 1844 1845 dev_dbg(&pdev->dev, "Using %s memcpy for RX\n", 1846 qp->rx_dma_chan ? "DMA" : "CPU"); 1847 1848 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 1849 entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); 1850 if (!entry) 1851 goto err1; 1852 1853 entry->qp = qp; 1854 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, 1855 &qp->rx_free_q); 1856 } 1857 qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES; 1858 1859 for (i = 0; i < qp->tx_max_entry; i++) { 1860 entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); 1861 if (!entry) 1862 goto err2; 1863 1864 entry->qp = qp; 1865 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1866 &qp->tx_free_q); 1867 } 1868 1869 ntb_db_clear(qp->ndev, qp_bit); 1870 ntb_db_clear_mask(qp->ndev, qp_bit); 1871 1872 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); 1873 1874 return qp; 1875 1876 err2: 1877 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 1878 kfree(entry); 1879 err1: 1880 qp->rx_alloc_entry = 0; 1881 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 1882 kfree(entry); 1883 if (qp->tx_dma_chan) 1884 dma_release_channel(qp->tx_dma_chan); 1885 if (qp->rx_dma_chan) 1886 dma_release_channel(qp->rx_dma_chan); 1887 nt->qp_bitmap_free |= qp_bit; 1888 err: 1889 return NULL; 1890 } 1891 EXPORT_SYMBOL_GPL(ntb_transport_create_queue); 1892 1893 /** 1894 * ntb_transport_free_queue - Frees NTB transport queue 1895 * @qp: NTB queue to be freed 1896 * 1897 * Frees NTB transport queue 1898 */ 1899 void ntb_transport_free_queue(struct ntb_transport_qp *qp) 1900 { 1901 struct pci_dev *pdev; 1902 struct ntb_queue_entry *entry; 1903 u64 qp_bit; 1904 1905 if (!qp) 1906 return; 1907 1908 pdev = qp->ndev->pdev; 1909 1910 qp->active = false; 1911 1912 if (qp->tx_dma_chan) { 1913 struct dma_chan *chan = qp->tx_dma_chan; 1914 /* Putting the dma_chan to NULL will force any new traffic to be 1915 * processed by the CPU instead of the DAM engine 1916 */ 1917 qp->tx_dma_chan = NULL; 1918 1919 /* Try to be nice and wait for any queued DMA engine 1920 * transactions to process before smashing it with a rock 1921 */ 1922 dma_sync_wait(chan, qp->last_cookie); 1923 dmaengine_terminate_all(chan); 1924 dma_release_channel(chan); 1925 } 1926 1927 if (qp->rx_dma_chan) { 1928 struct dma_chan *chan = qp->rx_dma_chan; 1929 /* Putting the dma_chan to NULL will force any new traffic to be 1930 * processed by the CPU instead of the DAM engine 1931 */ 1932 qp->rx_dma_chan = NULL; 1933 1934 /* Try to be nice and wait for any queued DMA engine 1935 * transactions to process before smashing it with a rock 1936 */ 1937 dma_sync_wait(chan, qp->last_cookie); 1938 dmaengine_terminate_all(chan); 1939 dma_release_channel(chan); 1940 } 1941 1942 qp_bit = BIT_ULL(qp->qp_num); 1943 1944 ntb_db_set_mask(qp->ndev, qp_bit); 1945 tasklet_kill(&qp->rxc_db_work); 1946 1947 cancel_delayed_work_sync(&qp->link_work); 1948 1949 qp->cb_data = NULL; 1950 qp->rx_handler = NULL; 1951 qp->tx_handler = NULL; 1952 qp->event_handler = NULL; 1953 1954 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 1955 kfree(entry); 1956 1957 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) { 1958 dev_warn(&pdev->dev, "Freeing item from non-empty rx_pend_q\n"); 1959 kfree(entry); 1960 } 1961 1962 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) { 1963 dev_warn(&pdev->dev, "Freeing item from non-empty rx_post_q\n"); 1964 kfree(entry); 1965 } 1966 1967 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 1968 kfree(entry); 1969 1970 qp->transport->qp_bitmap_free |= qp_bit; 1971 1972 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); 1973 } 1974 EXPORT_SYMBOL_GPL(ntb_transport_free_queue); 1975 1976 /** 1977 * ntb_transport_rx_remove - Dequeues enqueued rx packet 1978 * @qp: NTB queue to be freed 1979 * @len: pointer to variable to write enqueued buffers length 1980 * 1981 * Dequeues unused buffers from receive queue. Should only be used during 1982 * shutdown of qp. 1983 * 1984 * RETURNS: NULL error value on error, or void* for success. 1985 */ 1986 void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) 1987 { 1988 struct ntb_queue_entry *entry; 1989 void *buf; 1990 1991 if (!qp || qp->client_ready) 1992 return NULL; 1993 1994 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q); 1995 if (!entry) 1996 return NULL; 1997 1998 buf = entry->cb_data; 1999 *len = entry->len; 2000 2001 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q); 2002 2003 return buf; 2004 } 2005 EXPORT_SYMBOL_GPL(ntb_transport_rx_remove); 2006 2007 /** 2008 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry 2009 * @qp: NTB transport layer queue the entry is to be enqueued on 2010 * @cb: per buffer pointer for callback function to use 2011 * @data: pointer to data buffer that incoming packets will be copied into 2012 * @len: length of the data buffer 2013 * 2014 * Enqueue a new receive buffer onto the transport queue into which a NTB 2015 * payload can be received into. 2016 * 2017 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 2018 */ 2019 int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 2020 unsigned int len) 2021 { 2022 struct ntb_queue_entry *entry; 2023 2024 if (!qp) 2025 return -EINVAL; 2026 2027 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q); 2028 if (!entry) 2029 return -ENOMEM; 2030 2031 entry->cb_data = cb; 2032 entry->buf = data; 2033 entry->len = len; 2034 entry->flags = 0; 2035 entry->retries = 0; 2036 entry->errors = 0; 2037 entry->rx_index = 0; 2038 2039 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q); 2040 2041 if (qp->active) 2042 tasklet_schedule(&qp->rxc_db_work); 2043 2044 return 0; 2045 } 2046 EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue); 2047 2048 /** 2049 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 2050 * @qp: NTB transport layer queue the entry is to be enqueued on 2051 * @cb: per buffer pointer for callback function to use 2052 * @data: pointer to data buffer that will be sent 2053 * @len: length of the data buffer 2054 * 2055 * Enqueue a new transmit buffer onto the transport queue from which a NTB 2056 * payload will be transmitted. This assumes that a lock is being held to 2057 * serialize access to the qp. 2058 * 2059 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 2060 */ 2061 int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 2062 unsigned int len) 2063 { 2064 struct ntb_queue_entry *entry; 2065 int rc; 2066 2067 if (!qp || !qp->link_is_up || !len) 2068 return -EINVAL; 2069 2070 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 2071 if (!entry) { 2072 qp->tx_err_no_buf++; 2073 return -EBUSY; 2074 } 2075 2076 entry->cb_data = cb; 2077 entry->buf = data; 2078 entry->len = len; 2079 entry->flags = 0; 2080 entry->errors = 0; 2081 entry->retries = 0; 2082 entry->tx_index = 0; 2083 2084 rc = ntb_process_tx(qp, entry); 2085 if (rc) 2086 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 2087 &qp->tx_free_q); 2088 2089 return rc; 2090 } 2091 EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue); 2092 2093 /** 2094 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 2095 * @qp: NTB transport layer queue to be enabled 2096 * 2097 * Notify NTB transport layer of client readiness to use queue 2098 */ 2099 void ntb_transport_link_up(struct ntb_transport_qp *qp) 2100 { 2101 if (!qp) 2102 return; 2103 2104 qp->client_ready = true; 2105 2106 if (qp->transport->link_is_up) 2107 schedule_delayed_work(&qp->link_work, 0); 2108 } 2109 EXPORT_SYMBOL_GPL(ntb_transport_link_up); 2110 2111 /** 2112 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 2113 * @qp: NTB transport layer queue to be disabled 2114 * 2115 * Notify NTB transport layer of client's desire to no longer receive data on 2116 * transport queue specified. It is the client's responsibility to ensure all 2117 * entries on queue are purged or otherwise handled appropriately. 2118 */ 2119 void ntb_transport_link_down(struct ntb_transport_qp *qp) 2120 { 2121 int val; 2122 2123 if (!qp) 2124 return; 2125 2126 qp->client_ready = false; 2127 2128 val = ntb_spad_read(qp->ndev, QP_LINKS); 2129 2130 ntb_peer_spad_write(qp->ndev, QP_LINKS, 2131 val & ~BIT(qp->qp_num)); 2132 2133 if (qp->link_is_up) 2134 ntb_send_link_down(qp); 2135 else 2136 cancel_delayed_work_sync(&qp->link_work); 2137 } 2138 EXPORT_SYMBOL_GPL(ntb_transport_link_down); 2139 2140 /** 2141 * ntb_transport_link_query - Query transport link state 2142 * @qp: NTB transport layer queue to be queried 2143 * 2144 * Query connectivity to the remote system of the NTB transport queue 2145 * 2146 * RETURNS: true for link up or false for link down 2147 */ 2148 bool ntb_transport_link_query(struct ntb_transport_qp *qp) 2149 { 2150 if (!qp) 2151 return false; 2152 2153 return qp->link_is_up; 2154 } 2155 EXPORT_SYMBOL_GPL(ntb_transport_link_query); 2156 2157 /** 2158 * ntb_transport_qp_num - Query the qp number 2159 * @qp: NTB transport layer queue to be queried 2160 * 2161 * Query qp number of the NTB transport queue 2162 * 2163 * RETURNS: a zero based number specifying the qp number 2164 */ 2165 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) 2166 { 2167 if (!qp) 2168 return 0; 2169 2170 return qp->qp_num; 2171 } 2172 EXPORT_SYMBOL_GPL(ntb_transport_qp_num); 2173 2174 /** 2175 * ntb_transport_max_size - Query the max payload size of a qp 2176 * @qp: NTB transport layer queue to be queried 2177 * 2178 * Query the maximum payload size permissible on the given qp 2179 * 2180 * RETURNS: the max payload size of a qp 2181 */ 2182 unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) 2183 { 2184 unsigned int max_size; 2185 unsigned int copy_align; 2186 struct dma_chan *rx_chan, *tx_chan; 2187 2188 if (!qp) 2189 return 0; 2190 2191 rx_chan = qp->rx_dma_chan; 2192 tx_chan = qp->tx_dma_chan; 2193 2194 copy_align = max(rx_chan ? rx_chan->device->copy_align : 0, 2195 tx_chan ? tx_chan->device->copy_align : 0); 2196 2197 /* If DMA engine usage is possible, try to find the max size for that */ 2198 max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header); 2199 max_size = round_down(max_size, 1 << copy_align); 2200 2201 return max_size; 2202 } 2203 EXPORT_SYMBOL_GPL(ntb_transport_max_size); 2204 2205 unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp) 2206 { 2207 unsigned int head = qp->tx_index; 2208 unsigned int tail = qp->remote_rx_info->entry; 2209 2210 return tail > head ? tail - head : qp->tx_max_entry + tail - head; 2211 } 2212 EXPORT_SYMBOL_GPL(ntb_transport_tx_free_entry); 2213 2214 static void ntb_transport_doorbell_callback(void *data, int vector) 2215 { 2216 struct ntb_transport_ctx *nt = data; 2217 struct ntb_transport_qp *qp; 2218 u64 db_bits; 2219 unsigned int qp_num; 2220 2221 db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free & 2222 ntb_db_vector_mask(nt->ndev, vector)); 2223 2224 while (db_bits) { 2225 qp_num = __ffs(db_bits); 2226 qp = &nt->qp_vec[qp_num]; 2227 2228 if (qp->active) 2229 tasklet_schedule(&qp->rxc_db_work); 2230 2231 db_bits &= ~BIT_ULL(qp_num); 2232 } 2233 } 2234 2235 static const struct ntb_ctx_ops ntb_transport_ops = { 2236 .link_event = ntb_transport_event_callback, 2237 .db_event = ntb_transport_doorbell_callback, 2238 }; 2239 2240 static struct ntb_client ntb_transport_client = { 2241 .ops = { 2242 .probe = ntb_transport_probe, 2243 .remove = ntb_transport_free, 2244 }, 2245 }; 2246 2247 static int __init ntb_transport_init(void) 2248 { 2249 int rc; 2250 2251 pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER); 2252 2253 if (debugfs_initialized()) 2254 nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 2255 2256 rc = bus_register(&ntb_transport_bus); 2257 if (rc) 2258 goto err_bus; 2259 2260 rc = ntb_register_client(&ntb_transport_client); 2261 if (rc) 2262 goto err_client; 2263 2264 return 0; 2265 2266 err_client: 2267 bus_unregister(&ntb_transport_bus); 2268 err_bus: 2269 debugfs_remove_recursive(nt_debugfs_dir); 2270 return rc; 2271 } 2272 module_init(ntb_transport_init); 2273 2274 static void __exit ntb_transport_exit(void) 2275 { 2276 ntb_unregister_client(&ntb_transport_client); 2277 bus_unregister(&ntb_transport_bus); 2278 debugfs_remove_recursive(nt_debugfs_dir); 2279 } 2280 module_exit(ntb_transport_exit); 2281