1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2015 Intel Corporation. All rights reserved. 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * BSD LICENSE 14 * 15 * Copyright(c) 2015 Intel Corporation. All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 21 * * Redistributions of source code must retain the above copyright 22 * notice, this list of conditions and the following disclaimer. 23 * * Redistributions in binary form must reproduce the above copy 24 * notice, this list of conditions and the following disclaimer in 25 * the documentation and/or other materials provided with the 26 * distribution. 27 * * Neither the name of Intel Corporation nor the names of its 28 * contributors may be used to endorse or promote products derived 29 * from this software without specific prior written permission. 30 * 31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * PCIe NTB Perf Linux driver 44 */ 45 46 #include <linux/init.h> 47 #include <linux/kernel.h> 48 #include <linux/module.h> 49 #include <linux/kthread.h> 50 #include <linux/time.h> 51 #include <linux/timer.h> 52 #include <linux/dma-mapping.h> 53 #include <linux/pci.h> 54 #include <linux/slab.h> 55 #include <linux/spinlock.h> 56 #include <linux/debugfs.h> 57 #include <linux/dmaengine.h> 58 #include <linux/delay.h> 59 #include <linux/sizes.h> 60 #include <linux/ntb.h> 61 #include <linux/mutex.h> 62 63 #define DRIVER_NAME "ntb_perf" 64 #define DRIVER_DESCRIPTION "PCIe NTB Performance Measurement Tool" 65 66 #define DRIVER_LICENSE "Dual BSD/GPL" 67 #define DRIVER_VERSION "1.0" 68 #define DRIVER_AUTHOR "Dave Jiang <dave.jiang@intel.com>" 69 70 #define PERF_LINK_DOWN_TIMEOUT 10 71 #define PERF_VERSION 0xffff0001 72 #define MAX_THREADS 32 73 #define MAX_TEST_SIZE SZ_1M 74 #define MAX_SRCS 32 75 #define DMA_OUT_RESOURCE_TO 50 76 #define DMA_RETRIES 20 77 #define SZ_4G (1ULL << 32) 78 #define MAX_SEG_ORDER 20 /* no larger than 1M for kmalloc buffer */ 79 80 MODULE_LICENSE(DRIVER_LICENSE); 81 MODULE_VERSION(DRIVER_VERSION); 82 MODULE_AUTHOR(DRIVER_AUTHOR); 83 MODULE_DESCRIPTION(DRIVER_DESCRIPTION); 84 85 static struct dentry *perf_debugfs_dir; 86 87 static unsigned long max_mw_size; 88 module_param(max_mw_size, ulong, 0644); 89 MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); 90 91 static unsigned int seg_order = 19; /* 512K */ 92 module_param(seg_order, uint, 0644); 93 MODULE_PARM_DESC(seg_order, "size order [n^2] of buffer segment for testing"); 94 95 static unsigned int run_order = 32; /* 4G */ 96 module_param(run_order, uint, 0644); 97 MODULE_PARM_DESC(run_order, "size order [n^2] of total data to transfer"); 98 99 static bool use_dma; /* default to 0 */ 100 module_param(use_dma, bool, 0644); 101 MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance"); 102 103 struct perf_mw { 104 phys_addr_t phys_addr; 105 resource_size_t phys_size; 106 resource_size_t xlat_align; 107 resource_size_t xlat_align_size; 108 void __iomem *vbase; 109 size_t xlat_size; 110 size_t buf_size; 111 void *virt_addr; 112 dma_addr_t dma_addr; 113 }; 114 115 struct perf_ctx; 116 117 struct pthr_ctx { 118 struct task_struct *thread; 119 struct perf_ctx *perf; 120 atomic_t dma_sync; 121 struct dma_chan *dma_chan; 122 int dma_prep_err; 123 int src_idx; 124 void *srcs[MAX_SRCS]; 125 wait_queue_head_t *wq; 126 int status; 127 u64 copied; 128 u64 diff_us; 129 }; 130 131 struct perf_ctx { 132 struct ntb_dev *ntb; 133 spinlock_t db_lock; 134 struct perf_mw mw; 135 bool link_is_up; 136 struct delayed_work link_work; 137 wait_queue_head_t link_wq; 138 struct dentry *debugfs_node_dir; 139 struct dentry *debugfs_run; 140 struct dentry *debugfs_threads; 141 u8 perf_threads; 142 /* mutex ensures only one set of threads run at once */ 143 struct mutex run_mutex; 144 struct pthr_ctx pthr_ctx[MAX_THREADS]; 145 atomic_t tsync; 146 atomic_t tdone; 147 }; 148 149 enum { 150 VERSION = 0, 151 MW_SZ_HIGH, 152 MW_SZ_LOW, 153 MAX_SPAD 154 }; 155 156 static void perf_link_event(void *ctx) 157 { 158 struct perf_ctx *perf = ctx; 159 160 if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1) { 161 schedule_delayed_work(&perf->link_work, 2*HZ); 162 } else { 163 dev_dbg(&perf->ntb->pdev->dev, "link down\n"); 164 165 if (!perf->link_is_up) 166 cancel_delayed_work_sync(&perf->link_work); 167 168 perf->link_is_up = false; 169 } 170 } 171 172 static void perf_db_event(void *ctx, int vec) 173 { 174 struct perf_ctx *perf = ctx; 175 u64 db_bits, db_mask; 176 177 db_mask = ntb_db_vector_mask(perf->ntb, vec); 178 db_bits = ntb_db_read(perf->ntb); 179 180 dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n", 181 vec, db_mask, db_bits); 182 } 183 184 static const struct ntb_ctx_ops perf_ops = { 185 .link_event = perf_link_event, 186 .db_event = perf_db_event, 187 }; 188 189 static void perf_copy_callback(void *data) 190 { 191 struct pthr_ctx *pctx = data; 192 193 atomic_dec(&pctx->dma_sync); 194 } 195 196 static ssize_t perf_copy(struct pthr_ctx *pctx, char __iomem *dst, 197 char *src, size_t size) 198 { 199 struct perf_ctx *perf = pctx->perf; 200 struct dma_async_tx_descriptor *txd; 201 struct dma_chan *chan = pctx->dma_chan; 202 struct dma_device *device; 203 struct dmaengine_unmap_data *unmap; 204 dma_cookie_t cookie; 205 size_t src_off, dst_off; 206 struct perf_mw *mw = &perf->mw; 207 void __iomem *vbase; 208 void __iomem *dst_vaddr; 209 dma_addr_t dst_phys; 210 int retries = 0; 211 212 if (!use_dma) { 213 memcpy_toio(dst, src, size); 214 return size; 215 } 216 217 if (!chan) { 218 dev_err(&perf->ntb->dev, "DMA engine does not exist\n"); 219 return -EINVAL; 220 } 221 222 device = chan->device; 223 src_off = (uintptr_t)src & ~PAGE_MASK; 224 dst_off = (uintptr_t __force)dst & ~PAGE_MASK; 225 226 if (!is_dma_copy_aligned(device, src_off, dst_off, size)) 227 return -ENODEV; 228 229 vbase = mw->vbase; 230 dst_vaddr = dst; 231 dst_phys = mw->phys_addr + (dst_vaddr - vbase); 232 233 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); 234 if (!unmap) 235 return -ENOMEM; 236 237 unmap->len = size; 238 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src), 239 src_off, size, DMA_TO_DEVICE); 240 if (dma_mapping_error(device->dev, unmap->addr[0])) 241 goto err_get_unmap; 242 243 unmap->to_cnt = 1; 244 245 do { 246 txd = device->device_prep_dma_memcpy(chan, dst_phys, 247 unmap->addr[0], 248 size, DMA_PREP_INTERRUPT); 249 if (!txd) { 250 set_current_state(TASK_INTERRUPTIBLE); 251 schedule_timeout(DMA_OUT_RESOURCE_TO); 252 } 253 } while (!txd && (++retries < DMA_RETRIES)); 254 255 if (!txd) { 256 pctx->dma_prep_err++; 257 goto err_get_unmap; 258 } 259 260 txd->callback = perf_copy_callback; 261 txd->callback_param = pctx; 262 dma_set_unmap(txd, unmap); 263 264 cookie = dmaengine_submit(txd); 265 if (dma_submit_error(cookie)) 266 goto err_set_unmap; 267 268 atomic_inc(&pctx->dma_sync); 269 dma_async_issue_pending(chan); 270 271 return size; 272 273 err_set_unmap: 274 dmaengine_unmap_put(unmap); 275 err_get_unmap: 276 dmaengine_unmap_put(unmap); 277 return 0; 278 } 279 280 static int perf_move_data(struct pthr_ctx *pctx, char __iomem *dst, char *src, 281 u64 buf_size, u64 win_size, u64 total) 282 { 283 int chunks, total_chunks, i; 284 int copied_chunks = 0; 285 u64 copied = 0, result; 286 char __iomem *tmp = dst; 287 u64 perf, diff_us; 288 ktime_t kstart, kstop, kdiff; 289 unsigned long last_sleep = jiffies; 290 291 chunks = div64_u64(win_size, buf_size); 292 total_chunks = div64_u64(total, buf_size); 293 kstart = ktime_get(); 294 295 for (i = 0; i < total_chunks; i++) { 296 result = perf_copy(pctx, tmp, src, buf_size); 297 copied += result; 298 copied_chunks++; 299 if (copied_chunks == chunks) { 300 tmp = dst; 301 copied_chunks = 0; 302 } else 303 tmp += buf_size; 304 305 /* Probably should schedule every 5s to prevent soft hang. */ 306 if (unlikely((jiffies - last_sleep) > 5 * HZ)) { 307 last_sleep = jiffies; 308 set_current_state(TASK_INTERRUPTIBLE); 309 schedule_timeout(1); 310 } 311 312 if (unlikely(kthread_should_stop())) 313 break; 314 } 315 316 if (use_dma) { 317 pr_debug("%s: All DMA descriptors submitted\n", current->comm); 318 while (atomic_read(&pctx->dma_sync) != 0) { 319 if (kthread_should_stop()) 320 break; 321 msleep(20); 322 } 323 } 324 325 kstop = ktime_get(); 326 kdiff = ktime_sub(kstop, kstart); 327 diff_us = ktime_to_us(kdiff); 328 329 pr_debug("%s: copied %llu bytes\n", current->comm, copied); 330 331 pr_debug("%s: lasted %llu usecs\n", current->comm, diff_us); 332 333 perf = div64_u64(copied, diff_us); 334 335 pr_debug("%s: MBytes/s: %llu\n", current->comm, perf); 336 337 pctx->copied = copied; 338 pctx->diff_us = diff_us; 339 340 return 0; 341 } 342 343 static bool perf_dma_filter_fn(struct dma_chan *chan, void *node) 344 { 345 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; 346 } 347 348 static int ntb_perf_thread(void *data) 349 { 350 struct pthr_ctx *pctx = data; 351 struct perf_ctx *perf = pctx->perf; 352 struct pci_dev *pdev = perf->ntb->pdev; 353 struct perf_mw *mw = &perf->mw; 354 char __iomem *dst; 355 u64 win_size, buf_size, total; 356 void *src; 357 int rc, node, i; 358 struct dma_chan *dma_chan = NULL; 359 360 pr_debug("kthread %s starting...\n", current->comm); 361 362 node = dev_to_node(&pdev->dev); 363 364 if (use_dma && !pctx->dma_chan) { 365 dma_cap_mask_t dma_mask; 366 367 dma_cap_zero(dma_mask); 368 dma_cap_set(DMA_MEMCPY, dma_mask); 369 dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn, 370 (void *)(unsigned long)node); 371 if (!dma_chan) { 372 pr_warn("%s: cannot acquire DMA channel, quitting\n", 373 current->comm); 374 return -ENODEV; 375 } 376 pctx->dma_chan = dma_chan; 377 } 378 379 for (i = 0; i < MAX_SRCS; i++) { 380 pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node); 381 if (!pctx->srcs[i]) { 382 rc = -ENOMEM; 383 goto err; 384 } 385 } 386 387 win_size = mw->phys_size; 388 buf_size = 1ULL << seg_order; 389 total = 1ULL << run_order; 390 391 if (buf_size > MAX_TEST_SIZE) 392 buf_size = MAX_TEST_SIZE; 393 394 dst = (char __iomem *)mw->vbase; 395 396 atomic_inc(&perf->tsync); 397 while (atomic_read(&perf->tsync) != perf->perf_threads) 398 schedule(); 399 400 src = pctx->srcs[pctx->src_idx]; 401 pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1); 402 403 rc = perf_move_data(pctx, dst, src, buf_size, win_size, total); 404 405 atomic_dec(&perf->tsync); 406 407 if (rc < 0) { 408 pr_err("%s: failed\n", current->comm); 409 rc = -ENXIO; 410 goto err; 411 } 412 413 for (i = 0; i < MAX_SRCS; i++) { 414 kfree(pctx->srcs[i]); 415 pctx->srcs[i] = NULL; 416 } 417 418 atomic_inc(&perf->tdone); 419 wake_up(pctx->wq); 420 rc = 0; 421 goto done; 422 423 err: 424 for (i = 0; i < MAX_SRCS; i++) { 425 kfree(pctx->srcs[i]); 426 pctx->srcs[i] = NULL; 427 } 428 429 if (dma_chan) { 430 dma_release_channel(dma_chan); 431 pctx->dma_chan = NULL; 432 } 433 434 done: 435 /* Wait until we are told to stop */ 436 for (;;) { 437 set_current_state(TASK_INTERRUPTIBLE); 438 if (kthread_should_stop()) 439 break; 440 schedule(); 441 } 442 __set_current_state(TASK_RUNNING); 443 444 return rc; 445 } 446 447 static void perf_free_mw(struct perf_ctx *perf) 448 { 449 struct perf_mw *mw = &perf->mw; 450 struct pci_dev *pdev = perf->ntb->pdev; 451 452 if (!mw->virt_addr) 453 return; 454 455 ntb_mw_clear_trans(perf->ntb, 0); 456 dma_free_coherent(&pdev->dev, mw->buf_size, 457 mw->virt_addr, mw->dma_addr); 458 mw->xlat_size = 0; 459 mw->buf_size = 0; 460 mw->virt_addr = NULL; 461 } 462 463 static int perf_set_mw(struct perf_ctx *perf, resource_size_t size) 464 { 465 struct perf_mw *mw = &perf->mw; 466 size_t xlat_size, buf_size; 467 int rc; 468 469 if (!size) 470 return -EINVAL; 471 472 xlat_size = round_up(size, mw->xlat_align_size); 473 buf_size = round_up(size, mw->xlat_align); 474 475 if (mw->xlat_size == xlat_size) 476 return 0; 477 478 if (mw->buf_size) 479 perf_free_mw(perf); 480 481 mw->xlat_size = xlat_size; 482 mw->buf_size = buf_size; 483 484 mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size, 485 &mw->dma_addr, GFP_KERNEL); 486 if (!mw->virt_addr) { 487 mw->xlat_size = 0; 488 mw->buf_size = 0; 489 } 490 491 rc = ntb_mw_set_trans(perf->ntb, 0, mw->dma_addr, mw->xlat_size); 492 if (rc) { 493 dev_err(&perf->ntb->dev, "Unable to set mw0 translation\n"); 494 perf_free_mw(perf); 495 return -EIO; 496 } 497 498 return 0; 499 } 500 501 static void perf_link_work(struct work_struct *work) 502 { 503 struct perf_ctx *perf = 504 container_of(work, struct perf_ctx, link_work.work); 505 struct ntb_dev *ndev = perf->ntb; 506 struct pci_dev *pdev = ndev->pdev; 507 u32 val; 508 u64 size; 509 int rc; 510 511 dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__); 512 513 size = perf->mw.phys_size; 514 515 if (max_mw_size && size > max_mw_size) 516 size = max_mw_size; 517 518 ntb_peer_spad_write(ndev, MW_SZ_HIGH, upper_32_bits(size)); 519 ntb_peer_spad_write(ndev, MW_SZ_LOW, lower_32_bits(size)); 520 ntb_peer_spad_write(ndev, VERSION, PERF_VERSION); 521 522 /* now read what peer wrote */ 523 val = ntb_spad_read(ndev, VERSION); 524 if (val != PERF_VERSION) { 525 dev_dbg(&pdev->dev, "Remote version = %#x\n", val); 526 goto out; 527 } 528 529 val = ntb_spad_read(ndev, MW_SZ_HIGH); 530 size = (u64)val << 32; 531 532 val = ntb_spad_read(ndev, MW_SZ_LOW); 533 size |= val; 534 535 dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size); 536 537 rc = perf_set_mw(perf, size); 538 if (rc) 539 goto out1; 540 541 perf->link_is_up = true; 542 wake_up(&perf->link_wq); 543 544 return; 545 546 out1: 547 perf_free_mw(perf); 548 549 out: 550 if (ntb_link_is_up(ndev, NULL, NULL) == 1) 551 schedule_delayed_work(&perf->link_work, 552 msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT)); 553 } 554 555 static int perf_setup_mw(struct ntb_dev *ntb, struct perf_ctx *perf) 556 { 557 struct perf_mw *mw; 558 int rc; 559 560 mw = &perf->mw; 561 562 rc = ntb_mw_get_range(ntb, 0, &mw->phys_addr, &mw->phys_size, 563 &mw->xlat_align, &mw->xlat_align_size); 564 if (rc) 565 return rc; 566 567 perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size); 568 if (!mw->vbase) 569 return -ENOMEM; 570 571 return 0; 572 } 573 574 static ssize_t debugfs_run_read(struct file *filp, char __user *ubuf, 575 size_t count, loff_t *offp) 576 { 577 struct perf_ctx *perf = filp->private_data; 578 char *buf; 579 ssize_t ret, out_off = 0; 580 struct pthr_ctx *pctx; 581 int i; 582 u64 rate; 583 584 if (!perf) 585 return 0; 586 587 buf = kmalloc(1024, GFP_KERNEL); 588 if (!buf) 589 return -ENOMEM; 590 591 if (mutex_is_locked(&perf->run_mutex)) { 592 out_off = snprintf(buf, 64, "running\n"); 593 goto read_from_buf; 594 } 595 596 for (i = 0; i < MAX_THREADS; i++) { 597 pctx = &perf->pthr_ctx[i]; 598 599 if (pctx->status == -ENODATA) 600 break; 601 602 if (pctx->status) { 603 out_off += snprintf(buf + out_off, 1024 - out_off, 604 "%d: error %d\n", i, 605 pctx->status); 606 continue; 607 } 608 609 rate = div64_u64(pctx->copied, pctx->diff_us); 610 out_off += snprintf(buf + out_off, 1024 - out_off, 611 "%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n", 612 i, pctx->copied, pctx->diff_us, rate); 613 } 614 615 read_from_buf: 616 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_off); 617 kfree(buf); 618 619 return ret; 620 } 621 622 static void threads_cleanup(struct perf_ctx *perf) 623 { 624 struct pthr_ctx *pctx; 625 int i; 626 627 for (i = 0; i < MAX_THREADS; i++) { 628 pctx = &perf->pthr_ctx[i]; 629 if (pctx->thread) { 630 pctx->status = kthread_stop(pctx->thread); 631 pctx->thread = NULL; 632 } 633 } 634 } 635 636 static void perf_clear_thread_status(struct perf_ctx *perf) 637 { 638 int i; 639 640 for (i = 0; i < MAX_THREADS; i++) 641 perf->pthr_ctx[i].status = -ENODATA; 642 } 643 644 static ssize_t debugfs_run_write(struct file *filp, const char __user *ubuf, 645 size_t count, loff_t *offp) 646 { 647 struct perf_ctx *perf = filp->private_data; 648 int node, i; 649 DECLARE_WAIT_QUEUE_HEAD(wq); 650 651 if (wait_event_interruptible(perf->link_wq, perf->link_is_up)) 652 return -ENOLINK; 653 654 if (perf->perf_threads == 0) 655 return -EINVAL; 656 657 if (!mutex_trylock(&perf->run_mutex)) 658 return -EBUSY; 659 660 perf_clear_thread_status(perf); 661 662 if (perf->perf_threads > MAX_THREADS) { 663 perf->perf_threads = MAX_THREADS; 664 pr_info("Reset total threads to: %u\n", MAX_THREADS); 665 } 666 667 /* no greater than 1M */ 668 if (seg_order > MAX_SEG_ORDER) { 669 seg_order = MAX_SEG_ORDER; 670 pr_info("Fix seg_order to %u\n", seg_order); 671 } 672 673 if (run_order < seg_order) { 674 run_order = seg_order; 675 pr_info("Fix run_order to %u\n", run_order); 676 } 677 678 node = dev_to_node(&perf->ntb->pdev->dev); 679 atomic_set(&perf->tdone, 0); 680 681 /* launch kernel thread */ 682 for (i = 0; i < perf->perf_threads; i++) { 683 struct pthr_ctx *pctx; 684 685 pctx = &perf->pthr_ctx[i]; 686 atomic_set(&pctx->dma_sync, 0); 687 pctx->perf = perf; 688 pctx->wq = &wq; 689 pctx->thread = 690 kthread_create_on_node(ntb_perf_thread, 691 (void *)pctx, 692 node, "ntb_perf %d", i); 693 if (IS_ERR(pctx->thread)) { 694 pctx->thread = NULL; 695 goto err; 696 } else { 697 wake_up_process(pctx->thread); 698 } 699 } 700 701 wait_event_interruptible(wq, 702 atomic_read(&perf->tdone) == perf->perf_threads); 703 704 threads_cleanup(perf); 705 mutex_unlock(&perf->run_mutex); 706 return count; 707 708 err: 709 threads_cleanup(perf); 710 mutex_unlock(&perf->run_mutex); 711 return -ENXIO; 712 } 713 714 static const struct file_operations ntb_perf_debugfs_run = { 715 .owner = THIS_MODULE, 716 .open = simple_open, 717 .read = debugfs_run_read, 718 .write = debugfs_run_write, 719 }; 720 721 static int perf_debugfs_setup(struct perf_ctx *perf) 722 { 723 struct pci_dev *pdev = perf->ntb->pdev; 724 725 if (!debugfs_initialized()) 726 return -ENODEV; 727 728 if (!perf_debugfs_dir) { 729 perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 730 if (!perf_debugfs_dir) 731 return -ENODEV; 732 } 733 734 perf->debugfs_node_dir = debugfs_create_dir(pci_name(pdev), 735 perf_debugfs_dir); 736 if (!perf->debugfs_node_dir) 737 return -ENODEV; 738 739 perf->debugfs_run = debugfs_create_file("run", S_IRUSR | S_IWUSR, 740 perf->debugfs_node_dir, perf, 741 &ntb_perf_debugfs_run); 742 if (!perf->debugfs_run) 743 return -ENODEV; 744 745 perf->debugfs_threads = debugfs_create_u8("threads", S_IRUSR | S_IWUSR, 746 perf->debugfs_node_dir, 747 &perf->perf_threads); 748 if (!perf->debugfs_threads) 749 return -ENODEV; 750 751 return 0; 752 } 753 754 static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb) 755 { 756 struct pci_dev *pdev = ntb->pdev; 757 struct perf_ctx *perf; 758 int node; 759 int rc = 0; 760 761 if (ntb_spad_count(ntb) < MAX_SPAD) { 762 dev_err(&ntb->dev, "Not enough scratch pad registers for %s", 763 DRIVER_NAME); 764 return -EIO; 765 } 766 767 node = dev_to_node(&pdev->dev); 768 769 perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node); 770 if (!perf) { 771 rc = -ENOMEM; 772 goto err_perf; 773 } 774 775 perf->ntb = ntb; 776 perf->perf_threads = 1; 777 atomic_set(&perf->tsync, 0); 778 mutex_init(&perf->run_mutex); 779 spin_lock_init(&perf->db_lock); 780 perf_setup_mw(ntb, perf); 781 init_waitqueue_head(&perf->link_wq); 782 INIT_DELAYED_WORK(&perf->link_work, perf_link_work); 783 784 rc = ntb_set_ctx(ntb, perf, &perf_ops); 785 if (rc) 786 goto err_ctx; 787 788 perf->link_is_up = false; 789 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 790 ntb_link_event(ntb); 791 792 rc = perf_debugfs_setup(perf); 793 if (rc) 794 goto err_ctx; 795 796 perf_clear_thread_status(perf); 797 798 return 0; 799 800 err_ctx: 801 cancel_delayed_work_sync(&perf->link_work); 802 kfree(perf); 803 err_perf: 804 return rc; 805 } 806 807 static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb) 808 { 809 struct perf_ctx *perf = ntb->ctx; 810 int i; 811 812 dev_dbg(&perf->ntb->dev, "%s called\n", __func__); 813 814 mutex_lock(&perf->run_mutex); 815 816 cancel_delayed_work_sync(&perf->link_work); 817 818 ntb_clear_ctx(ntb); 819 ntb_link_disable(ntb); 820 821 debugfs_remove_recursive(perf_debugfs_dir); 822 perf_debugfs_dir = NULL; 823 824 if (use_dma) { 825 for (i = 0; i < MAX_THREADS; i++) { 826 struct pthr_ctx *pctx = &perf->pthr_ctx[i]; 827 828 if (pctx->dma_chan) 829 dma_release_channel(pctx->dma_chan); 830 } 831 } 832 833 kfree(perf); 834 } 835 836 static struct ntb_client perf_client = { 837 .ops = { 838 .probe = perf_probe, 839 .remove = perf_remove, 840 }, 841 }; 842 module_ntb_client(perf_client); 843