xref: /openbmc/linux/drivers/ntb/test/ntb_perf.c (revision 80483c3a)
1 /*
2  * This file is provided under a dual BSD/GPLv2 license.  When using or
3  *   redistributing this file, you may do so under either license.
4  *
5  *   GPL LICENSE SUMMARY
6  *
7  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
8  *
9  *   This program is free software; you can redistribute it and/or modify
10  *   it under the terms of version 2 of the GNU General Public License as
11  *   published by the Free Software Foundation.
12  *
13  *   BSD LICENSE
14  *
15  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
16  *
17  *   Redistribution and use in source and binary forms, with or without
18  *   modification, are permitted provided that the following conditions
19  *   are met:
20  *
21  *     * Redistributions of source code must retain the above copyright
22  *       notice, this list of conditions and the following disclaimer.
23  *     * Redistributions in binary form must reproduce the above copy
24  *       notice, this list of conditions and the following disclaimer in
25  *       the documentation and/or other materials provided with the
26  *       distribution.
27  *     * Neither the name of Intel Corporation nor the names of its
28  *       contributors may be used to endorse or promote products derived
29  *       from this software without specific prior written permission.
30  *
31  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42  *
43  *   PCIe NTB Perf Linux driver
44  */
45 
46 #include <linux/init.h>
47 #include <linux/kernel.h>
48 #include <linux/module.h>
49 #include <linux/kthread.h>
50 #include <linux/time.h>
51 #include <linux/timer.h>
52 #include <linux/dma-mapping.h>
53 #include <linux/pci.h>
54 #include <linux/slab.h>
55 #include <linux/spinlock.h>
56 #include <linux/debugfs.h>
57 #include <linux/dmaengine.h>
58 #include <linux/delay.h>
59 #include <linux/sizes.h>
60 #include <linux/ntb.h>
61 #include <linux/mutex.h>
62 
63 #define DRIVER_NAME		"ntb_perf"
64 #define DRIVER_DESCRIPTION	"PCIe NTB Performance Measurement Tool"
65 
66 #define DRIVER_LICENSE		"Dual BSD/GPL"
67 #define DRIVER_VERSION		"1.0"
68 #define DRIVER_AUTHOR		"Dave Jiang <dave.jiang@intel.com>"
69 
70 #define PERF_LINK_DOWN_TIMEOUT	10
71 #define PERF_VERSION		0xffff0001
72 #define MAX_THREADS		32
73 #define MAX_TEST_SIZE		SZ_1M
74 #define MAX_SRCS		32
75 #define DMA_OUT_RESOURCE_TO	50
76 #define DMA_RETRIES		20
77 #define SZ_4G			(1ULL << 32)
78 #define MAX_SEG_ORDER		20 /* no larger than 1M for kmalloc buffer */
79 
80 MODULE_LICENSE(DRIVER_LICENSE);
81 MODULE_VERSION(DRIVER_VERSION);
82 MODULE_AUTHOR(DRIVER_AUTHOR);
83 MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
84 
85 static struct dentry *perf_debugfs_dir;
86 
87 static unsigned long max_mw_size;
88 module_param(max_mw_size, ulong, 0644);
89 MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
90 
91 static unsigned int seg_order = 19; /* 512K */
92 module_param(seg_order, uint, 0644);
93 MODULE_PARM_DESC(seg_order, "size order [n^2] of buffer segment for testing");
94 
95 static unsigned int run_order = 32; /* 4G */
96 module_param(run_order, uint, 0644);
97 MODULE_PARM_DESC(run_order, "size order [n^2] of total data to transfer");
98 
99 static bool use_dma; /* default to 0 */
100 module_param(use_dma, bool, 0644);
101 MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance");
102 
103 struct perf_mw {
104 	phys_addr_t	phys_addr;
105 	resource_size_t	phys_size;
106 	resource_size_t	xlat_align;
107 	resource_size_t	xlat_align_size;
108 	void __iomem	*vbase;
109 	size_t		xlat_size;
110 	size_t		buf_size;
111 	void		*virt_addr;
112 	dma_addr_t	dma_addr;
113 };
114 
115 struct perf_ctx;
116 
117 struct pthr_ctx {
118 	struct task_struct	*thread;
119 	struct perf_ctx		*perf;
120 	atomic_t		dma_sync;
121 	struct dma_chan		*dma_chan;
122 	int			dma_prep_err;
123 	int			src_idx;
124 	void			*srcs[MAX_SRCS];
125 	wait_queue_head_t       *wq;
126 	int			status;
127 	u64			copied;
128 	u64			diff_us;
129 };
130 
131 struct perf_ctx {
132 	struct ntb_dev		*ntb;
133 	spinlock_t		db_lock;
134 	struct perf_mw		mw;
135 	bool			link_is_up;
136 	struct delayed_work	link_work;
137 	wait_queue_head_t	link_wq;
138 	struct dentry		*debugfs_node_dir;
139 	struct dentry		*debugfs_run;
140 	struct dentry		*debugfs_threads;
141 	u8			perf_threads;
142 	/* mutex ensures only one set of threads run at once */
143 	struct mutex		run_mutex;
144 	struct pthr_ctx		pthr_ctx[MAX_THREADS];
145 	atomic_t		tsync;
146 	atomic_t                tdone;
147 };
148 
149 enum {
150 	VERSION = 0,
151 	MW_SZ_HIGH,
152 	MW_SZ_LOW,
153 	MAX_SPAD
154 };
155 
156 static void perf_link_event(void *ctx)
157 {
158 	struct perf_ctx *perf = ctx;
159 
160 	if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1) {
161 		schedule_delayed_work(&perf->link_work, 2*HZ);
162 	} else {
163 		dev_dbg(&perf->ntb->pdev->dev, "link down\n");
164 
165 		if (!perf->link_is_up)
166 			cancel_delayed_work_sync(&perf->link_work);
167 
168 		perf->link_is_up = false;
169 	}
170 }
171 
172 static void perf_db_event(void *ctx, int vec)
173 {
174 	struct perf_ctx *perf = ctx;
175 	u64 db_bits, db_mask;
176 
177 	db_mask = ntb_db_vector_mask(perf->ntb, vec);
178 	db_bits = ntb_db_read(perf->ntb);
179 
180 	dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
181 		vec, db_mask, db_bits);
182 }
183 
184 static const struct ntb_ctx_ops perf_ops = {
185 	.link_event = perf_link_event,
186 	.db_event = perf_db_event,
187 };
188 
189 static void perf_copy_callback(void *data)
190 {
191 	struct pthr_ctx *pctx = data;
192 
193 	atomic_dec(&pctx->dma_sync);
194 }
195 
196 static ssize_t perf_copy(struct pthr_ctx *pctx, char __iomem *dst,
197 			 char *src, size_t size)
198 {
199 	struct perf_ctx *perf = pctx->perf;
200 	struct dma_async_tx_descriptor *txd;
201 	struct dma_chan *chan = pctx->dma_chan;
202 	struct dma_device *device;
203 	struct dmaengine_unmap_data *unmap;
204 	dma_cookie_t cookie;
205 	size_t src_off, dst_off;
206 	struct perf_mw *mw = &perf->mw;
207 	void __iomem *vbase;
208 	void __iomem *dst_vaddr;
209 	dma_addr_t dst_phys;
210 	int retries = 0;
211 
212 	if (!use_dma) {
213 		memcpy_toio(dst, src, size);
214 		return size;
215 	}
216 
217 	if (!chan) {
218 		dev_err(&perf->ntb->dev, "DMA engine does not exist\n");
219 		return -EINVAL;
220 	}
221 
222 	device = chan->device;
223 	src_off = (uintptr_t)src & ~PAGE_MASK;
224 	dst_off = (uintptr_t __force)dst & ~PAGE_MASK;
225 
226 	if (!is_dma_copy_aligned(device, src_off, dst_off, size))
227 		return -ENODEV;
228 
229 	vbase = mw->vbase;
230 	dst_vaddr = dst;
231 	dst_phys = mw->phys_addr + (dst_vaddr - vbase);
232 
233 	unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT);
234 	if (!unmap)
235 		return -ENOMEM;
236 
237 	unmap->len = size;
238 	unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src),
239 				      src_off, size, DMA_TO_DEVICE);
240 	if (dma_mapping_error(device->dev, unmap->addr[0]))
241 		goto err_get_unmap;
242 
243 	unmap->to_cnt = 1;
244 
245 	do {
246 		txd = device->device_prep_dma_memcpy(chan, dst_phys,
247 						     unmap->addr[0],
248 						     size, DMA_PREP_INTERRUPT);
249 		if (!txd) {
250 			set_current_state(TASK_INTERRUPTIBLE);
251 			schedule_timeout(DMA_OUT_RESOURCE_TO);
252 		}
253 	} while (!txd && (++retries < DMA_RETRIES));
254 
255 	if (!txd) {
256 		pctx->dma_prep_err++;
257 		goto err_get_unmap;
258 	}
259 
260 	txd->callback = perf_copy_callback;
261 	txd->callback_param = pctx;
262 	dma_set_unmap(txd, unmap);
263 
264 	cookie = dmaengine_submit(txd);
265 	if (dma_submit_error(cookie))
266 		goto err_set_unmap;
267 
268 	atomic_inc(&pctx->dma_sync);
269 	dma_async_issue_pending(chan);
270 
271 	return size;
272 
273 err_set_unmap:
274 	dmaengine_unmap_put(unmap);
275 err_get_unmap:
276 	dmaengine_unmap_put(unmap);
277 	return 0;
278 }
279 
280 static int perf_move_data(struct pthr_ctx *pctx, char __iomem *dst, char *src,
281 			  u64 buf_size, u64 win_size, u64 total)
282 {
283 	int chunks, total_chunks, i;
284 	int copied_chunks = 0;
285 	u64 copied = 0, result;
286 	char __iomem *tmp = dst;
287 	u64 perf, diff_us;
288 	ktime_t kstart, kstop, kdiff;
289 	unsigned long last_sleep = jiffies;
290 
291 	chunks = div64_u64(win_size, buf_size);
292 	total_chunks = div64_u64(total, buf_size);
293 	kstart = ktime_get();
294 
295 	for (i = 0; i < total_chunks; i++) {
296 		result = perf_copy(pctx, tmp, src, buf_size);
297 		copied += result;
298 		copied_chunks++;
299 		if (copied_chunks == chunks) {
300 			tmp = dst;
301 			copied_chunks = 0;
302 		} else
303 			tmp += buf_size;
304 
305 		/* Probably should schedule every 5s to prevent soft hang. */
306 		if (unlikely((jiffies - last_sleep) > 5 * HZ)) {
307 			last_sleep = jiffies;
308 			set_current_state(TASK_INTERRUPTIBLE);
309 			schedule_timeout(1);
310 		}
311 
312 		if (unlikely(kthread_should_stop()))
313 			break;
314 	}
315 
316 	if (use_dma) {
317 		pr_debug("%s: All DMA descriptors submitted\n", current->comm);
318 		while (atomic_read(&pctx->dma_sync) != 0) {
319 			if (kthread_should_stop())
320 				break;
321 			msleep(20);
322 		}
323 	}
324 
325 	kstop = ktime_get();
326 	kdiff = ktime_sub(kstop, kstart);
327 	diff_us = ktime_to_us(kdiff);
328 
329 	pr_debug("%s: copied %llu bytes\n", current->comm, copied);
330 
331 	pr_debug("%s: lasted %llu usecs\n", current->comm, diff_us);
332 
333 	perf = div64_u64(copied, diff_us);
334 
335 	pr_debug("%s: MBytes/s: %llu\n", current->comm, perf);
336 
337 	pctx->copied = copied;
338 	pctx->diff_us = diff_us;
339 
340 	return 0;
341 }
342 
343 static bool perf_dma_filter_fn(struct dma_chan *chan, void *node)
344 {
345 	return dev_to_node(&chan->dev->device) == (int)(unsigned long)node;
346 }
347 
348 static int ntb_perf_thread(void *data)
349 {
350 	struct pthr_ctx *pctx = data;
351 	struct perf_ctx *perf = pctx->perf;
352 	struct pci_dev *pdev = perf->ntb->pdev;
353 	struct perf_mw *mw = &perf->mw;
354 	char __iomem *dst;
355 	u64 win_size, buf_size, total;
356 	void *src;
357 	int rc, node, i;
358 	struct dma_chan *dma_chan = NULL;
359 
360 	pr_debug("kthread %s starting...\n", current->comm);
361 
362 	node = dev_to_node(&pdev->dev);
363 
364 	if (use_dma && !pctx->dma_chan) {
365 		dma_cap_mask_t dma_mask;
366 
367 		dma_cap_zero(dma_mask);
368 		dma_cap_set(DMA_MEMCPY, dma_mask);
369 		dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn,
370 					       (void *)(unsigned long)node);
371 		if (!dma_chan) {
372 			pr_warn("%s: cannot acquire DMA channel, quitting\n",
373 				current->comm);
374 			return -ENODEV;
375 		}
376 		pctx->dma_chan = dma_chan;
377 	}
378 
379 	for (i = 0; i < MAX_SRCS; i++) {
380 		pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node);
381 		if (!pctx->srcs[i]) {
382 			rc = -ENOMEM;
383 			goto err;
384 		}
385 	}
386 
387 	win_size = mw->phys_size;
388 	buf_size = 1ULL << seg_order;
389 	total = 1ULL << run_order;
390 
391 	if (buf_size > MAX_TEST_SIZE)
392 		buf_size = MAX_TEST_SIZE;
393 
394 	dst = (char __iomem *)mw->vbase;
395 
396 	atomic_inc(&perf->tsync);
397 	while (atomic_read(&perf->tsync) != perf->perf_threads)
398 		schedule();
399 
400 	src = pctx->srcs[pctx->src_idx];
401 	pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1);
402 
403 	rc = perf_move_data(pctx, dst, src, buf_size, win_size, total);
404 
405 	atomic_dec(&perf->tsync);
406 
407 	if (rc < 0) {
408 		pr_err("%s: failed\n", current->comm);
409 		rc = -ENXIO;
410 		goto err;
411 	}
412 
413 	for (i = 0; i < MAX_SRCS; i++) {
414 		kfree(pctx->srcs[i]);
415 		pctx->srcs[i] = NULL;
416 	}
417 
418 	atomic_inc(&perf->tdone);
419 	wake_up(pctx->wq);
420 	rc = 0;
421 	goto done;
422 
423 err:
424 	for (i = 0; i < MAX_SRCS; i++) {
425 		kfree(pctx->srcs[i]);
426 		pctx->srcs[i] = NULL;
427 	}
428 
429 	if (dma_chan) {
430 		dma_release_channel(dma_chan);
431 		pctx->dma_chan = NULL;
432 	}
433 
434 done:
435 	/* Wait until we are told to stop */
436 	for (;;) {
437 		set_current_state(TASK_INTERRUPTIBLE);
438 		if (kthread_should_stop())
439 			break;
440 		schedule();
441 	}
442 	__set_current_state(TASK_RUNNING);
443 
444 	return rc;
445 }
446 
447 static void perf_free_mw(struct perf_ctx *perf)
448 {
449 	struct perf_mw *mw = &perf->mw;
450 	struct pci_dev *pdev = perf->ntb->pdev;
451 
452 	if (!mw->virt_addr)
453 		return;
454 
455 	ntb_mw_clear_trans(perf->ntb, 0);
456 	dma_free_coherent(&pdev->dev, mw->buf_size,
457 			  mw->virt_addr, mw->dma_addr);
458 	mw->xlat_size = 0;
459 	mw->buf_size = 0;
460 	mw->virt_addr = NULL;
461 }
462 
463 static int perf_set_mw(struct perf_ctx *perf, resource_size_t size)
464 {
465 	struct perf_mw *mw = &perf->mw;
466 	size_t xlat_size, buf_size;
467 	int rc;
468 
469 	if (!size)
470 		return -EINVAL;
471 
472 	xlat_size = round_up(size, mw->xlat_align_size);
473 	buf_size = round_up(size, mw->xlat_align);
474 
475 	if (mw->xlat_size == xlat_size)
476 		return 0;
477 
478 	if (mw->buf_size)
479 		perf_free_mw(perf);
480 
481 	mw->xlat_size = xlat_size;
482 	mw->buf_size = buf_size;
483 
484 	mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size,
485 					   &mw->dma_addr, GFP_KERNEL);
486 	if (!mw->virt_addr) {
487 		mw->xlat_size = 0;
488 		mw->buf_size = 0;
489 	}
490 
491 	rc = ntb_mw_set_trans(perf->ntb, 0, mw->dma_addr, mw->xlat_size);
492 	if (rc) {
493 		dev_err(&perf->ntb->dev, "Unable to set mw0 translation\n");
494 		perf_free_mw(perf);
495 		return -EIO;
496 	}
497 
498 	return 0;
499 }
500 
501 static void perf_link_work(struct work_struct *work)
502 {
503 	struct perf_ctx *perf =
504 		container_of(work, struct perf_ctx, link_work.work);
505 	struct ntb_dev *ndev = perf->ntb;
506 	struct pci_dev *pdev = ndev->pdev;
507 	u32 val;
508 	u64 size;
509 	int rc;
510 
511 	dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__);
512 
513 	size = perf->mw.phys_size;
514 
515 	if (max_mw_size && size > max_mw_size)
516 		size = max_mw_size;
517 
518 	ntb_peer_spad_write(ndev, MW_SZ_HIGH, upper_32_bits(size));
519 	ntb_peer_spad_write(ndev, MW_SZ_LOW, lower_32_bits(size));
520 	ntb_peer_spad_write(ndev, VERSION, PERF_VERSION);
521 
522 	/* now read what peer wrote */
523 	val = ntb_spad_read(ndev, VERSION);
524 	if (val != PERF_VERSION) {
525 		dev_dbg(&pdev->dev, "Remote version = %#x\n", val);
526 		goto out;
527 	}
528 
529 	val = ntb_spad_read(ndev, MW_SZ_HIGH);
530 	size = (u64)val << 32;
531 
532 	val = ntb_spad_read(ndev, MW_SZ_LOW);
533 	size |= val;
534 
535 	dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size);
536 
537 	rc = perf_set_mw(perf, size);
538 	if (rc)
539 		goto out1;
540 
541 	perf->link_is_up = true;
542 	wake_up(&perf->link_wq);
543 
544 	return;
545 
546 out1:
547 	perf_free_mw(perf);
548 
549 out:
550 	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
551 		schedule_delayed_work(&perf->link_work,
552 				      msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT));
553 }
554 
555 static int perf_setup_mw(struct ntb_dev *ntb, struct perf_ctx *perf)
556 {
557 	struct perf_mw *mw;
558 	int rc;
559 
560 	mw = &perf->mw;
561 
562 	rc = ntb_mw_get_range(ntb, 0, &mw->phys_addr, &mw->phys_size,
563 			      &mw->xlat_align, &mw->xlat_align_size);
564 	if (rc)
565 		return rc;
566 
567 	perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size);
568 	if (!mw->vbase)
569 		return -ENOMEM;
570 
571 	return 0;
572 }
573 
574 static ssize_t debugfs_run_read(struct file *filp, char __user *ubuf,
575 				size_t count, loff_t *offp)
576 {
577 	struct perf_ctx *perf = filp->private_data;
578 	char *buf;
579 	ssize_t ret, out_off = 0;
580 	struct pthr_ctx *pctx;
581 	int i;
582 	u64 rate;
583 
584 	if (!perf)
585 		return 0;
586 
587 	buf = kmalloc(1024, GFP_KERNEL);
588 	if (!buf)
589 		return -ENOMEM;
590 
591 	if (mutex_is_locked(&perf->run_mutex)) {
592 		out_off = snprintf(buf, 64, "running\n");
593 		goto read_from_buf;
594 	}
595 
596 	for (i = 0; i < MAX_THREADS; i++) {
597 		pctx = &perf->pthr_ctx[i];
598 
599 		if (pctx->status == -ENODATA)
600 			break;
601 
602 		if (pctx->status) {
603 			out_off += snprintf(buf + out_off, 1024 - out_off,
604 					    "%d: error %d\n", i,
605 					    pctx->status);
606 			continue;
607 		}
608 
609 		rate = div64_u64(pctx->copied, pctx->diff_us);
610 		out_off += snprintf(buf + out_off, 1024 - out_off,
611 			"%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n",
612 			i, pctx->copied, pctx->diff_us, rate);
613 	}
614 
615 read_from_buf:
616 	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_off);
617 	kfree(buf);
618 
619 	return ret;
620 }
621 
622 static void threads_cleanup(struct perf_ctx *perf)
623 {
624 	struct pthr_ctx *pctx;
625 	int i;
626 
627 	for (i = 0; i < MAX_THREADS; i++) {
628 		pctx = &perf->pthr_ctx[i];
629 		if (pctx->thread) {
630 			pctx->status = kthread_stop(pctx->thread);
631 			pctx->thread = NULL;
632 		}
633 	}
634 }
635 
636 static void perf_clear_thread_status(struct perf_ctx *perf)
637 {
638 	int i;
639 
640 	for (i = 0; i < MAX_THREADS; i++)
641 		perf->pthr_ctx[i].status = -ENODATA;
642 }
643 
644 static ssize_t debugfs_run_write(struct file *filp, const char __user *ubuf,
645 				 size_t count, loff_t *offp)
646 {
647 	struct perf_ctx *perf = filp->private_data;
648 	int node, i;
649 	DECLARE_WAIT_QUEUE_HEAD(wq);
650 
651 	if (wait_event_interruptible(perf->link_wq, perf->link_is_up))
652 		return -ENOLINK;
653 
654 	if (perf->perf_threads == 0)
655 		return -EINVAL;
656 
657 	if (!mutex_trylock(&perf->run_mutex))
658 		return -EBUSY;
659 
660 	perf_clear_thread_status(perf);
661 
662 	if (perf->perf_threads > MAX_THREADS) {
663 		perf->perf_threads = MAX_THREADS;
664 		pr_info("Reset total threads to: %u\n", MAX_THREADS);
665 	}
666 
667 	/* no greater than 1M */
668 	if (seg_order > MAX_SEG_ORDER) {
669 		seg_order = MAX_SEG_ORDER;
670 		pr_info("Fix seg_order to %u\n", seg_order);
671 	}
672 
673 	if (run_order < seg_order) {
674 		run_order = seg_order;
675 		pr_info("Fix run_order to %u\n", run_order);
676 	}
677 
678 	node = dev_to_node(&perf->ntb->pdev->dev);
679 	atomic_set(&perf->tdone, 0);
680 
681 	/* launch kernel thread */
682 	for (i = 0; i < perf->perf_threads; i++) {
683 		struct pthr_ctx *pctx;
684 
685 		pctx = &perf->pthr_ctx[i];
686 		atomic_set(&pctx->dma_sync, 0);
687 		pctx->perf = perf;
688 		pctx->wq = &wq;
689 		pctx->thread =
690 			kthread_create_on_node(ntb_perf_thread,
691 					       (void *)pctx,
692 					       node, "ntb_perf %d", i);
693 		if (IS_ERR(pctx->thread)) {
694 			pctx->thread = NULL;
695 			goto err;
696 		} else {
697 			wake_up_process(pctx->thread);
698 		}
699 	}
700 
701 	wait_event_interruptible(wq,
702 		atomic_read(&perf->tdone) == perf->perf_threads);
703 
704 	threads_cleanup(perf);
705 	mutex_unlock(&perf->run_mutex);
706 	return count;
707 
708 err:
709 	threads_cleanup(perf);
710 	mutex_unlock(&perf->run_mutex);
711 	return -ENXIO;
712 }
713 
714 static const struct file_operations ntb_perf_debugfs_run = {
715 	.owner = THIS_MODULE,
716 	.open = simple_open,
717 	.read = debugfs_run_read,
718 	.write = debugfs_run_write,
719 };
720 
721 static int perf_debugfs_setup(struct perf_ctx *perf)
722 {
723 	struct pci_dev *pdev = perf->ntb->pdev;
724 
725 	if (!debugfs_initialized())
726 		return -ENODEV;
727 
728 	if (!perf_debugfs_dir) {
729 		perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
730 		if (!perf_debugfs_dir)
731 			return -ENODEV;
732 	}
733 
734 	perf->debugfs_node_dir = debugfs_create_dir(pci_name(pdev),
735 						    perf_debugfs_dir);
736 	if (!perf->debugfs_node_dir)
737 		return -ENODEV;
738 
739 	perf->debugfs_run = debugfs_create_file("run", S_IRUSR | S_IWUSR,
740 						perf->debugfs_node_dir, perf,
741 						&ntb_perf_debugfs_run);
742 	if (!perf->debugfs_run)
743 		return -ENODEV;
744 
745 	perf->debugfs_threads = debugfs_create_u8("threads", S_IRUSR | S_IWUSR,
746 						  perf->debugfs_node_dir,
747 						  &perf->perf_threads);
748 	if (!perf->debugfs_threads)
749 		return -ENODEV;
750 
751 	return 0;
752 }
753 
754 static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)
755 {
756 	struct pci_dev *pdev = ntb->pdev;
757 	struct perf_ctx *perf;
758 	int node;
759 	int rc = 0;
760 
761 	if (ntb_spad_count(ntb) < MAX_SPAD) {
762 		dev_err(&ntb->dev, "Not enough scratch pad registers for %s",
763 			DRIVER_NAME);
764 		return -EIO;
765 	}
766 
767 	node = dev_to_node(&pdev->dev);
768 
769 	perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node);
770 	if (!perf) {
771 		rc = -ENOMEM;
772 		goto err_perf;
773 	}
774 
775 	perf->ntb = ntb;
776 	perf->perf_threads = 1;
777 	atomic_set(&perf->tsync, 0);
778 	mutex_init(&perf->run_mutex);
779 	spin_lock_init(&perf->db_lock);
780 	perf_setup_mw(ntb, perf);
781 	init_waitqueue_head(&perf->link_wq);
782 	INIT_DELAYED_WORK(&perf->link_work, perf_link_work);
783 
784 	rc = ntb_set_ctx(ntb, perf, &perf_ops);
785 	if (rc)
786 		goto err_ctx;
787 
788 	perf->link_is_up = false;
789 	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
790 	ntb_link_event(ntb);
791 
792 	rc = perf_debugfs_setup(perf);
793 	if (rc)
794 		goto err_ctx;
795 
796 	perf_clear_thread_status(perf);
797 
798 	return 0;
799 
800 err_ctx:
801 	cancel_delayed_work_sync(&perf->link_work);
802 	kfree(perf);
803 err_perf:
804 	return rc;
805 }
806 
807 static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb)
808 {
809 	struct perf_ctx *perf = ntb->ctx;
810 	int i;
811 
812 	dev_dbg(&perf->ntb->dev, "%s called\n", __func__);
813 
814 	mutex_lock(&perf->run_mutex);
815 
816 	cancel_delayed_work_sync(&perf->link_work);
817 
818 	ntb_clear_ctx(ntb);
819 	ntb_link_disable(ntb);
820 
821 	debugfs_remove_recursive(perf_debugfs_dir);
822 	perf_debugfs_dir = NULL;
823 
824 	if (use_dma) {
825 		for (i = 0; i < MAX_THREADS; i++) {
826 			struct pthr_ctx *pctx = &perf->pthr_ctx[i];
827 
828 			if (pctx->dma_chan)
829 				dma_release_channel(pctx->dma_chan);
830 		}
831 	}
832 
833 	kfree(perf);
834 }
835 
836 static struct ntb_client perf_client = {
837 	.ops = {
838 		.probe = perf_probe,
839 		.remove = perf_remove,
840 	},
841 };
842 module_ntb_client(perf_client);
843