xref: /openbmc/linux/drivers/ntb/ntb_transport.c (revision 170d35a5)
1 /*
2  * This file is provided under a dual BSD/GPLv2 license.  When using or
3  *   redistributing this file, you may do so under either license.
4  *
5  *   GPL LICENSE SUMMARY
6  *
7  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
8  *
9  *   This program is free software; you can redistribute it and/or modify
10  *   it under the terms of version 2 of the GNU General Public License as
11  *   published by the Free Software Foundation.
12  *
13  *   BSD LICENSE
14  *
15  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
16  *
17  *   Redistribution and use in source and binary forms, with or without
18  *   modification, are permitted provided that the following conditions
19  *   are met:
20  *
21  *     * Redistributions of source code must retain the above copyright
22  *       notice, this list of conditions and the following disclaimer.
23  *     * Redistributions in binary form must reproduce the above copy
24  *       notice, this list of conditions and the following disclaimer in
25  *       the documentation and/or other materials provided with the
26  *       distribution.
27  *     * Neither the name of Intel Corporation nor the names of its
28  *       contributors may be used to endorse or promote products derived
29  *       from this software without specific prior written permission.
30  *
31  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42  *
43  * Intel PCIe NTB Linux driver
44  *
45  * Contact Information:
46  * Jon Mason <jon.mason@intel.com>
47  */
48 #include <linux/debugfs.h>
49 #include <linux/delay.h>
50 #include <linux/dma-mapping.h>
51 #include <linux/errno.h>
52 #include <linux/export.h>
53 #include <linux/interrupt.h>
54 #include <linux/module.h>
55 #include <linux/pci.h>
56 #include <linux/slab.h>
57 #include <linux/types.h>
58 #include <linux/ntb.h>
59 #include "ntb_hw.h"
60 
61 #define NTB_TRANSPORT_VERSION	1
62 
63 static unsigned int transport_mtu = 0x401E;
64 module_param(transport_mtu, uint, 0644);
65 MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");
66 
67 static unsigned char max_num_clients = 2;
68 module_param(max_num_clients, byte, 0644);
69 MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");
70 
71 struct ntb_queue_entry {
72 	/* ntb_queue list reference */
73 	struct list_head entry;
74 	/* pointers to data to be transfered */
75 	void *cb_data;
76 	void *buf;
77 	unsigned int len;
78 	unsigned int flags;
79 };
80 
81 struct ntb_transport_qp {
82 	struct ntb_transport *transport;
83 	struct ntb_device *ndev;
84 	void *cb_data;
85 
86 	bool client_ready;
87 	bool qp_link;
88 	u8 qp_num;	/* Only 64 QP's are allowed.  0-63 */
89 
90 	void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
91 			    void *data, int len);
92 	struct list_head tx_free_q;
93 	spinlock_t ntb_tx_free_q_lock;
94 	void *tx_mw_begin;
95 	void *tx_mw_end;
96 	void *tx_offset;
97 	unsigned int tx_max_frame;
98 
99 	void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
100 			    void *data, int len);
101 	struct tasklet_struct rx_work;
102 	struct list_head rx_pend_q;
103 	struct list_head rx_free_q;
104 	spinlock_t ntb_rx_pend_q_lock;
105 	spinlock_t ntb_rx_free_q_lock;
106 	void *rx_buff_begin;
107 	void *rx_buff_end;
108 	void *rx_offset;
109 	unsigned int rx_max_frame;
110 
111 	void (*event_handler) (void *data, int status);
112 	struct delayed_work link_work;
113 	struct work_struct link_cleanup;
114 
115 	struct dentry *debugfs_dir;
116 	struct dentry *debugfs_stats;
117 
118 	/* Stats */
119 	u64 rx_bytes;
120 	u64 rx_pkts;
121 	u64 rx_ring_empty;
122 	u64 rx_err_no_buf;
123 	u64 rx_err_oflow;
124 	u64 rx_err_ver;
125 	u64 tx_bytes;
126 	u64 tx_pkts;
127 	u64 tx_ring_full;
128 };
129 
130 struct ntb_transport_mw {
131 	size_t size;
132 	void *virt_addr;
133 	dma_addr_t dma_addr;
134 };
135 
136 struct ntb_transport_client_dev {
137 	struct list_head entry;
138 	struct device dev;
139 };
140 
141 struct ntb_transport {
142 	struct list_head entry;
143 	struct list_head client_devs;
144 
145 	struct ntb_device *ndev;
146 	struct ntb_transport_mw mw[NTB_NUM_MW];
147 	struct ntb_transport_qp *qps;
148 	unsigned int max_qps;
149 	unsigned long qp_bitmap;
150 	bool transport_link;
151 	struct delayed_work link_work;
152 	struct work_struct link_cleanup;
153 	struct dentry *debugfs_dir;
154 };
155 
156 enum {
157 	DESC_DONE_FLAG = 1 << 0,
158 	LINK_DOWN_FLAG = 1 << 1,
159 };
160 
161 struct ntb_payload_header {
162 	u64 ver;
163 	unsigned int len;
164 	unsigned int flags;
165 };
166 
167 enum {
168 	VERSION = 0,
169 	MW0_SZ,
170 	MW1_SZ,
171 	NUM_QPS,
172 	QP_LINKS,
173 	MAX_SPAD,
174 };
175 
176 #define QP_TO_MW(qp)		((qp) % NTB_NUM_MW)
177 #define NTB_QP_DEF_NUM_ENTRIES	100
178 #define NTB_LINK_DOWN_TIMEOUT	10
179 
180 static int ntb_match_bus(struct device *dev, struct device_driver *drv)
181 {
182 	return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
183 }
184 
185 static int ntb_client_probe(struct device *dev)
186 {
187 	const struct ntb_client *drv = container_of(dev->driver,
188 						    struct ntb_client, driver);
189 	struct pci_dev *pdev = container_of(dev->parent, struct pci_dev, dev);
190 	int rc = -EINVAL;
191 
192 	get_device(dev);
193 	if (drv && drv->probe)
194 		rc = drv->probe(pdev);
195 	if (rc)
196 		put_device(dev);
197 
198 	return rc;
199 }
200 
201 static int ntb_client_remove(struct device *dev)
202 {
203 	const struct ntb_client *drv = container_of(dev->driver,
204 						    struct ntb_client, driver);
205 	struct pci_dev *pdev = container_of(dev->parent, struct pci_dev, dev);
206 
207 	if (drv && drv->remove)
208 		drv->remove(pdev);
209 
210 	put_device(dev);
211 
212 	return 0;
213 }
214 
215 static struct bus_type ntb_bus_type = {
216 	.name = "ntb_bus",
217 	.match = ntb_match_bus,
218 	.probe = ntb_client_probe,
219 	.remove = ntb_client_remove,
220 };
221 
222 static LIST_HEAD(ntb_transport_list);
223 
224 static int ntb_bus_init(struct ntb_transport *nt)
225 {
226 	if (list_empty(&ntb_transport_list)) {
227 		int rc = bus_register(&ntb_bus_type);
228 		if (rc)
229 			return rc;
230 	}
231 
232 	list_add(&nt->entry, &ntb_transport_list);
233 
234 	return 0;
235 }
236 
237 static void ntb_bus_remove(struct ntb_transport *nt)
238 {
239 	struct ntb_transport_client_dev *client_dev, *cd;
240 
241 	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
242 		dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
243 			dev_name(&client_dev->dev));
244 		list_del(&client_dev->entry);
245 		device_unregister(&client_dev->dev);
246 	}
247 
248 	list_del(&nt->entry);
249 
250 	if (list_empty(&ntb_transport_list))
251 		bus_unregister(&ntb_bus_type);
252 }
253 
254 static void ntb_client_release(struct device *dev)
255 {
256 	struct ntb_transport_client_dev *client_dev;
257 	client_dev = container_of(dev, struct ntb_transport_client_dev, dev);
258 
259 	kfree(client_dev);
260 }
261 
262 /**
263  * ntb_unregister_client_dev - Unregister NTB client device
264  * @device_name: Name of NTB client device
265  *
266  * Unregister an NTB client device with the NTB transport layer
267  */
268 void ntb_unregister_client_dev(char *device_name)
269 {
270 	struct ntb_transport_client_dev *client, *cd;
271 	struct ntb_transport *nt;
272 
273 	list_for_each_entry(nt, &ntb_transport_list, entry)
274 		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
275 			if (!strncmp(dev_name(&client->dev), device_name,
276 				     strlen(device_name))) {
277 				list_del(&client->entry);
278 				device_unregister(&client->dev);
279 			}
280 }
281 EXPORT_SYMBOL_GPL(ntb_unregister_client_dev);
282 
283 /**
284  * ntb_register_client_dev - Register NTB client device
285  * @device_name: Name of NTB client device
286  *
287  * Register an NTB client device with the NTB transport layer
288  */
289 int ntb_register_client_dev(char *device_name)
290 {
291 	struct ntb_transport_client_dev *client_dev;
292 	struct ntb_transport *nt;
293 	int rc;
294 
295 	if (list_empty(&ntb_transport_list))
296 		return -ENODEV;
297 
298 	list_for_each_entry(nt, &ntb_transport_list, entry) {
299 		struct device *dev;
300 
301 		client_dev = kzalloc(sizeof(struct ntb_transport_client_dev),
302 				     GFP_KERNEL);
303 		if (!client_dev) {
304 			rc = -ENOMEM;
305 			goto err;
306 		}
307 
308 		dev = &client_dev->dev;
309 
310 		/* setup and register client devices */
311 		dev_set_name(dev, "%s", device_name);
312 		dev->bus = &ntb_bus_type;
313 		dev->release = ntb_client_release;
314 		dev->parent = &ntb_query_pdev(nt->ndev)->dev;
315 
316 		rc = device_register(dev);
317 		if (rc) {
318 			kfree(client_dev);
319 			goto err;
320 		}
321 
322 		list_add_tail(&client_dev->entry, &nt->client_devs);
323 	}
324 
325 	return 0;
326 
327 err:
328 	ntb_unregister_client_dev(device_name);
329 
330 	return rc;
331 }
332 EXPORT_SYMBOL_GPL(ntb_register_client_dev);
333 
334 /**
335  * ntb_register_client - Register NTB client driver
336  * @drv: NTB client driver to be registered
337  *
338  * Register an NTB client driver with the NTB transport layer
339  *
340  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
341  */
342 int ntb_register_client(struct ntb_client *drv)
343 {
344 	drv->driver.bus = &ntb_bus_type;
345 
346 	if (list_empty(&ntb_transport_list))
347 		return -ENODEV;
348 
349 	return driver_register(&drv->driver);
350 }
351 EXPORT_SYMBOL_GPL(ntb_register_client);
352 
353 /**
354  * ntb_unregister_client - Unregister NTB client driver
355  * @drv: NTB client driver to be unregistered
356  *
357  * Unregister an NTB client driver with the NTB transport layer
358  *
359  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
360  */
361 void ntb_unregister_client(struct ntb_client *drv)
362 {
363 	driver_unregister(&drv->driver);
364 }
365 EXPORT_SYMBOL_GPL(ntb_unregister_client);
366 
367 static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
368 			    loff_t *offp)
369 {
370 	struct ntb_transport_qp *qp;
371 	char buf[1024];
372 	ssize_t ret, out_offset, out_count;
373 
374 	out_count = 1024;
375 
376 	qp = filp->private_data;
377 	out_offset = 0;
378 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
379 			       "NTB QP stats\n");
380 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
381 			       "rx_bytes - \t%llu\n", qp->rx_bytes);
382 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
383 			       "rx_pkts - \t%llu\n", qp->rx_pkts);
384 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
385 			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);
386 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
387 			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
388 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
389 			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
390 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
391 			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);
392 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
393 			       "rx_buff_begin - %p\n", qp->rx_buff_begin);
394 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
395 			       "rx_offset - \t%p\n", qp->rx_offset);
396 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
397 			       "rx_buff_end - \t%p\n", qp->rx_buff_end);
398 
399 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
400 			       "tx_bytes - \t%llu\n", qp->tx_bytes);
401 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
402 			       "tx_pkts - \t%llu\n", qp->tx_pkts);
403 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
404 			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);
405 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
406 			       "tx_mw_begin - \t%p\n", qp->tx_mw_begin);
407 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
408 			       "tx_offset - \t%p\n", qp->tx_offset);
409 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
410 			       "tx_mw_end - \t%p\n", qp->tx_mw_end);
411 
412 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
413 			       "QP Link %s\n", (qp->qp_link == NTB_LINK_UP) ?
414 			       "Up" : "Down");
415 
416 	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
417 	return ret;
418 }
419 
420 static const struct file_operations ntb_qp_debugfs_stats = {
421 	.owner = THIS_MODULE,
422 	.open = simple_open,
423 	.read = debugfs_read,
424 };
425 
426 static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
427 			 struct list_head *list)
428 {
429 	unsigned long flags;
430 
431 	spin_lock_irqsave(lock, flags);
432 	list_add_tail(entry, list);
433 	spin_unlock_irqrestore(lock, flags);
434 }
435 
436 static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
437 						struct list_head *list)
438 {
439 	struct ntb_queue_entry *entry;
440 	unsigned long flags;
441 
442 	spin_lock_irqsave(lock, flags);
443 	if (list_empty(list)) {
444 		entry = NULL;
445 		goto out;
446 	}
447 	entry = list_first_entry(list, struct ntb_queue_entry, entry);
448 	list_del(&entry->entry);
449 out:
450 	spin_unlock_irqrestore(lock, flags);
451 
452 	return entry;
453 }
454 
455 static void ntb_transport_setup_qp_mw(struct ntb_transport *nt,
456 				      unsigned int qp_num)
457 {
458 	struct ntb_transport_qp *qp = &nt->qps[qp_num];
459 	unsigned int rx_size, num_qps_mw;
460 	u8 mw_num = QP_TO_MW(qp_num);
461 	void *offset;
462 
463 	WARN_ON(nt->mw[mw_num].virt_addr == 0);
464 
465 	if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW)
466 		num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
467 	else
468 		num_qps_mw = nt->max_qps / NTB_NUM_MW;
469 
470 	rx_size = nt->mw[mw_num].size / num_qps_mw;
471 	qp->rx_buff_begin = nt->mw[mw_num].virt_addr +
472 			    (qp_num / NTB_NUM_MW * rx_size);
473 	qp->rx_buff_end = qp->rx_buff_begin + rx_size;
474 	qp->rx_offset = qp->rx_buff_begin;
475 	qp->rx_max_frame = min(transport_mtu, rx_size);
476 
477 	/* setup the hdr offsets with 0's */
478 	for (offset = qp->rx_buff_begin + qp->rx_max_frame -
479 		      sizeof(struct ntb_payload_header);
480 	     offset < qp->rx_buff_end; offset += qp->rx_max_frame)
481 		memset(offset, 0, sizeof(struct ntb_payload_header));
482 
483 	qp->rx_pkts = 0;
484 	qp->tx_pkts = 0;
485 }
486 
487 static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size)
488 {
489 	struct ntb_transport_mw *mw = &nt->mw[num_mw];
490 	struct pci_dev *pdev = ntb_query_pdev(nt->ndev);
491 
492 	/* Alloc memory for receiving data.  Must be 4k aligned */
493 	mw->size = ALIGN(size, 4096);
494 
495 	mw->virt_addr = dma_alloc_coherent(&pdev->dev, mw->size, &mw->dma_addr,
496 					   GFP_KERNEL);
497 	if (!mw->virt_addr) {
498 		dev_err(&pdev->dev, "Unable to allocate MW buffer of size %d\n",
499 		       (int) mw->size);
500 		return -ENOMEM;
501 	}
502 
503 	/* Notify HW the memory location of the receive buffer */
504 	ntb_set_mw_addr(nt->ndev, num_mw, mw->dma_addr);
505 
506 	return 0;
507 }
508 
509 static void ntb_qp_link_cleanup(struct work_struct *work)
510 {
511 	struct ntb_transport_qp *qp = container_of(work,
512 						   struct ntb_transport_qp,
513 						   link_cleanup);
514 	struct ntb_transport *nt = qp->transport;
515 	struct pci_dev *pdev = ntb_query_pdev(nt->ndev);
516 
517 	if (qp->qp_link == NTB_LINK_DOWN) {
518 		cancel_delayed_work_sync(&qp->link_work);
519 		return;
520 	}
521 
522 	if (qp->event_handler)
523 		qp->event_handler(qp->cb_data, NTB_LINK_DOWN);
524 
525 	dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num);
526 	qp->qp_link = NTB_LINK_DOWN;
527 
528 	if (nt->transport_link == NTB_LINK_UP)
529 		schedule_delayed_work(&qp->link_work,
530 				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
531 }
532 
533 static void ntb_qp_link_down(struct ntb_transport_qp *qp)
534 {
535 	schedule_work(&qp->link_cleanup);
536 }
537 
538 static void ntb_transport_link_cleanup(struct work_struct *work)
539 {
540 	struct ntb_transport *nt = container_of(work, struct ntb_transport,
541 						link_cleanup);
542 	int i;
543 
544 	if (nt->transport_link == NTB_LINK_DOWN)
545 		cancel_delayed_work_sync(&nt->link_work);
546 	else
547 		nt->transport_link = NTB_LINK_DOWN;
548 
549 	/* Pass along the info to any clients */
550 	for (i = 0; i < nt->max_qps; i++)
551 		if (!test_bit(i, &nt->qp_bitmap))
552 			ntb_qp_link_down(&nt->qps[i]);
553 
554 	/* The scratchpad registers keep the values if the remote side
555 	 * goes down, blast them now to give them a sane value the next
556 	 * time they are accessed
557 	 */
558 	for (i = 0; i < MAX_SPAD; i++)
559 		ntb_write_local_spad(nt->ndev, i, 0);
560 }
561 
562 static void ntb_transport_event_callback(void *data, enum ntb_hw_event event)
563 {
564 	struct ntb_transport *nt = data;
565 
566 	switch (event) {
567 	case NTB_EVENT_HW_LINK_UP:
568 		schedule_delayed_work(&nt->link_work, 0);
569 		break;
570 	case NTB_EVENT_HW_LINK_DOWN:
571 		schedule_work(&nt->link_cleanup);
572 		break;
573 	default:
574 		BUG();
575 	}
576 }
577 
578 static void ntb_transport_link_work(struct work_struct *work)
579 {
580 	struct ntb_transport *nt = container_of(work, struct ntb_transport,
581 						link_work.work);
582 	struct ntb_device *ndev = nt->ndev;
583 	struct pci_dev *pdev = ntb_query_pdev(ndev);
584 	u32 val;
585 	int rc, i;
586 
587 	/* send the local info */
588 	rc = ntb_write_remote_spad(ndev, VERSION, NTB_TRANSPORT_VERSION);
589 	if (rc) {
590 		dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
591 			0, VERSION);
592 		goto out;
593 	}
594 
595 	rc = ntb_write_remote_spad(ndev, MW0_SZ, ntb_get_mw_size(ndev, 0));
596 	if (rc) {
597 		dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
598 			(u32) ntb_get_mw_size(ndev, 0), MW0_SZ);
599 		goto out;
600 	}
601 
602 	rc = ntb_write_remote_spad(ndev, MW1_SZ, ntb_get_mw_size(ndev, 1));
603 	if (rc) {
604 		dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
605 			(u32) ntb_get_mw_size(ndev, 1), MW1_SZ);
606 		goto out;
607 	}
608 
609 	rc = ntb_write_remote_spad(ndev, NUM_QPS, nt->max_qps);
610 	if (rc) {
611 		dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
612 			nt->max_qps, NUM_QPS);
613 		goto out;
614 	}
615 
616 	rc = ntb_read_local_spad(nt->ndev, QP_LINKS, &val);
617 	if (rc) {
618 		dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS);
619 		goto out;
620 	}
621 
622 	rc = ntb_write_remote_spad(ndev, QP_LINKS, val);
623 	if (rc) {
624 		dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
625 			val, QP_LINKS);
626 		goto out;
627 	}
628 
629 	/* Query the remote side for its info */
630 	rc = ntb_read_remote_spad(ndev, VERSION, &val);
631 	if (rc) {
632 		dev_err(&pdev->dev, "Error reading remote spad %d\n", VERSION);
633 		goto out;
634 	}
635 
636 	if (val != NTB_TRANSPORT_VERSION)
637 		goto out;
638 	dev_dbg(&pdev->dev, "Remote version = %d\n", val);
639 
640 	rc = ntb_read_remote_spad(ndev, NUM_QPS, &val);
641 	if (rc) {
642 		dev_err(&pdev->dev, "Error reading remote spad %d\n", NUM_QPS);
643 		goto out;
644 	}
645 
646 	if (val != nt->max_qps)
647 		goto out;
648 	dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
649 
650 	rc = ntb_read_remote_spad(ndev, MW0_SZ, &val);
651 	if (rc) {
652 		dev_err(&pdev->dev, "Error reading remote spad %d\n", MW0_SZ);
653 		goto out;
654 	}
655 
656 	if (!val)
657 		goto out;
658 	dev_dbg(&pdev->dev, "Remote MW0 size = %d\n", val);
659 
660 	rc = ntb_set_mw(nt, 0, val);
661 	if (rc)
662 		goto out;
663 
664 	rc = ntb_read_remote_spad(ndev, MW1_SZ, &val);
665 	if (rc) {
666 		dev_err(&pdev->dev, "Error reading remote spad %d\n", MW1_SZ);
667 		goto out;
668 	}
669 
670 	if (!val)
671 		goto out;
672 	dev_dbg(&pdev->dev, "Remote MW1 size = %d\n", val);
673 
674 	rc = ntb_set_mw(nt, 1, val);
675 	if (rc)
676 		goto out;
677 
678 	nt->transport_link = NTB_LINK_UP;
679 
680 	for (i = 0; i < nt->max_qps; i++) {
681 		struct ntb_transport_qp *qp = &nt->qps[i];
682 
683 		ntb_transport_setup_qp_mw(nt, i);
684 
685 		if (qp->client_ready == NTB_LINK_UP)
686 			schedule_delayed_work(&qp->link_work, 0);
687 	}
688 
689 	return;
690 
691 out:
692 	if (ntb_hw_link_status(ndev))
693 		schedule_delayed_work(&nt->link_work,
694 				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
695 }
696 
697 static void ntb_qp_link_work(struct work_struct *work)
698 {
699 	struct ntb_transport_qp *qp = container_of(work,
700 						   struct ntb_transport_qp,
701 						   link_work.work);
702 	struct pci_dev *pdev = ntb_query_pdev(qp->ndev);
703 	struct ntb_transport *nt = qp->transport;
704 	int rc, val;
705 
706 	WARN_ON(nt->transport_link != NTB_LINK_UP);
707 
708 	rc = ntb_read_local_spad(nt->ndev, QP_LINKS, &val);
709 	if (rc) {
710 		dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS);
711 		return;
712 	}
713 
714 	rc = ntb_write_remote_spad(nt->ndev, QP_LINKS, val | 1 << qp->qp_num);
715 	if (rc)
716 		dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
717 			val | 1 << qp->qp_num, QP_LINKS);
718 
719 	/* query remote spad for qp ready bits */
720 	rc = ntb_read_remote_spad(nt->ndev, QP_LINKS, &val);
721 	if (rc)
722 		dev_err(&pdev->dev, "Error reading remote spad %d\n", QP_LINKS);
723 
724 	dev_dbg(&pdev->dev, "Remote QP link status = %x\n", val);
725 
726 	/* See if the remote side is up */
727 	if (1 << qp->qp_num & val) {
728 		qp->qp_link = NTB_LINK_UP;
729 
730 		dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);
731 		if (qp->event_handler)
732 			qp->event_handler(qp->cb_data, NTB_LINK_UP);
733 	} else if (nt->transport_link == NTB_LINK_UP)
734 		schedule_delayed_work(&qp->link_work,
735 				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
736 }
737 
738 static void ntb_transport_init_queue(struct ntb_transport *nt,
739 				     unsigned int qp_num)
740 {
741 	struct ntb_transport_qp *qp;
742 	unsigned int num_qps_mw, tx_size;
743 	u8 mw_num = QP_TO_MW(qp_num);
744 
745 	qp = &nt->qps[qp_num];
746 	qp->qp_num = qp_num;
747 	qp->transport = nt;
748 	qp->ndev = nt->ndev;
749 	qp->qp_link = NTB_LINK_DOWN;
750 	qp->client_ready = NTB_LINK_DOWN;
751 	qp->event_handler = NULL;
752 
753 	if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW)
754 		num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
755 	else
756 		num_qps_mw = nt->max_qps / NTB_NUM_MW;
757 
758 	tx_size = ntb_get_mw_size(qp->ndev, mw_num) / num_qps_mw;
759 	qp->tx_mw_begin = ntb_get_mw_vbase(nt->ndev, mw_num) +
760 			  (qp_num / NTB_NUM_MW * tx_size);
761 	qp->tx_mw_end = qp->tx_mw_begin + tx_size;
762 	qp->tx_offset = qp->tx_mw_begin;
763 	qp->tx_max_frame = min(transport_mtu, tx_size);
764 
765 	if (nt->debugfs_dir) {
766 		char debugfs_name[4];
767 
768 		snprintf(debugfs_name, 4, "qp%d", qp_num);
769 		qp->debugfs_dir = debugfs_create_dir(debugfs_name,
770 						     nt->debugfs_dir);
771 
772 		qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR,
773 							qp->debugfs_dir, qp,
774 							&ntb_qp_debugfs_stats);
775 	}
776 
777 	INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);
778 	INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup);
779 
780 	spin_lock_init(&qp->ntb_rx_pend_q_lock);
781 	spin_lock_init(&qp->ntb_rx_free_q_lock);
782 	spin_lock_init(&qp->ntb_tx_free_q_lock);
783 
784 	INIT_LIST_HEAD(&qp->rx_pend_q);
785 	INIT_LIST_HEAD(&qp->rx_free_q);
786 	INIT_LIST_HEAD(&qp->tx_free_q);
787 }
788 
789 int ntb_transport_init(struct pci_dev *pdev)
790 {
791 	struct ntb_transport *nt;
792 	int rc, i;
793 
794 	nt = kzalloc(sizeof(struct ntb_transport), GFP_KERNEL);
795 	if (!nt)
796 		return -ENOMEM;
797 
798 	if (debugfs_initialized())
799 		nt->debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
800 	else
801 		nt->debugfs_dir = NULL;
802 
803 	nt->ndev = ntb_register_transport(pdev, nt);
804 	if (!nt->ndev) {
805 		rc = -EIO;
806 		goto err;
807 	}
808 
809 	nt->max_qps = min(nt->ndev->max_cbs, max_num_clients);
810 
811 	nt->qps = kcalloc(nt->max_qps, sizeof(struct ntb_transport_qp),
812 			  GFP_KERNEL);
813 	if (!nt->qps) {
814 		rc = -ENOMEM;
815 		goto err1;
816 	}
817 
818 	nt->qp_bitmap = ((u64) 1 << nt->max_qps) - 1;
819 
820 	for (i = 0; i < nt->max_qps; i++)
821 		ntb_transport_init_queue(nt, i);
822 
823 	INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
824 	INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup);
825 
826 	rc = ntb_register_event_callback(nt->ndev,
827 					 ntb_transport_event_callback);
828 	if (rc)
829 		goto err2;
830 
831 	INIT_LIST_HEAD(&nt->client_devs);
832 	rc = ntb_bus_init(nt);
833 	if (rc)
834 		goto err3;
835 
836 	if (ntb_hw_link_status(nt->ndev))
837 		schedule_delayed_work(&nt->link_work, 0);
838 
839 	return 0;
840 
841 err3:
842 	ntb_unregister_event_callback(nt->ndev);
843 err2:
844 	kfree(nt->qps);
845 err1:
846 	ntb_unregister_transport(nt->ndev);
847 err:
848 	debugfs_remove_recursive(nt->debugfs_dir);
849 	kfree(nt);
850 	return rc;
851 }
852 
853 void ntb_transport_free(void *transport)
854 {
855 	struct ntb_transport *nt = transport;
856 	struct pci_dev *pdev;
857 	int i;
858 
859 	nt->transport_link = NTB_LINK_DOWN;
860 
861 	/* verify that all the qp's are freed */
862 	for (i = 0; i < nt->max_qps; i++)
863 		if (!test_bit(i, &nt->qp_bitmap))
864 			ntb_transport_free_queue(&nt->qps[i]);
865 
866 	ntb_bus_remove(nt);
867 
868 	cancel_delayed_work_sync(&nt->link_work);
869 
870 	debugfs_remove_recursive(nt->debugfs_dir);
871 
872 	ntb_unregister_event_callback(nt->ndev);
873 
874 	pdev = ntb_query_pdev(nt->ndev);
875 
876 	for (i = 0; i < NTB_NUM_MW; i++)
877 		if (nt->mw[i].virt_addr)
878 			dma_free_coherent(&pdev->dev, nt->mw[i].size,
879 					  nt->mw[i].virt_addr,
880 					  nt->mw[i].dma_addr);
881 
882 	kfree(nt->qps);
883 	ntb_unregister_transport(nt->ndev);
884 	kfree(nt);
885 }
886 
887 static void ntb_rx_copy_task(struct ntb_transport_qp *qp,
888 			     struct ntb_queue_entry *entry, void *offset)
889 {
890 
891 	struct ntb_payload_header *hdr;
892 
893 	BUG_ON(offset < qp->rx_buff_begin ||
894 	       offset + qp->rx_max_frame >= qp->rx_buff_end);
895 
896 	hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header);
897 	entry->len = hdr->len;
898 
899 	memcpy(entry->buf, offset, entry->len);
900 
901 	/* Ensure that the data is fully copied out before clearing the flag */
902 	wmb();
903 	hdr->flags = 0;
904 
905 	if (qp->rx_handler && qp->client_ready == NTB_LINK_UP)
906 		qp->rx_handler(qp, qp->cb_data, entry->cb_data, entry->len);
907 
908 	ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q);
909 }
910 
911 static int ntb_process_rxc(struct ntb_transport_qp *qp)
912 {
913 	struct ntb_payload_header *hdr;
914 	struct ntb_queue_entry *entry;
915 	void *offset;
916 
917 	entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
918 	if (!entry) {
919 		hdr = offset + qp->rx_max_frame -
920 		      sizeof(struct ntb_payload_header);
921 		dev_dbg(&ntb_query_pdev(qp->ndev)->dev,
922 			"no buffer - HDR ver %llu, len %d, flags %x\n",
923 			hdr->ver, hdr->len, hdr->flags);
924 		qp->rx_err_no_buf++;
925 		return -ENOMEM;
926 	}
927 
928 	offset = qp->rx_offset;
929 	hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header);
930 
931 	if (!(hdr->flags & DESC_DONE_FLAG)) {
932 		ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry,
933 				  &qp->rx_pend_q);
934 		qp->rx_ring_empty++;
935 		return -EAGAIN;
936 	}
937 
938 	if (hdr->ver != qp->rx_pkts) {
939 		dev_dbg(&ntb_query_pdev(qp->ndev)->dev,
940 			"qp %d: version mismatch, expected %llu - got %llu\n",
941 			qp->qp_num, qp->rx_pkts, hdr->ver);
942 		ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry,
943 				  &qp->rx_pend_q);
944 		qp->rx_err_ver++;
945 		return -EIO;
946 	}
947 
948 	if (hdr->flags & LINK_DOWN_FLAG) {
949 		ntb_qp_link_down(qp);
950 
951 		ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry,
952 				  &qp->rx_pend_q);
953 
954 		/* Ensure that the data is fully copied out before clearing the
955 		 * done flag
956 		 */
957 		wmb();
958 		hdr->flags = 0;
959 		goto out;
960 	}
961 
962 	dev_dbg(&ntb_query_pdev(qp->ndev)->dev,
963 		"rx offset %p, ver %llu - %d payload received, buf size %d\n",
964 		qp->rx_offset, hdr->ver, hdr->len, entry->len);
965 
966 	if (hdr->len <= entry->len)
967 		ntb_rx_copy_task(qp, entry, offset);
968 	else {
969 		ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry,
970 				  &qp->rx_pend_q);
971 
972 		/* Ensure that the data is fully copied out before clearing the
973 		 * done flag
974 		 */
975 		wmb();
976 		hdr->flags = 0;
977 		qp->rx_err_oflow++;
978 		dev_dbg(&ntb_query_pdev(qp->ndev)->dev,
979 			"RX overflow! Wanted %d got %d\n",
980 			hdr->len, entry->len);
981 	}
982 
983 	qp->rx_bytes += hdr->len;
984 	qp->rx_pkts++;
985 
986 out:
987 	qp->rx_offset += qp->rx_max_frame;
988 	if (qp->rx_offset + qp->rx_max_frame >= qp->rx_buff_end)
989 		qp->rx_offset = qp->rx_buff_begin;
990 
991 	return 0;
992 }
993 
994 static void ntb_transport_rx(unsigned long data)
995 {
996 	struct ntb_transport_qp *qp = (struct ntb_transport_qp *)data;
997 	int rc;
998 
999 	do {
1000 		rc = ntb_process_rxc(qp);
1001 	} while (!rc);
1002 }
1003 
1004 static void ntb_transport_rxc_db(void *data, int db_num)
1005 {
1006 	struct ntb_transport_qp *qp = data;
1007 
1008 	dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n",
1009 		__func__, db_num);
1010 
1011 	tasklet_schedule(&qp->rx_work);
1012 }
1013 
1014 static void ntb_tx_copy_task(struct ntb_transport_qp *qp,
1015 			     struct ntb_queue_entry *entry,
1016 			     void *offset)
1017 {
1018 	struct ntb_payload_header *hdr;
1019 
1020 	BUG_ON(offset < qp->tx_mw_begin ||
1021 	       offset + qp->tx_max_frame >= qp->tx_mw_end);
1022 
1023 	memcpy_toio(offset, entry->buf, entry->len);
1024 
1025 	hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);
1026 	hdr->len = entry->len;
1027 	hdr->ver = qp->tx_pkts;
1028 
1029 	/* Ensure that the data is fully copied out before setting the flag */
1030 	wmb();
1031 	hdr->flags = entry->flags | DESC_DONE_FLAG;
1032 
1033 	ntb_ring_sdb(qp->ndev, qp->qp_num);
1034 
1035 	/* The entry length can only be zero if the packet is intended to be a
1036 	 * "link down" or similar.  Since no payload is being sent in these
1037 	 * cases, there is nothing to add to the completion queue.
1038 	 */
1039 	if (entry->len > 0) {
1040 		qp->tx_bytes += entry->len;
1041 
1042 		if (qp->tx_handler)
1043 			qp->tx_handler(qp, qp->cb_data, entry->cb_data,
1044 				       entry->len);
1045 	}
1046 
1047 	ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);
1048 }
1049 
1050 static int ntb_process_tx(struct ntb_transport_qp *qp,
1051 			  struct ntb_queue_entry *entry)
1052 {
1053 	struct ntb_payload_header *hdr;
1054 	void *offset;
1055 
1056 	offset = qp->tx_offset;
1057 	hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);
1058 
1059 	dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%lld - offset %p, tx %p, entry len %d flags %x buff %p\n",
1060 		 qp->tx_pkts, offset, qp->tx_offset, entry->len, entry->flags,
1061 		 entry->buf);
1062 	if (hdr->flags) {
1063 		qp->tx_ring_full++;
1064 		return -EAGAIN;
1065 	}
1066 
1067 	if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {
1068 		if (qp->tx_handler)
1069 			qp->tx_handler(qp->cb_data, qp, NULL, -EIO);
1070 
1071 		ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
1072 			     &qp->tx_free_q);
1073 		return 0;
1074 	}
1075 
1076 	ntb_tx_copy_task(qp, entry, offset);
1077 
1078 	qp->tx_offset += qp->tx_max_frame;
1079 	if (qp->tx_offset + qp->tx_max_frame >= qp->tx_mw_end)
1080 		qp->tx_offset = qp->tx_mw_begin;
1081 
1082 	qp->tx_pkts++;
1083 
1084 	return 0;
1085 }
1086 
1087 static void ntb_send_link_down(struct ntb_transport_qp *qp)
1088 {
1089 	struct pci_dev *pdev = ntb_query_pdev(qp->ndev);
1090 	struct ntb_queue_entry *entry;
1091 	int i, rc;
1092 
1093 	if (qp->qp_link == NTB_LINK_DOWN)
1094 		return;
1095 
1096 	qp->qp_link = NTB_LINK_DOWN;
1097 	dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num);
1098 
1099 	for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) {
1100 		entry = ntb_list_rm(&qp->ntb_tx_free_q_lock,
1101 					 &qp->tx_free_q);
1102 		if (entry)
1103 			break;
1104 		msleep(100);
1105 	}
1106 
1107 	if (!entry)
1108 		return;
1109 
1110 	entry->cb_data = NULL;
1111 	entry->buf = NULL;
1112 	entry->len = 0;
1113 	entry->flags = LINK_DOWN_FLAG;
1114 
1115 	rc = ntb_process_tx(qp, entry);
1116 	if (rc)
1117 		dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n",
1118 			qp->qp_num);
1119 }
1120 
1121 /**
1122  * ntb_transport_create_queue - Create a new NTB transport layer queue
1123  * @rx_handler: receive callback function
1124  * @tx_handler: transmit callback function
1125  * @event_handler: event callback function
1126  *
1127  * Create a new NTB transport layer queue and provide the queue with a callback
1128  * routine for both transmit and receive.  The receive callback routine will be
1129  * used to pass up data when the transport has received it on the queue.   The
1130  * transmit callback routine will be called when the transport has completed the
1131  * transmission of the data on the queue and the data is ready to be freed.
1132  *
1133  * RETURNS: pointer to newly created ntb_queue, NULL on error.
1134  */
1135 struct ntb_transport_qp *
1136 ntb_transport_create_queue(void *data, struct pci_dev *pdev,
1137 			   const struct ntb_queue_handlers *handlers)
1138 {
1139 	struct ntb_queue_entry *entry;
1140 	struct ntb_transport_qp *qp;
1141 	struct ntb_transport *nt;
1142 	unsigned int free_queue;
1143 	int rc, i;
1144 
1145 	nt = ntb_find_transport(pdev);
1146 	if (!nt)
1147 		goto err;
1148 
1149 	free_queue = ffs(nt->qp_bitmap);
1150 	if (!free_queue)
1151 		goto err;
1152 
1153 	/* decrement free_queue to make it zero based */
1154 	free_queue--;
1155 
1156 	clear_bit(free_queue, &nt->qp_bitmap);
1157 
1158 	qp = &nt->qps[free_queue];
1159 	qp->cb_data = data;
1160 	qp->rx_handler = handlers->rx_handler;
1161 	qp->tx_handler = handlers->tx_handler;
1162 	qp->event_handler = handlers->event_handler;
1163 
1164 	for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
1165 		entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC);
1166 		if (!entry)
1167 			goto err1;
1168 
1169 		ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry,
1170 				  &qp->rx_free_q);
1171 	}
1172 
1173 	for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
1174 		entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC);
1175 		if (!entry)
1176 			goto err2;
1177 
1178 		ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
1179 				  &qp->tx_free_q);
1180 	}
1181 
1182 	tasklet_init(&qp->rx_work, ntb_transport_rx, (unsigned long) qp);
1183 
1184 	rc = ntb_register_db_callback(qp->ndev, free_queue, qp,
1185 				      ntb_transport_rxc_db);
1186 	if (rc)
1187 		goto err3;
1188 
1189 	dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num);
1190 
1191 	return qp;
1192 
1193 err3:
1194 	tasklet_disable(&qp->rx_work);
1195 err2:
1196 	while ((entry =
1197 		ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
1198 		kfree(entry);
1199 err1:
1200 	while ((entry =
1201 		ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
1202 		kfree(entry);
1203 	set_bit(free_queue, &nt->qp_bitmap);
1204 err:
1205 	return NULL;
1206 }
1207 EXPORT_SYMBOL_GPL(ntb_transport_create_queue);
1208 
1209 /**
1210  * ntb_transport_free_queue - Frees NTB transport queue
1211  * @qp: NTB queue to be freed
1212  *
1213  * Frees NTB transport queue
1214  */
1215 void ntb_transport_free_queue(struct ntb_transport_qp *qp)
1216 {
1217 	struct pci_dev *pdev = ntb_query_pdev(qp->ndev);
1218 	struct ntb_queue_entry *entry;
1219 
1220 	if (!qp)
1221 		return;
1222 
1223 	cancel_delayed_work_sync(&qp->link_work);
1224 
1225 	ntb_unregister_db_callback(qp->ndev, qp->qp_num);
1226 	tasklet_disable(&qp->rx_work);
1227 
1228 	while ((entry =
1229 		ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
1230 		kfree(entry);
1231 
1232 	while ((entry =
1233 		ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q))) {
1234 		dev_warn(&pdev->dev, "Freeing item from a non-empty queue\n");
1235 		kfree(entry);
1236 	}
1237 
1238 	while ((entry =
1239 		ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
1240 		kfree(entry);
1241 
1242 	set_bit(qp->qp_num, &qp->transport->qp_bitmap);
1243 
1244 	dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num);
1245 }
1246 EXPORT_SYMBOL_GPL(ntb_transport_free_queue);
1247 
1248 /**
1249  * ntb_transport_rx_remove - Dequeues enqueued rx packet
1250  * @qp: NTB queue to be freed
1251  * @len: pointer to variable to write enqueued buffers length
1252  *
1253  * Dequeues unused buffers from receive queue.  Should only be used during
1254  * shutdown of qp.
1255  *
1256  * RETURNS: NULL error value on error, or void* for success.
1257  */
1258 void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len)
1259 {
1260 	struct ntb_queue_entry *entry;
1261 	void *buf;
1262 
1263 	if (!qp || qp->client_ready == NTB_LINK_UP)
1264 		return NULL;
1265 
1266 	entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
1267 	if (!entry)
1268 		return NULL;
1269 
1270 	buf = entry->cb_data;
1271 	*len = entry->len;
1272 
1273 	ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry,
1274 			  &qp->rx_free_q);
1275 
1276 	return buf;
1277 }
1278 EXPORT_SYMBOL_GPL(ntb_transport_rx_remove);
1279 
1280 /**
1281  * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry
1282  * @qp: NTB transport layer queue the entry is to be enqueued on
1283  * @cb: per buffer pointer for callback function to use
1284  * @data: pointer to data buffer that incoming packets will be copied into
1285  * @len: length of the data buffer
1286  *
1287  * Enqueue a new receive buffer onto the transport queue into which a NTB
1288  * payload can be received into.
1289  *
1290  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1291  */
1292 int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
1293 			     unsigned int len)
1294 {
1295 	struct ntb_queue_entry *entry;
1296 
1297 	if (!qp)
1298 		return -EINVAL;
1299 
1300 	entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q);
1301 	if (!entry)
1302 		return -ENOMEM;
1303 
1304 	entry->cb_data = cb;
1305 	entry->buf = data;
1306 	entry->len = len;
1307 
1308 	ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry,
1309 			  &qp->rx_pend_q);
1310 
1311 	return 0;
1312 }
1313 EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue);
1314 
1315 /**
1316  * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry
1317  * @qp: NTB transport layer queue the entry is to be enqueued on
1318  * @cb: per buffer pointer for callback function to use
1319  * @data: pointer to data buffer that will be sent
1320  * @len: length of the data buffer
1321  *
1322  * Enqueue a new transmit buffer onto the transport queue from which a NTB
1323  * payload will be transmitted.  This assumes that a lock is behing held to
1324  * serialize access to the qp.
1325  *
1326  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1327  */
1328 int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
1329 			     unsigned int len)
1330 {
1331 	struct ntb_queue_entry *entry;
1332 	int rc;
1333 
1334 	if (!qp || qp->qp_link != NTB_LINK_UP || !len)
1335 		return -EINVAL;
1336 
1337 	entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
1338 	if (!entry)
1339 		return -ENOMEM;
1340 
1341 	entry->cb_data = cb;
1342 	entry->buf = data;
1343 	entry->len = len;
1344 	entry->flags = 0;
1345 
1346 	rc = ntb_process_tx(qp, entry);
1347 	if (rc)
1348 		ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
1349 			     &qp->tx_free_q);
1350 
1351 	return rc;
1352 }
1353 EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue);
1354 
1355 /**
1356  * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
1357  * @qp: NTB transport layer queue to be enabled
1358  *
1359  * Notify NTB transport layer of client readiness to use queue
1360  */
1361 void ntb_transport_link_up(struct ntb_transport_qp *qp)
1362 {
1363 	if (!qp)
1364 		return;
1365 
1366 	qp->client_ready = NTB_LINK_UP;
1367 
1368 	if (qp->transport->transport_link == NTB_LINK_UP)
1369 		schedule_delayed_work(&qp->link_work, 0);
1370 }
1371 EXPORT_SYMBOL_GPL(ntb_transport_link_up);
1372 
1373 /**
1374  * ntb_transport_link_down - Notify NTB transport to no longer enqueue data
1375  * @qp: NTB transport layer queue to be disabled
1376  *
1377  * Notify NTB transport layer of client's desire to no longer receive data on
1378  * transport queue specified.  It is the client's responsibility to ensure all
1379  * entries on queue are purged or otherwise handled appropraitely.
1380  */
1381 void ntb_transport_link_down(struct ntb_transport_qp *qp)
1382 {
1383 	struct pci_dev *pdev = ntb_query_pdev(qp->ndev);
1384 	int rc, val;
1385 
1386 	if (!qp)
1387 		return;
1388 
1389 	qp->client_ready = NTB_LINK_DOWN;
1390 
1391 	rc = ntb_read_local_spad(qp->ndev, QP_LINKS, &val);
1392 	if (rc) {
1393 		dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS);
1394 		return;
1395 	}
1396 
1397 	rc = ntb_write_remote_spad(qp->ndev, QP_LINKS,
1398 				   val & ~(1 << qp->qp_num));
1399 	if (rc)
1400 		dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
1401 			val & ~(1 << qp->qp_num), QP_LINKS);
1402 
1403 	if (qp->qp_link == NTB_LINK_UP)
1404 		ntb_send_link_down(qp);
1405 	else
1406 		cancel_delayed_work_sync(&qp->link_work);
1407 }
1408 EXPORT_SYMBOL_GPL(ntb_transport_link_down);
1409 
1410 /**
1411  * ntb_transport_link_query - Query transport link state
1412  * @qp: NTB transport layer queue to be queried
1413  *
1414  * Query connectivity to the remote system of the NTB transport queue
1415  *
1416  * RETURNS: true for link up or false for link down
1417  */
1418 bool ntb_transport_link_query(struct ntb_transport_qp *qp)
1419 {
1420 	return qp->qp_link == NTB_LINK_UP;
1421 }
1422 EXPORT_SYMBOL_GPL(ntb_transport_link_query);
1423 
1424 /**
1425  * ntb_transport_qp_num - Query the qp number
1426  * @qp: NTB transport layer queue to be queried
1427  *
1428  * Query qp number of the NTB transport queue
1429  *
1430  * RETURNS: a zero based number specifying the qp number
1431  */
1432 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp)
1433 {
1434 	return qp->qp_num;
1435 }
1436 EXPORT_SYMBOL_GPL(ntb_transport_qp_num);
1437 
1438 /**
1439  * ntb_transport_max_size - Query the max payload size of a qp
1440  * @qp: NTB transport layer queue to be queried
1441  *
1442  * Query the maximum payload size permissible on the given qp
1443  *
1444  * RETURNS: the max payload size of a qp
1445  */
1446 unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp)
1447 {
1448 	return qp->tx_max_frame - sizeof(struct ntb_payload_header);
1449 }
1450 EXPORT_SYMBOL_GPL(ntb_transport_max_size);
1451