120 	struct ntb_transport_qp *qp;  member
148 	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
160 	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
272 #define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)  argument
279 static int ntb_async_tx_submit(struct ntb_transport_qp *qp,
466 	struct ntb_transport_qp *qp;  in debugfs_read()  local
470 	qp = filp->private_data;  in debugfs_read()
472 	if (!qp || !qp->link_is_up)  in debugfs_read()
485 			       "rx_bytes - \t%llu\n", qp->rx_bytes);  in debugfs_read()
487 			       "rx_pkts - \t%llu\n", qp->rx_pkts);  in debugfs_read()
489 			       "rx_memcpy - \t%llu\n", qp->rx_memcpy);  in debugfs_read()
491 			       "rx_async - \t%llu\n", qp->rx_async);  in debugfs_read()
493 			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);  in debugfs_read()
495 			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);  in debugfs_read()
497 			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);  in debugfs_read()
499 			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);  in debugfs_read()
501 			       "rx_buff - \t0x%p\n", qp->rx_buff);  in debugfs_read()
503 			       "rx_index - \t%u\n", qp->rx_index);  in debugfs_read()
505 			       "rx_max_entry - \t%u\n", qp->rx_max_entry);  in debugfs_read()
507 			       "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);  in debugfs_read()
510 			       "tx_bytes - \t%llu\n", qp->tx_bytes);  in debugfs_read()
512 			       "tx_pkts - \t%llu\n", qp->tx_pkts);  in debugfs_read()
514 			       "tx_memcpy - \t%llu\n", qp->tx_memcpy);  in debugfs_read()
516 			       "tx_async - \t%llu\n", qp->tx_async);  in debugfs_read()
518 			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);  in debugfs_read()
520 			       "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);  in debugfs_read()
522 			       "tx_mw - \t0x%p\n", qp->tx_mw);  in debugfs_read()
524 			       "tx_index (H) - \t%u\n", qp->tx_index);  in debugfs_read()
527 			       qp->remote_rx_info->entry);  in debugfs_read()
529 			       "tx_max_entry - \t%u\n", qp->tx_max_entry);  in debugfs_read()
532 			       ntb_transport_tx_free_entry(qp));  in debugfs_read()
538 			       qp->tx_dma_chan ? "Yes" : "No");  in debugfs_read()
541 			       qp->rx_dma_chan ? "Yes" : "No");  in debugfs_read()
544 			       qp->link_is_up ? "Up" : "Down");  in debugfs_read()
616 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];  in ntb_transport_setup_qp_mw()  local
640 	qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count);  in ntb_transport_setup_qp_mw()
643 	qp->remote_rx_info = qp->rx_buff + rx_size;  in ntb_transport_setup_qp_mw()
646 	qp->rx_max_frame = min(transport_mtu, rx_size / 2);  in ntb_transport_setup_qp_mw()
647 	qp->rx_max_entry = rx_size / qp->rx_max_frame;  in ntb_transport_setup_qp_mw()
648 	qp->rx_index = 0;  in ntb_transport_setup_qp_mw()
656 	for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) {  in ntb_transport_setup_qp_mw()
661 		entry->qp = qp;  in ntb_transport_setup_qp_mw()
662 		ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,  in ntb_transport_setup_qp_mw()
663 			     &qp->rx_free_q);  in ntb_transport_setup_qp_mw()
664 		qp->rx_alloc_entry++;  in ntb_transport_setup_qp_mw()
667 	qp->remote_rx_info->entry = qp->rx_max_entry - 1;  in ntb_transport_setup_qp_mw()
670 	for (i = 0; i < qp->rx_max_entry; i++) {  in ntb_transport_setup_qp_mw()
671 		void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -  in ntb_transport_setup_qp_mw()
676 	qp->rx_pkts = 0;  in ntb_transport_setup_qp_mw()
677 	qp->tx_pkts = 0;  in ntb_transport_setup_qp_mw()
678 	qp->tx_index = 0;  in ntb_transport_setup_qp_mw()
685 	struct ntb_transport_qp *qp = dev;  in ntb_transport_isr()  local
687 	tasklet_schedule(&qp->rxc_db_work);  in ntb_transport_isr()
695 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];  in ntb_transport_setup_qp_peer_msi()  local
704 	qp->peer_msi_desc.addr_offset =  in ntb_transport_setup_qp_peer_msi()
705 		ntb_peer_spad_read(qp->ndev, PIDX, spad);  in ntb_transport_setup_qp_peer_msi()
706 	qp->peer_msi_desc.data =  in ntb_transport_setup_qp_peer_msi()
707 		ntb_peer_spad_read(qp->ndev, PIDX, spad + 1);  in ntb_transport_setup_qp_peer_msi()
709 	dev_dbg(&qp->ndev->pdev->dev, "QP%d Peer MSI addr=%x data=%x\n",  in ntb_transport_setup_qp_peer_msi()
710 		qp_num, qp->peer_msi_desc.addr_offset, qp->peer_msi_desc.data);  in ntb_transport_setup_qp_peer_msi()
712 	if (qp->peer_msi_desc.addr_offset) {  in ntb_transport_setup_qp_peer_msi()
713 		qp->use_msi = true;  in ntb_transport_setup_qp_peer_msi()
714 		dev_info(&qp->ndev->pdev->dev,  in ntb_transport_setup_qp_peer_msi()
722 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];  in ntb_transport_setup_qp_msi()  local
730 		dev_warn_once(&qp->ndev->pdev->dev,  in ntb_transport_setup_qp_msi()
735 	ntb_spad_write(qp->ndev, spad, 0);  in ntb_transport_setup_qp_msi()
736 	ntb_spad_write(qp->ndev, spad + 1, 0);  in ntb_transport_setup_qp_msi()
738 	if (!qp->msi_irq) {  in ntb_transport_setup_qp_msi()
739 		qp->msi_irq = ntbm_msi_request_irq(qp->ndev, ntb_transport_isr,  in ntb_transport_setup_qp_msi()
740 						   KBUILD_MODNAME, qp,  in ntb_transport_setup_qp_msi()
741 						   &qp->msi_desc);  in ntb_transport_setup_qp_msi()
742 		if (qp->msi_irq < 0) {  in ntb_transport_setup_qp_msi()
743 			dev_warn(&qp->ndev->pdev->dev,  in ntb_transport_setup_qp_msi()
750 	rc = ntb_spad_write(qp->ndev, spad, qp->msi_desc.addr_offset);  in ntb_transport_setup_qp_msi()
754 	rc = ntb_spad_write(qp->ndev, spad + 1, qp->msi_desc.data);  in ntb_transport_setup_qp_msi()
758 	dev_dbg(&qp->ndev->pdev->dev, "QP%d MSI %d addr=%x data=%x\n",  in ntb_transport_setup_qp_msi()
759 		qp_num, qp->msi_irq, qp->msi_desc.addr_offset,  in ntb_transport_setup_qp_msi()
760 		qp->msi_desc.data);  in ntb_transport_setup_qp_msi()
765 	devm_free_irq(&nt->ndev->dev, qp->msi_irq, qp);  in ntb_transport_setup_qp_msi()
925 static void ntb_qp_link_context_reset(struct ntb_transport_qp *qp)  in ntb_qp_link_context_reset()  argument
927 	qp->link_is_up = false;  in ntb_qp_link_context_reset()
928 	qp->active = false;  in ntb_qp_link_context_reset()
930 	qp->tx_index = 0;  in ntb_qp_link_context_reset()
931 	qp->rx_index = 0;  in ntb_qp_link_context_reset()
932 	qp->rx_bytes = 0;  in ntb_qp_link_context_reset()
933 	qp->rx_pkts = 0;  in ntb_qp_link_context_reset()
934 	qp->rx_ring_empty = 0;  in ntb_qp_link_context_reset()
935 	qp->rx_err_no_buf = 0;  in ntb_qp_link_context_reset()
936 	qp->rx_err_oflow = 0;  in ntb_qp_link_context_reset()
937 	qp->rx_err_ver = 0;  in ntb_qp_link_context_reset()
938 	qp->rx_memcpy = 0;  in ntb_qp_link_context_reset()
939 	qp->rx_async = 0;  in ntb_qp_link_context_reset()
940 	qp->tx_bytes = 0;  in ntb_qp_link_context_reset()
941 	qp->tx_pkts = 0;  in ntb_qp_link_context_reset()
942 	qp->tx_ring_full = 0;  in ntb_qp_link_context_reset()
943 	qp->tx_err_no_buf = 0;  in ntb_qp_link_context_reset()
944 	qp->tx_memcpy = 0;  in ntb_qp_link_context_reset()
945 	qp->tx_async = 0;  in ntb_qp_link_context_reset()
948 static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)  in ntb_qp_link_down_reset()  argument
950 	ntb_qp_link_context_reset(qp);  in ntb_qp_link_down_reset()
951 	if (qp->remote_rx_info)  in ntb_qp_link_down_reset()
952 		qp->remote_rx_info->entry = qp->rx_max_entry - 1;  in ntb_qp_link_down_reset()
955 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)  in ntb_qp_link_cleanup()  argument
957 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_cleanup()
960 	dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);  in ntb_qp_link_cleanup()
962 	cancel_delayed_work_sync(&qp->link_work);  in ntb_qp_link_cleanup()
963 	ntb_qp_link_down_reset(qp);  in ntb_qp_link_cleanup()
965 	if (qp->event_handler)  in ntb_qp_link_cleanup()
966 		qp->event_handler(qp->cb_data, qp->link_is_up);  in ntb_qp_link_cleanup()
971 	struct ntb_transport_qp *qp = container_of(work,  in ntb_qp_link_cleanup_work()  local
974 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_cleanup_work()
976 	ntb_qp_link_cleanup(qp);  in ntb_qp_link_cleanup_work()
979 		schedule_delayed_work(&qp->link_work,  in ntb_qp_link_cleanup_work()
983 static void ntb_qp_link_down(struct ntb_transport_qp *qp)  in ntb_qp_link_down()  argument
985 	schedule_work(&qp->link_cleanup);  in ntb_qp_link_down()
990 	struct ntb_transport_qp *qp;  in ntb_transport_link_cleanup()  local
999 			qp = &nt->qp_vec[i];  in ntb_transport_link_cleanup()
1000 			ntb_qp_link_cleanup(qp);  in ntb_transport_link_cleanup()
1001 			cancel_work_sync(&qp->link_cleanup);  in ntb_transport_link_cleanup()
1002 			cancel_delayed_work_sync(&qp->link_work);  in ntb_transport_link_cleanup()
1117 		struct ntb_transport_qp *qp = &nt->qp_vec[i];  in ntb_transport_link_work()  local
1122 		if (qp->client_ready)  in ntb_transport_link_work()
1123 			schedule_delayed_work(&qp->link_work, 0);  in ntb_transport_link_work()
1144 	struct ntb_transport_qp *qp = container_of(work,  in ntb_qp_link_work()  local
1147 	struct pci_dev *pdev = qp->ndev->pdev;  in ntb_qp_link_work()
1148 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_work()
1155 	ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num));  in ntb_qp_link_work()
1161 	if (val & BIT(qp->qp_num)) {  in ntb_qp_link_work()
1162 		dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);  in ntb_qp_link_work()
1163 		qp->link_is_up = true;  in ntb_qp_link_work()
1164 		qp->active = true;  in ntb_qp_link_work()
1166 		if (qp->event_handler)  in ntb_qp_link_work()
1167 			qp->event_handler(qp->cb_data, qp->link_is_up);  in ntb_qp_link_work()
1169 		if (qp->active)  in ntb_qp_link_work()
1170 			tasklet_schedule(&qp->rxc_db_work);  in ntb_qp_link_work()
1172 		schedule_delayed_work(&qp->link_work,  in ntb_qp_link_work()
1179 	struct ntb_transport_qp *qp;  in ntb_transport_init_queue()  local
1191 	qp = &nt->qp_vec[qp_num];  in ntb_transport_init_queue()
1192 	qp->qp_num = qp_num;  in ntb_transport_init_queue()
1193 	qp->transport = nt;  in ntb_transport_init_queue()
1194 	qp->ndev = nt->ndev;  in ntb_transport_init_queue()
1195 	qp->client_ready = false;  in ntb_transport_init_queue()
1196 	qp->event_handler = NULL;  in ntb_transport_init_queue()
1197 	ntb_qp_link_context_reset(qp);  in ntb_transport_init_queue()
1213 	qp->tx_mw_size = tx_size;  in ntb_transport_init_queue()
1214 	qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;  in ntb_transport_init_queue()
1215 	if (!qp->tx_mw)  in ntb_transport_init_queue()
1218 	qp->tx_mw_phys = mw_base + qp_offset;  in ntb_transport_init_queue()
1219 	if (!qp->tx_mw_phys)  in ntb_transport_init_queue()
1223 	qp->rx_info = qp->tx_mw + tx_size;  in ntb_transport_init_queue()
1226 	qp->tx_max_frame = min(transport_mtu, tx_size / 2);  in ntb_transport_init_queue()
1227 	qp->tx_max_entry = tx_size / qp->tx_max_frame;  in ntb_transport_init_queue()
1233 		qp->debugfs_dir = debugfs_create_dir(debugfs_name,  in ntb_transport_init_queue()
1236 		qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR,  in ntb_transport_init_queue()
1237 							qp->debugfs_dir, qp,  in ntb_transport_init_queue()
1240 		qp->debugfs_dir = NULL;  in ntb_transport_init_queue()
1241 		qp->debugfs_stats = NULL;  in ntb_transport_init_queue()
1244 	INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);  in ntb_transport_init_queue()
1245 	INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work);  in ntb_transport_init_queue()
1247 	spin_lock_init(&qp->ntb_rx_q_lock);  in ntb_transport_init_queue()
1248 	spin_lock_init(&qp->ntb_tx_free_q_lock);  in ntb_transport_init_queue()
1250 	INIT_LIST_HEAD(&qp->rx_post_q);  in ntb_transport_init_queue()
1251 	INIT_LIST_HEAD(&qp->rx_pend_q);  in ntb_transport_init_queue()
1252 	INIT_LIST_HEAD(&qp->rx_free_q);  in ntb_transport_init_queue()
1253 	INIT_LIST_HEAD(&qp->tx_free_q);  in ntb_transport_init_queue()
1255 	tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db,  in ntb_transport_init_queue()
1256 		     (unsigned long)qp);  in ntb_transport_init_queue()
1424 	struct ntb_transport_qp *qp;  in ntb_transport_free()  local
1436 		qp = &nt->qp_vec[i];  in ntb_transport_free()
1438 			ntb_transport_free_queue(qp);  in ntb_transport_free()
1439 		debugfs_remove_recursive(qp->debugfs_dir);  in ntb_transport_free()
1457 static void ntb_complete_rxc(struct ntb_transport_qp *qp)  in ntb_complete_rxc()  argument
1464 	spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags);  in ntb_complete_rxc()
1466 	while (!list_empty(&qp->rx_post_q)) {  in ntb_complete_rxc()
1467 		entry = list_first_entry(&qp->rx_post_q,  in ntb_complete_rxc()
1473 		iowrite32(entry->rx_index, &qp->rx_info->entry);  in ntb_complete_rxc()
1478 		list_move_tail(&entry->entry, &qp->rx_free_q);  in ntb_complete_rxc()
1480 		spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags);  in ntb_complete_rxc()
1482 		if (qp->rx_handler && qp->client_ready)  in ntb_complete_rxc()
1483 			qp->rx_handler(qp, qp->cb_data, cb_data, len);  in ntb_complete_rxc()
1485 		spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags);  in ntb_complete_rxc()
1488 	spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags);  in ntb_complete_rxc()
1507 			struct ntb_transport_qp *qp = entry->qp;  in ntb_rx_copy_callback()  local
1508 			void *offset = qp->rx_buff + qp->rx_max_frame *  in ntb_rx_copy_callback()
1509 					qp->rx_index;  in ntb_rx_copy_callback()
1512 			qp->rx_memcpy++;  in ntb_rx_copy_callback()
1524 	ntb_complete_rxc(entry->qp);  in ntb_rx_copy_callback()
1543 	struct ntb_transport_qp *qp = entry->qp;  in ntb_async_rx_submit()  local
1544 	struct dma_chan *chan = qp->rx_dma_chan;  in ntb_async_rx_submit()
1594 	qp->last_cookie = cookie;  in ntb_async_rx_submit()
1596 	qp->rx_async++;  in ntb_async_rx_submit()
1610 	struct ntb_transport_qp *qp = entry->qp;  in ntb_async_rx()  local
1611 	struct dma_chan *chan = qp->rx_dma_chan;  in ntb_async_rx()
1625 		qp->rx_async++;  in ntb_async_rx()
1631 	qp->rx_memcpy++;  in ntb_async_rx()
1634 static int ntb_process_rxc(struct ntb_transport_qp *qp)  in ntb_process_rxc()  argument
1640 	offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index;  in ntb_process_rxc()
1641 	hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header);  in ntb_process_rxc()
1643 	dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n",  in ntb_process_rxc()
1644 		qp->qp_num, hdr->ver, hdr->len, hdr->flags);  in ntb_process_rxc()
1647 		dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n");  in ntb_process_rxc()
1648 		qp->rx_ring_empty++;  in ntb_process_rxc()
1653 		dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n");  in ntb_process_rxc()
1654 		ntb_qp_link_down(qp);  in ntb_process_rxc()
1659 	if (hdr->ver != (u32)qp->rx_pkts) {  in ntb_process_rxc()
1660 		dev_dbg(&qp->ndev->pdev->dev,  in ntb_process_rxc()
1662 			qp->rx_pkts, hdr->ver);  in ntb_process_rxc()
1663 		qp->rx_err_ver++;  in ntb_process_rxc()
1667 	entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q);  in ntb_process_rxc()
1669 		dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n");  in ntb_process_rxc()
1670 		qp->rx_err_no_buf++;  in ntb_process_rxc()
1675 	entry->rx_index = qp->rx_index;  in ntb_process_rxc()
1678 		dev_dbg(&qp->ndev->pdev->dev,  in ntb_process_rxc()
1681 		qp->rx_err_oflow++;  in ntb_process_rxc()
1686 		ntb_complete_rxc(qp);  in ntb_process_rxc()
1688 		dev_dbg(&qp->ndev->pdev->dev,  in ntb_process_rxc()
1690 			qp->rx_index, hdr->ver, hdr->len, entry->len);  in ntb_process_rxc()
1692 		qp->rx_bytes += hdr->len;  in ntb_process_rxc()
1693 		qp->rx_pkts++;  in ntb_process_rxc()
1700 	qp->rx_index++;  in ntb_process_rxc()
1701 	qp->rx_index %= qp->rx_max_entry;  in ntb_process_rxc()
1708 	struct ntb_transport_qp *qp = (void *)data;  in ntb_transport_rxc_db()  local
1711 	dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n",  in ntb_transport_rxc_db()
1712 		__func__, qp->qp_num);  in ntb_transport_rxc_db()
1717 	for (i = 0; i < qp->rx_max_entry; i++) {  in ntb_transport_rxc_db()
1718 		rc = ntb_process_rxc(qp);  in ntb_transport_rxc_db()
1723 	if (i && qp->rx_dma_chan)  in ntb_transport_rxc_db()
1724 		dma_async_issue_pending(qp->rx_dma_chan);  in ntb_transport_rxc_db()
1726 	if (i == qp->rx_max_entry) {  in ntb_transport_rxc_db()
1728 		if (qp->active)  in ntb_transport_rxc_db()
1729 			tasklet_schedule(&qp->rxc_db_work);  in ntb_transport_rxc_db()
1730 	} else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) {  in ntb_transport_rxc_db()
1732 		ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num));  in ntb_transport_rxc_db()
1734 		ntb_db_read(qp->ndev);  in ntb_transport_rxc_db()
1740 		if (qp->active)  in ntb_transport_rxc_db()
1741 			tasklet_schedule(&qp->rxc_db_work);  in ntb_transport_rxc_db()
1749 	struct ntb_transport_qp *qp = entry->qp;  in ntb_tx_copy_callback()  local
1764 				qp->tx_mw + qp->tx_max_frame *  in ntb_tx_copy_callback()
1769 			qp->tx_memcpy++;  in ntb_tx_copy_callback()
1781 	if (qp->use_msi)  in ntb_tx_copy_callback()
1782 		ntb_msi_peer_trigger(qp->ndev, PIDX, &qp->peer_msi_desc);  in ntb_tx_copy_callback()
1784 		ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num));  in ntb_tx_copy_callback()
1791 		qp->tx_bytes += entry->len;  in ntb_tx_copy_callback()
1793 		if (qp->tx_handler)  in ntb_tx_copy_callback()
1794 			qp->tx_handler(qp, qp->cb_data, entry->cb_data,  in ntb_tx_copy_callback()
1798 	ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);  in ntb_tx_copy_callback()
1819 static int ntb_async_tx_submit(struct ntb_transport_qp *qp,  in ntb_async_tx_submit()  argument
1823 	struct dma_chan *chan = qp->tx_dma_chan;  in ntb_async_tx_submit()
1833 	dest = qp->tx_mw_dma_addr + qp->tx_max_frame * entry->tx_index;  in ntb_async_tx_submit()
1878 static void ntb_async_tx(struct ntb_transport_qp *qp,  in ntb_async_tx()  argument
1882 	struct dma_chan *chan = qp->tx_dma_chan;  in ntb_async_tx()
1886 	entry->tx_index = qp->tx_index;  in ntb_async_tx()
1887 	offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index;  in ntb_async_tx()
1888 	hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);  in ntb_async_tx()
1892 	iowrite32((u32)qp->tx_pkts, &hdr->ver);  in ntb_async_tx()
1900 	res = ntb_async_tx_submit(qp, entry);  in ntb_async_tx()
1905 		qp->tx_async++;  in ntb_async_tx()
1911 	qp->tx_memcpy++;  in ntb_async_tx()
1914 static int ntb_process_tx(struct ntb_transport_qp *qp,  in ntb_process_tx()  argument
1917 	if (!ntb_transport_tx_free_entry(qp)) {  in ntb_process_tx()
1918 		qp->tx_ring_full++;  in ntb_process_tx()
1922 	if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {  in ntb_process_tx()
1923 		if (qp->tx_handler)  in ntb_process_tx()
1924 			qp->tx_handler(qp, qp->cb_data, NULL, -EIO);  in ntb_process_tx()
1926 		ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,  in ntb_process_tx()
1927 			     &qp->tx_free_q);  in ntb_process_tx()
1931 	ntb_async_tx(qp, entry);  in ntb_process_tx()
1933 	qp->tx_index++;  in ntb_process_tx()
1934 	qp->tx_index %= qp->tx_max_entry;  in ntb_process_tx()
1936 	qp->tx_pkts++;  in ntb_process_tx()
1941 static void ntb_send_link_down(struct ntb_transport_qp *qp)  in ntb_send_link_down()  argument
1943 	struct pci_dev *pdev = qp->ndev->pdev;  in ntb_send_link_down()
1947 	if (!qp->link_is_up)  in ntb_send_link_down()
1950 	dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num);  in ntb_send_link_down()
1953 		entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);  in ntb_send_link_down()
1967 	rc = ntb_process_tx(qp, entry);  in ntb_send_link_down()
1970 			qp->qp_num);  in ntb_send_link_down()
1972 	ntb_qp_link_down_reset(qp);  in ntb_send_link_down()
2002 	struct ntb_transport_qp *qp;  in ntb_transport_create_queue()  local
2022 	qp = &nt->qp_vec[free_queue];  in ntb_transport_create_queue()
2023 	qp_bit = BIT_ULL(qp->qp_num);  in ntb_transport_create_queue()
2027 	qp->cb_data = data;  in ntb_transport_create_queue()
2028 	qp->rx_handler = handlers->rx_handler;  in ntb_transport_create_queue()
2029 	qp->tx_handler = handlers->tx_handler;  in ntb_transport_create_queue()
2030 	qp->event_handler = handlers->event_handler;  in ntb_transport_create_queue()
2036 		qp->tx_dma_chan =  in ntb_transport_create_queue()
2039 		if (!qp->tx_dma_chan)  in ntb_transport_create_queue()
2042 		qp->rx_dma_chan =  in ntb_transport_create_queue()
2045 		if (!qp->rx_dma_chan)  in ntb_transport_create_queue()
2048 		qp->tx_dma_chan = NULL;  in ntb_transport_create_queue()
2049 		qp->rx_dma_chan = NULL;  in ntb_transport_create_queue()
2052 	qp->tx_mw_dma_addr = 0;  in ntb_transport_create_queue()
2053 	if (qp->tx_dma_chan) {  in ntb_transport_create_queue()
2054 		qp->tx_mw_dma_addr =  in ntb_transport_create_queue()
2055 			dma_map_resource(qp->tx_dma_chan->device->dev,  in ntb_transport_create_queue()
2056 					 qp->tx_mw_phys, qp->tx_mw_size,  in ntb_transport_create_queue()
2058 		if (dma_mapping_error(qp->tx_dma_chan->device->dev,  in ntb_transport_create_queue()
2059 				      qp->tx_mw_dma_addr)) {  in ntb_transport_create_queue()
2060 			qp->tx_mw_dma_addr = 0;  in ntb_transport_create_queue()
2066 		qp->tx_dma_chan ? "DMA" : "CPU");  in ntb_transport_create_queue()
2069 		qp->rx_dma_chan ? "DMA" : "CPU");  in ntb_transport_create_queue()
2076 		entry->qp = qp;  in ntb_transport_create_queue()
2077 		ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,  in ntb_transport_create_queue()
2078 			     &qp->rx_free_q);  in ntb_transport_create_queue()
2080 	qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES;  in ntb_transport_create_queue()
2082 	for (i = 0; i < qp->tx_max_entry; i++) {  in ntb_transport_create_queue()
2087 		entry->qp = qp;  in ntb_transport_create_queue()
2088 		ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,  in ntb_transport_create_queue()
2089 			     &qp->tx_free_q);  in ntb_transport_create_queue()
2092 	ntb_db_clear(qp->ndev, qp_bit);  in ntb_transport_create_queue()
2093 	ntb_db_clear_mask(qp->ndev, qp_bit);  in ntb_transport_create_queue()
2095 	dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num);  in ntb_transport_create_queue()
2097 	return qp;  in ntb_transport_create_queue()
2100 	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))  in ntb_transport_create_queue()
2103 	qp->rx_alloc_entry = 0;  in ntb_transport_create_queue()
2104 	while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q)))  in ntb_transport_create_queue()
2106 	if (qp->tx_mw_dma_addr)  in ntb_transport_create_queue()
2107 		dma_unmap_resource(qp->tx_dma_chan->device->dev,  in ntb_transport_create_queue()
2108 				   qp->tx_mw_dma_addr, qp->tx_mw_size,  in ntb_transport_create_queue()
2110 	if (qp->tx_dma_chan)  in ntb_transport_create_queue()
2111 		dma_release_channel(qp->tx_dma_chan);  in ntb_transport_create_queue()
2112 	if (qp->rx_dma_chan)  in ntb_transport_create_queue()
2113 		dma_release_channel(qp->rx_dma_chan);  in ntb_transport_create_queue()
2126 void ntb_transport_free_queue(struct ntb_transport_qp *qp)  in ntb_transport_free_queue()  argument
2132 	if (!qp)  in ntb_transport_free_queue()
2135 	pdev = qp->ndev->pdev;  in ntb_transport_free_queue()
2137 	qp->active = false;  in ntb_transport_free_queue()
2139 	if (qp->tx_dma_chan) {  in ntb_transport_free_queue()
2140 		struct dma_chan *chan = qp->tx_dma_chan;  in ntb_transport_free_queue()
2144 		qp->tx_dma_chan = NULL;  in ntb_transport_free_queue()
2149 		dma_sync_wait(chan, qp->last_cookie);  in ntb_transport_free_queue()
2153 				   qp->tx_mw_dma_addr, qp->tx_mw_size,  in ntb_transport_free_queue()
2159 	if (qp->rx_dma_chan) {  in ntb_transport_free_queue()
2160 		struct dma_chan *chan = qp->rx_dma_chan;  in ntb_transport_free_queue()
2164 		qp->rx_dma_chan = NULL;  in ntb_transport_free_queue()
2169 		dma_sync_wait(chan, qp->last_cookie);  in ntb_transport_free_queue()
2174 	qp_bit = BIT_ULL(qp->qp_num);  in ntb_transport_free_queue()
2176 	ntb_db_set_mask(qp->ndev, qp_bit);  in ntb_transport_free_queue()
2177 	tasklet_kill(&qp->rxc_db_work);  in ntb_transport_free_queue()
2179 	cancel_delayed_work_sync(&qp->link_work);  in ntb_transport_free_queue()
2181 	qp->cb_data = NULL;  in ntb_transport_free_queue()
2182 	qp->rx_handler = NULL;  in ntb_transport_free_queue()
2183 	qp->tx_handler = NULL;  in ntb_transport_free_queue()
2184 	qp->event_handler = NULL;  in ntb_transport_free_queue()
2186 	while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q)))  in ntb_transport_free_queue()
2189 	while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) {  in ntb_transport_free_queue()
2194 	while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) {  in ntb_transport_free_queue()
2199 	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))  in ntb_transport_free_queue()
2202 	qp->transport->qp_bitmap_free |= qp_bit;  in ntb_transport_free_queue()
2204 	dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num);  in ntb_transport_free_queue()
2218 void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len)  in ntb_transport_rx_remove()  argument
2223 	if (!qp || qp->client_ready)  in ntb_transport_rx_remove()
2226 	entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q);  in ntb_transport_rx_remove()
2233 	ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);  in ntb_transport_rx_remove()
2251 int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,  in ntb_transport_rx_enqueue()  argument
2256 	if (!qp)  in ntb_transport_rx_enqueue()
2259 	entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q);  in ntb_transport_rx_enqueue()
2271 	ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);  in ntb_transport_rx_enqueue()
2273 	if (qp->active)  in ntb_transport_rx_enqueue()
2274 		tasklet_schedule(&qp->rxc_db_work);  in ntb_transport_rx_enqueue()
2293 int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,  in ntb_transport_tx_enqueue()  argument
2299 	if (!qp || !len)  in ntb_transport_tx_enqueue()
2303 	if (!qp->link_is_up)  in ntb_transport_tx_enqueue()
2306 	entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);  in ntb_transport_tx_enqueue()
2308 		qp->tx_err_no_buf++;  in ntb_transport_tx_enqueue()
2320 	rc = ntb_process_tx(qp, entry);  in ntb_transport_tx_enqueue()
2322 		ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,  in ntb_transport_tx_enqueue()
2323 			     &qp->tx_free_q);  in ntb_transport_tx_enqueue()
2335 void ntb_transport_link_up(struct ntb_transport_qp *qp)  in ntb_transport_link_up()  argument
2337 	if (!qp)  in ntb_transport_link_up()
2340 	qp->client_ready = true;  in ntb_transport_link_up()
2342 	if (qp->transport->link_is_up)  in ntb_transport_link_up()
2343 		schedule_delayed_work(&qp->link_work, 0);  in ntb_transport_link_up()
2355 void ntb_transport_link_down(struct ntb_transport_qp *qp)  in ntb_transport_link_down()  argument
2359 	if (!qp)  in ntb_transport_link_down()
2362 	qp->client_ready = false;  in ntb_transport_link_down()
2364 	val = ntb_spad_read(qp->ndev, QP_LINKS);  in ntb_transport_link_down()
2366 	ntb_peer_spad_write(qp->ndev, PIDX, QP_LINKS, val & ~BIT(qp->qp_num));  in ntb_transport_link_down()
2368 	if (qp->link_is_up)  in ntb_transport_link_down()
2369 		ntb_send_link_down(qp);  in ntb_transport_link_down()
2371 		cancel_delayed_work_sync(&qp->link_work);  in ntb_transport_link_down()
2383 bool ntb_transport_link_query(struct ntb_transport_qp *qp)  in ntb_transport_link_query()  argument
2385 	if (!qp)  in ntb_transport_link_query()
2388 	return qp->link_is_up;  in ntb_transport_link_query()
2400 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp)  in ntb_transport_qp_num()  argument
2402 	if (!qp)  in ntb_transport_qp_num()
2405 	return qp->qp_num;  in ntb_transport_qp_num()
2417 unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp)  in ntb_transport_max_size()  argument
2423 	if (!qp)  in ntb_transport_max_size()
2426 	rx_chan = qp->rx_dma_chan;  in ntb_transport_max_size()
2427 	tx_chan = qp->tx_dma_chan;  in ntb_transport_max_size()
2433 	max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header);  in ntb_transport_max_size()
2440 unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp)  in ntb_transport_tx_free_entry()  argument
2442 	unsigned int head = qp->tx_index;  in ntb_transport_tx_free_entry()
2443 	unsigned int tail = qp->remote_rx_info->entry;  in ntb_transport_tx_free_entry()
2445 	return tail >= head ? tail - head : qp->tx_max_entry + tail - head;  in ntb_transport_tx_free_entry()
2452 	struct ntb_transport_qp *qp;  in ntb_transport_doorbell_callback()  local
2466 		qp = &nt->qp_vec[qp_num];  in ntb_transport_doorbell_callback()
2468 		if (qp->active)  in ntb_transport_doorbell_callback()
2469 			tasklet_schedule(&qp->rxc_db_work);  in ntb_transport_doorbell_callback()