12025cf9eSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
233dea5aaSLogan Gunthorpe /*
333dea5aaSLogan Gunthorpe * Microsemi Switchtec(tm) PCIe Management Driver
433dea5aaSLogan Gunthorpe * Copyright (c) 2017, Microsemi Corporation
533dea5aaSLogan Gunthorpe */
633dea5aaSLogan Gunthorpe
78fdf9062SLogan Gunthorpe #include <linux/interrupt.h>
88fdf9062SLogan Gunthorpe #include <linux/io-64-nonatomic-lo-hi.h>
9ec0467ccSLogan Gunthorpe #include <linux/delay.h>
10ec0467ccSLogan Gunthorpe #include <linux/kthread.h>
118fdf9062SLogan Gunthorpe #include <linux/module.h>
12e099b45bSLogan Gunthorpe #include <linux/ntb.h>
13cfdfc14eSDoug Meyer #include <linux/pci.h>
148fdf9062SLogan Gunthorpe #include <linux/switchtec.h>
1533dea5aaSLogan Gunthorpe
1633dea5aaSLogan Gunthorpe MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver");
1733dea5aaSLogan Gunthorpe MODULE_VERSION("0.1");
1833dea5aaSLogan Gunthorpe MODULE_LICENSE("GPL");
1933dea5aaSLogan Gunthorpe MODULE_AUTHOR("Microsemi Corporation");
2033dea5aaSLogan Gunthorpe
2187d11e64SLogan Gunthorpe static ulong max_mw_size = SZ_2M;
2287d11e64SLogan Gunthorpe module_param(max_mw_size, ulong, 0644);
2387d11e64SLogan Gunthorpe MODULE_PARM_DESC(max_mw_size,
2487d11e64SLogan Gunthorpe "Max memory window size reported to the upper layer");
2587d11e64SLogan Gunthorpe
26ec0467ccSLogan Gunthorpe static bool use_lut_mws;
27ec0467ccSLogan Gunthorpe module_param(use_lut_mws, bool, 0644);
28ec0467ccSLogan Gunthorpe MODULE_PARM_DESC(use_lut_mws,
29ec0467ccSLogan Gunthorpe "Enable the use of the LUT based memory windows");
30ec0467ccSLogan Gunthorpe
31ec0467ccSLogan Gunthorpe #define SWITCHTEC_NTB_MAGIC 0x45CC0001
32ec0467ccSLogan Gunthorpe #define MAX_MWS 128
33ec0467ccSLogan Gunthorpe
34ec0467ccSLogan Gunthorpe struct shared_mw {
35ec0467ccSLogan Gunthorpe u32 magic;
360ee28f26SLogan Gunthorpe u32 link_sta;
37ec0467ccSLogan Gunthorpe u32 partition_id;
38ec0467ccSLogan Gunthorpe u64 mw_sizes[MAX_MWS];
39b9a4acacSLogan Gunthorpe u32 spad[128];
40ec0467ccSLogan Gunthorpe };
41ec0467ccSLogan Gunthorpe
42ec0467ccSLogan Gunthorpe #define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry)
43ec0467ccSLogan Gunthorpe #define LUT_SIZE SZ_64K
44ec0467ccSLogan Gunthorpe
4533dea5aaSLogan Gunthorpe struct switchtec_ntb {
46e099b45bSLogan Gunthorpe struct ntb_dev ntb;
4733dea5aaSLogan Gunthorpe struct switchtec_dev *stdev;
48ec0467ccSLogan Gunthorpe
49ec0467ccSLogan Gunthorpe int self_partition;
50ec0467ccSLogan Gunthorpe int peer_partition;
51ec0467ccSLogan Gunthorpe
523dd4db47SLogan Gunthorpe int doorbell_irq;
533dd4db47SLogan Gunthorpe int message_irq;
543dd4db47SLogan Gunthorpe
55ec0467ccSLogan Gunthorpe struct ntb_info_regs __iomem *mmio_ntb;
56ec0467ccSLogan Gunthorpe struct ntb_ctrl_regs __iomem *mmio_ctrl;
57ec0467ccSLogan Gunthorpe struct ntb_dbmsg_regs __iomem *mmio_dbmsg;
58ec0467ccSLogan Gunthorpe struct ntb_ctrl_regs __iomem *mmio_self_ctrl;
59ec0467ccSLogan Gunthorpe struct ntb_ctrl_regs __iomem *mmio_peer_ctrl;
60ec0467ccSLogan Gunthorpe struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg;
61270d32e6SLogan Gunthorpe struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg;
62ec0467ccSLogan Gunthorpe
6301752501SLogan Gunthorpe void __iomem *mmio_xlink_win;
6401752501SLogan Gunthorpe
65ec0467ccSLogan Gunthorpe struct shared_mw *self_shared;
66ec0467ccSLogan Gunthorpe struct shared_mw __iomem *peer_shared;
67ec0467ccSLogan Gunthorpe dma_addr_t self_shared_dma;
68ec0467ccSLogan Gunthorpe
693dd4db47SLogan Gunthorpe u64 db_mask;
703dd4db47SLogan Gunthorpe u64 db_valid_mask;
713dd4db47SLogan Gunthorpe int db_shift;
723dd4db47SLogan Gunthorpe int db_peer_shift;
733dd4db47SLogan Gunthorpe
746619bf95SLogan Gunthorpe /* synchronize rmw access of db_mask and hw reg */
756619bf95SLogan Gunthorpe spinlock_t db_mask_lock;
766619bf95SLogan Gunthorpe
77ec0467ccSLogan Gunthorpe int nr_direct_mw;
78ec0467ccSLogan Gunthorpe int nr_lut_mw;
79c3585cd8SLogan Gunthorpe int nr_rsvd_luts;
80ec0467ccSLogan Gunthorpe int direct_mw_to_bar[MAX_DIRECT_MW];
81ec0467ccSLogan Gunthorpe
82ec0467ccSLogan Gunthorpe int peer_nr_direct_mw;
83ec0467ccSLogan Gunthorpe int peer_nr_lut_mw;
84ec0467ccSLogan Gunthorpe int peer_direct_mw_to_bar[MAX_DIRECT_MW];
850ee28f26SLogan Gunthorpe
860ee28f26SLogan Gunthorpe bool link_is_up;
870ee28f26SLogan Gunthorpe enum ntb_speed link_speed;
880ee28f26SLogan Gunthorpe enum ntb_width link_width;
89a944ccc3SJoey Zhang struct work_struct check_link_status_work;
90a944ccc3SJoey Zhang bool link_force_down;
9133dea5aaSLogan Gunthorpe };
9233dea5aaSLogan Gunthorpe
ntb_sndev(struct ntb_dev * ntb)930ee28f26SLogan Gunthorpe static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
940ee28f26SLogan Gunthorpe {
950ee28f26SLogan Gunthorpe return container_of(ntb, struct switchtec_ntb, ntb);
960ee28f26SLogan Gunthorpe }
970ee28f26SLogan Gunthorpe
switchtec_ntb_part_op(struct switchtec_ntb * sndev,struct ntb_ctrl_regs __iomem * ctl,u32 op,int wait_status)98ec0467ccSLogan Gunthorpe static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
99ec0467ccSLogan Gunthorpe struct ntb_ctrl_regs __iomem *ctl,
100ec0467ccSLogan Gunthorpe u32 op, int wait_status)
101ec0467ccSLogan Gunthorpe {
102ec0467ccSLogan Gunthorpe static const char * const op_text[] = {
103ec0467ccSLogan Gunthorpe [NTB_CTRL_PART_OP_LOCK] = "lock",
104ec0467ccSLogan Gunthorpe [NTB_CTRL_PART_OP_CFG] = "configure",
105ec0467ccSLogan Gunthorpe [NTB_CTRL_PART_OP_RESET] = "reset",
106ec0467ccSLogan Gunthorpe };
107ec0467ccSLogan Gunthorpe
108ec0467ccSLogan Gunthorpe int i;
109ec0467ccSLogan Gunthorpe u32 ps;
110ec0467ccSLogan Gunthorpe int status;
111ec0467ccSLogan Gunthorpe
112ec0467ccSLogan Gunthorpe switch (op) {
113ec0467ccSLogan Gunthorpe case NTB_CTRL_PART_OP_LOCK:
114ec0467ccSLogan Gunthorpe status = NTB_CTRL_PART_STATUS_LOCKING;
115ec0467ccSLogan Gunthorpe break;
116ec0467ccSLogan Gunthorpe case NTB_CTRL_PART_OP_CFG:
117ec0467ccSLogan Gunthorpe status = NTB_CTRL_PART_STATUS_CONFIGURING;
118ec0467ccSLogan Gunthorpe break;
119ec0467ccSLogan Gunthorpe case NTB_CTRL_PART_OP_RESET:
120ec0467ccSLogan Gunthorpe status = NTB_CTRL_PART_STATUS_RESETTING;
121ec0467ccSLogan Gunthorpe break;
122ec0467ccSLogan Gunthorpe default:
123ec0467ccSLogan Gunthorpe return -EINVAL;
124ec0467ccSLogan Gunthorpe }
125ec0467ccSLogan Gunthorpe
126ec0467ccSLogan Gunthorpe iowrite32(op, &ctl->partition_op);
127ec0467ccSLogan Gunthorpe
128ec0467ccSLogan Gunthorpe for (i = 0; i < 1000; i++) {
129ec0467ccSLogan Gunthorpe if (msleep_interruptible(50) != 0) {
130ec0467ccSLogan Gunthorpe iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op);
131ec0467ccSLogan Gunthorpe return -EINTR;
132ec0467ccSLogan Gunthorpe }
133ec0467ccSLogan Gunthorpe
134ec0467ccSLogan Gunthorpe ps = ioread32(&ctl->partition_status) & 0xFFFF;
135ec0467ccSLogan Gunthorpe
136ec0467ccSLogan Gunthorpe if (ps != status)
137ec0467ccSLogan Gunthorpe break;
138ec0467ccSLogan Gunthorpe }
139ec0467ccSLogan Gunthorpe
140ec0467ccSLogan Gunthorpe if (ps == wait_status)
141ec0467ccSLogan Gunthorpe return 0;
142ec0467ccSLogan Gunthorpe
143ec0467ccSLogan Gunthorpe if (ps == status) {
144ec0467ccSLogan Gunthorpe dev_err(&sndev->stdev->dev,
1452dd0f6a6SJon Mason "Timed out while performing %s (%d). (%08x)\n",
146ec0467ccSLogan Gunthorpe op_text[op], op,
147ec0467ccSLogan Gunthorpe ioread32(&ctl->partition_status));
148ec0467ccSLogan Gunthorpe
149ec0467ccSLogan Gunthorpe return -ETIMEDOUT;
150ec0467ccSLogan Gunthorpe }
151ec0467ccSLogan Gunthorpe
152ec0467ccSLogan Gunthorpe return -EIO;
153ec0467ccSLogan Gunthorpe }
154ec0467ccSLogan Gunthorpe
switchtec_ntb_send_msg(struct switchtec_ntb * sndev,int idx,u32 val)1550ee28f26SLogan Gunthorpe static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,
1560ee28f26SLogan Gunthorpe u32 val)
1570ee28f26SLogan Gunthorpe {
158270d32e6SLogan Gunthorpe if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))
1590ee28f26SLogan Gunthorpe return -EINVAL;
1600ee28f26SLogan Gunthorpe
161270d32e6SLogan Gunthorpe iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);
1620ee28f26SLogan Gunthorpe
1630ee28f26SLogan Gunthorpe return 0;
1640ee28f26SLogan Gunthorpe }
1650ee28f26SLogan Gunthorpe
switchtec_ntb_mw_count(struct ntb_dev * ntb,int pidx)166e099b45bSLogan Gunthorpe static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
167e099b45bSLogan Gunthorpe {
16887d11e64SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
16987d11e64SLogan Gunthorpe int nr_direct_mw = sndev->peer_nr_direct_mw;
170c3585cd8SLogan Gunthorpe int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;
17187d11e64SLogan Gunthorpe
17287d11e64SLogan Gunthorpe if (pidx != NTB_DEF_PEER_IDX)
17387d11e64SLogan Gunthorpe return -EINVAL;
17487d11e64SLogan Gunthorpe
17587d11e64SLogan Gunthorpe if (!use_lut_mws)
17687d11e64SLogan Gunthorpe nr_lut_mw = 0;
17787d11e64SLogan Gunthorpe
17887d11e64SLogan Gunthorpe return nr_direct_mw + nr_lut_mw;
17987d11e64SLogan Gunthorpe }
18087d11e64SLogan Gunthorpe
lut_index(struct switchtec_ntb * sndev,int mw_idx)18187d11e64SLogan Gunthorpe static int lut_index(struct switchtec_ntb *sndev, int mw_idx)
18287d11e64SLogan Gunthorpe {
183c3585cd8SLogan Gunthorpe return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;
18487d11e64SLogan Gunthorpe }
18587d11e64SLogan Gunthorpe
peer_lut_index(struct switchtec_ntb * sndev,int mw_idx)18687d11e64SLogan Gunthorpe static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)
18787d11e64SLogan Gunthorpe {
188c3585cd8SLogan Gunthorpe return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;
189e099b45bSLogan Gunthorpe }
190e099b45bSLogan Gunthorpe
switchtec_ntb_mw_get_align(struct ntb_dev * ntb,int pidx,int widx,resource_size_t * addr_align,resource_size_t * size_align,resource_size_t * size_max)191e099b45bSLogan Gunthorpe static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx,
192e099b45bSLogan Gunthorpe int widx, resource_size_t *addr_align,
193e099b45bSLogan Gunthorpe resource_size_t *size_align,
194e099b45bSLogan Gunthorpe resource_size_t *size_max)
195e099b45bSLogan Gunthorpe {
19687d11e64SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
19787d11e64SLogan Gunthorpe int lut;
19887d11e64SLogan Gunthorpe resource_size_t size;
19987d11e64SLogan Gunthorpe
20087d11e64SLogan Gunthorpe if (pidx != NTB_DEF_PEER_IDX)
20187d11e64SLogan Gunthorpe return -EINVAL;
20287d11e64SLogan Gunthorpe
20387d11e64SLogan Gunthorpe lut = widx >= sndev->peer_nr_direct_mw;
20487d11e64SLogan Gunthorpe size = ioread64(&sndev->peer_shared->mw_sizes[widx]);
20587d11e64SLogan Gunthorpe
20687d11e64SLogan Gunthorpe if (size == 0)
20787d11e64SLogan Gunthorpe return -EINVAL;
20887d11e64SLogan Gunthorpe
20987d11e64SLogan Gunthorpe if (addr_align)
21087d11e64SLogan Gunthorpe *addr_align = lut ? size : SZ_4K;
21187d11e64SLogan Gunthorpe
21287d11e64SLogan Gunthorpe if (size_align)
21387d11e64SLogan Gunthorpe *size_align = lut ? size : SZ_4K;
21487d11e64SLogan Gunthorpe
21587d11e64SLogan Gunthorpe if (size_max)
21687d11e64SLogan Gunthorpe *size_max = size;
21787d11e64SLogan Gunthorpe
218e099b45bSLogan Gunthorpe return 0;
219e099b45bSLogan Gunthorpe }
220e099b45bSLogan Gunthorpe
switchtec_ntb_mw_clr_direct(struct switchtec_ntb * sndev,int idx)22187d11e64SLogan Gunthorpe static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx)
22287d11e64SLogan Gunthorpe {
22387d11e64SLogan Gunthorpe struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
22487d11e64SLogan Gunthorpe int bar = sndev->peer_direct_mw_to_bar[idx];
22587d11e64SLogan Gunthorpe u32 ctl_val;
22687d11e64SLogan Gunthorpe
22787d11e64SLogan Gunthorpe ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
22887d11e64SLogan Gunthorpe ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN;
22987d11e64SLogan Gunthorpe iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
23087d11e64SLogan Gunthorpe iowrite32(0, &ctl->bar_entry[bar].win_size);
231a2585cdcSPaul Selles iowrite32(0, &ctl->bar_ext_entry[bar].win_size);
23287d11e64SLogan Gunthorpe iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr);
23387d11e64SLogan Gunthorpe }
23487d11e64SLogan Gunthorpe
switchtec_ntb_mw_clr_lut(struct switchtec_ntb * sndev,int idx)23587d11e64SLogan Gunthorpe static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx)
23687d11e64SLogan Gunthorpe {
23787d11e64SLogan Gunthorpe struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
23887d11e64SLogan Gunthorpe
23987d11e64SLogan Gunthorpe iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]);
24087d11e64SLogan Gunthorpe }
24187d11e64SLogan Gunthorpe
switchtec_ntb_mw_set_direct(struct switchtec_ntb * sndev,int idx,dma_addr_t addr,resource_size_t size)24287d11e64SLogan Gunthorpe static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx,
24387d11e64SLogan Gunthorpe dma_addr_t addr, resource_size_t size)
24487d11e64SLogan Gunthorpe {
24587d11e64SLogan Gunthorpe int xlate_pos = ilog2(size);
24687d11e64SLogan Gunthorpe int bar = sndev->peer_direct_mw_to_bar[idx];
24787d11e64SLogan Gunthorpe struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
24887d11e64SLogan Gunthorpe u32 ctl_val;
24987d11e64SLogan Gunthorpe
25087d11e64SLogan Gunthorpe ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
25187d11e64SLogan Gunthorpe ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
25287d11e64SLogan Gunthorpe
25387d11e64SLogan Gunthorpe iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
254a2585cdcSPaul Selles iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
255a2585cdcSPaul Selles &ctl->bar_entry[bar].win_size);
256a2585cdcSPaul Selles iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
25787d11e64SLogan Gunthorpe iowrite64(sndev->self_partition | addr,
25887d11e64SLogan Gunthorpe &ctl->bar_entry[bar].xlate_addr);
25987d11e64SLogan Gunthorpe }
26087d11e64SLogan Gunthorpe
switchtec_ntb_mw_set_lut(struct switchtec_ntb * sndev,int idx,dma_addr_t addr,resource_size_t size)26187d11e64SLogan Gunthorpe static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx,
26287d11e64SLogan Gunthorpe dma_addr_t addr, resource_size_t size)
26387d11e64SLogan Gunthorpe {
26487d11e64SLogan Gunthorpe struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
26587d11e64SLogan Gunthorpe
26687d11e64SLogan Gunthorpe iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr),
26787d11e64SLogan Gunthorpe &ctl->lut_entry[peer_lut_index(sndev, idx)]);
26887d11e64SLogan Gunthorpe }
26987d11e64SLogan Gunthorpe
switchtec_ntb_mw_set_trans(struct ntb_dev * ntb,int pidx,int widx,dma_addr_t addr,resource_size_t size)270e099b45bSLogan Gunthorpe static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
271e099b45bSLogan Gunthorpe dma_addr_t addr, resource_size_t size)
272e099b45bSLogan Gunthorpe {
27387d11e64SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
27487d11e64SLogan Gunthorpe struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
27587d11e64SLogan Gunthorpe int xlate_pos = ilog2(size);
27687d11e64SLogan Gunthorpe int nr_direct_mw = sndev->peer_nr_direct_mw;
27787d11e64SLogan Gunthorpe int rc;
27887d11e64SLogan Gunthorpe
27987d11e64SLogan Gunthorpe if (pidx != NTB_DEF_PEER_IDX)
28087d11e64SLogan Gunthorpe return -EINVAL;
28187d11e64SLogan Gunthorpe
2822dd0f6a6SJon Mason dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",
28387d11e64SLogan Gunthorpe widx, pidx, &addr, &size);
28487d11e64SLogan Gunthorpe
28587d11e64SLogan Gunthorpe if (widx >= switchtec_ntb_mw_count(ntb, pidx))
28687d11e64SLogan Gunthorpe return -EINVAL;
28787d11e64SLogan Gunthorpe
288788b041aSAlexander Fomichev if (size != 0 && xlate_pos < 12)
28987d11e64SLogan Gunthorpe return -EINVAL;
29087d11e64SLogan Gunthorpe
2911e2fd202SLogan Gunthorpe if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
2921e2fd202SLogan Gunthorpe /*
2931e2fd202SLogan Gunthorpe * In certain circumstances we can get a buffer that is
2941e2fd202SLogan Gunthorpe * not aligned to its size. (Most of the time
2951e2fd202SLogan Gunthorpe * dma_alloc_coherent ensures this). This can happen when
2961e2fd202SLogan Gunthorpe * using large buffers allocated by the CMA
2971e2fd202SLogan Gunthorpe * (see CMA_CONFIG_ALIGNMENT)
2981e2fd202SLogan Gunthorpe */
2991e2fd202SLogan Gunthorpe dev_err(&sndev->stdev->dev,
30078c5335bSRandy Dunlap "ERROR: Memory window address is not aligned to its size!\n");
3011e2fd202SLogan Gunthorpe return -EINVAL;
3021e2fd202SLogan Gunthorpe }
3031e2fd202SLogan Gunthorpe
30487d11e64SLogan Gunthorpe rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
30587d11e64SLogan Gunthorpe NTB_CTRL_PART_STATUS_LOCKED);
30687d11e64SLogan Gunthorpe if (rc)
30787d11e64SLogan Gunthorpe return rc;
30887d11e64SLogan Gunthorpe
309c16c6655SAlexander Fomichev if (size == 0) {
31087d11e64SLogan Gunthorpe if (widx < nr_direct_mw)
31187d11e64SLogan Gunthorpe switchtec_ntb_mw_clr_direct(sndev, widx);
31287d11e64SLogan Gunthorpe else
31387d11e64SLogan Gunthorpe switchtec_ntb_mw_clr_lut(sndev, widx);
31487d11e64SLogan Gunthorpe } else {
31587d11e64SLogan Gunthorpe if (widx < nr_direct_mw)
31687d11e64SLogan Gunthorpe switchtec_ntb_mw_set_direct(sndev, widx, addr, size);
31787d11e64SLogan Gunthorpe else
31887d11e64SLogan Gunthorpe switchtec_ntb_mw_set_lut(sndev, widx, addr, size);
31987d11e64SLogan Gunthorpe }
32087d11e64SLogan Gunthorpe
32187d11e64SLogan Gunthorpe rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
32287d11e64SLogan Gunthorpe NTB_CTRL_PART_STATUS_NORMAL);
32387d11e64SLogan Gunthorpe
32487d11e64SLogan Gunthorpe if (rc == -EIO) {
32587d11e64SLogan Gunthorpe dev_err(&sndev->stdev->dev,
3262dd0f6a6SJon Mason "Hardware reported an error configuring mw %d: %08x\n",
32787d11e64SLogan Gunthorpe widx, ioread32(&ctl->bar_error));
32887d11e64SLogan Gunthorpe
32987d11e64SLogan Gunthorpe if (widx < nr_direct_mw)
33087d11e64SLogan Gunthorpe switchtec_ntb_mw_clr_direct(sndev, widx);
33187d11e64SLogan Gunthorpe else
33287d11e64SLogan Gunthorpe switchtec_ntb_mw_clr_lut(sndev, widx);
33387d11e64SLogan Gunthorpe
33487d11e64SLogan Gunthorpe switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
33587d11e64SLogan Gunthorpe NTB_CTRL_PART_STATUS_NORMAL);
33687d11e64SLogan Gunthorpe }
33787d11e64SLogan Gunthorpe
33887d11e64SLogan Gunthorpe return rc;
339e099b45bSLogan Gunthorpe }
340e099b45bSLogan Gunthorpe
switchtec_ntb_peer_mw_count(struct ntb_dev * ntb)341e099b45bSLogan Gunthorpe static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb)
342e099b45bSLogan Gunthorpe {
34387d11e64SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
344c3585cd8SLogan Gunthorpe int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;
34587d11e64SLogan Gunthorpe
346c3585cd8SLogan Gunthorpe return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);
34787d11e64SLogan Gunthorpe }
34887d11e64SLogan Gunthorpe
switchtec_ntb_direct_get_addr(struct switchtec_ntb * sndev,int idx,phys_addr_t * base,resource_size_t * size)34987d11e64SLogan Gunthorpe static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,
35087d11e64SLogan Gunthorpe int idx, phys_addr_t *base,
35187d11e64SLogan Gunthorpe resource_size_t *size)
35287d11e64SLogan Gunthorpe {
35387d11e64SLogan Gunthorpe int bar = sndev->direct_mw_to_bar[idx];
35487d11e64SLogan Gunthorpe size_t offset = 0;
35587d11e64SLogan Gunthorpe
35687d11e64SLogan Gunthorpe if (bar < 0)
35787d11e64SLogan Gunthorpe return -EINVAL;
35887d11e64SLogan Gunthorpe
35987d11e64SLogan Gunthorpe if (idx == 0) {
36087d11e64SLogan Gunthorpe /*
36187d11e64SLogan Gunthorpe * This is the direct BAR shared with the LUTs
36287d11e64SLogan Gunthorpe * which means the actual window will be offset
36387d11e64SLogan Gunthorpe * by the size of all the LUT entries.
36487d11e64SLogan Gunthorpe */
36587d11e64SLogan Gunthorpe
36687d11e64SLogan Gunthorpe offset = LUT_SIZE * sndev->nr_lut_mw;
36787d11e64SLogan Gunthorpe }
36887d11e64SLogan Gunthorpe
36987d11e64SLogan Gunthorpe if (base)
37087d11e64SLogan Gunthorpe *base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
37187d11e64SLogan Gunthorpe
37287d11e64SLogan Gunthorpe if (size) {
37387d11e64SLogan Gunthorpe *size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
37487d11e64SLogan Gunthorpe if (offset && *size > offset)
37587d11e64SLogan Gunthorpe *size = offset;
37687d11e64SLogan Gunthorpe
37787d11e64SLogan Gunthorpe if (*size > max_mw_size)
37887d11e64SLogan Gunthorpe *size = max_mw_size;
37987d11e64SLogan Gunthorpe }
38087d11e64SLogan Gunthorpe
38187d11e64SLogan Gunthorpe return 0;
38287d11e64SLogan Gunthorpe }
38387d11e64SLogan Gunthorpe
switchtec_ntb_lut_get_addr(struct switchtec_ntb * sndev,int idx,phys_addr_t * base,resource_size_t * size)38487d11e64SLogan Gunthorpe static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev,
38587d11e64SLogan Gunthorpe int idx, phys_addr_t *base,
38687d11e64SLogan Gunthorpe resource_size_t *size)
38787d11e64SLogan Gunthorpe {
38887d11e64SLogan Gunthorpe int bar = sndev->direct_mw_to_bar[0];
38987d11e64SLogan Gunthorpe int offset;
39087d11e64SLogan Gunthorpe
39187d11e64SLogan Gunthorpe offset = LUT_SIZE * lut_index(sndev, idx);
39287d11e64SLogan Gunthorpe
39387d11e64SLogan Gunthorpe if (base)
39487d11e64SLogan Gunthorpe *base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
39587d11e64SLogan Gunthorpe
39687d11e64SLogan Gunthorpe if (size)
39787d11e64SLogan Gunthorpe *size = LUT_SIZE;
39887d11e64SLogan Gunthorpe
399e099b45bSLogan Gunthorpe return 0;
400e099b45bSLogan Gunthorpe }
401e099b45bSLogan Gunthorpe
switchtec_ntb_peer_mw_get_addr(struct ntb_dev * ntb,int idx,phys_addr_t * base,resource_size_t * size)402e099b45bSLogan Gunthorpe static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
403e099b45bSLogan Gunthorpe phys_addr_t *base,
404e099b45bSLogan Gunthorpe resource_size_t *size)
405e099b45bSLogan Gunthorpe {
40687d11e64SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
40787d11e64SLogan Gunthorpe
40887d11e64SLogan Gunthorpe if (idx < sndev->nr_direct_mw)
40987d11e64SLogan Gunthorpe return switchtec_ntb_direct_get_addr(sndev, idx, base, size);
41087d11e64SLogan Gunthorpe else if (idx < switchtec_ntb_peer_mw_count(ntb))
41187d11e64SLogan Gunthorpe return switchtec_ntb_lut_get_addr(sndev, idx, base, size);
41287d11e64SLogan Gunthorpe else
41387d11e64SLogan Gunthorpe return -EINVAL;
414e099b45bSLogan Gunthorpe }
415e099b45bSLogan Gunthorpe
switchtec_ntb_part_link_speed(struct switchtec_ntb * sndev,int partition,enum ntb_speed * speed,enum ntb_width * width)4160ee28f26SLogan Gunthorpe static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev,
4170ee28f26SLogan Gunthorpe int partition,
4180ee28f26SLogan Gunthorpe enum ntb_speed *speed,
4190ee28f26SLogan Gunthorpe enum ntb_width *width)
4200ee28f26SLogan Gunthorpe {
4210ee28f26SLogan Gunthorpe struct switchtec_dev *stdev = sndev->stdev;
4222f58265eSKelvin Cao struct part_cfg_regs __iomem *part_cfg =
4232f58265eSKelvin Cao &stdev->mmio_part_cfg_all[partition];
4242f58265eSKelvin Cao
4252f58265eSKelvin Cao u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
4260ee28f26SLogan Gunthorpe u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]);
4270ee28f26SLogan Gunthorpe
4280ee28f26SLogan Gunthorpe if (speed)
4290ee28f26SLogan Gunthorpe *speed = (linksta >> 16) & 0xF;
4300ee28f26SLogan Gunthorpe
4310ee28f26SLogan Gunthorpe if (width)
4320ee28f26SLogan Gunthorpe *width = (linksta >> 20) & 0x3F;
4330ee28f26SLogan Gunthorpe }
4340ee28f26SLogan Gunthorpe
switchtec_ntb_set_link_speed(struct switchtec_ntb * sndev)4350ee28f26SLogan Gunthorpe static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)
4360ee28f26SLogan Gunthorpe {
4370ee28f26SLogan Gunthorpe enum ntb_speed self_speed, peer_speed;
4380ee28f26SLogan Gunthorpe enum ntb_width self_width, peer_width;
4390ee28f26SLogan Gunthorpe
4400ee28f26SLogan Gunthorpe if (!sndev->link_is_up) {
4410ee28f26SLogan Gunthorpe sndev->link_speed = NTB_SPEED_NONE;
4420ee28f26SLogan Gunthorpe sndev->link_width = NTB_WIDTH_NONE;
4430ee28f26SLogan Gunthorpe return;
4440ee28f26SLogan Gunthorpe }
4450ee28f26SLogan Gunthorpe
4460ee28f26SLogan Gunthorpe switchtec_ntb_part_link_speed(sndev, sndev->self_partition,
4470ee28f26SLogan Gunthorpe &self_speed, &self_width);
4480ee28f26SLogan Gunthorpe switchtec_ntb_part_link_speed(sndev, sndev->peer_partition,
4490ee28f26SLogan Gunthorpe &peer_speed, &peer_width);
4500ee28f26SLogan Gunthorpe
4510ee28f26SLogan Gunthorpe sndev->link_speed = min(self_speed, peer_speed);
4520ee28f26SLogan Gunthorpe sndev->link_width = min(self_width, peer_width);
4530ee28f26SLogan Gunthorpe }
4540ee28f26SLogan Gunthorpe
crosslink_is_enabled(struct switchtec_ntb * sndev)45501752501SLogan Gunthorpe static int crosslink_is_enabled(struct switchtec_ntb *sndev)
45601752501SLogan Gunthorpe {
45701752501SLogan Gunthorpe struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;
45801752501SLogan Gunthorpe
45901752501SLogan Gunthorpe return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);
46001752501SLogan Gunthorpe }
46101752501SLogan Gunthorpe
crosslink_init_dbmsgs(struct switchtec_ntb * sndev)462270d32e6SLogan Gunthorpe static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)
463270d32e6SLogan Gunthorpe {
464270d32e6SLogan Gunthorpe int i;
465270d32e6SLogan Gunthorpe u32 msg_map = 0;
466270d32e6SLogan Gunthorpe
467270d32e6SLogan Gunthorpe if (!crosslink_is_enabled(sndev))
468270d32e6SLogan Gunthorpe return;
469270d32e6SLogan Gunthorpe
470270d32e6SLogan Gunthorpe for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {
471270d32e6SLogan Gunthorpe int m = i | sndev->self_partition << 2;
472270d32e6SLogan Gunthorpe
473270d32e6SLogan Gunthorpe msg_map |= m << i * 8;
474270d32e6SLogan Gunthorpe }
475270d32e6SLogan Gunthorpe
476270d32e6SLogan Gunthorpe iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);
477270d32e6SLogan Gunthorpe iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
478270d32e6SLogan Gunthorpe &sndev->mmio_peer_dbmsg->odb_mask);
479270d32e6SLogan Gunthorpe }
480270d32e6SLogan Gunthorpe
481d04be142SLogan Gunthorpe enum switchtec_msg {
4820ee28f26SLogan Gunthorpe LINK_MESSAGE = 0,
4830ee28f26SLogan Gunthorpe MSG_LINK_UP = 1,
4840ee28f26SLogan Gunthorpe MSG_LINK_DOWN = 2,
4850ee28f26SLogan Gunthorpe MSG_CHECK_LINK = 3,
486d04be142SLogan Gunthorpe MSG_LINK_FORCE_DOWN = 4,
4870ee28f26SLogan Gunthorpe };
4880ee28f26SLogan Gunthorpe
489d04be142SLogan Gunthorpe static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
490d04be142SLogan Gunthorpe
switchtec_ntb_link_status_update(struct switchtec_ntb * sndev)491a944ccc3SJoey Zhang static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev)
4920ee28f26SLogan Gunthorpe {
4930ee28f26SLogan Gunthorpe int link_sta;
4940ee28f26SLogan Gunthorpe int old = sndev->link_is_up;
4950ee28f26SLogan Gunthorpe
4960ee28f26SLogan Gunthorpe link_sta = sndev->self_shared->link_sta;
4970ee28f26SLogan Gunthorpe if (link_sta) {
4980ee28f26SLogan Gunthorpe u64 peer = ioread64(&sndev->peer_shared->magic);
4990ee28f26SLogan Gunthorpe
5000ee28f26SLogan Gunthorpe if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC)
5010ee28f26SLogan Gunthorpe link_sta = peer >> 32;
5020ee28f26SLogan Gunthorpe else
5030ee28f26SLogan Gunthorpe link_sta = 0;
5040ee28f26SLogan Gunthorpe }
5050ee28f26SLogan Gunthorpe
5060ee28f26SLogan Gunthorpe sndev->link_is_up = link_sta;
5070ee28f26SLogan Gunthorpe switchtec_ntb_set_link_speed(sndev);
5080ee28f26SLogan Gunthorpe
5090ee28f26SLogan Gunthorpe if (link_sta != old) {
5100ee28f26SLogan Gunthorpe switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK);
5110ee28f26SLogan Gunthorpe ntb_link_event(&sndev->ntb);
5122dd0f6a6SJon Mason dev_info(&sndev->stdev->dev, "ntb link %s\n",
5130ee28f26SLogan Gunthorpe link_sta ? "up" : "down");
514270d32e6SLogan Gunthorpe
515270d32e6SLogan Gunthorpe if (link_sta)
516270d32e6SLogan Gunthorpe crosslink_init_dbmsgs(sndev);
5170ee28f26SLogan Gunthorpe }
5180ee28f26SLogan Gunthorpe }
5190ee28f26SLogan Gunthorpe
check_link_status_work(struct work_struct * work)520a944ccc3SJoey Zhang static void check_link_status_work(struct work_struct *work)
521a944ccc3SJoey Zhang {
522a944ccc3SJoey Zhang struct switchtec_ntb *sndev;
523a944ccc3SJoey Zhang
524a944ccc3SJoey Zhang sndev = container_of(work, struct switchtec_ntb,
525a944ccc3SJoey Zhang check_link_status_work);
526a944ccc3SJoey Zhang
527a944ccc3SJoey Zhang if (sndev->link_force_down) {
528a944ccc3SJoey Zhang sndev->link_force_down = false;
529a944ccc3SJoey Zhang switchtec_ntb_reinit_peer(sndev);
530a944ccc3SJoey Zhang
531a944ccc3SJoey Zhang if (sndev->link_is_up) {
532a944ccc3SJoey Zhang sndev->link_is_up = 0;
533a944ccc3SJoey Zhang ntb_link_event(&sndev->ntb);
534a944ccc3SJoey Zhang dev_info(&sndev->stdev->dev, "ntb link forced down\n");
535a944ccc3SJoey Zhang }
536a944ccc3SJoey Zhang
537a944ccc3SJoey Zhang return;
538a944ccc3SJoey Zhang }
539a944ccc3SJoey Zhang
540a944ccc3SJoey Zhang switchtec_ntb_link_status_update(sndev);
541a944ccc3SJoey Zhang }
542a944ccc3SJoey Zhang
switchtec_ntb_check_link(struct switchtec_ntb * sndev,enum switchtec_msg msg)543a944ccc3SJoey Zhang static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
544a944ccc3SJoey Zhang enum switchtec_msg msg)
545a944ccc3SJoey Zhang {
546a944ccc3SJoey Zhang if (msg == MSG_LINK_FORCE_DOWN)
547a944ccc3SJoey Zhang sndev->link_force_down = true;
548a944ccc3SJoey Zhang
549a944ccc3SJoey Zhang schedule_work(&sndev->check_link_status_work);
550a944ccc3SJoey Zhang }
551a944ccc3SJoey Zhang
switchtec_ntb_link_notification(struct switchtec_dev * stdev)5520ee28f26SLogan Gunthorpe static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
5530ee28f26SLogan Gunthorpe {
5540ee28f26SLogan Gunthorpe struct switchtec_ntb *sndev = stdev->sndev;
5550ee28f26SLogan Gunthorpe
556d04be142SLogan Gunthorpe switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
5570ee28f26SLogan Gunthorpe }
5580ee28f26SLogan Gunthorpe
switchtec_ntb_link_is_up(struct ntb_dev * ntb,enum ntb_speed * speed,enum ntb_width * width)559e099b45bSLogan Gunthorpe static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb,
560e099b45bSLogan Gunthorpe enum ntb_speed *speed,
561e099b45bSLogan Gunthorpe enum ntb_width *width)
562e099b45bSLogan Gunthorpe {
5630ee28f26SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
5640ee28f26SLogan Gunthorpe
5650ee28f26SLogan Gunthorpe if (speed)
5660ee28f26SLogan Gunthorpe *speed = sndev->link_speed;
5670ee28f26SLogan Gunthorpe if (width)
5680ee28f26SLogan Gunthorpe *width = sndev->link_width;
5690ee28f26SLogan Gunthorpe
5700ee28f26SLogan Gunthorpe return sndev->link_is_up;
571e099b45bSLogan Gunthorpe }
572e099b45bSLogan Gunthorpe
switchtec_ntb_link_enable(struct ntb_dev * ntb,enum ntb_speed max_speed,enum ntb_width max_width)573e099b45bSLogan Gunthorpe static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
574e099b45bSLogan Gunthorpe enum ntb_speed max_speed,
575e099b45bSLogan Gunthorpe enum ntb_width max_width)
576e099b45bSLogan Gunthorpe {
5770ee28f26SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
5780ee28f26SLogan Gunthorpe
5792dd0f6a6SJon Mason dev_dbg(&sndev->stdev->dev, "enabling link\n");
5800ee28f26SLogan Gunthorpe
5810ee28f26SLogan Gunthorpe sndev->self_shared->link_sta = 1;
5820ee28f26SLogan Gunthorpe switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);
5830ee28f26SLogan Gunthorpe
584a944ccc3SJoey Zhang switchtec_ntb_link_status_update(sndev);
5850ee28f26SLogan Gunthorpe
586e099b45bSLogan Gunthorpe return 0;
587e099b45bSLogan Gunthorpe }
588e099b45bSLogan Gunthorpe
switchtec_ntb_link_disable(struct ntb_dev * ntb)589e099b45bSLogan Gunthorpe static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
590e099b45bSLogan Gunthorpe {
5910ee28f26SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
5920ee28f26SLogan Gunthorpe
5932dd0f6a6SJon Mason dev_dbg(&sndev->stdev->dev, "disabling link\n");
5940ee28f26SLogan Gunthorpe
5950ee28f26SLogan Gunthorpe sndev->self_shared->link_sta = 0;
596d04be142SLogan Gunthorpe switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN);
5970ee28f26SLogan Gunthorpe
598a944ccc3SJoey Zhang switchtec_ntb_link_status_update(sndev);
5990ee28f26SLogan Gunthorpe
600e099b45bSLogan Gunthorpe return 0;
601e099b45bSLogan Gunthorpe }
602e099b45bSLogan Gunthorpe
switchtec_ntb_db_valid_mask(struct ntb_dev * ntb)603e099b45bSLogan Gunthorpe static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb)
604e099b45bSLogan Gunthorpe {
6056619bf95SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
6066619bf95SLogan Gunthorpe
6076619bf95SLogan Gunthorpe return sndev->db_valid_mask;
608e099b45bSLogan Gunthorpe }
609e099b45bSLogan Gunthorpe
switchtec_ntb_db_vector_count(struct ntb_dev * ntb)610e099b45bSLogan Gunthorpe static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb)
611e099b45bSLogan Gunthorpe {
6126619bf95SLogan Gunthorpe return 1;
613e099b45bSLogan Gunthorpe }
614e099b45bSLogan Gunthorpe
switchtec_ntb_db_vector_mask(struct ntb_dev * ntb,int db_vector)615e099b45bSLogan Gunthorpe static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
616e099b45bSLogan Gunthorpe {
6176619bf95SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
6186619bf95SLogan Gunthorpe
6196619bf95SLogan Gunthorpe if (db_vector < 0 || db_vector > 1)
620e099b45bSLogan Gunthorpe return 0;
6216619bf95SLogan Gunthorpe
6226619bf95SLogan Gunthorpe return sndev->db_valid_mask;
623e099b45bSLogan Gunthorpe }
624e099b45bSLogan Gunthorpe
switchtec_ntb_db_read(struct ntb_dev * ntb)625e099b45bSLogan Gunthorpe static u64 switchtec_ntb_db_read(struct ntb_dev *ntb)
626e099b45bSLogan Gunthorpe {
6276619bf95SLogan Gunthorpe u64 ret;
6286619bf95SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
6296619bf95SLogan Gunthorpe
6306619bf95SLogan Gunthorpe ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift;
6316619bf95SLogan Gunthorpe
6326619bf95SLogan Gunthorpe return ret & sndev->db_valid_mask;
633e099b45bSLogan Gunthorpe }
634e099b45bSLogan Gunthorpe
switchtec_ntb_db_clear(struct ntb_dev * ntb,u64 db_bits)635e099b45bSLogan Gunthorpe static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
636e099b45bSLogan Gunthorpe {
6376619bf95SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
6386619bf95SLogan Gunthorpe
6396619bf95SLogan Gunthorpe iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb);
6406619bf95SLogan Gunthorpe
641e099b45bSLogan Gunthorpe return 0;
642e099b45bSLogan Gunthorpe }
643e099b45bSLogan Gunthorpe
switchtec_ntb_db_set_mask(struct ntb_dev * ntb,u64 db_bits)644e099b45bSLogan Gunthorpe static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
645e099b45bSLogan Gunthorpe {
6466619bf95SLogan Gunthorpe unsigned long irqflags;
6476619bf95SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
6486619bf95SLogan Gunthorpe
6496619bf95SLogan Gunthorpe if (db_bits & ~sndev->db_valid_mask)
6506619bf95SLogan Gunthorpe return -EINVAL;
6516619bf95SLogan Gunthorpe
6526619bf95SLogan Gunthorpe spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
6536619bf95SLogan Gunthorpe
6546619bf95SLogan Gunthorpe sndev->db_mask |= db_bits << sndev->db_shift;
6556619bf95SLogan Gunthorpe iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
6566619bf95SLogan Gunthorpe
6576619bf95SLogan Gunthorpe spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
6586619bf95SLogan Gunthorpe
659e099b45bSLogan Gunthorpe return 0;
660e099b45bSLogan Gunthorpe }
661e099b45bSLogan Gunthorpe
switchtec_ntb_db_clear_mask(struct ntb_dev * ntb,u64 db_bits)662e099b45bSLogan Gunthorpe static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
663e099b45bSLogan Gunthorpe {
6646619bf95SLogan Gunthorpe unsigned long irqflags;
6656619bf95SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
6666619bf95SLogan Gunthorpe
6676619bf95SLogan Gunthorpe if (db_bits & ~sndev->db_valid_mask)
6686619bf95SLogan Gunthorpe return -EINVAL;
6696619bf95SLogan Gunthorpe
6706619bf95SLogan Gunthorpe spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
6716619bf95SLogan Gunthorpe
6726619bf95SLogan Gunthorpe sndev->db_mask &= ~(db_bits << sndev->db_shift);
6736619bf95SLogan Gunthorpe iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
6746619bf95SLogan Gunthorpe
6756619bf95SLogan Gunthorpe spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
6766619bf95SLogan Gunthorpe
6776619bf95SLogan Gunthorpe return 0;
6786619bf95SLogan Gunthorpe }
6796619bf95SLogan Gunthorpe
switchtec_ntb_db_read_mask(struct ntb_dev * ntb)6806619bf95SLogan Gunthorpe static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb)
6816619bf95SLogan Gunthorpe {
6826619bf95SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
6836619bf95SLogan Gunthorpe
6846619bf95SLogan Gunthorpe return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask;
6856619bf95SLogan Gunthorpe }
6866619bf95SLogan Gunthorpe
switchtec_ntb_peer_db_addr(struct ntb_dev * ntb,phys_addr_t * db_addr,resource_size_t * db_size,u64 * db_data,int db_bit)6876619bf95SLogan Gunthorpe static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
6886619bf95SLogan Gunthorpe phys_addr_t *db_addr,
689ebb09b33SLeonid Ravich resource_size_t *db_size,
690ebb09b33SLeonid Ravich u64 *db_data,
691ebb09b33SLeonid Ravich int db_bit)
6926619bf95SLogan Gunthorpe {
6936619bf95SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
6946619bf95SLogan Gunthorpe unsigned long offset;
6956619bf95SLogan Gunthorpe
696ebb09b33SLeonid Ravich if (unlikely(db_bit >= BITS_PER_LONG_LONG))
697ebb09b33SLeonid Ravich return -EINVAL;
698ebb09b33SLeonid Ravich
699270d32e6SLogan Gunthorpe offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
7006619bf95SLogan Gunthorpe (unsigned long)sndev->stdev->mmio;
7016619bf95SLogan Gunthorpe
7026619bf95SLogan Gunthorpe offset += sndev->db_shift / 8;
7036619bf95SLogan Gunthorpe
7046619bf95SLogan Gunthorpe if (db_addr)
7056619bf95SLogan Gunthorpe *db_addr = pci_resource_start(ntb->pdev, 0) + offset;
7066619bf95SLogan Gunthorpe if (db_size)
7076619bf95SLogan Gunthorpe *db_size = sizeof(u32);
708ebb09b33SLeonid Ravich if (db_data)
709ebb09b33SLeonid Ravich *db_data = BIT_ULL(db_bit) << sndev->db_peer_shift;
7106619bf95SLogan Gunthorpe
711e099b45bSLogan Gunthorpe return 0;
712e099b45bSLogan Gunthorpe }
713e099b45bSLogan Gunthorpe
switchtec_ntb_peer_db_set(struct ntb_dev * ntb,u64 db_bits)714e099b45bSLogan Gunthorpe static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
715e099b45bSLogan Gunthorpe {
7166619bf95SLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
7176619bf95SLogan Gunthorpe
7186619bf95SLogan Gunthorpe iowrite64(db_bits << sndev->db_peer_shift,
719270d32e6SLogan Gunthorpe &sndev->mmio_peer_dbmsg->odb);
7206619bf95SLogan Gunthorpe
721e099b45bSLogan Gunthorpe return 0;
722e099b45bSLogan Gunthorpe }
723e099b45bSLogan Gunthorpe
switchtec_ntb_spad_count(struct ntb_dev * ntb)724e099b45bSLogan Gunthorpe static int switchtec_ntb_spad_count(struct ntb_dev *ntb)
725e099b45bSLogan Gunthorpe {
726b9a4acacSLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
727b9a4acacSLogan Gunthorpe
728b9a4acacSLogan Gunthorpe return ARRAY_SIZE(sndev->self_shared->spad);
729e099b45bSLogan Gunthorpe }
730e099b45bSLogan Gunthorpe
switchtec_ntb_spad_read(struct ntb_dev * ntb,int idx)731e099b45bSLogan Gunthorpe static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx)
732e099b45bSLogan Gunthorpe {
733b9a4acacSLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
734b9a4acacSLogan Gunthorpe
735b9a4acacSLogan Gunthorpe if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
736e099b45bSLogan Gunthorpe return 0;
737b9a4acacSLogan Gunthorpe
738b9a4acacSLogan Gunthorpe if (!sndev->self_shared)
739b9a4acacSLogan Gunthorpe return 0;
740b9a4acacSLogan Gunthorpe
741b9a4acacSLogan Gunthorpe return sndev->self_shared->spad[idx];
742e099b45bSLogan Gunthorpe }
743e099b45bSLogan Gunthorpe
switchtec_ntb_spad_write(struct ntb_dev * ntb,int idx,u32 val)744e099b45bSLogan Gunthorpe static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
745e099b45bSLogan Gunthorpe {
746b9a4acacSLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
747b9a4acacSLogan Gunthorpe
748b9a4acacSLogan Gunthorpe if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
749b9a4acacSLogan Gunthorpe return -EINVAL;
750b9a4acacSLogan Gunthorpe
751b9a4acacSLogan Gunthorpe if (!sndev->self_shared)
752b9a4acacSLogan Gunthorpe return -EIO;
753b9a4acacSLogan Gunthorpe
754b9a4acacSLogan Gunthorpe sndev->self_shared->spad[idx] = val;
755b9a4acacSLogan Gunthorpe
756e099b45bSLogan Gunthorpe return 0;
757e099b45bSLogan Gunthorpe }
758e099b45bSLogan Gunthorpe
switchtec_ntb_peer_spad_read(struct ntb_dev * ntb,int pidx,int sidx)759b9a4acacSLogan Gunthorpe static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx,
760b9a4acacSLogan Gunthorpe int sidx)
761b9a4acacSLogan Gunthorpe {
762b9a4acacSLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
763b9a4acacSLogan Gunthorpe
764b9a4acacSLogan Gunthorpe if (pidx != NTB_DEF_PEER_IDX)
765b9a4acacSLogan Gunthorpe return -EINVAL;
766b9a4acacSLogan Gunthorpe
767b9a4acacSLogan Gunthorpe if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
768b9a4acacSLogan Gunthorpe return 0;
769b9a4acacSLogan Gunthorpe
770b9a4acacSLogan Gunthorpe if (!sndev->peer_shared)
771b9a4acacSLogan Gunthorpe return 0;
772b9a4acacSLogan Gunthorpe
773b9a4acacSLogan Gunthorpe return ioread32(&sndev->peer_shared->spad[sidx]);
774b9a4acacSLogan Gunthorpe }
775b9a4acacSLogan Gunthorpe
switchtec_ntb_peer_spad_write(struct ntb_dev * ntb,int pidx,int sidx,u32 val)776e099b45bSLogan Gunthorpe static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
777e099b45bSLogan Gunthorpe int sidx, u32 val)
778e099b45bSLogan Gunthorpe {
779b9a4acacSLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
780b9a4acacSLogan Gunthorpe
781b9a4acacSLogan Gunthorpe if (pidx != NTB_DEF_PEER_IDX)
782b9a4acacSLogan Gunthorpe return -EINVAL;
783b9a4acacSLogan Gunthorpe
784b9a4acacSLogan Gunthorpe if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
785b9a4acacSLogan Gunthorpe return -EINVAL;
786b9a4acacSLogan Gunthorpe
787b9a4acacSLogan Gunthorpe if (!sndev->peer_shared)
788b9a4acacSLogan Gunthorpe return -EIO;
789b9a4acacSLogan Gunthorpe
790b9a4acacSLogan Gunthorpe iowrite32(val, &sndev->peer_shared->spad[sidx]);
791b9a4acacSLogan Gunthorpe
792b9a4acacSLogan Gunthorpe return 0;
793b9a4acacSLogan Gunthorpe }
794b9a4acacSLogan Gunthorpe
switchtec_ntb_peer_spad_addr(struct ntb_dev * ntb,int pidx,int sidx,phys_addr_t * spad_addr)795b9a4acacSLogan Gunthorpe static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx,
796b9a4acacSLogan Gunthorpe int sidx, phys_addr_t *spad_addr)
797b9a4acacSLogan Gunthorpe {
798b9a4acacSLogan Gunthorpe struct switchtec_ntb *sndev = ntb_sndev(ntb);
799b9a4acacSLogan Gunthorpe unsigned long offset;
800b9a4acacSLogan Gunthorpe
801b9a4acacSLogan Gunthorpe if (pidx != NTB_DEF_PEER_IDX)
802b9a4acacSLogan Gunthorpe return -EINVAL;
803b9a4acacSLogan Gunthorpe
804b9a4acacSLogan Gunthorpe offset = (unsigned long)&sndev->peer_shared->spad[sidx] -
805b9a4acacSLogan Gunthorpe (unsigned long)sndev->stdev->mmio;
806b9a4acacSLogan Gunthorpe
807b9a4acacSLogan Gunthorpe if (spad_addr)
808b9a4acacSLogan Gunthorpe *spad_addr = pci_resource_start(ntb->pdev, 0) + offset;
809b9a4acacSLogan Gunthorpe
810e099b45bSLogan Gunthorpe return 0;
811e099b45bSLogan Gunthorpe }
812e099b45bSLogan Gunthorpe
813e099b45bSLogan Gunthorpe static const struct ntb_dev_ops switchtec_ntb_ops = {
814e099b45bSLogan Gunthorpe .mw_count = switchtec_ntb_mw_count,
815e099b45bSLogan Gunthorpe .mw_get_align = switchtec_ntb_mw_get_align,
816e099b45bSLogan Gunthorpe .mw_set_trans = switchtec_ntb_mw_set_trans,
817e099b45bSLogan Gunthorpe .peer_mw_count = switchtec_ntb_peer_mw_count,
818e099b45bSLogan Gunthorpe .peer_mw_get_addr = switchtec_ntb_peer_mw_get_addr,
819e099b45bSLogan Gunthorpe .link_is_up = switchtec_ntb_link_is_up,
820e099b45bSLogan Gunthorpe .link_enable = switchtec_ntb_link_enable,
821e099b45bSLogan Gunthorpe .link_disable = switchtec_ntb_link_disable,
822e099b45bSLogan Gunthorpe .db_valid_mask = switchtec_ntb_db_valid_mask,
823e099b45bSLogan Gunthorpe .db_vector_count = switchtec_ntb_db_vector_count,
824e099b45bSLogan Gunthorpe .db_vector_mask = switchtec_ntb_db_vector_mask,
825e099b45bSLogan Gunthorpe .db_read = switchtec_ntb_db_read,
826e099b45bSLogan Gunthorpe .db_clear = switchtec_ntb_db_clear,
827e099b45bSLogan Gunthorpe .db_set_mask = switchtec_ntb_db_set_mask,
828e099b45bSLogan Gunthorpe .db_clear_mask = switchtec_ntb_db_clear_mask,
8296619bf95SLogan Gunthorpe .db_read_mask = switchtec_ntb_db_read_mask,
8306619bf95SLogan Gunthorpe .peer_db_addr = switchtec_ntb_peer_db_addr,
831e099b45bSLogan Gunthorpe .peer_db_set = switchtec_ntb_peer_db_set,
832e099b45bSLogan Gunthorpe .spad_count = switchtec_ntb_spad_count,
833e099b45bSLogan Gunthorpe .spad_read = switchtec_ntb_spad_read,
834e099b45bSLogan Gunthorpe .spad_write = switchtec_ntb_spad_write,
835b9a4acacSLogan Gunthorpe .peer_spad_read = switchtec_ntb_peer_spad_read,
836e099b45bSLogan Gunthorpe .peer_spad_write = switchtec_ntb_peer_spad_write,
837b9a4acacSLogan Gunthorpe .peer_spad_addr = switchtec_ntb_peer_spad_addr,
838e099b45bSLogan Gunthorpe };
839e099b45bSLogan Gunthorpe
switchtec_ntb_init_sndev(struct switchtec_ntb * sndev)8403df54c87SKelvin Cao static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
841ec0467ccSLogan Gunthorpe {
8423df54c87SKelvin Cao u64 tpart_vec;
8433df54c87SKelvin Cao int self;
844ec0467ccSLogan Gunthorpe u64 part_map;
845ec0467ccSLogan Gunthorpe
846e099b45bSLogan Gunthorpe sndev->ntb.pdev = sndev->stdev->pdev;
847e099b45bSLogan Gunthorpe sndev->ntb.topo = NTB_TOPO_SWITCH;
848e099b45bSLogan Gunthorpe sndev->ntb.ops = &switchtec_ntb_ops;
849e099b45bSLogan Gunthorpe
850a944ccc3SJoey Zhang INIT_WORK(&sndev->check_link_status_work, check_link_status_work);
851a944ccc3SJoey Zhang sndev->link_force_down = false;
852d04be142SLogan Gunthorpe
853ec0467ccSLogan Gunthorpe sndev->self_partition = sndev->stdev->partition;
854ec0467ccSLogan Gunthorpe
855ec0467ccSLogan Gunthorpe sndev->mmio_ntb = sndev->stdev->mmio_ntb;
8563df54c87SKelvin Cao
8573df54c87SKelvin Cao self = sndev->self_partition;
8583df54c87SKelvin Cao tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);
8593df54c87SKelvin Cao tpart_vec <<= 32;
8603df54c87SKelvin Cao tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);
8613df54c87SKelvin Cao
862ec0467ccSLogan Gunthorpe part_map = ioread64(&sndev->mmio_ntb->ep_map);
863857e239cSJeremy Pallotta tpart_vec &= part_map;
864ec0467ccSLogan Gunthorpe part_map &= ~(1 << sndev->self_partition);
865ec0467ccSLogan Gunthorpe
8667ff351c8SWesley Sheng if (!tpart_vec) {
8673df54c87SKelvin Cao if (sndev->stdev->partition_count != 2) {
8683df54c87SKelvin Cao dev_err(&sndev->stdev->dev,
8693df54c87SKelvin Cao "ntb target partition not defined\n");
8703df54c87SKelvin Cao return -ENODEV;
8713df54c87SKelvin Cao }
8723df54c87SKelvin Cao
8737ff351c8SWesley Sheng if (!part_map) {
8743df54c87SKelvin Cao dev_err(&sndev->stdev->dev,
8753df54c87SKelvin Cao "peer partition is not NT partition\n");
8763df54c87SKelvin Cao return -ENODEV;
8773df54c87SKelvin Cao }
8783df54c87SKelvin Cao
8797ff351c8SWesley Sheng sndev->peer_partition = __ffs64(part_map);
8803df54c87SKelvin Cao } else {
8817ff351c8SWesley Sheng if (__ffs64(tpart_vec) != (fls64(tpart_vec) - 1)) {
8823df54c87SKelvin Cao dev_err(&sndev->stdev->dev,
8833df54c87SKelvin Cao "ntb driver only supports 1 pair of 1-1 ntb mapping\n");
8843df54c87SKelvin Cao return -ENODEV;
8853df54c87SKelvin Cao }
8863df54c87SKelvin Cao
8877ff351c8SWesley Sheng sndev->peer_partition = __ffs64(tpart_vec);
888ff148d8aSDan Carpenter if (!(part_map & (1ULL << sndev->peer_partition))) {
8893df54c87SKelvin Cao dev_err(&sndev->stdev->dev,
8903df54c87SKelvin Cao "ntb target partition is not NT partition\n");
8913df54c87SKelvin Cao return -ENODEV;
8923df54c87SKelvin Cao }
8933df54c87SKelvin Cao }
8943df54c87SKelvin Cao
8953df54c87SKelvin Cao dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",
8963df54c87SKelvin Cao sndev->self_partition, sndev->stdev->partition_count);
897ec0467ccSLogan Gunthorpe
898ec0467ccSLogan Gunthorpe sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +
899ec0467ccSLogan Gunthorpe SWITCHTEC_NTB_REG_CTRL_OFFSET;
900ec0467ccSLogan Gunthorpe sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb +
901ec0467ccSLogan Gunthorpe SWITCHTEC_NTB_REG_DBMSG_OFFSET;
902ec0467ccSLogan Gunthorpe
903ec0467ccSLogan Gunthorpe sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];
904ec0467ccSLogan Gunthorpe sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];
905ec0467ccSLogan Gunthorpe sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];
906270d32e6SLogan Gunthorpe sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;
9073df54c87SKelvin Cao
9083df54c87SKelvin Cao return 0;
909ec0467ccSLogan Gunthorpe }
910ec0467ccSLogan Gunthorpe
config_rsvd_lut_win(struct switchtec_ntb * sndev,struct ntb_ctrl_regs __iomem * ctl,int lut_idx,int partition,u64 addr)91112cb203bSLogan Gunthorpe static int config_rsvd_lut_win(struct switchtec_ntb *sndev,
91212cb203bSLogan Gunthorpe struct ntb_ctrl_regs __iomem *ctl,
91301752501SLogan Gunthorpe int lut_idx, int partition, u64 addr)
91412cb203bSLogan Gunthorpe {
91512cb203bSLogan Gunthorpe int peer_bar = sndev->peer_direct_mw_to_bar[0];
91612cb203bSLogan Gunthorpe u32 ctl_val;
91712cb203bSLogan Gunthorpe int rc;
91812cb203bSLogan Gunthorpe
91912cb203bSLogan Gunthorpe rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
92012cb203bSLogan Gunthorpe NTB_CTRL_PART_STATUS_LOCKED);
92112cb203bSLogan Gunthorpe if (rc)
92212cb203bSLogan Gunthorpe return rc;
92312cb203bSLogan Gunthorpe
92412cb203bSLogan Gunthorpe ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl);
92512cb203bSLogan Gunthorpe ctl_val &= 0xFF;
92612cb203bSLogan Gunthorpe ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
92712cb203bSLogan Gunthorpe ctl_val |= ilog2(LUT_SIZE) << 8;
92812cb203bSLogan Gunthorpe ctl_val |= (sndev->nr_lut_mw - 1) << 14;
92912cb203bSLogan Gunthorpe iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl);
93012cb203bSLogan Gunthorpe
93112cb203bSLogan Gunthorpe iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr),
93212cb203bSLogan Gunthorpe &ctl->lut_entry[lut_idx]);
93312cb203bSLogan Gunthorpe
93412cb203bSLogan Gunthorpe rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
93512cb203bSLogan Gunthorpe NTB_CTRL_PART_STATUS_NORMAL);
93612cb203bSLogan Gunthorpe if (rc) {
93712cb203bSLogan Gunthorpe u32 bar_error, lut_error;
93812cb203bSLogan Gunthorpe
93912cb203bSLogan Gunthorpe bar_error = ioread32(&ctl->bar_error);
94012cb203bSLogan Gunthorpe lut_error = ioread32(&ctl->lut_error);
94112cb203bSLogan Gunthorpe dev_err(&sndev->stdev->dev,
94212cb203bSLogan Gunthorpe "Error setting up reserved lut window: %08x / %08x\n",
94312cb203bSLogan Gunthorpe bar_error, lut_error);
94412cb203bSLogan Gunthorpe return rc;
94512cb203bSLogan Gunthorpe }
94612cb203bSLogan Gunthorpe
94712cb203bSLogan Gunthorpe return 0;
94812cb203bSLogan Gunthorpe }
94912cb203bSLogan Gunthorpe
config_req_id_table(struct switchtec_ntb * sndev,struct ntb_ctrl_regs __iomem * mmio_ctrl,int * req_ids,int count)950bbe35ca5SLogan Gunthorpe static int config_req_id_table(struct switchtec_ntb *sndev,
951bbe35ca5SLogan Gunthorpe struct ntb_ctrl_regs __iomem *mmio_ctrl,
952bbe35ca5SLogan Gunthorpe int *req_ids, int count)
953bbe35ca5SLogan Gunthorpe {
954bbe35ca5SLogan Gunthorpe int i, rc = 0;
955bbe35ca5SLogan Gunthorpe u32 error;
956bbe35ca5SLogan Gunthorpe u32 proxy_id;
957bbe35ca5SLogan Gunthorpe
9588cd77865SKelvin Cao if (ioread16(&mmio_ctrl->req_id_table_size) < count) {
959bbe35ca5SLogan Gunthorpe dev_err(&sndev->stdev->dev,
960bbe35ca5SLogan Gunthorpe "Not enough requester IDs available.\n");
961bbe35ca5SLogan Gunthorpe return -EFAULT;
962bbe35ca5SLogan Gunthorpe }
963bbe35ca5SLogan Gunthorpe
964bbe35ca5SLogan Gunthorpe rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
965bbe35ca5SLogan Gunthorpe NTB_CTRL_PART_OP_LOCK,
966bbe35ca5SLogan Gunthorpe NTB_CTRL_PART_STATUS_LOCKED);
967bbe35ca5SLogan Gunthorpe if (rc)
968bbe35ca5SLogan Gunthorpe return rc;
969bbe35ca5SLogan Gunthorpe
970bbe35ca5SLogan Gunthorpe for (i = 0; i < count; i++) {
971bbe35ca5SLogan Gunthorpe iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN,
972bbe35ca5SLogan Gunthorpe &mmio_ctrl->req_id_table[i]);
973bbe35ca5SLogan Gunthorpe
974bbe35ca5SLogan Gunthorpe proxy_id = ioread32(&mmio_ctrl->req_id_table[i]);
975bbe35ca5SLogan Gunthorpe dev_dbg(&sndev->stdev->dev,
976bbe35ca5SLogan Gunthorpe "Requester ID %02X:%02X.%X -> BB:%02X.%X\n",
977bbe35ca5SLogan Gunthorpe req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F,
978bbe35ca5SLogan Gunthorpe req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F,
979bbe35ca5SLogan Gunthorpe (proxy_id >> 1) & 0x7);
980bbe35ca5SLogan Gunthorpe }
981bbe35ca5SLogan Gunthorpe
982bbe35ca5SLogan Gunthorpe rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
983bbe35ca5SLogan Gunthorpe NTB_CTRL_PART_OP_CFG,
984bbe35ca5SLogan Gunthorpe NTB_CTRL_PART_STATUS_NORMAL);
985bbe35ca5SLogan Gunthorpe
986bbe35ca5SLogan Gunthorpe if (rc == -EIO) {
987bbe35ca5SLogan Gunthorpe error = ioread32(&mmio_ctrl->req_id_error);
988bbe35ca5SLogan Gunthorpe dev_err(&sndev->stdev->dev,
989bbe35ca5SLogan Gunthorpe "Error setting up the requester ID table: %08x\n",
990bbe35ca5SLogan Gunthorpe error);
991bbe35ca5SLogan Gunthorpe }
992bbe35ca5SLogan Gunthorpe
993bbe35ca5SLogan Gunthorpe return 0;
994bbe35ca5SLogan Gunthorpe }
995bbe35ca5SLogan Gunthorpe
crosslink_setup_mws(struct switchtec_ntb * sndev,int ntb_lut_idx,u64 * mw_addrs,int mw_count)99601752501SLogan Gunthorpe static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,
99701752501SLogan Gunthorpe u64 *mw_addrs, int mw_count)
99801752501SLogan Gunthorpe {
99901752501SLogan Gunthorpe int rc, i;
100001752501SLogan Gunthorpe struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;
100101752501SLogan Gunthorpe u64 addr;
100201752501SLogan Gunthorpe size_t size, offset;
100301752501SLogan Gunthorpe int bar;
100401752501SLogan Gunthorpe int xlate_pos;
100501752501SLogan Gunthorpe u32 ctl_val;
100601752501SLogan Gunthorpe
100701752501SLogan Gunthorpe rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
100801752501SLogan Gunthorpe NTB_CTRL_PART_STATUS_LOCKED);
100901752501SLogan Gunthorpe if (rc)
101001752501SLogan Gunthorpe return rc;
101101752501SLogan Gunthorpe
101201752501SLogan Gunthorpe for (i = 0; i < sndev->nr_lut_mw; i++) {
101301752501SLogan Gunthorpe if (i == ntb_lut_idx)
101401752501SLogan Gunthorpe continue;
101501752501SLogan Gunthorpe
101601752501SLogan Gunthorpe addr = mw_addrs[0] + LUT_SIZE * i;
101701752501SLogan Gunthorpe
101801752501SLogan Gunthorpe iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |
101901752501SLogan Gunthorpe addr),
102001752501SLogan Gunthorpe &ctl->lut_entry[i]);
102101752501SLogan Gunthorpe }
102201752501SLogan Gunthorpe
102301752501SLogan Gunthorpe sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);
102401752501SLogan Gunthorpe
102501752501SLogan Gunthorpe for (i = 0; i < sndev->nr_direct_mw; i++) {
102601752501SLogan Gunthorpe bar = sndev->direct_mw_to_bar[i];
102701752501SLogan Gunthorpe offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;
102801752501SLogan Gunthorpe addr = mw_addrs[i] + offset;
102901752501SLogan Gunthorpe size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
103001752501SLogan Gunthorpe xlate_pos = ilog2(size);
103101752501SLogan Gunthorpe
103201752501SLogan Gunthorpe if (offset && size > offset)
103301752501SLogan Gunthorpe size = offset;
103401752501SLogan Gunthorpe
103501752501SLogan Gunthorpe ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
103601752501SLogan Gunthorpe ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
103701752501SLogan Gunthorpe
103801752501SLogan Gunthorpe iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
1039a2585cdcSPaul Selles iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
1040a2585cdcSPaul Selles &ctl->bar_entry[bar].win_size);
1041a2585cdcSPaul Selles iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
104201752501SLogan Gunthorpe iowrite64(sndev->peer_partition | addr,
104301752501SLogan Gunthorpe &ctl->bar_entry[bar].xlate_addr);
104401752501SLogan Gunthorpe }
104501752501SLogan Gunthorpe
104601752501SLogan Gunthorpe rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
104701752501SLogan Gunthorpe NTB_CTRL_PART_STATUS_NORMAL);
104801752501SLogan Gunthorpe if (rc) {
104901752501SLogan Gunthorpe u32 bar_error, lut_error;
105001752501SLogan Gunthorpe
105101752501SLogan Gunthorpe bar_error = ioread32(&ctl->bar_error);
105201752501SLogan Gunthorpe lut_error = ioread32(&ctl->lut_error);
105301752501SLogan Gunthorpe dev_err(&sndev->stdev->dev,
105401752501SLogan Gunthorpe "Error setting up cross link windows: %08x / %08x\n",
105501752501SLogan Gunthorpe bar_error, lut_error);
105601752501SLogan Gunthorpe return rc;
105701752501SLogan Gunthorpe }
105801752501SLogan Gunthorpe
105901752501SLogan Gunthorpe return 0;
106001752501SLogan Gunthorpe }
106101752501SLogan Gunthorpe
crosslink_setup_req_ids(struct switchtec_ntb * sndev,struct ntb_ctrl_regs __iomem * mmio_ctrl)106201752501SLogan Gunthorpe static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,
106301752501SLogan Gunthorpe struct ntb_ctrl_regs __iomem *mmio_ctrl)
106401752501SLogan Gunthorpe {
106501752501SLogan Gunthorpe int req_ids[16];
106601752501SLogan Gunthorpe int i;
106701752501SLogan Gunthorpe u32 proxy_id;
106801752501SLogan Gunthorpe
106901752501SLogan Gunthorpe for (i = 0; i < ARRAY_SIZE(req_ids); i++) {
107001752501SLogan Gunthorpe proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);
107101752501SLogan Gunthorpe
107201752501SLogan Gunthorpe if (!(proxy_id & NTB_CTRL_REQ_ID_EN))
107301752501SLogan Gunthorpe break;
107401752501SLogan Gunthorpe
107501752501SLogan Gunthorpe req_ids[i] = ((proxy_id >> 1) & 0xFF);
107601752501SLogan Gunthorpe }
107701752501SLogan Gunthorpe
107801752501SLogan Gunthorpe return config_req_id_table(sndev, mmio_ctrl, req_ids, i);
107901752501SLogan Gunthorpe }
108001752501SLogan Gunthorpe
108101752501SLogan Gunthorpe /*
108201752501SLogan Gunthorpe * In crosslink configuration there is a virtual partition in the
108301752501SLogan Gunthorpe * middle of the two switches. The BARs in this partition have to be
108401752501SLogan Gunthorpe * enumerated and assigned addresses.
108501752501SLogan Gunthorpe */
crosslink_enum_partition(struct switchtec_ntb * sndev,u64 * bar_addrs)108601752501SLogan Gunthorpe static int crosslink_enum_partition(struct switchtec_ntb *sndev,
108701752501SLogan Gunthorpe u64 *bar_addrs)
108801752501SLogan Gunthorpe {
108901752501SLogan Gunthorpe struct part_cfg_regs __iomem *part_cfg =
109001752501SLogan Gunthorpe &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];
10912f58265eSKelvin Cao u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
109201752501SLogan Gunthorpe struct pff_csr_regs __iomem *mmio_pff =
109301752501SLogan Gunthorpe &sndev->stdev->mmio_pff_csr[pff];
109401752501SLogan Gunthorpe const u64 bar_space = 0x1000000000LL;
109501752501SLogan Gunthorpe u64 bar_addr;
109601752501SLogan Gunthorpe int bar_cnt = 0;
109701752501SLogan Gunthorpe int i;
109801752501SLogan Gunthorpe
109901752501SLogan Gunthorpe iowrite16(0x6, &mmio_pff->pcicmd);
110001752501SLogan Gunthorpe
110101752501SLogan Gunthorpe for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) {
110201752501SLogan Gunthorpe iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]);
110301752501SLogan Gunthorpe bar_addr = ioread64(&mmio_pff->pci_bar64[i]);
110401752501SLogan Gunthorpe bar_addr &= ~0xf;
110501752501SLogan Gunthorpe
110601752501SLogan Gunthorpe dev_dbg(&sndev->stdev->dev,
110701752501SLogan Gunthorpe "Crosslink BAR%d addr: %llx\n",
1108cce8e04cSPaul Selles i*2, bar_addr);
110901752501SLogan Gunthorpe
111001752501SLogan Gunthorpe if (bar_addr != bar_space * i)
111101752501SLogan Gunthorpe continue;
111201752501SLogan Gunthorpe
111301752501SLogan Gunthorpe bar_addrs[bar_cnt++] = bar_addr;
111401752501SLogan Gunthorpe }
111501752501SLogan Gunthorpe
111601752501SLogan Gunthorpe return bar_cnt;
111701752501SLogan Gunthorpe }
111801752501SLogan Gunthorpe
switchtec_ntb_init_crosslink(struct switchtec_ntb * sndev)111901752501SLogan Gunthorpe static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)
112001752501SLogan Gunthorpe {
112101752501SLogan Gunthorpe int rc;
112201752501SLogan Gunthorpe int bar = sndev->direct_mw_to_bar[0];
112301752501SLogan Gunthorpe const int ntb_lut_idx = 1;
112401752501SLogan Gunthorpe u64 bar_addrs[6];
112501752501SLogan Gunthorpe u64 addr;
1126270d32e6SLogan Gunthorpe int offset;
112701752501SLogan Gunthorpe int bar_cnt;
112801752501SLogan Gunthorpe
112901752501SLogan Gunthorpe if (!crosslink_is_enabled(sndev))
113001752501SLogan Gunthorpe return 0;
113101752501SLogan Gunthorpe
113201752501SLogan Gunthorpe dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");
113301752501SLogan Gunthorpe sndev->ntb.topo = NTB_TOPO_CROSSLINK;
113401752501SLogan Gunthorpe
113501752501SLogan Gunthorpe bar_cnt = crosslink_enum_partition(sndev, bar_addrs);
113601752501SLogan Gunthorpe if (bar_cnt < sndev->nr_direct_mw + 1) {
113701752501SLogan Gunthorpe dev_err(&sndev->stdev->dev,
113801752501SLogan Gunthorpe "Error enumerating crosslink partition\n");
113901752501SLogan Gunthorpe return -EINVAL;
114001752501SLogan Gunthorpe }
114101752501SLogan Gunthorpe
1142270d32e6SLogan Gunthorpe addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET +
1143270d32e6SLogan Gunthorpe SWITCHTEC_NTB_REG_DBMSG_OFFSET +
1144270d32e6SLogan Gunthorpe sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);
1145270d32e6SLogan Gunthorpe
1146270d32e6SLogan Gunthorpe offset = addr & (LUT_SIZE - 1);
1147270d32e6SLogan Gunthorpe addr -= offset;
1148270d32e6SLogan Gunthorpe
114901752501SLogan Gunthorpe rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx,
115001752501SLogan Gunthorpe sndev->peer_partition, addr);
115101752501SLogan Gunthorpe if (rc)
115201752501SLogan Gunthorpe return rc;
115301752501SLogan Gunthorpe
115401752501SLogan Gunthorpe rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1],
115501752501SLogan Gunthorpe bar_cnt - 1);
115601752501SLogan Gunthorpe if (rc)
115701752501SLogan Gunthorpe return rc;
115801752501SLogan Gunthorpe
115901752501SLogan Gunthorpe rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl);
116001752501SLogan Gunthorpe if (rc)
116101752501SLogan Gunthorpe return rc;
116201752501SLogan Gunthorpe
116301752501SLogan Gunthorpe sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar,
116401752501SLogan Gunthorpe LUT_SIZE, LUT_SIZE);
116501752501SLogan Gunthorpe if (!sndev->mmio_xlink_win) {
116601752501SLogan Gunthorpe rc = -ENOMEM;
116701752501SLogan Gunthorpe return rc;
116801752501SLogan Gunthorpe }
116901752501SLogan Gunthorpe
1170270d32e6SLogan Gunthorpe sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;
117101752501SLogan Gunthorpe sndev->nr_rsvd_luts++;
117201752501SLogan Gunthorpe
1173270d32e6SLogan Gunthorpe crosslink_init_dbmsgs(sndev);
1174270d32e6SLogan Gunthorpe
117501752501SLogan Gunthorpe return 0;
117601752501SLogan Gunthorpe }
117701752501SLogan Gunthorpe
switchtec_ntb_deinit_crosslink(struct switchtec_ntb * sndev)117801752501SLogan Gunthorpe static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)
117901752501SLogan Gunthorpe {
118001752501SLogan Gunthorpe if (sndev->mmio_xlink_win)
118101752501SLogan Gunthorpe pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win);
118201752501SLogan Gunthorpe }
118301752501SLogan Gunthorpe
map_bars(int * map,struct ntb_ctrl_regs __iomem * ctrl)1184ec0467ccSLogan Gunthorpe static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl)
1185ec0467ccSLogan Gunthorpe {
1186ec0467ccSLogan Gunthorpe int i;
1187ec0467ccSLogan Gunthorpe int cnt = 0;
1188ec0467ccSLogan Gunthorpe
1189ec0467ccSLogan Gunthorpe for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) {
1190ec0467ccSLogan Gunthorpe u32 r = ioread32(&ctrl->bar_entry[i].ctl);
1191ec0467ccSLogan Gunthorpe
1192ec0467ccSLogan Gunthorpe if (r & NTB_CTRL_BAR_VALID)
1193ec0467ccSLogan Gunthorpe map[cnt++] = i;
1194ec0467ccSLogan Gunthorpe }
1195ec0467ccSLogan Gunthorpe
1196ec0467ccSLogan Gunthorpe return cnt;
1197ec0467ccSLogan Gunthorpe }
1198ec0467ccSLogan Gunthorpe
switchtec_ntb_init_mw(struct switchtec_ntb * sndev)1199ec0467ccSLogan Gunthorpe static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
1200ec0467ccSLogan Gunthorpe {
1201ec0467ccSLogan Gunthorpe sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar,
1202ec0467ccSLogan Gunthorpe sndev->mmio_self_ctrl);
1203ec0467ccSLogan Gunthorpe
1204ec0467ccSLogan Gunthorpe sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
1205ec0467ccSLogan Gunthorpe sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
1206ec0467ccSLogan Gunthorpe
12072dd0f6a6SJon Mason dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
1208ec0467ccSLogan Gunthorpe sndev->nr_direct_mw, sndev->nr_lut_mw);
1209ec0467ccSLogan Gunthorpe
1210ec0467ccSLogan Gunthorpe sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar,
1211ec0467ccSLogan Gunthorpe sndev->mmio_peer_ctrl);
1212ec0467ccSLogan Gunthorpe
1213ec0467ccSLogan Gunthorpe sndev->peer_nr_lut_mw =
1214ec0467ccSLogan Gunthorpe ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
1215ec0467ccSLogan Gunthorpe sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
1216ec0467ccSLogan Gunthorpe
12172dd0f6a6SJon Mason dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
1218ec0467ccSLogan Gunthorpe sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
1219ec0467ccSLogan Gunthorpe
1220ec0467ccSLogan Gunthorpe }
1221ec0467ccSLogan Gunthorpe
12223dd4db47SLogan Gunthorpe /*
12233dd4db47SLogan Gunthorpe * There are 64 doorbells in the switch hardware but this is
12243dd4db47SLogan Gunthorpe * shared among all partitions. So we must split them in half
12253dd4db47SLogan Gunthorpe * (32 for each partition). However, the message interrupts are
12263dd4db47SLogan Gunthorpe * also shared with the top 4 doorbells so we just limit this to
1227270d32e6SLogan Gunthorpe * 28 doorbells per partition.
1228270d32e6SLogan Gunthorpe *
1229270d32e6SLogan Gunthorpe * In crosslink mode, each side has it's own dbmsg register so
1230270d32e6SLogan Gunthorpe * they can each use all 60 of the available doorbells.
12313dd4db47SLogan Gunthorpe */
switchtec_ntb_init_db(struct switchtec_ntb * sndev)12323dd4db47SLogan Gunthorpe static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)
12333dd4db47SLogan Gunthorpe {
1234270d32e6SLogan Gunthorpe sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
12353dd4db47SLogan Gunthorpe
1236270d32e6SLogan Gunthorpe if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {
1237270d32e6SLogan Gunthorpe sndev->db_shift = 0;
1238270d32e6SLogan Gunthorpe sndev->db_peer_shift = 0;
1239270d32e6SLogan Gunthorpe sndev->db_valid_mask = sndev->db_mask;
1240270d32e6SLogan Gunthorpe } else if (sndev->self_partition < sndev->peer_partition) {
12413dd4db47SLogan Gunthorpe sndev->db_shift = 0;
12423dd4db47SLogan Gunthorpe sndev->db_peer_shift = 32;
1243270d32e6SLogan Gunthorpe sndev->db_valid_mask = 0x0FFFFFFF;
12443dd4db47SLogan Gunthorpe } else {
12453dd4db47SLogan Gunthorpe sndev->db_shift = 32;
12463dd4db47SLogan Gunthorpe sndev->db_peer_shift = 0;
1247270d32e6SLogan Gunthorpe sndev->db_valid_mask = 0x0FFFFFFF;
12483dd4db47SLogan Gunthorpe }
12493dd4db47SLogan Gunthorpe
12503dd4db47SLogan Gunthorpe iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
12513dd4db47SLogan Gunthorpe iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
1252270d32e6SLogan Gunthorpe &sndev->mmio_peer_dbmsg->odb_mask);
1253270d32e6SLogan Gunthorpe
1254270d32e6SLogan Gunthorpe dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",
1255270d32e6SLogan Gunthorpe sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);
12563dd4db47SLogan Gunthorpe }
12573dd4db47SLogan Gunthorpe
switchtec_ntb_init_msgs(struct switchtec_ntb * sndev)12583dd4db47SLogan Gunthorpe static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
12593dd4db47SLogan Gunthorpe {
12603dd4db47SLogan Gunthorpe int i;
12613dd4db47SLogan Gunthorpe u32 msg_map = 0;
12623dd4db47SLogan Gunthorpe
12633dd4db47SLogan Gunthorpe for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
12643dd4db47SLogan Gunthorpe int m = i | sndev->peer_partition << 2;
12653dd4db47SLogan Gunthorpe
12663dd4db47SLogan Gunthorpe msg_map |= m << i * 8;
12673dd4db47SLogan Gunthorpe }
12683dd4db47SLogan Gunthorpe
12693dd4db47SLogan Gunthorpe iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map);
12703dd4db47SLogan Gunthorpe
12713dd4db47SLogan Gunthorpe for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++)
12723dd4db47SLogan Gunthorpe iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK,
12733dd4db47SLogan Gunthorpe &sndev->mmio_self_dbmsg->imsg[i]);
12743dd4db47SLogan Gunthorpe }
12753dd4db47SLogan Gunthorpe
1276bbe35ca5SLogan Gunthorpe static int
switchtec_ntb_init_req_id_table(struct switchtec_ntb * sndev)1277bbe35ca5SLogan Gunthorpe switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
1278ec0467ccSLogan Gunthorpe {
1279bbe35ca5SLogan Gunthorpe int req_ids[2];
1280ec0467ccSLogan Gunthorpe
1281ec0467ccSLogan Gunthorpe /*
1282ec0467ccSLogan Gunthorpe * Root Complex Requester ID (which is 0:00.0)
1283ec0467ccSLogan Gunthorpe */
1284bbe35ca5SLogan Gunthorpe req_ids[0] = 0;
1285ec0467ccSLogan Gunthorpe
1286ec0467ccSLogan Gunthorpe /*
1287ec0467ccSLogan Gunthorpe * Host Bridge Requester ID (as read from the mmap address)
1288ec0467ccSLogan Gunthorpe */
1289bbe35ca5SLogan Gunthorpe req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);
1290ec0467ccSLogan Gunthorpe
1291bbe35ca5SLogan Gunthorpe return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids,
1292bbe35ca5SLogan Gunthorpe ARRAY_SIZE(req_ids));
1293ec0467ccSLogan Gunthorpe }
1294ec0467ccSLogan Gunthorpe
switchtec_ntb_init_shared(struct switchtec_ntb * sndev)1295ec0467ccSLogan Gunthorpe static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
1296ec0467ccSLogan Gunthorpe {
1297ec0467ccSLogan Gunthorpe int i;
1298ec0467ccSLogan Gunthorpe
1299ec0467ccSLogan Gunthorpe memset(sndev->self_shared, 0, LUT_SIZE);
1300ec0467ccSLogan Gunthorpe sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC;
1301ec0467ccSLogan Gunthorpe sndev->self_shared->partition_id = sndev->stdev->partition;
1302ec0467ccSLogan Gunthorpe
1303ec0467ccSLogan Gunthorpe for (i = 0; i < sndev->nr_direct_mw; i++) {
1304ec0467ccSLogan Gunthorpe int bar = sndev->direct_mw_to_bar[i];
1305ec0467ccSLogan Gunthorpe resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar);
1306ec0467ccSLogan Gunthorpe
1307ec0467ccSLogan Gunthorpe if (i == 0)
1308ec0467ccSLogan Gunthorpe sz = min_t(resource_size_t, sz,
1309ec0467ccSLogan Gunthorpe LUT_SIZE * sndev->nr_lut_mw);
1310ec0467ccSLogan Gunthorpe
1311ec0467ccSLogan Gunthorpe sndev->self_shared->mw_sizes[i] = sz;
1312ec0467ccSLogan Gunthorpe }
1313ec0467ccSLogan Gunthorpe
1314ec0467ccSLogan Gunthorpe for (i = 0; i < sndev->nr_lut_mw; i++) {
1315ec0467ccSLogan Gunthorpe int idx = sndev->nr_direct_mw + i;
1316ec0467ccSLogan Gunthorpe
1317ec0467ccSLogan Gunthorpe sndev->self_shared->mw_sizes[idx] = LUT_SIZE;
1318ec0467ccSLogan Gunthorpe }
1319ec0467ccSLogan Gunthorpe }
1320ec0467ccSLogan Gunthorpe
switchtec_ntb_init_shared_mw(struct switchtec_ntb * sndev)1321ec0467ccSLogan Gunthorpe static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)
1322ec0467ccSLogan Gunthorpe {
1323140eb522SDoug Meyer int self_bar = sndev->direct_mw_to_bar[0];
1324ec0467ccSLogan Gunthorpe int rc;
1325ec0467ccSLogan Gunthorpe
1326c3585cd8SLogan Gunthorpe sndev->nr_rsvd_luts++;
1327750afb08SLuis Chamberlain sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev,
1328ec0467ccSLogan Gunthorpe LUT_SIZE,
1329ec0467ccSLogan Gunthorpe &sndev->self_shared_dma,
1330ec0467ccSLogan Gunthorpe GFP_KERNEL);
1331ec0467ccSLogan Gunthorpe if (!sndev->self_shared) {
1332ec0467ccSLogan Gunthorpe dev_err(&sndev->stdev->dev,
13332dd0f6a6SJon Mason "unable to allocate memory for shared mw\n");
1334ec0467ccSLogan Gunthorpe return -ENOMEM;
1335ec0467ccSLogan Gunthorpe }
1336ec0467ccSLogan Gunthorpe
1337ec0467ccSLogan Gunthorpe switchtec_ntb_init_shared(sndev);
1338ec0467ccSLogan Gunthorpe
133912cb203bSLogan Gunthorpe rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
134012cb203bSLogan Gunthorpe sndev->self_partition,
134112cb203bSLogan Gunthorpe sndev->self_shared_dma);
1342ec0467ccSLogan Gunthorpe if (rc)
1343ec0467ccSLogan Gunthorpe goto unalloc_and_exit;
1344ec0467ccSLogan Gunthorpe
1345140eb522SDoug Meyer sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE);
1346ec0467ccSLogan Gunthorpe if (!sndev->peer_shared) {
1347ec0467ccSLogan Gunthorpe rc = -ENOMEM;
1348ec0467ccSLogan Gunthorpe goto unalloc_and_exit;
1349ec0467ccSLogan Gunthorpe }
1350ec0467ccSLogan Gunthorpe
13512dd0f6a6SJon Mason dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");
1352ec0467ccSLogan Gunthorpe return 0;
1353ec0467ccSLogan Gunthorpe
1354ec0467ccSLogan Gunthorpe unalloc_and_exit:
1355ec0467ccSLogan Gunthorpe dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1356ec0467ccSLogan Gunthorpe sndev->self_shared, sndev->self_shared_dma);
1357ec0467ccSLogan Gunthorpe
1358ec0467ccSLogan Gunthorpe return rc;
1359ec0467ccSLogan Gunthorpe }
1360ec0467ccSLogan Gunthorpe
switchtec_ntb_deinit_shared_mw(struct switchtec_ntb * sndev)1361ec0467ccSLogan Gunthorpe static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)
1362ec0467ccSLogan Gunthorpe {
1363ec0467ccSLogan Gunthorpe if (sndev->peer_shared)
1364ec0467ccSLogan Gunthorpe pci_iounmap(sndev->stdev->pdev, sndev->peer_shared);
1365ec0467ccSLogan Gunthorpe
1366ec0467ccSLogan Gunthorpe if (sndev->self_shared)
1367ec0467ccSLogan Gunthorpe dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1368ec0467ccSLogan Gunthorpe sndev->self_shared,
1369ec0467ccSLogan Gunthorpe sndev->self_shared_dma);
1370c3585cd8SLogan Gunthorpe sndev->nr_rsvd_luts--;
1371ec0467ccSLogan Gunthorpe }
1372ec0467ccSLogan Gunthorpe
switchtec_ntb_doorbell_isr(int irq,void * dev)13733dd4db47SLogan Gunthorpe static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev)
13743dd4db47SLogan Gunthorpe {
13753dd4db47SLogan Gunthorpe struct switchtec_ntb *sndev = dev;
13763dd4db47SLogan Gunthorpe
13773dd4db47SLogan Gunthorpe dev_dbg(&sndev->stdev->dev, "doorbell\n");
13783dd4db47SLogan Gunthorpe
13796619bf95SLogan Gunthorpe ntb_db_event(&sndev->ntb, 0);
13806619bf95SLogan Gunthorpe
13813dd4db47SLogan Gunthorpe return IRQ_HANDLED;
13823dd4db47SLogan Gunthorpe }
13833dd4db47SLogan Gunthorpe
switchtec_ntb_message_isr(int irq,void * dev)13843dd4db47SLogan Gunthorpe static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev)
13853dd4db47SLogan Gunthorpe {
13863dd4db47SLogan Gunthorpe int i;
13873dd4db47SLogan Gunthorpe struct switchtec_ntb *sndev = dev;
13883dd4db47SLogan Gunthorpe
13893dd4db47SLogan Gunthorpe for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
13903dd4db47SLogan Gunthorpe u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]);
13913dd4db47SLogan Gunthorpe
13923dd4db47SLogan Gunthorpe if (msg & NTB_DBMSG_IMSG_STATUS) {
13932dd0f6a6SJon Mason dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",
13942dd0f6a6SJon Mason i, (u32)msg);
13953dd4db47SLogan Gunthorpe iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);
13960ee28f26SLogan Gunthorpe
13970ee28f26SLogan Gunthorpe if (i == LINK_MESSAGE)
1398d04be142SLogan Gunthorpe switchtec_ntb_check_link(sndev, msg);
13993dd4db47SLogan Gunthorpe }
14003dd4db47SLogan Gunthorpe }
14013dd4db47SLogan Gunthorpe
14023dd4db47SLogan Gunthorpe return IRQ_HANDLED;
14033dd4db47SLogan Gunthorpe }
14043dd4db47SLogan Gunthorpe
switchtec_ntb_init_db_msg_irq(struct switchtec_ntb * sndev)14053dd4db47SLogan Gunthorpe static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)
14063dd4db47SLogan Gunthorpe {
14073dd4db47SLogan Gunthorpe int i;
14083dd4db47SLogan Gunthorpe int rc;
14093dd4db47SLogan Gunthorpe int doorbell_irq = 0;
14103dd4db47SLogan Gunthorpe int message_irq = 0;
14113dd4db47SLogan Gunthorpe int event_irq;
14123dd4db47SLogan Gunthorpe int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map);
14133dd4db47SLogan Gunthorpe
14143dd4db47SLogan Gunthorpe event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number);
14153dd4db47SLogan Gunthorpe
14163dd4db47SLogan Gunthorpe while (doorbell_irq == event_irq)
14173dd4db47SLogan Gunthorpe doorbell_irq++;
14183dd4db47SLogan Gunthorpe while (message_irq == doorbell_irq ||
14193dd4db47SLogan Gunthorpe message_irq == event_irq)
14203dd4db47SLogan Gunthorpe message_irq++;
14213dd4db47SLogan Gunthorpe
14222dd0f6a6SJon Mason dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",
14233dd4db47SLogan Gunthorpe event_irq, doorbell_irq, message_irq);
14243dd4db47SLogan Gunthorpe
14253dd4db47SLogan Gunthorpe for (i = 0; i < idb_vecs - 4; i++)
14263dd4db47SLogan Gunthorpe iowrite8(doorbell_irq,
14273dd4db47SLogan Gunthorpe &sndev->mmio_self_dbmsg->idb_vec_map[i]);
14283dd4db47SLogan Gunthorpe
14293dd4db47SLogan Gunthorpe for (; i < idb_vecs; i++)
14303dd4db47SLogan Gunthorpe iowrite8(message_irq,
14313dd4db47SLogan Gunthorpe &sndev->mmio_self_dbmsg->idb_vec_map[i]);
14323dd4db47SLogan Gunthorpe
14333dd4db47SLogan Gunthorpe sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq);
14343dd4db47SLogan Gunthorpe sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq);
14353dd4db47SLogan Gunthorpe
14363dd4db47SLogan Gunthorpe rc = request_irq(sndev->doorbell_irq,
14373dd4db47SLogan Gunthorpe switchtec_ntb_doorbell_isr, 0,
14383dd4db47SLogan Gunthorpe "switchtec_ntb_doorbell", sndev);
14393dd4db47SLogan Gunthorpe if (rc)
14403dd4db47SLogan Gunthorpe return rc;
14413dd4db47SLogan Gunthorpe
14423dd4db47SLogan Gunthorpe rc = request_irq(sndev->message_irq,
14433dd4db47SLogan Gunthorpe switchtec_ntb_message_isr, 0,
14443dd4db47SLogan Gunthorpe "switchtec_ntb_message", sndev);
14453dd4db47SLogan Gunthorpe if (rc) {
14463dd4db47SLogan Gunthorpe free_irq(sndev->doorbell_irq, sndev);
14473dd4db47SLogan Gunthorpe return rc;
14483dd4db47SLogan Gunthorpe }
14493dd4db47SLogan Gunthorpe
14503dd4db47SLogan Gunthorpe return 0;
14513dd4db47SLogan Gunthorpe }
14523dd4db47SLogan Gunthorpe
switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb * sndev)14533dd4db47SLogan Gunthorpe static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
14543dd4db47SLogan Gunthorpe {
14553dd4db47SLogan Gunthorpe free_irq(sndev->doorbell_irq, sndev);
14563dd4db47SLogan Gunthorpe free_irq(sndev->message_irq, sndev);
14573dd4db47SLogan Gunthorpe }
14583dd4db47SLogan Gunthorpe
switchtec_ntb_reinit_peer(struct switchtec_ntb * sndev)1459d04be142SLogan Gunthorpe static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
1460d04be142SLogan Gunthorpe {
146128e339f1SJoey Zhang int rc;
146228e339f1SJoey Zhang
1463f0f43e76SWesley Sheng if (crosslink_is_enabled(sndev))
1464f0f43e76SWesley Sheng return 0;
1465f0f43e76SWesley Sheng
146628e339f1SJoey Zhang dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n");
146728e339f1SJoey Zhang rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
146828e339f1SJoey Zhang sndev->self_partition,
146928e339f1SJoey Zhang sndev->self_shared_dma);
147028e339f1SJoey Zhang return rc;
1471d04be142SLogan Gunthorpe }
1472d04be142SLogan Gunthorpe
switchtec_ntb_add(struct device * dev)1473*2243acd5SGreg Kroah-Hartman static int switchtec_ntb_add(struct device *dev)
147433dea5aaSLogan Gunthorpe {
147533dea5aaSLogan Gunthorpe struct switchtec_dev *stdev = to_stdev(dev);
147633dea5aaSLogan Gunthorpe struct switchtec_ntb *sndev;
1477ec0467ccSLogan Gunthorpe int rc;
147833dea5aaSLogan Gunthorpe
147933dea5aaSLogan Gunthorpe stdev->sndev = NULL;
148033dea5aaSLogan Gunthorpe
1481cfdfc14eSDoug Meyer if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8))
148233dea5aaSLogan Gunthorpe return -ENODEV;
148333dea5aaSLogan Gunthorpe
148433dea5aaSLogan Gunthorpe sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev));
148533dea5aaSLogan Gunthorpe if (!sndev)
148633dea5aaSLogan Gunthorpe return -ENOMEM;
148733dea5aaSLogan Gunthorpe
148833dea5aaSLogan Gunthorpe sndev->stdev = stdev;
14893df54c87SKelvin Cao rc = switchtec_ntb_init_sndev(sndev);
14903df54c87SKelvin Cao if (rc)
14913df54c87SKelvin Cao goto free_and_exit;
14923df54c87SKelvin Cao
1493ec0467ccSLogan Gunthorpe switchtec_ntb_init_mw(sndev);
1494ec0467ccSLogan Gunthorpe
1495ec0467ccSLogan Gunthorpe rc = switchtec_ntb_init_req_id_table(sndev);
1496ec0467ccSLogan Gunthorpe if (rc)
1497ec0467ccSLogan Gunthorpe goto free_and_exit;
1498ec0467ccSLogan Gunthorpe
149901752501SLogan Gunthorpe rc = switchtec_ntb_init_crosslink(sndev);
1500ec0467ccSLogan Gunthorpe if (rc)
1501ec0467ccSLogan Gunthorpe goto free_and_exit;
1502ec0467ccSLogan Gunthorpe
150301752501SLogan Gunthorpe switchtec_ntb_init_db(sndev);
150401752501SLogan Gunthorpe switchtec_ntb_init_msgs(sndev);
150501752501SLogan Gunthorpe
150601752501SLogan Gunthorpe rc = switchtec_ntb_init_shared_mw(sndev);
150701752501SLogan Gunthorpe if (rc)
150801752501SLogan Gunthorpe goto deinit_crosslink;
150901752501SLogan Gunthorpe
15103dd4db47SLogan Gunthorpe rc = switchtec_ntb_init_db_msg_irq(sndev);
15113dd4db47SLogan Gunthorpe if (rc)
15123dd4db47SLogan Gunthorpe goto deinit_shared_and_exit;
15133dd4db47SLogan Gunthorpe
1514d04be142SLogan Gunthorpe /*
1515d04be142SLogan Gunthorpe * If this host crashed, the other host may think the link is
1516d04be142SLogan Gunthorpe * still up. Tell them to force it down (it will go back up
1517d04be142SLogan Gunthorpe * once we register the ntb device).
1518d04be142SLogan Gunthorpe */
1519d04be142SLogan Gunthorpe switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN);
1520d04be142SLogan Gunthorpe
1521e099b45bSLogan Gunthorpe rc = ntb_register_device(&sndev->ntb);
1522e099b45bSLogan Gunthorpe if (rc)
1523e099b45bSLogan Gunthorpe goto deinit_and_exit;
1524e099b45bSLogan Gunthorpe
152533dea5aaSLogan Gunthorpe stdev->sndev = sndev;
15260ee28f26SLogan Gunthorpe stdev->link_notifier = switchtec_ntb_link_notification;
15272dd0f6a6SJon Mason dev_info(dev, "NTB device registered\n");
152833dea5aaSLogan Gunthorpe
152933dea5aaSLogan Gunthorpe return 0;
1530ec0467ccSLogan Gunthorpe
1531e099b45bSLogan Gunthorpe deinit_and_exit:
1532e099b45bSLogan Gunthorpe switchtec_ntb_deinit_db_msg_irq(sndev);
15333dd4db47SLogan Gunthorpe deinit_shared_and_exit:
15343dd4db47SLogan Gunthorpe switchtec_ntb_deinit_shared_mw(sndev);
153501752501SLogan Gunthorpe deinit_crosslink:
153601752501SLogan Gunthorpe switchtec_ntb_deinit_crosslink(sndev);
1537ec0467ccSLogan Gunthorpe free_and_exit:
1538ec0467ccSLogan Gunthorpe kfree(sndev);
15392dd0f6a6SJon Mason dev_err(dev, "failed to register ntb device: %d\n", rc);
1540ec0467ccSLogan Gunthorpe return rc;
154133dea5aaSLogan Gunthorpe }
154233dea5aaSLogan Gunthorpe
switchtec_ntb_remove(struct device * dev)1543*2243acd5SGreg Kroah-Hartman static void switchtec_ntb_remove(struct device *dev)
154433dea5aaSLogan Gunthorpe {
154533dea5aaSLogan Gunthorpe struct switchtec_dev *stdev = to_stdev(dev);
154633dea5aaSLogan Gunthorpe struct switchtec_ntb *sndev = stdev->sndev;
154733dea5aaSLogan Gunthorpe
154833dea5aaSLogan Gunthorpe if (!sndev)
154933dea5aaSLogan Gunthorpe return;
155033dea5aaSLogan Gunthorpe
15510ee28f26SLogan Gunthorpe stdev->link_notifier = NULL;
155233dea5aaSLogan Gunthorpe stdev->sndev = NULL;
1553e099b45bSLogan Gunthorpe ntb_unregister_device(&sndev->ntb);
15543dd4db47SLogan Gunthorpe switchtec_ntb_deinit_db_msg_irq(sndev);
1555ec0467ccSLogan Gunthorpe switchtec_ntb_deinit_shared_mw(sndev);
155601752501SLogan Gunthorpe switchtec_ntb_deinit_crosslink(sndev);
155733dea5aaSLogan Gunthorpe kfree(sndev);
15582dd0f6a6SJon Mason dev_info(dev, "ntb device unregistered\n");
155933dea5aaSLogan Gunthorpe }
156033dea5aaSLogan Gunthorpe
156133dea5aaSLogan Gunthorpe static struct class_interface switchtec_interface = {
156233dea5aaSLogan Gunthorpe .add_dev = switchtec_ntb_add,
156333dea5aaSLogan Gunthorpe .remove_dev = switchtec_ntb_remove,
156433dea5aaSLogan Gunthorpe };
156533dea5aaSLogan Gunthorpe
switchtec_ntb_init(void)156633dea5aaSLogan Gunthorpe static int __init switchtec_ntb_init(void)
156733dea5aaSLogan Gunthorpe {
156833dea5aaSLogan Gunthorpe switchtec_interface.class = switchtec_class;
156933dea5aaSLogan Gunthorpe return class_interface_register(&switchtec_interface);
157033dea5aaSLogan Gunthorpe }
157133dea5aaSLogan Gunthorpe module_init(switchtec_ntb_init);
157233dea5aaSLogan Gunthorpe
switchtec_ntb_exit(void)157333dea5aaSLogan Gunthorpe static void __exit switchtec_ntb_exit(void)
157433dea5aaSLogan Gunthorpe {
157533dea5aaSLogan Gunthorpe class_interface_unregister(&switchtec_interface);
157633dea5aaSLogan Gunthorpe }
157733dea5aaSLogan Gunthorpe module_exit(switchtec_ntb_exit);
1578