1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Microsemi Switchtec(tm) PCIe Management Driver
4  * Copyright (c) 2017, Microsemi Corporation
5  */
6 
7 #include <linux/interrupt.h>
8 #include <linux/io-64-nonatomic-lo-hi.h>
9 #include <linux/delay.h>
10 #include <linux/kthread.h>
11 #include <linux/module.h>
12 #include <linux/ntb.h>
13 #include <linux/pci.h>
14 #include <linux/switchtec.h>
15 
16 MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver");
17 MODULE_VERSION("0.1");
18 MODULE_LICENSE("GPL");
19 MODULE_AUTHOR("Microsemi Corporation");
20 
21 static ulong max_mw_size = SZ_2M;
22 module_param(max_mw_size, ulong, 0644);
23 MODULE_PARM_DESC(max_mw_size,
24 	"Max memory window size reported to the upper layer");
25 
26 static bool use_lut_mws;
27 module_param(use_lut_mws, bool, 0644);
28 MODULE_PARM_DESC(use_lut_mws,
29 		 "Enable the use of the LUT based memory windows");
30 
31 #define SWITCHTEC_NTB_MAGIC 0x45CC0001
32 #define MAX_MWS     128
33 
34 struct shared_mw {
35 	u32 magic;
36 	u32 link_sta;
37 	u32 partition_id;
38 	u64 mw_sizes[MAX_MWS];
39 	u32 spad[128];
40 };
41 
42 #define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry)
43 #define LUT_SIZE SZ_64K
44 
45 struct switchtec_ntb {
46 	struct ntb_dev ntb;
47 	struct switchtec_dev *stdev;
48 
49 	int self_partition;
50 	int peer_partition;
51 
52 	int doorbell_irq;
53 	int message_irq;
54 
55 	struct ntb_info_regs __iomem *mmio_ntb;
56 	struct ntb_ctrl_regs __iomem *mmio_ctrl;
57 	struct ntb_dbmsg_regs __iomem *mmio_dbmsg;
58 	struct ntb_ctrl_regs __iomem *mmio_self_ctrl;
59 	struct ntb_ctrl_regs __iomem *mmio_peer_ctrl;
60 	struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg;
61 	struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg;
62 
63 	void __iomem *mmio_xlink_win;
64 
65 	struct shared_mw *self_shared;
66 	struct shared_mw __iomem *peer_shared;
67 	dma_addr_t self_shared_dma;
68 
69 	u64 db_mask;
70 	u64 db_valid_mask;
71 	int db_shift;
72 	int db_peer_shift;
73 
74 	/* synchronize rmw access of db_mask and hw reg */
75 	spinlock_t db_mask_lock;
76 
77 	int nr_direct_mw;
78 	int nr_lut_mw;
79 	int nr_rsvd_luts;
80 	int direct_mw_to_bar[MAX_DIRECT_MW];
81 
82 	int peer_nr_direct_mw;
83 	int peer_nr_lut_mw;
84 	int peer_direct_mw_to_bar[MAX_DIRECT_MW];
85 
86 	bool link_is_up;
87 	enum ntb_speed link_speed;
88 	enum ntb_width link_width;
89 	struct work_struct link_reinit_work;
90 };
91 
92 static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
93 {
94 	return container_of(ntb, struct switchtec_ntb, ntb);
95 }
96 
97 static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
98 				 struct ntb_ctrl_regs __iomem *ctl,
99 				 u32 op, int wait_status)
100 {
101 	static const char * const op_text[] = {
102 		[NTB_CTRL_PART_OP_LOCK] = "lock",
103 		[NTB_CTRL_PART_OP_CFG] = "configure",
104 		[NTB_CTRL_PART_OP_RESET] = "reset",
105 	};
106 
107 	int i;
108 	u32 ps;
109 	int status;
110 
111 	switch (op) {
112 	case NTB_CTRL_PART_OP_LOCK:
113 		status = NTB_CTRL_PART_STATUS_LOCKING;
114 		break;
115 	case NTB_CTRL_PART_OP_CFG:
116 		status = NTB_CTRL_PART_STATUS_CONFIGURING;
117 		break;
118 	case NTB_CTRL_PART_OP_RESET:
119 		status = NTB_CTRL_PART_STATUS_RESETTING;
120 		break;
121 	default:
122 		return -EINVAL;
123 	}
124 
125 	iowrite32(op, &ctl->partition_op);
126 
127 	for (i = 0; i < 1000; i++) {
128 		if (msleep_interruptible(50) != 0) {
129 			iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op);
130 			return -EINTR;
131 		}
132 
133 		ps = ioread32(&ctl->partition_status) & 0xFFFF;
134 
135 		if (ps != status)
136 			break;
137 	}
138 
139 	if (ps == wait_status)
140 		return 0;
141 
142 	if (ps == status) {
143 		dev_err(&sndev->stdev->dev,
144 			"Timed out while performing %s (%d). (%08x)\n",
145 			op_text[op], op,
146 			ioread32(&ctl->partition_status));
147 
148 		return -ETIMEDOUT;
149 	}
150 
151 	return -EIO;
152 }
153 
154 static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,
155 				  u32 val)
156 {
157 	if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))
158 		return -EINVAL;
159 
160 	iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);
161 
162 	return 0;
163 }
164 
165 static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
166 {
167 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
168 	int nr_direct_mw = sndev->peer_nr_direct_mw;
169 	int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;
170 
171 	if (pidx != NTB_DEF_PEER_IDX)
172 		return -EINVAL;
173 
174 	if (!use_lut_mws)
175 		nr_lut_mw = 0;
176 
177 	return nr_direct_mw + nr_lut_mw;
178 }
179 
180 static int lut_index(struct switchtec_ntb *sndev, int mw_idx)
181 {
182 	return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;
183 }
184 
185 static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)
186 {
187 	return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;
188 }
189 
190 static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx,
191 				      int widx, resource_size_t *addr_align,
192 				      resource_size_t *size_align,
193 				      resource_size_t *size_max)
194 {
195 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
196 	int lut;
197 	resource_size_t size;
198 
199 	if (pidx != NTB_DEF_PEER_IDX)
200 		return -EINVAL;
201 
202 	lut = widx >= sndev->peer_nr_direct_mw;
203 	size = ioread64(&sndev->peer_shared->mw_sizes[widx]);
204 
205 	if (size == 0)
206 		return -EINVAL;
207 
208 	if (addr_align)
209 		*addr_align = lut ? size : SZ_4K;
210 
211 	if (size_align)
212 		*size_align = lut ? size : SZ_4K;
213 
214 	if (size_max)
215 		*size_max = size;
216 
217 	return 0;
218 }
219 
220 static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx)
221 {
222 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
223 	int bar = sndev->peer_direct_mw_to_bar[idx];
224 	u32 ctl_val;
225 
226 	ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
227 	ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN;
228 	iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
229 	iowrite32(0, &ctl->bar_entry[bar].win_size);
230 	iowrite32(0, &ctl->bar_ext_entry[bar].win_size);
231 	iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr);
232 }
233 
234 static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx)
235 {
236 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
237 
238 	iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]);
239 }
240 
241 static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx,
242 					dma_addr_t addr, resource_size_t size)
243 {
244 	int xlate_pos = ilog2(size);
245 	int bar = sndev->peer_direct_mw_to_bar[idx];
246 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
247 	u32 ctl_val;
248 
249 	ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
250 	ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
251 
252 	iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
253 	iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
254 		  &ctl->bar_entry[bar].win_size);
255 	iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
256 	iowrite64(sndev->self_partition | addr,
257 		  &ctl->bar_entry[bar].xlate_addr);
258 }
259 
260 static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx,
261 				     dma_addr_t addr, resource_size_t size)
262 {
263 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
264 
265 	iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr),
266 		  &ctl->lut_entry[peer_lut_index(sndev, idx)]);
267 }
268 
269 static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
270 				      dma_addr_t addr, resource_size_t size)
271 {
272 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
273 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
274 	int xlate_pos = ilog2(size);
275 	int nr_direct_mw = sndev->peer_nr_direct_mw;
276 	int rc;
277 
278 	if (pidx != NTB_DEF_PEER_IDX)
279 		return -EINVAL;
280 
281 	dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",
282 		widx, pidx, &addr, &size);
283 
284 	if (widx >= switchtec_ntb_mw_count(ntb, pidx))
285 		return -EINVAL;
286 
287 	if (xlate_pos < 12)
288 		return -EINVAL;
289 
290 	if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
291 		/*
292 		 * In certain circumstances we can get a buffer that is
293 		 * not aligned to its size. (Most of the time
294 		 * dma_alloc_coherent ensures this). This can happen when
295 		 * using large buffers allocated by the CMA
296 		 * (see CMA_CONFIG_ALIGNMENT)
297 		 */
298 		dev_err(&sndev->stdev->dev,
299 			"ERROR: Memory window address is not aligned to it's size!\n");
300 		return -EINVAL;
301 	}
302 
303 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
304 				   NTB_CTRL_PART_STATUS_LOCKED);
305 	if (rc)
306 		return rc;
307 
308 	if (addr == 0 || size == 0) {
309 		if (widx < nr_direct_mw)
310 			switchtec_ntb_mw_clr_direct(sndev, widx);
311 		else
312 			switchtec_ntb_mw_clr_lut(sndev, widx);
313 	} else {
314 		if (widx < nr_direct_mw)
315 			switchtec_ntb_mw_set_direct(sndev, widx, addr, size);
316 		else
317 			switchtec_ntb_mw_set_lut(sndev, widx, addr, size);
318 	}
319 
320 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
321 				   NTB_CTRL_PART_STATUS_NORMAL);
322 
323 	if (rc == -EIO) {
324 		dev_err(&sndev->stdev->dev,
325 			"Hardware reported an error configuring mw %d: %08x\n",
326 			widx, ioread32(&ctl->bar_error));
327 
328 		if (widx < nr_direct_mw)
329 			switchtec_ntb_mw_clr_direct(sndev, widx);
330 		else
331 			switchtec_ntb_mw_clr_lut(sndev, widx);
332 
333 		switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
334 				      NTB_CTRL_PART_STATUS_NORMAL);
335 	}
336 
337 	return rc;
338 }
339 
340 static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb)
341 {
342 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
343 	int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;
344 
345 	return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);
346 }
347 
348 static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,
349 					 int idx, phys_addr_t *base,
350 					 resource_size_t *size)
351 {
352 	int bar = sndev->direct_mw_to_bar[idx];
353 	size_t offset = 0;
354 
355 	if (bar < 0)
356 		return -EINVAL;
357 
358 	if (idx == 0) {
359 		/*
360 		 * This is the direct BAR shared with the LUTs
361 		 * which means the actual window will be offset
362 		 * by the size of all the LUT entries.
363 		 */
364 
365 		offset = LUT_SIZE * sndev->nr_lut_mw;
366 	}
367 
368 	if (base)
369 		*base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
370 
371 	if (size) {
372 		*size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
373 		if (offset && *size > offset)
374 			*size = offset;
375 
376 		if (*size > max_mw_size)
377 			*size = max_mw_size;
378 	}
379 
380 	return 0;
381 }
382 
383 static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev,
384 				      int idx, phys_addr_t *base,
385 				      resource_size_t *size)
386 {
387 	int bar = sndev->direct_mw_to_bar[0];
388 	int offset;
389 
390 	offset = LUT_SIZE * lut_index(sndev, idx);
391 
392 	if (base)
393 		*base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
394 
395 	if (size)
396 		*size = LUT_SIZE;
397 
398 	return 0;
399 }
400 
401 static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
402 					  phys_addr_t *base,
403 					  resource_size_t *size)
404 {
405 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
406 
407 	if (idx < sndev->nr_direct_mw)
408 		return switchtec_ntb_direct_get_addr(sndev, idx, base, size);
409 	else if (idx < switchtec_ntb_peer_mw_count(ntb))
410 		return switchtec_ntb_lut_get_addr(sndev, idx, base, size);
411 	else
412 		return -EINVAL;
413 }
414 
415 static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev,
416 					  int partition,
417 					  enum ntb_speed *speed,
418 					  enum ntb_width *width)
419 {
420 	struct switchtec_dev *stdev = sndev->stdev;
421 
422 	u32 pff = ioread32(&stdev->mmio_part_cfg[partition].vep_pff_inst_id);
423 	u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]);
424 
425 	if (speed)
426 		*speed = (linksta >> 16) & 0xF;
427 
428 	if (width)
429 		*width = (linksta >> 20) & 0x3F;
430 }
431 
432 static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)
433 {
434 	enum ntb_speed self_speed, peer_speed;
435 	enum ntb_width self_width, peer_width;
436 
437 	if (!sndev->link_is_up) {
438 		sndev->link_speed = NTB_SPEED_NONE;
439 		sndev->link_width = NTB_WIDTH_NONE;
440 		return;
441 	}
442 
443 	switchtec_ntb_part_link_speed(sndev, sndev->self_partition,
444 				      &self_speed, &self_width);
445 	switchtec_ntb_part_link_speed(sndev, sndev->peer_partition,
446 				      &peer_speed, &peer_width);
447 
448 	sndev->link_speed = min(self_speed, peer_speed);
449 	sndev->link_width = min(self_width, peer_width);
450 }
451 
452 static int crosslink_is_enabled(struct switchtec_ntb *sndev)
453 {
454 	struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;
455 
456 	return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);
457 }
458 
459 static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)
460 {
461 	int i;
462 	u32 msg_map = 0;
463 
464 	if (!crosslink_is_enabled(sndev))
465 		return;
466 
467 	for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {
468 		int m = i | sndev->self_partition << 2;
469 
470 		msg_map |= m << i * 8;
471 	}
472 
473 	iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);
474 	iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
475 		  &sndev->mmio_peer_dbmsg->odb_mask);
476 }
477 
478 enum switchtec_msg {
479 	LINK_MESSAGE = 0,
480 	MSG_LINK_UP = 1,
481 	MSG_LINK_DOWN = 2,
482 	MSG_CHECK_LINK = 3,
483 	MSG_LINK_FORCE_DOWN = 4,
484 };
485 
486 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
487 
488 static void link_reinit_work(struct work_struct *work)
489 {
490 	struct switchtec_ntb *sndev;
491 
492 	sndev = container_of(work, struct switchtec_ntb, link_reinit_work);
493 
494 	switchtec_ntb_reinit_peer(sndev);
495 }
496 
497 static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
498 				     enum switchtec_msg msg)
499 {
500 	int link_sta;
501 	int old = sndev->link_is_up;
502 
503 	if (msg == MSG_LINK_FORCE_DOWN) {
504 		schedule_work(&sndev->link_reinit_work);
505 
506 		if (sndev->link_is_up) {
507 			sndev->link_is_up = 0;
508 			ntb_link_event(&sndev->ntb);
509 			dev_info(&sndev->stdev->dev, "ntb link forced down\n");
510 		}
511 
512 		return;
513 	}
514 
515 	link_sta = sndev->self_shared->link_sta;
516 	if (link_sta) {
517 		u64 peer = ioread64(&sndev->peer_shared->magic);
518 
519 		if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC)
520 			link_sta = peer >> 32;
521 		else
522 			link_sta = 0;
523 	}
524 
525 	sndev->link_is_up = link_sta;
526 	switchtec_ntb_set_link_speed(sndev);
527 
528 	if (link_sta != old) {
529 		switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK);
530 		ntb_link_event(&sndev->ntb);
531 		dev_info(&sndev->stdev->dev, "ntb link %s\n",
532 			 link_sta ? "up" : "down");
533 
534 		if (link_sta)
535 			crosslink_init_dbmsgs(sndev);
536 	}
537 }
538 
539 static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
540 {
541 	struct switchtec_ntb *sndev = stdev->sndev;
542 
543 	switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
544 }
545 
546 static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb,
547 				    enum ntb_speed *speed,
548 				    enum ntb_width *width)
549 {
550 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
551 
552 	if (speed)
553 		*speed = sndev->link_speed;
554 	if (width)
555 		*width = sndev->link_width;
556 
557 	return sndev->link_is_up;
558 }
559 
560 static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
561 				     enum ntb_speed max_speed,
562 				     enum ntb_width max_width)
563 {
564 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
565 
566 	dev_dbg(&sndev->stdev->dev, "enabling link\n");
567 
568 	sndev->self_shared->link_sta = 1;
569 	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);
570 
571 	switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
572 
573 	return 0;
574 }
575 
576 static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
577 {
578 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
579 
580 	dev_dbg(&sndev->stdev->dev, "disabling link\n");
581 
582 	sndev->self_shared->link_sta = 0;
583 	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN);
584 
585 	switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
586 
587 	return 0;
588 }
589 
590 static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb)
591 {
592 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
593 
594 	return sndev->db_valid_mask;
595 }
596 
597 static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb)
598 {
599 	return 1;
600 }
601 
602 static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
603 {
604 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
605 
606 	if (db_vector < 0 || db_vector > 1)
607 		return 0;
608 
609 	return sndev->db_valid_mask;
610 }
611 
612 static u64 switchtec_ntb_db_read(struct ntb_dev *ntb)
613 {
614 	u64 ret;
615 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
616 
617 	ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift;
618 
619 	return ret & sndev->db_valid_mask;
620 }
621 
622 static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
623 {
624 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
625 
626 	iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb);
627 
628 	return 0;
629 }
630 
631 static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
632 {
633 	unsigned long irqflags;
634 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
635 
636 	if (db_bits & ~sndev->db_valid_mask)
637 		return -EINVAL;
638 
639 	spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
640 
641 	sndev->db_mask |= db_bits << sndev->db_shift;
642 	iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
643 
644 	spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
645 
646 	return 0;
647 }
648 
649 static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
650 {
651 	unsigned long irqflags;
652 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
653 
654 	if (db_bits & ~sndev->db_valid_mask)
655 		return -EINVAL;
656 
657 	spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
658 
659 	sndev->db_mask &= ~(db_bits << sndev->db_shift);
660 	iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
661 
662 	spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
663 
664 	return 0;
665 }
666 
667 static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb)
668 {
669 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
670 
671 	return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask;
672 }
673 
674 static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
675 				      phys_addr_t *db_addr,
676 				      resource_size_t *db_size,
677 				      u64 *db_data,
678 				      int db_bit)
679 {
680 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
681 	unsigned long offset;
682 
683 	if (unlikely(db_bit >= BITS_PER_LONG_LONG))
684 		return -EINVAL;
685 
686 	offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
687 		(unsigned long)sndev->stdev->mmio;
688 
689 	offset += sndev->db_shift / 8;
690 
691 	if (db_addr)
692 		*db_addr = pci_resource_start(ntb->pdev, 0) + offset;
693 	if (db_size)
694 		*db_size = sizeof(u32);
695 	if (db_data)
696 		*db_data = BIT_ULL(db_bit) << sndev->db_peer_shift;
697 
698 	return 0;
699 }
700 
701 static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
702 {
703 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
704 
705 	iowrite64(db_bits << sndev->db_peer_shift,
706 		  &sndev->mmio_peer_dbmsg->odb);
707 
708 	return 0;
709 }
710 
711 static int switchtec_ntb_spad_count(struct ntb_dev *ntb)
712 {
713 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
714 
715 	return ARRAY_SIZE(sndev->self_shared->spad);
716 }
717 
718 static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx)
719 {
720 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
721 
722 	if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
723 		return 0;
724 
725 	if (!sndev->self_shared)
726 		return 0;
727 
728 	return sndev->self_shared->spad[idx];
729 }
730 
731 static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
732 {
733 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
734 
735 	if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
736 		return -EINVAL;
737 
738 	if (!sndev->self_shared)
739 		return -EIO;
740 
741 	sndev->self_shared->spad[idx] = val;
742 
743 	return 0;
744 }
745 
746 static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx,
747 					int sidx)
748 {
749 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
750 
751 	if (pidx != NTB_DEF_PEER_IDX)
752 		return -EINVAL;
753 
754 	if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
755 		return 0;
756 
757 	if (!sndev->peer_shared)
758 		return 0;
759 
760 	return ioread32(&sndev->peer_shared->spad[sidx]);
761 }
762 
763 static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
764 					 int sidx, u32 val)
765 {
766 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
767 
768 	if (pidx != NTB_DEF_PEER_IDX)
769 		return -EINVAL;
770 
771 	if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
772 		return -EINVAL;
773 
774 	if (!sndev->peer_shared)
775 		return -EIO;
776 
777 	iowrite32(val, &sndev->peer_shared->spad[sidx]);
778 
779 	return 0;
780 }
781 
782 static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx,
783 					int sidx, phys_addr_t *spad_addr)
784 {
785 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
786 	unsigned long offset;
787 
788 	if (pidx != NTB_DEF_PEER_IDX)
789 		return -EINVAL;
790 
791 	offset = (unsigned long)&sndev->peer_shared->spad[sidx] -
792 		(unsigned long)sndev->stdev->mmio;
793 
794 	if (spad_addr)
795 		*spad_addr = pci_resource_start(ntb->pdev, 0) + offset;
796 
797 	return 0;
798 }
799 
800 static const struct ntb_dev_ops switchtec_ntb_ops = {
801 	.mw_count		= switchtec_ntb_mw_count,
802 	.mw_get_align		= switchtec_ntb_mw_get_align,
803 	.mw_set_trans		= switchtec_ntb_mw_set_trans,
804 	.peer_mw_count		= switchtec_ntb_peer_mw_count,
805 	.peer_mw_get_addr	= switchtec_ntb_peer_mw_get_addr,
806 	.link_is_up		= switchtec_ntb_link_is_up,
807 	.link_enable		= switchtec_ntb_link_enable,
808 	.link_disable		= switchtec_ntb_link_disable,
809 	.db_valid_mask		= switchtec_ntb_db_valid_mask,
810 	.db_vector_count	= switchtec_ntb_db_vector_count,
811 	.db_vector_mask		= switchtec_ntb_db_vector_mask,
812 	.db_read		= switchtec_ntb_db_read,
813 	.db_clear		= switchtec_ntb_db_clear,
814 	.db_set_mask		= switchtec_ntb_db_set_mask,
815 	.db_clear_mask		= switchtec_ntb_db_clear_mask,
816 	.db_read_mask		= switchtec_ntb_db_read_mask,
817 	.peer_db_addr		= switchtec_ntb_peer_db_addr,
818 	.peer_db_set		= switchtec_ntb_peer_db_set,
819 	.spad_count		= switchtec_ntb_spad_count,
820 	.spad_read		= switchtec_ntb_spad_read,
821 	.spad_write		= switchtec_ntb_spad_write,
822 	.peer_spad_read		= switchtec_ntb_peer_spad_read,
823 	.peer_spad_write	= switchtec_ntb_peer_spad_write,
824 	.peer_spad_addr		= switchtec_ntb_peer_spad_addr,
825 };
826 
827 static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
828 {
829 	u64 tpart_vec;
830 	int self;
831 	u64 part_map;
832 	int bit;
833 
834 	sndev->ntb.pdev = sndev->stdev->pdev;
835 	sndev->ntb.topo = NTB_TOPO_SWITCH;
836 	sndev->ntb.ops = &switchtec_ntb_ops;
837 
838 	INIT_WORK(&sndev->link_reinit_work, link_reinit_work);
839 
840 	sndev->self_partition = sndev->stdev->partition;
841 
842 	sndev->mmio_ntb = sndev->stdev->mmio_ntb;
843 
844 	self = sndev->self_partition;
845 	tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);
846 	tpart_vec <<= 32;
847 	tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);
848 
849 	part_map = ioread64(&sndev->mmio_ntb->ep_map);
850 	part_map &= ~(1 << sndev->self_partition);
851 
852 	if (!ffs(tpart_vec)) {
853 		if (sndev->stdev->partition_count != 2) {
854 			dev_err(&sndev->stdev->dev,
855 				"ntb target partition not defined\n");
856 			return -ENODEV;
857 		}
858 
859 		bit = ffs(part_map);
860 		if (!bit) {
861 			dev_err(&sndev->stdev->dev,
862 				"peer partition is not NT partition\n");
863 			return -ENODEV;
864 		}
865 
866 		sndev->peer_partition = bit - 1;
867 	} else {
868 		if (ffs(tpart_vec) != fls(tpart_vec)) {
869 			dev_err(&sndev->stdev->dev,
870 				"ntb driver only supports 1 pair of 1-1 ntb mapping\n");
871 			return -ENODEV;
872 		}
873 
874 		sndev->peer_partition = ffs(tpart_vec) - 1;
875 		if (!(part_map & (1 << sndev->peer_partition))) {
876 			dev_err(&sndev->stdev->dev,
877 				"ntb target partition is not NT partition\n");
878 			return -ENODEV;
879 		}
880 	}
881 
882 	dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",
883 		sndev->self_partition, sndev->stdev->partition_count);
884 
885 	sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +
886 		SWITCHTEC_NTB_REG_CTRL_OFFSET;
887 	sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb +
888 		SWITCHTEC_NTB_REG_DBMSG_OFFSET;
889 
890 	sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];
891 	sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];
892 	sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];
893 	sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;
894 
895 	return 0;
896 }
897 
898 static int config_rsvd_lut_win(struct switchtec_ntb *sndev,
899 			       struct ntb_ctrl_regs __iomem *ctl,
900 			       int lut_idx, int partition, u64 addr)
901 {
902 	int peer_bar = sndev->peer_direct_mw_to_bar[0];
903 	u32 ctl_val;
904 	int rc;
905 
906 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
907 				   NTB_CTRL_PART_STATUS_LOCKED);
908 	if (rc)
909 		return rc;
910 
911 	ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl);
912 	ctl_val &= 0xFF;
913 	ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
914 	ctl_val |= ilog2(LUT_SIZE) << 8;
915 	ctl_val |= (sndev->nr_lut_mw - 1) << 14;
916 	iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl);
917 
918 	iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr),
919 		  &ctl->lut_entry[lut_idx]);
920 
921 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
922 				   NTB_CTRL_PART_STATUS_NORMAL);
923 	if (rc) {
924 		u32 bar_error, lut_error;
925 
926 		bar_error = ioread32(&ctl->bar_error);
927 		lut_error = ioread32(&ctl->lut_error);
928 		dev_err(&sndev->stdev->dev,
929 			"Error setting up reserved lut window: %08x / %08x\n",
930 			bar_error, lut_error);
931 		return rc;
932 	}
933 
934 	return 0;
935 }
936 
937 static int config_req_id_table(struct switchtec_ntb *sndev,
938 			       struct ntb_ctrl_regs __iomem *mmio_ctrl,
939 			       int *req_ids, int count)
940 {
941 	int i, rc = 0;
942 	u32 error;
943 	u32 proxy_id;
944 
945 	if (ioread32(&mmio_ctrl->req_id_table_size) < count) {
946 		dev_err(&sndev->stdev->dev,
947 			"Not enough requester IDs available.\n");
948 		return -EFAULT;
949 	}
950 
951 	rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
952 				   NTB_CTRL_PART_OP_LOCK,
953 				   NTB_CTRL_PART_STATUS_LOCKED);
954 	if (rc)
955 		return rc;
956 
957 	iowrite32(NTB_PART_CTRL_ID_PROT_DIS,
958 		  &mmio_ctrl->partition_ctrl);
959 
960 	for (i = 0; i < count; i++) {
961 		iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN,
962 			  &mmio_ctrl->req_id_table[i]);
963 
964 		proxy_id = ioread32(&mmio_ctrl->req_id_table[i]);
965 		dev_dbg(&sndev->stdev->dev,
966 			"Requester ID %02X:%02X.%X -> BB:%02X.%X\n",
967 			req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F,
968 			req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F,
969 			(proxy_id >> 1) & 0x7);
970 	}
971 
972 	rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
973 				   NTB_CTRL_PART_OP_CFG,
974 				   NTB_CTRL_PART_STATUS_NORMAL);
975 
976 	if (rc == -EIO) {
977 		error = ioread32(&mmio_ctrl->req_id_error);
978 		dev_err(&sndev->stdev->dev,
979 			"Error setting up the requester ID table: %08x\n",
980 			error);
981 	}
982 
983 	return 0;
984 }
985 
986 static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,
987 			       u64 *mw_addrs, int mw_count)
988 {
989 	int rc, i;
990 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;
991 	u64 addr;
992 	size_t size, offset;
993 	int bar;
994 	int xlate_pos;
995 	u32 ctl_val;
996 
997 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
998 				   NTB_CTRL_PART_STATUS_LOCKED);
999 	if (rc)
1000 		return rc;
1001 
1002 	for (i = 0; i < sndev->nr_lut_mw; i++) {
1003 		if (i == ntb_lut_idx)
1004 			continue;
1005 
1006 		addr = mw_addrs[0] + LUT_SIZE * i;
1007 
1008 		iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |
1009 			   addr),
1010 			  &ctl->lut_entry[i]);
1011 	}
1012 
1013 	sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);
1014 
1015 	for (i = 0; i < sndev->nr_direct_mw; i++) {
1016 		bar = sndev->direct_mw_to_bar[i];
1017 		offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;
1018 		addr = mw_addrs[i] + offset;
1019 		size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
1020 		xlate_pos = ilog2(size);
1021 
1022 		if (offset && size > offset)
1023 			size = offset;
1024 
1025 		ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
1026 		ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
1027 
1028 		iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
1029 		iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
1030 			  &ctl->bar_entry[bar].win_size);
1031 		iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
1032 		iowrite64(sndev->peer_partition | addr,
1033 			  &ctl->bar_entry[bar].xlate_addr);
1034 	}
1035 
1036 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
1037 				   NTB_CTRL_PART_STATUS_NORMAL);
1038 	if (rc) {
1039 		u32 bar_error, lut_error;
1040 
1041 		bar_error = ioread32(&ctl->bar_error);
1042 		lut_error = ioread32(&ctl->lut_error);
1043 		dev_err(&sndev->stdev->dev,
1044 			"Error setting up cross link windows: %08x / %08x\n",
1045 			bar_error, lut_error);
1046 		return rc;
1047 	}
1048 
1049 	return 0;
1050 }
1051 
1052 static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,
1053 	struct ntb_ctrl_regs __iomem *mmio_ctrl)
1054 {
1055 	int req_ids[16];
1056 	int i;
1057 	u32 proxy_id;
1058 
1059 	for (i = 0; i < ARRAY_SIZE(req_ids); i++) {
1060 		proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);
1061 
1062 		if (!(proxy_id & NTB_CTRL_REQ_ID_EN))
1063 			break;
1064 
1065 		req_ids[i] = ((proxy_id >> 1) & 0xFF);
1066 	}
1067 
1068 	return config_req_id_table(sndev, mmio_ctrl, req_ids, i);
1069 }
1070 
1071 /*
1072  * In crosslink configuration there is a virtual partition in the
1073  * middle of the two switches. The BARs in this partition have to be
1074  * enumerated and assigned addresses.
1075  */
1076 static int crosslink_enum_partition(struct switchtec_ntb *sndev,
1077 				    u64 *bar_addrs)
1078 {
1079 	struct part_cfg_regs __iomem *part_cfg =
1080 		&sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];
1081 	u32 pff = ioread32(&part_cfg->vep_pff_inst_id);
1082 	struct pff_csr_regs __iomem *mmio_pff =
1083 		&sndev->stdev->mmio_pff_csr[pff];
1084 	const u64 bar_space = 0x1000000000LL;
1085 	u64 bar_addr;
1086 	int bar_cnt = 0;
1087 	int i;
1088 
1089 	iowrite16(0x6, &mmio_pff->pcicmd);
1090 
1091 	for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) {
1092 		iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]);
1093 		bar_addr = ioread64(&mmio_pff->pci_bar64[i]);
1094 		bar_addr &= ~0xf;
1095 
1096 		dev_dbg(&sndev->stdev->dev,
1097 			"Crosslink BAR%d addr: %llx\n",
1098 			i*2, bar_addr);
1099 
1100 		if (bar_addr != bar_space * i)
1101 			continue;
1102 
1103 		bar_addrs[bar_cnt++] = bar_addr;
1104 	}
1105 
1106 	return bar_cnt;
1107 }
1108 
1109 static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)
1110 {
1111 	int rc;
1112 	int bar = sndev->direct_mw_to_bar[0];
1113 	const int ntb_lut_idx = 1;
1114 	u64 bar_addrs[6];
1115 	u64 addr;
1116 	int offset;
1117 	int bar_cnt;
1118 
1119 	if (!crosslink_is_enabled(sndev))
1120 		return 0;
1121 
1122 	dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");
1123 	sndev->ntb.topo = NTB_TOPO_CROSSLINK;
1124 
1125 	bar_cnt = crosslink_enum_partition(sndev, bar_addrs);
1126 	if (bar_cnt < sndev->nr_direct_mw + 1) {
1127 		dev_err(&sndev->stdev->dev,
1128 			"Error enumerating crosslink partition\n");
1129 		return -EINVAL;
1130 	}
1131 
1132 	addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET +
1133 		SWITCHTEC_NTB_REG_DBMSG_OFFSET +
1134 		sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);
1135 
1136 	offset = addr & (LUT_SIZE - 1);
1137 	addr -= offset;
1138 
1139 	rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx,
1140 				 sndev->peer_partition, addr);
1141 	if (rc)
1142 		return rc;
1143 
1144 	rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1],
1145 				 bar_cnt - 1);
1146 	if (rc)
1147 		return rc;
1148 
1149 	rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl);
1150 	if (rc)
1151 		return rc;
1152 
1153 	sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar,
1154 						LUT_SIZE, LUT_SIZE);
1155 	if (!sndev->mmio_xlink_win) {
1156 		rc = -ENOMEM;
1157 		return rc;
1158 	}
1159 
1160 	sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;
1161 	sndev->nr_rsvd_luts++;
1162 
1163 	crosslink_init_dbmsgs(sndev);
1164 
1165 	return 0;
1166 }
1167 
1168 static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)
1169 {
1170 	if (sndev->mmio_xlink_win)
1171 		pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win);
1172 }
1173 
1174 static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl)
1175 {
1176 	int i;
1177 	int cnt = 0;
1178 
1179 	for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) {
1180 		u32 r = ioread32(&ctrl->bar_entry[i].ctl);
1181 
1182 		if (r & NTB_CTRL_BAR_VALID)
1183 			map[cnt++] = i;
1184 	}
1185 
1186 	return cnt;
1187 }
1188 
1189 static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
1190 {
1191 	sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar,
1192 				       sndev->mmio_self_ctrl);
1193 
1194 	sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
1195 	sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
1196 
1197 	dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
1198 		sndev->nr_direct_mw, sndev->nr_lut_mw);
1199 
1200 	sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar,
1201 					    sndev->mmio_peer_ctrl);
1202 
1203 	sndev->peer_nr_lut_mw =
1204 		ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
1205 	sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
1206 
1207 	dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
1208 		sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
1209 
1210 }
1211 
1212 /*
1213  * There are 64 doorbells in the switch hardware but this is
1214  * shared among all partitions. So we must split them in half
1215  * (32 for each partition). However, the message interrupts are
1216  * also shared with the top 4 doorbells so we just limit this to
1217  * 28 doorbells per partition.
1218  *
1219  * In crosslink mode, each side has it's own dbmsg register so
1220  * they can each use all 60 of the available doorbells.
1221  */
1222 static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)
1223 {
1224 	sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
1225 
1226 	if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {
1227 		sndev->db_shift = 0;
1228 		sndev->db_peer_shift = 0;
1229 		sndev->db_valid_mask = sndev->db_mask;
1230 	} else if (sndev->self_partition < sndev->peer_partition) {
1231 		sndev->db_shift = 0;
1232 		sndev->db_peer_shift = 32;
1233 		sndev->db_valid_mask = 0x0FFFFFFF;
1234 	} else {
1235 		sndev->db_shift = 32;
1236 		sndev->db_peer_shift = 0;
1237 		sndev->db_valid_mask = 0x0FFFFFFF;
1238 	}
1239 
1240 	iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
1241 	iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
1242 		  &sndev->mmio_peer_dbmsg->odb_mask);
1243 
1244 	dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",
1245 		sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);
1246 }
1247 
1248 static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
1249 {
1250 	int i;
1251 	u32 msg_map = 0;
1252 
1253 	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1254 		int m = i | sndev->peer_partition << 2;
1255 
1256 		msg_map |= m << i * 8;
1257 	}
1258 
1259 	iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map);
1260 
1261 	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++)
1262 		iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK,
1263 			  &sndev->mmio_self_dbmsg->imsg[i]);
1264 }
1265 
1266 static int
1267 switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
1268 {
1269 	int req_ids[2];
1270 
1271 	/*
1272 	 * Root Complex Requester ID (which is 0:00.0)
1273 	 */
1274 	req_ids[0] = 0;
1275 
1276 	/*
1277 	 * Host Bridge Requester ID (as read from the mmap address)
1278 	 */
1279 	req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);
1280 
1281 	return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids,
1282 				   ARRAY_SIZE(req_ids));
1283 }
1284 
1285 static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
1286 {
1287 	int i;
1288 
1289 	memset(sndev->self_shared, 0, LUT_SIZE);
1290 	sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC;
1291 	sndev->self_shared->partition_id = sndev->stdev->partition;
1292 
1293 	for (i = 0; i < sndev->nr_direct_mw; i++) {
1294 		int bar = sndev->direct_mw_to_bar[i];
1295 		resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar);
1296 
1297 		if (i == 0)
1298 			sz = min_t(resource_size_t, sz,
1299 				   LUT_SIZE * sndev->nr_lut_mw);
1300 
1301 		sndev->self_shared->mw_sizes[i] = sz;
1302 	}
1303 
1304 	for (i = 0; i < sndev->nr_lut_mw; i++) {
1305 		int idx = sndev->nr_direct_mw + i;
1306 
1307 		sndev->self_shared->mw_sizes[idx] = LUT_SIZE;
1308 	}
1309 }
1310 
1311 static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)
1312 {
1313 	int self_bar = sndev->direct_mw_to_bar[0];
1314 	int rc;
1315 
1316 	sndev->nr_rsvd_luts++;
1317 	sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev,
1318 						LUT_SIZE,
1319 						&sndev->self_shared_dma,
1320 						GFP_KERNEL);
1321 	if (!sndev->self_shared) {
1322 		dev_err(&sndev->stdev->dev,
1323 			"unable to allocate memory for shared mw\n");
1324 		return -ENOMEM;
1325 	}
1326 
1327 	switchtec_ntb_init_shared(sndev);
1328 
1329 	rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
1330 				 sndev->self_partition,
1331 				 sndev->self_shared_dma);
1332 	if (rc)
1333 		goto unalloc_and_exit;
1334 
1335 	sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE);
1336 	if (!sndev->peer_shared) {
1337 		rc = -ENOMEM;
1338 		goto unalloc_and_exit;
1339 	}
1340 
1341 	dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");
1342 	return 0;
1343 
1344 unalloc_and_exit:
1345 	dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1346 			  sndev->self_shared, sndev->self_shared_dma);
1347 
1348 	return rc;
1349 }
1350 
1351 static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)
1352 {
1353 	if (sndev->peer_shared)
1354 		pci_iounmap(sndev->stdev->pdev, sndev->peer_shared);
1355 
1356 	if (sndev->self_shared)
1357 		dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1358 				  sndev->self_shared,
1359 				  sndev->self_shared_dma);
1360 	sndev->nr_rsvd_luts--;
1361 }
1362 
1363 static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev)
1364 {
1365 	struct switchtec_ntb *sndev = dev;
1366 
1367 	dev_dbg(&sndev->stdev->dev, "doorbell\n");
1368 
1369 	ntb_db_event(&sndev->ntb, 0);
1370 
1371 	return IRQ_HANDLED;
1372 }
1373 
1374 static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev)
1375 {
1376 	int i;
1377 	struct switchtec_ntb *sndev = dev;
1378 
1379 	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1380 		u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]);
1381 
1382 		if (msg & NTB_DBMSG_IMSG_STATUS) {
1383 			dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",
1384 				i, (u32)msg);
1385 			iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);
1386 
1387 			if (i == LINK_MESSAGE)
1388 				switchtec_ntb_check_link(sndev, msg);
1389 		}
1390 	}
1391 
1392 	return IRQ_HANDLED;
1393 }
1394 
1395 static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)
1396 {
1397 	int i;
1398 	int rc;
1399 	int doorbell_irq = 0;
1400 	int message_irq = 0;
1401 	int event_irq;
1402 	int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map);
1403 
1404 	event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number);
1405 
1406 	while (doorbell_irq == event_irq)
1407 		doorbell_irq++;
1408 	while (message_irq == doorbell_irq ||
1409 	       message_irq == event_irq)
1410 		message_irq++;
1411 
1412 	dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",
1413 		event_irq, doorbell_irq, message_irq);
1414 
1415 	for (i = 0; i < idb_vecs - 4; i++)
1416 		iowrite8(doorbell_irq,
1417 			 &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1418 
1419 	for (; i < idb_vecs; i++)
1420 		iowrite8(message_irq,
1421 			 &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1422 
1423 	sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq);
1424 	sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq);
1425 
1426 	rc = request_irq(sndev->doorbell_irq,
1427 			 switchtec_ntb_doorbell_isr, 0,
1428 			 "switchtec_ntb_doorbell", sndev);
1429 	if (rc)
1430 		return rc;
1431 
1432 	rc = request_irq(sndev->message_irq,
1433 			 switchtec_ntb_message_isr, 0,
1434 			 "switchtec_ntb_message", sndev);
1435 	if (rc) {
1436 		free_irq(sndev->doorbell_irq, sndev);
1437 		return rc;
1438 	}
1439 
1440 	return 0;
1441 }
1442 
1443 static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
1444 {
1445 	free_irq(sndev->doorbell_irq, sndev);
1446 	free_irq(sndev->message_irq, sndev);
1447 }
1448 
1449 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
1450 {
1451 	dev_info(&sndev->stdev->dev, "peer reinitialized\n");
1452 	switchtec_ntb_deinit_shared_mw(sndev);
1453 	switchtec_ntb_init_mw(sndev);
1454 	return switchtec_ntb_init_shared_mw(sndev);
1455 }
1456 
1457 static int switchtec_ntb_add(struct device *dev,
1458 			     struct class_interface *class_intf)
1459 {
1460 	struct switchtec_dev *stdev = to_stdev(dev);
1461 	struct switchtec_ntb *sndev;
1462 	int rc;
1463 
1464 	stdev->sndev = NULL;
1465 
1466 	if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8))
1467 		return -ENODEV;
1468 
1469 	sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev));
1470 	if (!sndev)
1471 		return -ENOMEM;
1472 
1473 	sndev->stdev = stdev;
1474 	rc = switchtec_ntb_init_sndev(sndev);
1475 	if (rc)
1476 		goto free_and_exit;
1477 
1478 	switchtec_ntb_init_mw(sndev);
1479 
1480 	rc = switchtec_ntb_init_req_id_table(sndev);
1481 	if (rc)
1482 		goto free_and_exit;
1483 
1484 	rc = switchtec_ntb_init_crosslink(sndev);
1485 	if (rc)
1486 		goto free_and_exit;
1487 
1488 	switchtec_ntb_init_db(sndev);
1489 	switchtec_ntb_init_msgs(sndev);
1490 
1491 	rc = switchtec_ntb_init_shared_mw(sndev);
1492 	if (rc)
1493 		goto deinit_crosslink;
1494 
1495 	rc = switchtec_ntb_init_db_msg_irq(sndev);
1496 	if (rc)
1497 		goto deinit_shared_and_exit;
1498 
1499 	/*
1500 	 * If this host crashed, the other host may think the link is
1501 	 * still up. Tell them to force it down (it will go back up
1502 	 * once we register the ntb device).
1503 	 */
1504 	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN);
1505 
1506 	rc = ntb_register_device(&sndev->ntb);
1507 	if (rc)
1508 		goto deinit_and_exit;
1509 
1510 	stdev->sndev = sndev;
1511 	stdev->link_notifier = switchtec_ntb_link_notification;
1512 	dev_info(dev, "NTB device registered\n");
1513 
1514 	return 0;
1515 
1516 deinit_and_exit:
1517 	switchtec_ntb_deinit_db_msg_irq(sndev);
1518 deinit_shared_and_exit:
1519 	switchtec_ntb_deinit_shared_mw(sndev);
1520 deinit_crosslink:
1521 	switchtec_ntb_deinit_crosslink(sndev);
1522 free_and_exit:
1523 	kfree(sndev);
1524 	dev_err(dev, "failed to register ntb device: %d\n", rc);
1525 	return rc;
1526 }
1527 
1528 static void switchtec_ntb_remove(struct device *dev,
1529 				 struct class_interface *class_intf)
1530 {
1531 	struct switchtec_dev *stdev = to_stdev(dev);
1532 	struct switchtec_ntb *sndev = stdev->sndev;
1533 
1534 	if (!sndev)
1535 		return;
1536 
1537 	stdev->link_notifier = NULL;
1538 	stdev->sndev = NULL;
1539 	ntb_unregister_device(&sndev->ntb);
1540 	switchtec_ntb_deinit_db_msg_irq(sndev);
1541 	switchtec_ntb_deinit_shared_mw(sndev);
1542 	switchtec_ntb_deinit_crosslink(sndev);
1543 	kfree(sndev);
1544 	dev_info(dev, "ntb device unregistered\n");
1545 }
1546 
1547 static struct class_interface switchtec_interface  = {
1548 	.add_dev = switchtec_ntb_add,
1549 	.remove_dev = switchtec_ntb_remove,
1550 };
1551 
1552 static int __init switchtec_ntb_init(void)
1553 {
1554 	switchtec_interface.class = switchtec_class;
1555 	return class_interface_register(&switchtec_interface);
1556 }
1557 module_init(switchtec_ntb_init);
1558 
1559 static void __exit switchtec_ntb_exit(void)
1560 {
1561 	class_interface_unregister(&switchtec_interface);
1562 }
1563 module_exit(switchtec_ntb_exit);
1564