xref: /openbmc/linux/drivers/ntb/hw/amd/ntb_hw_amd.c (revision 8795a739)
1 /*
2  * This file is provided under a dual BSD/GPLv2 license.  When using or
3  *   redistributing this file, you may do so under either license.
4  *
5  *   GPL LICENSE SUMMARY
6  *
7  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
8  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
9  *
10  *   This program is free software; you can redistribute it and/or modify
11  *   it under the terms of version 2 of the GNU General Public License as
12  *   published by the Free Software Foundation.
13  *
14  *   BSD LICENSE
15  *
16  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
17  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
18  *
19  *   Redistribution and use in source and binary forms, with or without
20  *   modification, are permitted provided that the following conditions
21  *   are met:
22  *
23  *     * Redistributions of source code must retain the above copyright
24  *       notice, this list of conditions and the following disclaimer.
25  *     * Redistributions in binary form must reproduce the above copy
26  *       notice, this list of conditions and the following disclaimer in
27  *       the documentation and/or other materials provided with the
28  *       distribution.
29  *     * Neither the name of AMD Corporation nor the names of its
30  *       contributors may be used to endorse or promote products derived
31  *       from this software without specific prior written permission.
32  *
33  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
34  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
35  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
36  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
37  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
38  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
40  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
41  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
42  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
43  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44  *
45  * AMD PCIe NTB Linux driver
46  *
47  * Contact Information:
48  * Xiangliang Yu <Xiangliang.Yu@amd.com>
49  */
50 
51 #include <linux/debugfs.h>
52 #include <linux/delay.h>
53 #include <linux/init.h>
54 #include <linux/interrupt.h>
55 #include <linux/module.h>
56 #include <linux/acpi.h>
57 #include <linux/pci.h>
58 #include <linux/random.h>
59 #include <linux/slab.h>
60 #include <linux/ntb.h>
61 
62 #include "ntb_hw_amd.h"
63 
64 #define NTB_NAME	"ntb_hw_amd"
65 #define NTB_DESC	"AMD(R) PCI-E Non-Transparent Bridge Driver"
66 #define NTB_VER		"1.0"
67 
68 MODULE_DESCRIPTION(NTB_DESC);
69 MODULE_VERSION(NTB_VER);
70 MODULE_LICENSE("Dual BSD/GPL");
71 MODULE_AUTHOR("AMD Inc.");
72 
73 static const struct file_operations amd_ntb_debugfs_info;
74 static struct dentry *debugfs_dir;
75 
76 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
77 {
78 	if (idx < 0 || idx > ndev->mw_count)
79 		return -EINVAL;
80 
81 	return ndev->dev_data->mw_idx << idx;
82 }
83 
84 static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
85 {
86 	if (pidx != NTB_DEF_PEER_IDX)
87 		return -EINVAL;
88 
89 	return ntb_ndev(ntb)->mw_count;
90 }
91 
92 static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
93 				resource_size_t *addr_align,
94 				resource_size_t *size_align,
95 				resource_size_t *size_max)
96 {
97 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
98 	int bar;
99 
100 	if (pidx != NTB_DEF_PEER_IDX)
101 		return -EINVAL;
102 
103 	bar = ndev_mw_to_bar(ndev, idx);
104 	if (bar < 0)
105 		return bar;
106 
107 	if (addr_align)
108 		*addr_align = SZ_4K;
109 
110 	if (size_align)
111 		*size_align = 1;
112 
113 	if (size_max)
114 		*size_max = pci_resource_len(ndev->ntb.pdev, bar);
115 
116 	return 0;
117 }
118 
119 static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
120 				dma_addr_t addr, resource_size_t size)
121 {
122 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
123 	unsigned long xlat_reg, limit_reg = 0;
124 	resource_size_t mw_size;
125 	void __iomem *mmio, *peer_mmio;
126 	u64 base_addr, limit, reg_val;
127 	int bar;
128 
129 	if (pidx != NTB_DEF_PEER_IDX)
130 		return -EINVAL;
131 
132 	bar = ndev_mw_to_bar(ndev, idx);
133 	if (bar < 0)
134 		return bar;
135 
136 	mw_size = pci_resource_len(ntb->pdev, bar);
137 
138 	/* make sure the range fits in the usable mw size */
139 	if (size > mw_size)
140 		return -EINVAL;
141 
142 	mmio = ndev->self_mmio;
143 	peer_mmio = ndev->peer_mmio;
144 
145 	base_addr = pci_resource_start(ntb->pdev, bar);
146 
147 	if (bar != 1) {
148 		xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2);
149 		limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2);
150 
151 		/* Set the limit if supported */
152 		limit = size;
153 
154 		/* set and verify setting the translation address */
155 		write64(addr, peer_mmio + xlat_reg);
156 		reg_val = read64(peer_mmio + xlat_reg);
157 		if (reg_val != addr) {
158 			write64(0, peer_mmio + xlat_reg);
159 			return -EIO;
160 		}
161 
162 		/* set and verify setting the limit */
163 		write64(limit, peer_mmio + limit_reg);
164 		reg_val = read64(peer_mmio + limit_reg);
165 		if (reg_val != limit) {
166 			write64(base_addr, mmio + limit_reg);
167 			write64(0, peer_mmio + xlat_reg);
168 			return -EIO;
169 		}
170 	} else {
171 		xlat_reg = AMD_BAR1XLAT_OFFSET;
172 		limit_reg = AMD_BAR1LMT_OFFSET;
173 
174 		/* Set the limit if supported */
175 		limit = size;
176 
177 		/* set and verify setting the translation address */
178 		write64(addr, peer_mmio + xlat_reg);
179 		reg_val = read64(peer_mmio + xlat_reg);
180 		if (reg_val != addr) {
181 			write64(0, peer_mmio + xlat_reg);
182 			return -EIO;
183 		}
184 
185 		/* set and verify setting the limit */
186 		writel(limit, peer_mmio + limit_reg);
187 		reg_val = readl(peer_mmio + limit_reg);
188 		if (reg_val != limit) {
189 			writel(base_addr, mmio + limit_reg);
190 			writel(0, peer_mmio + xlat_reg);
191 			return -EIO;
192 		}
193 	}
194 
195 	return 0;
196 }
197 
198 static int amd_link_is_up(struct amd_ntb_dev *ndev)
199 {
200 	if (!ndev->peer_sta)
201 		return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
202 
203 	if (ndev->peer_sta & AMD_LINK_UP_EVENT) {
204 		ndev->peer_sta = 0;
205 		return 1;
206 	}
207 
208 	/* If peer_sta is reset or D0 event, the ISR has
209 	 * started a timer to check link status of hardware.
210 	 * So here just clear status bit. And if peer_sta is
211 	 * D3 or PME_TO, D0/reset event will be happened when
212 	 * system wakeup/poweron, so do nothing here.
213 	 */
214 	if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
215 		ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
216 	else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT))
217 		ndev->peer_sta = 0;
218 
219 	return 0;
220 }
221 
222 static u64 amd_ntb_link_is_up(struct ntb_dev *ntb,
223 			      enum ntb_speed *speed,
224 			      enum ntb_width *width)
225 {
226 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
227 	int ret = 0;
228 
229 	if (amd_link_is_up(ndev)) {
230 		if (speed)
231 			*speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
232 		if (width)
233 			*width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
234 
235 		dev_dbg(&ntb->pdev->dev, "link is up.\n");
236 
237 		ret = 1;
238 	} else {
239 		if (speed)
240 			*speed = NTB_SPEED_NONE;
241 		if (width)
242 			*width = NTB_WIDTH_NONE;
243 
244 		dev_dbg(&ntb->pdev->dev, "link is down.\n");
245 	}
246 
247 	return ret;
248 }
249 
250 static int amd_ntb_link_enable(struct ntb_dev *ntb,
251 			       enum ntb_speed max_speed,
252 			       enum ntb_width max_width)
253 {
254 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
255 	void __iomem *mmio = ndev->self_mmio;
256 	u32 ntb_ctl;
257 
258 	/* Enable event interrupt */
259 	ndev->int_mask &= ~AMD_EVENT_INTMASK;
260 	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
261 
262 	if (ndev->ntb.topo == NTB_TOPO_SEC)
263 		return -EINVAL;
264 	dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
265 
266 	ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
267 	ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
268 	writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
269 
270 	return 0;
271 }
272 
273 static int amd_ntb_link_disable(struct ntb_dev *ntb)
274 {
275 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
276 	void __iomem *mmio = ndev->self_mmio;
277 	u32 ntb_ctl;
278 
279 	/* Disable event interrupt */
280 	ndev->int_mask |= AMD_EVENT_INTMASK;
281 	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
282 
283 	if (ndev->ntb.topo == NTB_TOPO_SEC)
284 		return -EINVAL;
285 	dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
286 
287 	ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
288 	ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
289 	writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
290 
291 	return 0;
292 }
293 
294 static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
295 {
296 	/* The same as for inbound MWs */
297 	return ntb_ndev(ntb)->mw_count;
298 }
299 
300 static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
301 				    phys_addr_t *base, resource_size_t *size)
302 {
303 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
304 	int bar;
305 
306 	bar = ndev_mw_to_bar(ndev, idx);
307 	if (bar < 0)
308 		return bar;
309 
310 	if (base)
311 		*base = pci_resource_start(ndev->ntb.pdev, bar);
312 
313 	if (size)
314 		*size = pci_resource_len(ndev->ntb.pdev, bar);
315 
316 	return 0;
317 }
318 
319 static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
320 {
321 	return ntb_ndev(ntb)->db_valid_mask;
322 }
323 
324 static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
325 {
326 	return ntb_ndev(ntb)->db_count;
327 }
328 
329 static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
330 {
331 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
332 
333 	if (db_vector < 0 || db_vector > ndev->db_count)
334 		return 0;
335 
336 	return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector);
337 }
338 
339 static u64 amd_ntb_db_read(struct ntb_dev *ntb)
340 {
341 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
342 	void __iomem *mmio = ndev->self_mmio;
343 
344 	return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
345 }
346 
347 static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
348 {
349 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
350 	void __iomem *mmio = ndev->self_mmio;
351 
352 	writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
353 
354 	return 0;
355 }
356 
357 static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
358 {
359 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
360 	void __iomem *mmio = ndev->self_mmio;
361 	unsigned long flags;
362 
363 	if (db_bits & ~ndev->db_valid_mask)
364 		return -EINVAL;
365 
366 	spin_lock_irqsave(&ndev->db_mask_lock, flags);
367 	ndev->db_mask |= db_bits;
368 	writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
369 	spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
370 
371 	return 0;
372 }
373 
374 static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
375 {
376 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
377 	void __iomem *mmio = ndev->self_mmio;
378 	unsigned long flags;
379 
380 	if (db_bits & ~ndev->db_valid_mask)
381 		return -EINVAL;
382 
383 	spin_lock_irqsave(&ndev->db_mask_lock, flags);
384 	ndev->db_mask &= ~db_bits;
385 	writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
386 	spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
387 
388 	return 0;
389 }
390 
391 static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
392 {
393 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
394 	void __iomem *mmio = ndev->self_mmio;
395 
396 	writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
397 
398 	return 0;
399 }
400 
401 static int amd_ntb_spad_count(struct ntb_dev *ntb)
402 {
403 	return ntb_ndev(ntb)->spad_count;
404 }
405 
406 static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
407 {
408 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
409 	void __iomem *mmio = ndev->self_mmio;
410 	u32 offset;
411 
412 	if (idx < 0 || idx >= ndev->spad_count)
413 		return 0;
414 
415 	offset = ndev->self_spad + (idx << 2);
416 	return readl(mmio + AMD_SPAD_OFFSET + offset);
417 }
418 
419 static int amd_ntb_spad_write(struct ntb_dev *ntb,
420 			      int idx, u32 val)
421 {
422 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
423 	void __iomem *mmio = ndev->self_mmio;
424 	u32 offset;
425 
426 	if (idx < 0 || idx >= ndev->spad_count)
427 		return -EINVAL;
428 
429 	offset = ndev->self_spad + (idx << 2);
430 	writel(val, mmio + AMD_SPAD_OFFSET + offset);
431 
432 	return 0;
433 }
434 
435 static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
436 {
437 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
438 	void __iomem *mmio = ndev->self_mmio;
439 	u32 offset;
440 
441 	if (sidx < 0 || sidx >= ndev->spad_count)
442 		return -EINVAL;
443 
444 	offset = ndev->peer_spad + (sidx << 2);
445 	return readl(mmio + AMD_SPAD_OFFSET + offset);
446 }
447 
448 static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
449 				   int sidx, u32 val)
450 {
451 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
452 	void __iomem *mmio = ndev->self_mmio;
453 	u32 offset;
454 
455 	if (sidx < 0 || sidx >= ndev->spad_count)
456 		return -EINVAL;
457 
458 	offset = ndev->peer_spad + (sidx << 2);
459 	writel(val, mmio + AMD_SPAD_OFFSET + offset);
460 
461 	return 0;
462 }
463 
464 static const struct ntb_dev_ops amd_ntb_ops = {
465 	.mw_count		= amd_ntb_mw_count,
466 	.mw_get_align		= amd_ntb_mw_get_align,
467 	.mw_set_trans		= amd_ntb_mw_set_trans,
468 	.peer_mw_count		= amd_ntb_peer_mw_count,
469 	.peer_mw_get_addr	= amd_ntb_peer_mw_get_addr,
470 	.link_is_up		= amd_ntb_link_is_up,
471 	.link_enable		= amd_ntb_link_enable,
472 	.link_disable		= amd_ntb_link_disable,
473 	.db_valid_mask		= amd_ntb_db_valid_mask,
474 	.db_vector_count	= amd_ntb_db_vector_count,
475 	.db_vector_mask		= amd_ntb_db_vector_mask,
476 	.db_read		= amd_ntb_db_read,
477 	.db_clear		= amd_ntb_db_clear,
478 	.db_set_mask		= amd_ntb_db_set_mask,
479 	.db_clear_mask		= amd_ntb_db_clear_mask,
480 	.peer_db_set		= amd_ntb_peer_db_set,
481 	.spad_count		= amd_ntb_spad_count,
482 	.spad_read		= amd_ntb_spad_read,
483 	.spad_write		= amd_ntb_spad_write,
484 	.peer_spad_read		= amd_ntb_peer_spad_read,
485 	.peer_spad_write	= amd_ntb_peer_spad_write,
486 };
487 
488 static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
489 {
490 	void __iomem *mmio = ndev->self_mmio;
491 	int reg;
492 
493 	reg = readl(mmio + AMD_SMUACK_OFFSET);
494 	reg |= bit;
495 	writel(reg, mmio + AMD_SMUACK_OFFSET);
496 
497 	ndev->peer_sta |= bit;
498 }
499 
500 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
501 {
502 	void __iomem *mmio = ndev->self_mmio;
503 	struct device *dev = &ndev->ntb.pdev->dev;
504 	u32 status;
505 
506 	status = readl(mmio + AMD_INTSTAT_OFFSET);
507 	if (!(status & AMD_EVENT_INTMASK))
508 		return;
509 
510 	dev_dbg(dev, "status = 0x%x and vec = %d\n", status, vec);
511 
512 	status &= AMD_EVENT_INTMASK;
513 	switch (status) {
514 	case AMD_PEER_FLUSH_EVENT:
515 		dev_info(dev, "Flush is done.\n");
516 		break;
517 	case AMD_PEER_RESET_EVENT:
518 		amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
519 
520 		/* link down first */
521 		ntb_link_event(&ndev->ntb);
522 		/* polling peer status */
523 		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
524 
525 		break;
526 	case AMD_PEER_D3_EVENT:
527 	case AMD_PEER_PMETO_EVENT:
528 	case AMD_LINK_UP_EVENT:
529 	case AMD_LINK_DOWN_EVENT:
530 		amd_ack_smu(ndev, status);
531 
532 		/* link down */
533 		ntb_link_event(&ndev->ntb);
534 
535 		break;
536 	case AMD_PEER_D0_EVENT:
537 		mmio = ndev->peer_mmio;
538 		status = readl(mmio + AMD_PMESTAT_OFFSET);
539 		/* check if this is WAKEUP event */
540 		if (status & 0x1)
541 			dev_info(dev, "Wakeup is done.\n");
542 
543 		amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
544 
545 		/* start a timer to poll link status */
546 		schedule_delayed_work(&ndev->hb_timer,
547 				      AMD_LINK_HB_TIMEOUT);
548 		break;
549 	default:
550 		dev_info(dev, "event status = 0x%x.\n", status);
551 		break;
552 	}
553 }
554 
555 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
556 {
557 	dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec);
558 
559 	if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
560 		amd_handle_event(ndev, vec);
561 
562 	if (vec < AMD_DB_CNT)
563 		ntb_db_event(&ndev->ntb, vec);
564 
565 	return IRQ_HANDLED;
566 }
567 
568 static irqreturn_t ndev_vec_isr(int irq, void *dev)
569 {
570 	struct amd_ntb_vec *nvec = dev;
571 
572 	return ndev_interrupt(nvec->ndev, nvec->num);
573 }
574 
575 static irqreturn_t ndev_irq_isr(int irq, void *dev)
576 {
577 	struct amd_ntb_dev *ndev = dev;
578 
579 	return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
580 }
581 
582 static int ndev_init_isr(struct amd_ntb_dev *ndev,
583 			 int msix_min, int msix_max)
584 {
585 	struct pci_dev *pdev;
586 	int rc, i, msix_count, node;
587 
588 	pdev = ndev->ntb.pdev;
589 
590 	node = dev_to_node(&pdev->dev);
591 
592 	ndev->db_mask = ndev->db_valid_mask;
593 
594 	/* Try to set up msix irq */
595 	ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
596 				 GFP_KERNEL, node);
597 	if (!ndev->vec)
598 		goto err_msix_vec_alloc;
599 
600 	ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
601 				  GFP_KERNEL, node);
602 	if (!ndev->msix)
603 		goto err_msix_alloc;
604 
605 	for (i = 0; i < msix_max; ++i)
606 		ndev->msix[i].entry = i;
607 
608 	msix_count = pci_enable_msix_range(pdev, ndev->msix,
609 					   msix_min, msix_max);
610 	if (msix_count < 0)
611 		goto err_msix_enable;
612 
613 	/* NOTE: Disable MSIX if msix count is less than 16 because of
614 	 * hardware limitation.
615 	 */
616 	if (msix_count < msix_min) {
617 		pci_disable_msix(pdev);
618 		goto err_msix_enable;
619 	}
620 
621 	for (i = 0; i < msix_count; ++i) {
622 		ndev->vec[i].ndev = ndev;
623 		ndev->vec[i].num = i;
624 		rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
625 				 "ndev_vec_isr", &ndev->vec[i]);
626 		if (rc)
627 			goto err_msix_request;
628 	}
629 
630 	dev_dbg(&pdev->dev, "Using msix interrupts\n");
631 	ndev->db_count = msix_min;
632 	ndev->msix_vec_count = msix_max;
633 	return 0;
634 
635 err_msix_request:
636 	while (i-- > 0)
637 		free_irq(ndev->msix[i].vector, &ndev->vec[i]);
638 	pci_disable_msix(pdev);
639 err_msix_enable:
640 	kfree(ndev->msix);
641 err_msix_alloc:
642 	kfree(ndev->vec);
643 err_msix_vec_alloc:
644 	ndev->msix = NULL;
645 	ndev->vec = NULL;
646 
647 	/* Try to set up msi irq */
648 	rc = pci_enable_msi(pdev);
649 	if (rc)
650 		goto err_msi_enable;
651 
652 	rc = request_irq(pdev->irq, ndev_irq_isr, 0,
653 			 "ndev_irq_isr", ndev);
654 	if (rc)
655 		goto err_msi_request;
656 
657 	dev_dbg(&pdev->dev, "Using msi interrupts\n");
658 	ndev->db_count = 1;
659 	ndev->msix_vec_count = 1;
660 	return 0;
661 
662 err_msi_request:
663 	pci_disable_msi(pdev);
664 err_msi_enable:
665 
666 	/* Try to set up intx irq */
667 	pci_intx(pdev, 1);
668 
669 	rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
670 			 "ndev_irq_isr", ndev);
671 	if (rc)
672 		goto err_intx_request;
673 
674 	dev_dbg(&pdev->dev, "Using intx interrupts\n");
675 	ndev->db_count = 1;
676 	ndev->msix_vec_count = 1;
677 	return 0;
678 
679 err_intx_request:
680 	return rc;
681 }
682 
683 static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
684 {
685 	struct pci_dev *pdev;
686 	void __iomem *mmio = ndev->self_mmio;
687 	int i;
688 
689 	pdev = ndev->ntb.pdev;
690 
691 	/* Mask all doorbell interrupts */
692 	ndev->db_mask = ndev->db_valid_mask;
693 	writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
694 
695 	if (ndev->msix) {
696 		i = ndev->msix_vec_count;
697 		while (i--)
698 			free_irq(ndev->msix[i].vector, &ndev->vec[i]);
699 		pci_disable_msix(pdev);
700 		kfree(ndev->msix);
701 		kfree(ndev->vec);
702 	} else {
703 		free_irq(pdev->irq, ndev);
704 		if (pci_dev_msi_enabled(pdev))
705 			pci_disable_msi(pdev);
706 		else
707 			pci_intx(pdev, 0);
708 	}
709 }
710 
711 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
712 				 size_t count, loff_t *offp)
713 {
714 	struct amd_ntb_dev *ndev;
715 	void __iomem *mmio;
716 	char *buf;
717 	size_t buf_size;
718 	ssize_t ret, off;
719 	union { u64 v64; u32 v32; u16 v16; } u;
720 
721 	ndev = filp->private_data;
722 	mmio = ndev->self_mmio;
723 
724 	buf_size = min(count, 0x800ul);
725 
726 	buf = kmalloc(buf_size, GFP_KERNEL);
727 	if (!buf)
728 		return -ENOMEM;
729 
730 	off = 0;
731 
732 	off += scnprintf(buf + off, buf_size - off,
733 			 "NTB Device Information:\n");
734 
735 	off += scnprintf(buf + off, buf_size - off,
736 			 "Connection Topology -\t%s\n",
737 			 ntb_topo_string(ndev->ntb.topo));
738 
739 	off += scnprintf(buf + off, buf_size - off,
740 			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
741 
742 	if (!amd_link_is_up(ndev)) {
743 		off += scnprintf(buf + off, buf_size - off,
744 				 "Link Status -\t\tDown\n");
745 	} else {
746 		off += scnprintf(buf + off, buf_size - off,
747 				 "Link Status -\t\tUp\n");
748 		off += scnprintf(buf + off, buf_size - off,
749 				 "Link Speed -\t\tPCI-E Gen %u\n",
750 				 NTB_LNK_STA_SPEED(ndev->lnk_sta));
751 		off += scnprintf(buf + off, buf_size - off,
752 				 "Link Width -\t\tx%u\n",
753 				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
754 	}
755 
756 	off += scnprintf(buf + off, buf_size - off,
757 			 "Memory Window Count -\t%u\n", ndev->mw_count);
758 	off += scnprintf(buf + off, buf_size - off,
759 			 "Scratchpad Count -\t%u\n", ndev->spad_count);
760 	off += scnprintf(buf + off, buf_size - off,
761 			 "Doorbell Count -\t%u\n", ndev->db_count);
762 	off += scnprintf(buf + off, buf_size - off,
763 			 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
764 
765 	off += scnprintf(buf + off, buf_size - off,
766 			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
767 
768 	u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
769 	off += scnprintf(buf + off, buf_size - off,
770 			 "Doorbell Mask -\t\t\t%#06x\n", u.v32);
771 
772 	u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
773 	off += scnprintf(buf + off, buf_size - off,
774 			 "Doorbell Bell -\t\t\t%#06x\n", u.v32);
775 
776 	off += scnprintf(buf + off, buf_size - off,
777 			 "\nNTB Incoming XLAT:\n");
778 
779 	u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
780 	off += scnprintf(buf + off, buf_size - off,
781 			 "XLAT1 -\t\t%#018llx\n", u.v64);
782 
783 	u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
784 	off += scnprintf(buf + off, buf_size - off,
785 			 "XLAT23 -\t\t%#018llx\n", u.v64);
786 
787 	u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
788 	off += scnprintf(buf + off, buf_size - off,
789 			 "XLAT45 -\t\t%#018llx\n", u.v64);
790 
791 	u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
792 	off += scnprintf(buf + off, buf_size - off,
793 			 "LMT1 -\t\t\t%#06x\n", u.v32);
794 
795 	u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
796 	off += scnprintf(buf + off, buf_size - off,
797 			 "LMT23 -\t\t\t%#018llx\n", u.v64);
798 
799 	u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
800 	off += scnprintf(buf + off, buf_size - off,
801 			 "LMT45 -\t\t\t%#018llx\n", u.v64);
802 
803 	ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
804 	kfree(buf);
805 	return ret;
806 }
807 
808 static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
809 {
810 	if (!debugfs_dir) {
811 		ndev->debugfs_dir = NULL;
812 		ndev->debugfs_info = NULL;
813 	} else {
814 		ndev->debugfs_dir =
815 			debugfs_create_dir(pci_name(ndev->ntb.pdev),
816 					   debugfs_dir);
817 		if (!ndev->debugfs_dir)
818 			ndev->debugfs_info = NULL;
819 		else
820 			ndev->debugfs_info =
821 				debugfs_create_file("info", S_IRUSR,
822 						    ndev->debugfs_dir, ndev,
823 						    &amd_ntb_debugfs_info);
824 	}
825 }
826 
827 static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
828 {
829 	debugfs_remove_recursive(ndev->debugfs_dir);
830 }
831 
832 static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
833 				    struct pci_dev *pdev)
834 {
835 	ndev->ntb.pdev = pdev;
836 	ndev->ntb.topo = NTB_TOPO_NONE;
837 	ndev->ntb.ops = &amd_ntb_ops;
838 	ndev->int_mask = AMD_EVENT_INTMASK;
839 	spin_lock_init(&ndev->db_mask_lock);
840 }
841 
842 static int amd_poll_link(struct amd_ntb_dev *ndev)
843 {
844 	void __iomem *mmio = ndev->peer_mmio;
845 	u32 reg, stat;
846 	int rc;
847 
848 	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
849 	reg &= NTB_LIN_STA_ACTIVE_BIT;
850 
851 	dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg);
852 
853 	if (reg == ndev->cntl_sta)
854 		return 0;
855 
856 	ndev->cntl_sta = reg;
857 
858 	rc = pci_read_config_dword(ndev->ntb.pdev,
859 				   AMD_LINK_STATUS_OFFSET, &stat);
860 	if (rc)
861 		return 0;
862 	ndev->lnk_sta = stat;
863 
864 	return 1;
865 }
866 
867 static void amd_link_hb(struct work_struct *work)
868 {
869 	struct amd_ntb_dev *ndev = hb_ndev(work);
870 
871 	if (amd_poll_link(ndev))
872 		ntb_link_event(&ndev->ntb);
873 
874 	if (!amd_link_is_up(ndev))
875 		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
876 }
877 
878 static int amd_init_isr(struct amd_ntb_dev *ndev)
879 {
880 	return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
881 }
882 
883 static void amd_init_side_info(struct amd_ntb_dev *ndev)
884 {
885 	void __iomem *mmio = ndev->self_mmio;
886 	unsigned int reg;
887 
888 	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
889 	if (!(reg & AMD_SIDE_READY)) {
890 		reg |= AMD_SIDE_READY;
891 		writel(reg, mmio + AMD_SIDEINFO_OFFSET);
892 	}
893 }
894 
895 static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
896 {
897 	void __iomem *mmio = ndev->self_mmio;
898 	unsigned int reg;
899 
900 	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
901 	if (reg & AMD_SIDE_READY) {
902 		reg &= ~AMD_SIDE_READY;
903 		writel(reg, mmio + AMD_SIDEINFO_OFFSET);
904 		readl(mmio + AMD_SIDEINFO_OFFSET);
905 	}
906 }
907 
908 static int amd_init_ntb(struct amd_ntb_dev *ndev)
909 {
910 	void __iomem *mmio = ndev->self_mmio;
911 
912 	ndev->mw_count = ndev->dev_data->mw_count;
913 	ndev->spad_count = AMD_SPADS_CNT;
914 	ndev->db_count = AMD_DB_CNT;
915 
916 	switch (ndev->ntb.topo) {
917 	case NTB_TOPO_PRI:
918 	case NTB_TOPO_SEC:
919 		ndev->spad_count >>= 1;
920 		if (ndev->ntb.topo == NTB_TOPO_PRI) {
921 			ndev->self_spad = 0;
922 			ndev->peer_spad = 0x20;
923 		} else {
924 			ndev->self_spad = 0x20;
925 			ndev->peer_spad = 0;
926 		}
927 
928 		INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
929 		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
930 
931 		break;
932 	default:
933 		dev_err(&ndev->ntb.pdev->dev,
934 			"AMD NTB does not support B2B mode.\n");
935 		return -EINVAL;
936 	}
937 
938 	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
939 
940 	/* Mask event interrupts */
941 	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
942 
943 	return 0;
944 }
945 
946 static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
947 {
948 	void __iomem *mmio = ndev->self_mmio;
949 	u32 info;
950 
951 	info = readl(mmio + AMD_SIDEINFO_OFFSET);
952 	if (info & AMD_SIDE_MASK)
953 		return NTB_TOPO_SEC;
954 	else
955 		return NTB_TOPO_PRI;
956 }
957 
958 static int amd_init_dev(struct amd_ntb_dev *ndev)
959 {
960 	struct pci_dev *pdev;
961 	int rc = 0;
962 
963 	pdev = ndev->ntb.pdev;
964 
965 	ndev->ntb.topo = amd_get_topo(ndev);
966 	dev_dbg(&pdev->dev, "AMD NTB topo is %s\n",
967 		ntb_topo_string(ndev->ntb.topo));
968 
969 	rc = amd_init_ntb(ndev);
970 	if (rc)
971 		return rc;
972 
973 	rc = amd_init_isr(ndev);
974 	if (rc) {
975 		dev_err(&pdev->dev, "fail to init isr.\n");
976 		return rc;
977 	}
978 
979 	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
980 
981 	return 0;
982 }
983 
984 static void amd_deinit_dev(struct amd_ntb_dev *ndev)
985 {
986 	cancel_delayed_work_sync(&ndev->hb_timer);
987 
988 	ndev_deinit_isr(ndev);
989 }
990 
991 static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
992 			    struct pci_dev *pdev)
993 {
994 	int rc;
995 
996 	pci_set_drvdata(pdev, ndev);
997 
998 	rc = pci_enable_device(pdev);
999 	if (rc)
1000 		goto err_pci_enable;
1001 
1002 	rc = pci_request_regions(pdev, NTB_NAME);
1003 	if (rc)
1004 		goto err_pci_regions;
1005 
1006 	pci_set_master(pdev);
1007 
1008 	rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1009 	if (rc) {
1010 		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1011 		if (rc)
1012 			goto err_dma_mask;
1013 		dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1014 	}
1015 
1016 	rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1017 	if (rc) {
1018 		rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1019 		if (rc)
1020 			goto err_dma_mask;
1021 		dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1022 	}
1023 	rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
1024 					  dma_get_mask(&pdev->dev));
1025 	if (rc)
1026 		goto err_dma_mask;
1027 
1028 	ndev->self_mmio = pci_iomap(pdev, 0, 0);
1029 	if (!ndev->self_mmio) {
1030 		rc = -EIO;
1031 		goto err_dma_mask;
1032 	}
1033 	ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
1034 
1035 	return 0;
1036 
1037 err_dma_mask:
1038 	pci_clear_master(pdev);
1039 err_pci_regions:
1040 	pci_disable_device(pdev);
1041 err_pci_enable:
1042 	pci_set_drvdata(pdev, NULL);
1043 	return rc;
1044 }
1045 
1046 static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
1047 {
1048 	struct pci_dev *pdev = ndev->ntb.pdev;
1049 
1050 	pci_iounmap(pdev, ndev->self_mmio);
1051 
1052 	pci_clear_master(pdev);
1053 	pci_release_regions(pdev);
1054 	pci_disable_device(pdev);
1055 	pci_set_drvdata(pdev, NULL);
1056 }
1057 
1058 static int amd_ntb_pci_probe(struct pci_dev *pdev,
1059 			     const struct pci_device_id *id)
1060 {
1061 	struct amd_ntb_dev *ndev;
1062 	int rc, node;
1063 
1064 	node = dev_to_node(&pdev->dev);
1065 
1066 	ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1067 	if (!ndev) {
1068 		rc = -ENOMEM;
1069 		goto err_ndev;
1070 	}
1071 
1072 	ndev->dev_data = (struct ntb_dev_data *)id->driver_data;
1073 
1074 	ndev_init_struct(ndev, pdev);
1075 
1076 	rc = amd_ntb_init_pci(ndev, pdev);
1077 	if (rc)
1078 		goto err_init_pci;
1079 
1080 	rc = amd_init_dev(ndev);
1081 	if (rc)
1082 		goto err_init_dev;
1083 
1084 	/* write side info */
1085 	amd_init_side_info(ndev);
1086 
1087 	amd_poll_link(ndev);
1088 
1089 	ndev_init_debugfs(ndev);
1090 
1091 	rc = ntb_register_device(&ndev->ntb);
1092 	if (rc)
1093 		goto err_register;
1094 
1095 	dev_info(&pdev->dev, "NTB device registered.\n");
1096 
1097 	return 0;
1098 
1099 err_register:
1100 	ndev_deinit_debugfs(ndev);
1101 	amd_deinit_dev(ndev);
1102 err_init_dev:
1103 	amd_ntb_deinit_pci(ndev);
1104 err_init_pci:
1105 	kfree(ndev);
1106 err_ndev:
1107 	return rc;
1108 }
1109 
1110 static void amd_ntb_pci_remove(struct pci_dev *pdev)
1111 {
1112 	struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1113 
1114 	ntb_unregister_device(&ndev->ntb);
1115 	ndev_deinit_debugfs(ndev);
1116 	amd_deinit_side_info(ndev);
1117 	amd_deinit_dev(ndev);
1118 	amd_ntb_deinit_pci(ndev);
1119 	kfree(ndev);
1120 }
1121 
1122 static const struct file_operations amd_ntb_debugfs_info = {
1123 	.owner = THIS_MODULE,
1124 	.open = simple_open,
1125 	.read = ndev_debugfs_read,
1126 };
1127 
1128 static const struct ntb_dev_data dev_data[] = {
1129 	{ /* for device 145b */
1130 		.mw_count = 3,
1131 		.mw_idx = 1,
1132 	},
1133 	{ /* for device 148b */
1134 		.mw_count = 2,
1135 		.mw_idx = 2,
1136 	},
1137 };
1138 
1139 static const struct pci_device_id amd_ntb_pci_tbl[] = {
1140 	{ PCI_VDEVICE(AMD, 0x145b), (kernel_ulong_t)&dev_data[0] },
1141 	{ PCI_VDEVICE(AMD, 0x148b), (kernel_ulong_t)&dev_data[1] },
1142 	{ 0, }
1143 };
1144 MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
1145 
1146 static struct pci_driver amd_ntb_pci_driver = {
1147 	.name		= KBUILD_MODNAME,
1148 	.id_table	= amd_ntb_pci_tbl,
1149 	.probe		= amd_ntb_pci_probe,
1150 	.remove		= amd_ntb_pci_remove,
1151 };
1152 
1153 static int __init amd_ntb_pci_driver_init(void)
1154 {
1155 	pr_info("%s %s\n", NTB_DESC, NTB_VER);
1156 
1157 	if (debugfs_initialized())
1158 		debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1159 
1160 	return pci_register_driver(&amd_ntb_pci_driver);
1161 }
1162 module_init(amd_ntb_pci_driver_init);
1163 
1164 static void __exit amd_ntb_pci_driver_exit(void)
1165 {
1166 	pci_unregister_driver(&amd_ntb_pci_driver);
1167 	debugfs_remove_recursive(debugfs_dir);
1168 }
1169 module_exit(amd_ntb_pci_driver_exit);
1170