xref: /openbmc/linux/drivers/edac/thunderx_edac.c (revision 8ee90c5c)
1 /*
2  * Cavium ThunderX memory controller kernel module
3  *
4  * This file is subject to the terms and conditions of the GNU General Public
5  * License.  See the file "COPYING" in the main directory of this archive
6  * for more details.
7  *
8  * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved.
9  *
10  */
11 
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/edac.h>
15 #include <linux/interrupt.h>
16 #include <linux/string.h>
17 #include <linux/stop_machine.h>
18 #include <linux/delay.h>
19 #include <linux/sizes.h>
20 #include <linux/atomic.h>
21 #include <linux/bitfield.h>
22 #include <linux/circ_buf.h>
23 
24 #include <asm/page.h>
25 
26 #include "edac_module.h"
27 
28 #define phys_to_pfn(phys)	(PFN_DOWN(phys))
29 
30 #define THUNDERX_NODE		GENMASK(45, 44)
31 
32 enum {
33 	ERR_CORRECTED	= 1,
34 	ERR_UNCORRECTED	= 2,
35 	ERR_UNKNOWN	= 3,
36 };
37 
38 #define MAX_SYNDROME_REGS 4
39 
40 struct error_syndrome {
41 	u64 reg[MAX_SYNDROME_REGS];
42 };
43 
44 struct error_descr {
45 	int	type;
46 	u64	mask;
47 	char	*descr;
48 };
49 
50 static void decode_register(char *str, size_t size,
51 			   const struct error_descr *descr,
52 			   const uint64_t reg)
53 {
54 	int ret = 0;
55 
56 	while (descr->type && descr->mask && descr->descr) {
57 		if (reg & descr->mask) {
58 			ret = snprintf(str, size, "\n\t%s, %s",
59 				       descr->type == ERR_CORRECTED ?
60 					 "Corrected" : "Uncorrected",
61 				       descr->descr);
62 			str += ret;
63 			size -= ret;
64 		}
65 		descr++;
66 	}
67 }
68 
69 static unsigned long get_bits(unsigned long data, int pos, int width)
70 {
71 	return (data >> pos) & ((1 << width) - 1);
72 }
73 
74 #define L2C_CTL			0x87E080800000
75 #define L2C_CTL_DISIDXALIAS	BIT(0)
76 
77 #define PCI_DEVICE_ID_THUNDER_LMC 0xa022
78 
79 #define LMC_FADR		0x20
80 #define LMC_FADR_FDIMM(x)	((x >> 37) & 0x1)
81 #define LMC_FADR_FBUNK(x)	((x >> 36) & 0x1)
82 #define LMC_FADR_FBANK(x)	((x >> 32) & 0xf)
83 #define LMC_FADR_FROW(x)	((x >> 14) & 0xffff)
84 #define LMC_FADR_FCOL(x)	((x >> 0) & 0x1fff)
85 
86 #define LMC_NXM_FADR		0x28
87 #define LMC_ECC_SYND		0x38
88 
89 #define LMC_ECC_PARITY_TEST	0x108
90 
91 #define LMC_INT_W1S		0x150
92 
93 #define LMC_INT_ENA_W1C		0x158
94 #define LMC_INT_ENA_W1S		0x160
95 
96 #define LMC_CONFIG		0x188
97 
98 #define LMC_CONFIG_BG2		BIT(62)
99 #define LMC_CONFIG_RANK_ENA	BIT(42)
100 #define LMC_CONFIG_PBANK_LSB(x)	(((x) >> 5) & 0xF)
101 #define LMC_CONFIG_ROW_LSB(x)	(((x) >> 2) & 0x7)
102 
103 #define LMC_CONTROL		0x190
104 #define LMC_CONTROL_XOR_BANK	BIT(16)
105 
106 #define LMC_INT			0x1F0
107 
108 #define LMC_INT_DDR_ERR		BIT(11)
109 #define LMC_INT_DED_ERR		(0xFUL << 5)
110 #define LMC_INT_SEC_ERR         (0xFUL << 1)
111 #define LMC_INT_NXM_WR_MASK	BIT(0)
112 
113 #define LMC_DDR_PLL_CTL		0x258
114 #define LMC_DDR_PLL_CTL_DDR4	BIT(29)
115 
116 #define LMC_FADR_SCRAMBLED	0x330
117 
118 #define LMC_INT_UE              (LMC_INT_DDR_ERR | LMC_INT_DED_ERR | \
119 				 LMC_INT_NXM_WR_MASK)
120 
121 #define LMC_INT_CE		(LMC_INT_SEC_ERR)
122 
123 static const struct error_descr lmc_errors[] = {
124 	{
125 		.type  = ERR_CORRECTED,
126 		.mask  = LMC_INT_SEC_ERR,
127 		.descr = "Single-bit ECC error",
128 	},
129 	{
130 		.type  = ERR_UNCORRECTED,
131 		.mask  = LMC_INT_DDR_ERR,
132 		.descr = "DDR chip error",
133 	},
134 	{
135 		.type  = ERR_UNCORRECTED,
136 		.mask  = LMC_INT_DED_ERR,
137 		.descr = "Double-bit ECC error",
138 	},
139 	{
140 		.type = ERR_UNCORRECTED,
141 		.mask = LMC_INT_NXM_WR_MASK,
142 		.descr = "Non-existent memory write",
143 	},
144 	{0, 0, NULL},
145 };
146 
147 #define LMC_INT_EN_DDR_ERROR_ALERT_ENA	BIT(5)
148 #define LMC_INT_EN_DLCRAM_DED_ERR	BIT(4)
149 #define LMC_INT_EN_DLCRAM_SEC_ERR	BIT(3)
150 #define LMC_INT_INTR_DED_ENA		BIT(2)
151 #define LMC_INT_INTR_SEC_ENA		BIT(1)
152 #define LMC_INT_INTR_NXM_WR_ENA		BIT(0)
153 
154 #define LMC_INT_ENA_ALL			GENMASK(5, 0)
155 
156 #define LMC_DDR_PLL_CTL		0x258
157 #define LMC_DDR_PLL_CTL_DDR4	BIT(29)
158 
159 #define LMC_CONTROL		0x190
160 #define LMC_CONTROL_RDIMM	BIT(0)
161 
162 #define LMC_SCRAM_FADR		0x330
163 
164 #define LMC_CHAR_MASK0		0x228
165 #define LMC_CHAR_MASK2		0x238
166 
167 #define RING_ENTRIES	8
168 
169 struct debugfs_entry {
170 	const char *name;
171 	umode_t mode;
172 	const struct file_operations fops;
173 };
174 
175 struct lmc_err_ctx {
176 	u64 reg_int;
177 	u64 reg_fadr;
178 	u64 reg_nxm_fadr;
179 	u64 reg_scram_fadr;
180 	u64 reg_ecc_synd;
181 };
182 
183 struct thunderx_lmc {
184 	void __iomem *regs;
185 	struct pci_dev *pdev;
186 	struct msix_entry msix_ent;
187 
188 	atomic_t ecc_int;
189 
190 	u64 mask0;
191 	u64 mask2;
192 	u64 parity_test;
193 	u64 node;
194 
195 	int xbits;
196 	int bank_width;
197 	int pbank_lsb;
198 	int dimm_lsb;
199 	int rank_lsb;
200 	int bank_lsb;
201 	int row_lsb;
202 	int col_hi_lsb;
203 
204 	int xor_bank;
205 	int l2c_alias;
206 
207 	struct page *mem;
208 
209 	struct lmc_err_ctx err_ctx[RING_ENTRIES];
210 	unsigned long ring_head;
211 	unsigned long ring_tail;
212 };
213 
214 #define ring_pos(pos, size) ((pos) & (size - 1))
215 
216 #define DEBUGFS_STRUCT(_name, _mode, _write, _read)			    \
217 static struct debugfs_entry debugfs_##_name = {				    \
218 	.name = __stringify(_name),					    \
219 	.mode = VERIFY_OCTAL_PERMISSIONS(_mode),			    \
220 	.fops = {							    \
221 		.open = simple_open,					    \
222 		.write = _write,					    \
223 		.read  = _read,						    \
224 		.llseek = generic_file_llseek,				    \
225 	},								    \
226 }
227 
228 #define DEBUGFS_FIELD_ATTR(_type, _field)				    \
229 static ssize_t thunderx_##_type##_##_field##_read(struct file *file,	    \
230 					    char __user *data,		    \
231 					    size_t count, loff_t *ppos)	    \
232 {									    \
233 	struct thunderx_##_type *pdata = file->private_data;		    \
234 	char buf[20];							    \
235 									    \
236 	snprintf(buf, count, "0x%016llx", pdata->_field);		    \
237 	return simple_read_from_buffer(data, count, ppos,		    \
238 				       buf, sizeof(buf));		    \
239 }									    \
240 									    \
241 static ssize_t thunderx_##_type##_##_field##_write(struct file *file,	    \
242 					     const char __user *data,	    \
243 					     size_t count, loff_t *ppos)    \
244 {									    \
245 	struct thunderx_##_type *pdata = file->private_data;		    \
246 	int res;							    \
247 									    \
248 	res = kstrtoull_from_user(data, count, 0, &pdata->_field);	    \
249 									    \
250 	return res ? res : count;					    \
251 }									    \
252 									    \
253 DEBUGFS_STRUCT(_field, 0600,						    \
254 		   thunderx_##_type##_##_field##_write,			    \
255 		   thunderx_##_type##_##_field##_read)			    \
256 
257 #define DEBUGFS_REG_ATTR(_type, _name, _reg)				    \
258 static ssize_t thunderx_##_type##_##_name##_read(struct file *file,	    \
259 					   char __user *data,		    \
260 					   size_t count, loff_t *ppos)      \
261 {									    \
262 	struct thunderx_##_type *pdata = file->private_data;		    \
263 	char buf[20];							    \
264 									    \
265 	sprintf(buf, "0x%016llx", readq(pdata->regs + _reg));		    \
266 	return simple_read_from_buffer(data, count, ppos,		    \
267 				       buf, sizeof(buf));		    \
268 }									    \
269 									    \
270 static ssize_t thunderx_##_type##_##_name##_write(struct file *file,	    \
271 					    const char __user *data,	    \
272 					    size_t count, loff_t *ppos)     \
273 {									    \
274 	struct thunderx_##_type *pdata = file->private_data;		    \
275 	u64 val;							    \
276 	int res;							    \
277 									    \
278 	res = kstrtoull_from_user(data, count, 0, &val);		    \
279 									    \
280 	if (!res) {							    \
281 		writeq(val, pdata->regs + _reg);			    \
282 		res = count;						    \
283 	}								    \
284 									    \
285 	return res;							    \
286 }									    \
287 									    \
288 DEBUGFS_STRUCT(_name, 0600,						    \
289 	       thunderx_##_type##_##_name##_write,			    \
290 	       thunderx_##_type##_##_name##_read)
291 
292 #define LMC_DEBUGFS_ENT(_field)	DEBUGFS_FIELD_ATTR(lmc, _field)
293 
294 /*
295  * To get an ECC error injected, the following steps are needed:
296  * - Setup the ECC injection by writing the appropriate parameters:
297  *	echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask0
298  *	echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask2
299  *	echo 0x802 > /sys/kernel/debug/<device number>/ecc_parity_test
300  * - Do the actual injection:
301  *	echo 1 > /sys/kernel/debug/<device number>/inject_ecc
302  */
303 static ssize_t thunderx_lmc_inject_int_write(struct file *file,
304 					     const char __user *data,
305 					     size_t count, loff_t *ppos)
306 {
307 	struct thunderx_lmc *lmc = file->private_data;
308 	u64 val;
309 	int res;
310 
311 	res = kstrtoull_from_user(data, count, 0, &val);
312 
313 	if (!res) {
314 		/* Trigger the interrupt */
315 		writeq(val, lmc->regs + LMC_INT_W1S);
316 		res = count;
317 	}
318 
319 	return res;
320 }
321 
322 static ssize_t thunderx_lmc_int_read(struct file *file,
323 				     char __user *data,
324 				     size_t count, loff_t *ppos)
325 {
326 	struct thunderx_lmc *lmc = file->private_data;
327 	char buf[20];
328 	u64 lmc_int = readq(lmc->regs + LMC_INT);
329 
330 	snprintf(buf, sizeof(buf), "0x%016llx", lmc_int);
331 	return simple_read_from_buffer(data, count, ppos, buf, sizeof(buf));
332 }
333 
334 #define TEST_PATTERN 0xa5
335 
336 static int inject_ecc_fn(void *arg)
337 {
338 	struct thunderx_lmc *lmc = arg;
339 	uintptr_t addr, phys;
340 	unsigned int cline_size = cache_line_size();
341 	const unsigned int lines = PAGE_SIZE / cline_size;
342 	unsigned int i, cl_idx;
343 
344 	addr = (uintptr_t)page_address(lmc->mem);
345 	phys = (uintptr_t)page_to_phys(lmc->mem);
346 
347 	cl_idx = (phys & 0x7f) >> 4;
348 	lmc->parity_test &= ~(7ULL << 8);
349 	lmc->parity_test |= (cl_idx << 8);
350 
351 	writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);
352 	writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);
353 	writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);
354 
355 	readq(lmc->regs + LMC_CHAR_MASK0);
356 	readq(lmc->regs + LMC_CHAR_MASK2);
357 	readq(lmc->regs + LMC_ECC_PARITY_TEST);
358 
359 	for (i = 0; i < lines; i++) {
360 		memset((void *)addr, TEST_PATTERN, cline_size);
361 		barrier();
362 
363 		/*
364 		 * Flush L1 cachelines to the PoC (L2).
365 		 * This will cause cacheline eviction to the L2.
366 		 */
367 		asm volatile("dc civac, %0\n"
368 			     "dsb sy\n"
369 			     : : "r"(addr + i * cline_size));
370 	}
371 
372 	for (i = 0; i < lines; i++) {
373 		/*
374 		 * Flush L2 cachelines to the DRAM.
375 		 * This will cause cacheline eviction to the DRAM
376 		 * and ECC corruption according to the masks set.
377 		 */
378 		__asm__ volatile("sys #0,c11,C1,#2, %0\n"
379 				 : : "r"(phys + i * cline_size));
380 	}
381 
382 	for (i = 0; i < lines; i++) {
383 		/*
384 		 * Invalidate L2 cachelines.
385 		 * The subsequent load will cause cacheline fetch
386 		 * from the DRAM and an error interrupt
387 		 */
388 		__asm__ volatile("sys #0,c11,C1,#1, %0"
389 				 : : "r"(phys + i * cline_size));
390 	}
391 
392 	for (i = 0; i < lines; i++) {
393 		/*
394 		 * Invalidate L1 cachelines.
395 		 * The subsequent load will cause cacheline fetch
396 		 * from the L2 and/or DRAM
397 		 */
398 		asm volatile("dc ivac, %0\n"
399 			     "dsb sy\n"
400 			     : : "r"(addr + i * cline_size));
401 	}
402 
403 	return 0;
404 }
405 
406 static ssize_t thunderx_lmc_inject_ecc_write(struct file *file,
407 					     const char __user *data,
408 					     size_t count, loff_t *ppos)
409 {
410 	struct thunderx_lmc *lmc = file->private_data;
411 
412 	unsigned int cline_size = cache_line_size();
413 
414 	u8 tmp[cline_size];
415 	void __iomem *addr;
416 	unsigned int offs, timeout = 100000;
417 
418 	atomic_set(&lmc->ecc_int, 0);
419 
420 	lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);
421 
422 	if (!lmc->mem)
423 		return -ENOMEM;
424 
425 	addr = page_address(lmc->mem);
426 
427 	while (!atomic_read(&lmc->ecc_int) && timeout--) {
428 		stop_machine(inject_ecc_fn, lmc, NULL);
429 
430 		for (offs = 0; offs < PAGE_SIZE; offs += sizeof(tmp)) {
431 			/*
432 			 * Do a load from the previously rigged location
433 			 * This should generate an error interrupt.
434 			 */
435 			memcpy(tmp, addr + offs, cline_size);
436 			asm volatile("dsb ld\n");
437 		}
438 	}
439 
440 	__free_pages(lmc->mem, 0);
441 
442 	return count;
443 }
444 
445 LMC_DEBUGFS_ENT(mask0);
446 LMC_DEBUGFS_ENT(mask2);
447 LMC_DEBUGFS_ENT(parity_test);
448 
449 DEBUGFS_STRUCT(inject_int, 0200, thunderx_lmc_inject_int_write, NULL);
450 DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL);
451 DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read);
452 
453 struct debugfs_entry *lmc_dfs_ents[] = {
454 	&debugfs_mask0,
455 	&debugfs_mask2,
456 	&debugfs_parity_test,
457 	&debugfs_inject_ecc,
458 	&debugfs_inject_int,
459 	&debugfs_int_w1c,
460 };
461 
462 static int thunderx_create_debugfs_nodes(struct dentry *parent,
463 					  struct debugfs_entry *attrs[],
464 					  void *data,
465 					  size_t num)
466 {
467 	int i;
468 	struct dentry *ent;
469 
470 	if (!IS_ENABLED(CONFIG_EDAC_DEBUG))
471 		return 0;
472 
473 	if (!parent)
474 		return -ENOENT;
475 
476 	for (i = 0; i < num; i++) {
477 		ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode,
478 					       parent, data, &attrs[i]->fops);
479 
480 		if (!ent)
481 			break;
482 	}
483 
484 	return i;
485 }
486 
487 static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc)
488 {
489 	phys_addr_t addr = 0;
490 	int bank, xbits;
491 
492 	addr |= lmc->node << 40;
493 	addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;
494 	addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;
495 	addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;
496 	addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;
497 
498 	bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;
499 
500 	if (lmc->xor_bank)
501 		bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);
502 
503 	addr |= bank << lmc->bank_lsb;
504 
505 	xbits = PCI_FUNC(lmc->pdev->devfn);
506 
507 	if (lmc->l2c_alias)
508 		xbits ^= get_bits(addr, 20, lmc->xbits) ^
509 			 get_bits(addr, 12, lmc->xbits);
510 
511 	addr |= xbits << 7;
512 
513 	return addr;
514 }
515 
516 static unsigned int thunderx_get_num_lmcs(unsigned int node)
517 {
518 	unsigned int number = 0;
519 	struct pci_dev *pdev = NULL;
520 
521 	do {
522 		pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
523 				      PCI_DEVICE_ID_THUNDER_LMC,
524 				      pdev);
525 		if (pdev) {
526 #ifdef CONFIG_NUMA
527 			if (pdev->dev.numa_node == node)
528 				number++;
529 #else
530 			number++;
531 #endif
532 		}
533 	} while (pdev);
534 
535 	return number;
536 }
537 
538 #define LMC_MESSAGE_SIZE	120
539 #define LMC_OTHER_SIZE		(50 * ARRAY_SIZE(lmc_errors))
540 
541 static irqreturn_t thunderx_lmc_err_isr(int irq, void *dev_id)
542 {
543 	struct mem_ctl_info *mci = dev_id;
544 	struct thunderx_lmc *lmc = mci->pvt_info;
545 
546 	unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));
547 	struct lmc_err_ctx *ctx = &lmc->err_ctx[head];
548 
549 	writeq(0, lmc->regs + LMC_CHAR_MASK0);
550 	writeq(0, lmc->regs + LMC_CHAR_MASK2);
551 	writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);
552 
553 	ctx->reg_int = readq(lmc->regs + LMC_INT);
554 	ctx->reg_fadr = readq(lmc->regs + LMC_FADR);
555 	ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);
556 	ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);
557 	ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);
558 
559 	lmc->ring_head++;
560 
561 	atomic_set(&lmc->ecc_int, 1);
562 
563 	/* Clear the interrupt */
564 	writeq(ctx->reg_int, lmc->regs + LMC_INT);
565 
566 	return IRQ_WAKE_THREAD;
567 }
568 
569 static irqreturn_t thunderx_lmc_threaded_isr(int irq, void *dev_id)
570 {
571 	struct mem_ctl_info *mci = dev_id;
572 	struct thunderx_lmc *lmc = mci->pvt_info;
573 	phys_addr_t phys_addr;
574 
575 	unsigned long tail;
576 	struct lmc_err_ctx *ctx;
577 
578 	irqreturn_t ret = IRQ_NONE;
579 
580 	char *msg;
581 	char *other;
582 
583 	msg = kmalloc(LMC_MESSAGE_SIZE, GFP_KERNEL);
584 	other =  kmalloc(LMC_OTHER_SIZE, GFP_KERNEL);
585 
586 	if (!msg || !other)
587 		goto err_free;
588 
589 	while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,
590 		ARRAY_SIZE(lmc->err_ctx))) {
591 		tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));
592 
593 		ctx = &lmc->err_ctx[tail];
594 
595 		dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",
596 			ctx->reg_int);
597 		dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",
598 			ctx->reg_fadr);
599 		dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",
600 			ctx->reg_nxm_fadr);
601 		dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",
602 			ctx->reg_scram_fadr);
603 		dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",
604 			ctx->reg_ecc_synd);
605 
606 		snprintf(msg, LMC_MESSAGE_SIZE,
607 			 "DIMM %lld rank %lld bank %lld row %lld col %lld",
608 			 LMC_FADR_FDIMM(ctx->reg_scram_fadr),
609 			 LMC_FADR_FBUNK(ctx->reg_scram_fadr),
610 			 LMC_FADR_FBANK(ctx->reg_scram_fadr),
611 			 LMC_FADR_FROW(ctx->reg_scram_fadr),
612 			 LMC_FADR_FCOL(ctx->reg_scram_fadr));
613 
614 		decode_register(other, LMC_OTHER_SIZE, lmc_errors,
615 				ctx->reg_int);
616 
617 		phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);
618 
619 		if (ctx->reg_int & LMC_INT_UE)
620 			edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
621 					     phys_to_pfn(phys_addr),
622 					     offset_in_page(phys_addr),
623 					     0, -1, -1, -1, msg, other);
624 		else if (ctx->reg_int & LMC_INT_CE)
625 			edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
626 					     phys_to_pfn(phys_addr),
627 					     offset_in_page(phys_addr),
628 					     0, -1, -1, -1, msg, other);
629 
630 		lmc->ring_tail++;
631 	}
632 
633 	ret = IRQ_HANDLED;
634 
635 err_free:
636 	kfree(msg);
637 	kfree(other);
638 
639 	return ret;
640 }
641 
642 #ifdef CONFIG_PM
643 static int thunderx_lmc_suspend(struct pci_dev *pdev, pm_message_t state)
644 {
645 	pci_save_state(pdev);
646 	pci_disable_device(pdev);
647 
648 	pci_set_power_state(pdev, pci_choose_state(pdev, state));
649 
650 	return 0;
651 }
652 
653 static int thunderx_lmc_resume(struct pci_dev *pdev)
654 {
655 	pci_set_power_state(pdev, PCI_D0);
656 	pci_enable_wake(pdev, PCI_D0, 0);
657 	pci_restore_state(pdev);
658 
659 	return 0;
660 }
661 #endif
662 
663 static const struct pci_device_id thunderx_lmc_pci_tbl[] = {
664 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_LMC) },
665 	{ 0, },
666 };
667 
668 static inline int pci_dev_to_mc_idx(struct pci_dev *pdev)
669 {
670 	int node = dev_to_node(&pdev->dev);
671 	int ret = PCI_FUNC(pdev->devfn);
672 
673 	ret += max(node, 0) << 3;
674 
675 	return ret;
676 }
677 
678 static int thunderx_lmc_probe(struct pci_dev *pdev,
679 				const struct pci_device_id *id)
680 {
681 	struct thunderx_lmc *lmc;
682 	struct edac_mc_layer layer;
683 	struct mem_ctl_info *mci;
684 	u64 lmc_control, lmc_ddr_pll_ctl, lmc_config;
685 	int ret;
686 	u64 lmc_int;
687 	void *l2c_ioaddr;
688 
689 	layer.type = EDAC_MC_LAYER_SLOT;
690 	layer.size = 2;
691 	layer.is_virt_csrow = false;
692 
693 	ret = pcim_enable_device(pdev);
694 	if (ret) {
695 		dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
696 		return ret;
697 	}
698 
699 	ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_lmc");
700 	if (ret) {
701 		dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
702 		return ret;
703 	}
704 
705 	mci = edac_mc_alloc(pci_dev_to_mc_idx(pdev), 1, &layer,
706 			    sizeof(struct thunderx_lmc));
707 	if (!mci)
708 		return -ENOMEM;
709 
710 	mci->pdev = &pdev->dev;
711 	lmc = mci->pvt_info;
712 
713 	pci_set_drvdata(pdev, mci);
714 
715 	lmc->regs = pcim_iomap_table(pdev)[0];
716 
717 	lmc_control = readq(lmc->regs + LMC_CONTROL);
718 	lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);
719 	lmc_config = readq(lmc->regs + LMC_CONFIG);
720 
721 	if (lmc_control & LMC_CONTROL_RDIMM) {
722 		mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
723 					   lmc_ddr_pll_ctl) ?
724 				MEM_RDDR4 : MEM_RDDR3;
725 	} else {
726 		mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
727 					   lmc_ddr_pll_ctl) ?
728 				MEM_DDR4 : MEM_DDR3;
729 	}
730 
731 	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
732 	mci->edac_cap = EDAC_FLAG_SECDED;
733 
734 	mci->mod_name = "thunderx-lmc";
735 	mci->ctl_name = "thunderx-lmc";
736 	mci->dev_name = dev_name(&pdev->dev);
737 	mci->scrub_mode = SCRUB_NONE;
738 
739 	lmc->pdev = pdev;
740 	lmc->msix_ent.entry = 0;
741 
742 	lmc->ring_head = 0;
743 	lmc->ring_tail = 0;
744 
745 	ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);
746 	if (ret) {
747 		dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
748 		goto err_free;
749 	}
750 
751 	ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,
752 					thunderx_lmc_err_isr,
753 					thunderx_lmc_threaded_isr, 0,
754 					"[EDAC] ThunderX LMC", mci);
755 	if (ret) {
756 		dev_err(&pdev->dev, "Cannot set ISR: %d\n", ret);
757 		goto err_free;
758 	}
759 
760 	lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));
761 
762 	lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;
763 	lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&
764 			   FIELD_GET(LMC_CONFIG_BG2, lmc_config)) ? 4 : 3;
765 
766 	lmc->pbank_lsb = (lmc_config >> 5) & 0xf;
767 	lmc->dimm_lsb  = 28 + lmc->pbank_lsb + lmc->xbits;
768 	lmc->rank_lsb = lmc->dimm_lsb;
769 	lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;
770 	lmc->bank_lsb = 7 + lmc->xbits;
771 	lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;
772 
773 	lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;
774 
775 	lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;
776 
777 	l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE);
778 	if (!l2c_ioaddr) {
779 		dev_err(&pdev->dev, "Cannot map L2C_CTL\n");
780 		ret = -ENOMEM;
781 		goto err_free;
782 	}
783 
784 	lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);
785 
786 	iounmap(l2c_ioaddr);
787 
788 	ret = edac_mc_add_mc(mci);
789 	if (ret) {
790 		dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret);
791 		goto err_free;
792 	}
793 
794 	lmc_int = readq(lmc->regs + LMC_INT);
795 	writeq(lmc_int, lmc->regs + LMC_INT);
796 
797 	writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);
798 
799 	if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
800 		ret = thunderx_create_debugfs_nodes(mci->debugfs,
801 						    lmc_dfs_ents,
802 						    lmc,
803 						    ARRAY_SIZE(lmc_dfs_ents));
804 
805 		if (ret != ARRAY_SIZE(lmc_dfs_ents)) {
806 			dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
807 				 ret, ret >= 0 ? " created" : "");
808 		}
809 	}
810 
811 	return 0;
812 
813 err_free:
814 	pci_set_drvdata(pdev, NULL);
815 	edac_mc_free(mci);
816 
817 	return ret;
818 }
819 
820 static void thunderx_lmc_remove(struct pci_dev *pdev)
821 {
822 	struct mem_ctl_info *mci = pci_get_drvdata(pdev);
823 	struct thunderx_lmc *lmc = mci->pvt_info;
824 
825 	writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);
826 
827 	edac_mc_del_mc(&pdev->dev);
828 	edac_mc_free(mci);
829 }
830 
831 MODULE_DEVICE_TABLE(pci, thunderx_lmc_pci_tbl);
832 
833 static struct pci_driver thunderx_lmc_driver = {
834 	.name     = "thunderx_lmc_edac",
835 	.probe    = thunderx_lmc_probe,
836 	.remove   = thunderx_lmc_remove,
837 #ifdef CONFIG_PM
838 	.suspend  = thunderx_lmc_suspend,
839 	.resume   = thunderx_lmc_resume,
840 #endif
841 	.id_table = thunderx_lmc_pci_tbl,
842 };
843 
844 /*---------------------- OCX driver ---------------------------------*/
845 
846 #define PCI_DEVICE_ID_THUNDER_OCX 0xa013
847 
848 #define OCX_LINK_INTS		3
849 #define OCX_INTS		(OCX_LINK_INTS + 1)
850 #define OCX_RX_LANES		24
851 #define OCX_RX_LANE_STATS	15
852 
853 #define OCX_COM_INT		0x100
854 #define OCX_COM_INT_W1S		0x108
855 #define OCX_COM_INT_ENA_W1S	0x110
856 #define OCX_COM_INT_ENA_W1C	0x118
857 
858 #define OCX_COM_IO_BADID		BIT(54)
859 #define OCX_COM_MEM_BADID		BIT(53)
860 #define OCX_COM_COPR_BADID		BIT(52)
861 #define OCX_COM_WIN_REQ_BADID		BIT(51)
862 #define OCX_COM_WIN_REQ_TOUT		BIT(50)
863 #define OCX_COM_RX_LANE			GENMASK(23, 0)
864 
865 #define OCX_COM_INT_CE			(OCX_COM_IO_BADID      | \
866 					 OCX_COM_MEM_BADID     | \
867 					 OCX_COM_COPR_BADID    | \
868 					 OCX_COM_WIN_REQ_BADID | \
869 					 OCX_COM_WIN_REQ_TOUT)
870 
871 static const struct error_descr ocx_com_errors[] = {
872 	{
873 		.type  = ERR_CORRECTED,
874 		.mask  = OCX_COM_IO_BADID,
875 		.descr = "Invalid IO transaction node ID",
876 	},
877 	{
878 		.type  = ERR_CORRECTED,
879 		.mask  = OCX_COM_MEM_BADID,
880 		.descr = "Invalid memory transaction node ID",
881 	},
882 	{
883 		.type  = ERR_CORRECTED,
884 		.mask  = OCX_COM_COPR_BADID,
885 		.descr = "Invalid coprocessor transaction node ID",
886 	},
887 	{
888 		.type  = ERR_CORRECTED,
889 		.mask  = OCX_COM_WIN_REQ_BADID,
890 		.descr = "Invalid SLI transaction node ID",
891 	},
892 	{
893 		.type  = ERR_CORRECTED,
894 		.mask  = OCX_COM_WIN_REQ_TOUT,
895 		.descr = "Window/core request timeout",
896 	},
897 	{0, 0, NULL},
898 };
899 
900 #define OCX_COM_LINKX_INT(x)		(0x120 + (x) * 8)
901 #define OCX_COM_LINKX_INT_W1S(x)	(0x140 + (x) * 8)
902 #define OCX_COM_LINKX_INT_ENA_W1S(x)	(0x160 + (x) * 8)
903 #define OCX_COM_LINKX_INT_ENA_W1C(x)	(0x180 + (x) * 8)
904 
905 #define OCX_COM_LINK_BAD_WORD			BIT(13)
906 #define OCX_COM_LINK_ALIGN_FAIL			BIT(12)
907 #define OCX_COM_LINK_ALIGN_DONE			BIT(11)
908 #define OCX_COM_LINK_UP				BIT(10)
909 #define OCX_COM_LINK_STOP			BIT(9)
910 #define OCX_COM_LINK_BLK_ERR			BIT(8)
911 #define OCX_COM_LINK_REINIT			BIT(7)
912 #define OCX_COM_LINK_LNK_DATA			BIT(6)
913 #define OCX_COM_LINK_RXFIFO_DBE			BIT(5)
914 #define OCX_COM_LINK_RXFIFO_SBE			BIT(4)
915 #define OCX_COM_LINK_TXFIFO_DBE			BIT(3)
916 #define OCX_COM_LINK_TXFIFO_SBE			BIT(2)
917 #define OCX_COM_LINK_REPLAY_DBE			BIT(1)
918 #define OCX_COM_LINK_REPLAY_SBE			BIT(0)
919 
920 static const struct error_descr ocx_com_link_errors[] = {
921 	{
922 		.type  = ERR_CORRECTED,
923 		.mask  = OCX_COM_LINK_REPLAY_SBE,
924 		.descr = "Replay buffer single-bit error",
925 	},
926 	{
927 		.type  = ERR_CORRECTED,
928 		.mask  = OCX_COM_LINK_TXFIFO_SBE,
929 		.descr = "TX FIFO single-bit error",
930 	},
931 	{
932 		.type  = ERR_CORRECTED,
933 		.mask  = OCX_COM_LINK_RXFIFO_SBE,
934 		.descr = "RX FIFO single-bit error",
935 	},
936 	{
937 		.type  = ERR_CORRECTED,
938 		.mask  = OCX_COM_LINK_BLK_ERR,
939 		.descr = "Block code error",
940 	},
941 	{
942 		.type  = ERR_CORRECTED,
943 		.mask  = OCX_COM_LINK_ALIGN_FAIL,
944 		.descr = "Link alignment failure",
945 	},
946 	{
947 		.type  = ERR_CORRECTED,
948 		.mask  = OCX_COM_LINK_BAD_WORD,
949 		.descr = "Bad code word",
950 	},
951 	{
952 		.type  = ERR_UNCORRECTED,
953 		.mask  = OCX_COM_LINK_REPLAY_DBE,
954 		.descr = "Replay buffer double-bit error",
955 	},
956 	{
957 		.type  = ERR_UNCORRECTED,
958 		.mask  = OCX_COM_LINK_TXFIFO_DBE,
959 		.descr = "TX FIFO double-bit error",
960 	},
961 	{
962 		.type  = ERR_UNCORRECTED,
963 		.mask  = OCX_COM_LINK_RXFIFO_DBE,
964 		.descr = "RX FIFO double-bit error",
965 	},
966 	{
967 		.type  = ERR_UNCORRECTED,
968 		.mask  = OCX_COM_LINK_STOP,
969 		.descr = "Link stopped",
970 	},
971 	{0, 0, NULL},
972 };
973 
974 #define OCX_COM_LINK_INT_UE       (OCX_COM_LINK_REPLAY_DBE | \
975 				   OCX_COM_LINK_TXFIFO_DBE | \
976 				   OCX_COM_LINK_RXFIFO_DBE | \
977 				   OCX_COM_LINK_STOP)
978 
979 #define OCX_COM_LINK_INT_CE       (OCX_COM_LINK_REPLAY_SBE | \
980 				   OCX_COM_LINK_TXFIFO_SBE | \
981 				   OCX_COM_LINK_RXFIFO_SBE | \
982 				   OCX_COM_LINK_BLK_ERR    | \
983 				   OCX_COM_LINK_ALIGN_FAIL | \
984 				   OCX_COM_LINK_BAD_WORD)
985 
986 #define OCX_LNE_INT(x)			(0x8018 + (x) * 0x100)
987 #define OCX_LNE_INT_EN(x)		(0x8020 + (x) * 0x100)
988 #define OCX_LNE_BAD_CNT(x)		(0x8028 + (x) * 0x100)
989 #define OCX_LNE_CFG(x)			(0x8000 + (x) * 0x100)
990 #define OCX_LNE_STAT(x, y)		(0x8040 + (x) * 0x100 + (y) * 8)
991 
992 #define OCX_LNE_CFG_RX_BDRY_LOCK_DIS		BIT(8)
993 #define OCX_LNE_CFG_RX_STAT_WRAP_DIS		BIT(2)
994 #define OCX_LNE_CFG_RX_STAT_RDCLR		BIT(1)
995 #define OCX_LNE_CFG_RX_STAT_ENA			BIT(0)
996 
997 
998 #define OCX_LANE_BAD_64B67B			BIT(8)
999 #define OCX_LANE_DSKEW_FIFO_OVFL		BIT(5)
1000 #define OCX_LANE_SCRM_SYNC_LOSS			BIT(4)
1001 #define OCX_LANE_UKWN_CNTL_WORD			BIT(3)
1002 #define OCX_LANE_CRC32_ERR			BIT(2)
1003 #define OCX_LANE_BDRY_SYNC_LOSS			BIT(1)
1004 #define OCX_LANE_SERDES_LOCK_LOSS		BIT(0)
1005 
1006 #define OCX_COM_LANE_INT_UE       (0)
1007 #define OCX_COM_LANE_INT_CE       (OCX_LANE_SERDES_LOCK_LOSS | \
1008 				   OCX_LANE_BDRY_SYNC_LOSS   | \
1009 				   OCX_LANE_CRC32_ERR        | \
1010 				   OCX_LANE_UKWN_CNTL_WORD   | \
1011 				   OCX_LANE_SCRM_SYNC_LOSS   | \
1012 				   OCX_LANE_DSKEW_FIFO_OVFL  | \
1013 				   OCX_LANE_BAD_64B67B)
1014 
1015 static const struct error_descr ocx_lane_errors[] = {
1016 	{
1017 		.type  = ERR_CORRECTED,
1018 		.mask  = OCX_LANE_SERDES_LOCK_LOSS,
1019 		.descr = "RX SerDes lock lost",
1020 	},
1021 	{
1022 		.type  = ERR_CORRECTED,
1023 		.mask  = OCX_LANE_BDRY_SYNC_LOSS,
1024 		.descr = "RX word boundary lost",
1025 	},
1026 	{
1027 		.type  = ERR_CORRECTED,
1028 		.mask  = OCX_LANE_CRC32_ERR,
1029 		.descr = "CRC32 error",
1030 	},
1031 	{
1032 		.type  = ERR_CORRECTED,
1033 		.mask  = OCX_LANE_UKWN_CNTL_WORD,
1034 		.descr = "Unknown control word",
1035 	},
1036 	{
1037 		.type  = ERR_CORRECTED,
1038 		.mask  = OCX_LANE_SCRM_SYNC_LOSS,
1039 		.descr = "Scrambler synchronization lost",
1040 	},
1041 	{
1042 		.type  = ERR_CORRECTED,
1043 		.mask  = OCX_LANE_DSKEW_FIFO_OVFL,
1044 		.descr = "RX deskew FIFO overflow",
1045 	},
1046 	{
1047 		.type  = ERR_CORRECTED,
1048 		.mask  = OCX_LANE_BAD_64B67B,
1049 		.descr = "Bad 64B/67B codeword",
1050 	},
1051 	{0, 0, NULL},
1052 };
1053 
1054 #define OCX_LNE_INT_ENA_ALL		(GENMASK(9, 8) | GENMASK(6, 0))
1055 #define OCX_COM_INT_ENA_ALL		(GENMASK(54, 50) | GENMASK(23, 0))
1056 #define OCX_COM_LINKX_INT_ENA_ALL	(GENMASK(13, 12) | \
1057 					 GENMASK(9, 7) | GENMASK(5, 0))
1058 
1059 #define OCX_TLKX_ECC_CTL(x)		(0x10018 + (x) * 0x2000)
1060 #define OCX_RLKX_ECC_CTL(x)		(0x18018 + (x) * 0x2000)
1061 
1062 struct ocx_com_err_ctx {
1063 	u64 reg_com_int;
1064 	u64 reg_lane_int[OCX_RX_LANES];
1065 	u64 reg_lane_stat11[OCX_RX_LANES];
1066 };
1067 
1068 struct ocx_link_err_ctx {
1069 	u64 reg_com_link_int;
1070 	int link;
1071 };
1072 
1073 struct thunderx_ocx {
1074 	void __iomem *regs;
1075 	int com_link;
1076 	struct pci_dev *pdev;
1077 	struct edac_device_ctl_info *edac_dev;
1078 
1079 	struct dentry *debugfs;
1080 	struct msix_entry msix_ent[OCX_INTS];
1081 
1082 	struct ocx_com_err_ctx com_err_ctx[RING_ENTRIES];
1083 	struct ocx_link_err_ctx link_err_ctx[RING_ENTRIES];
1084 
1085 	unsigned long com_ring_head;
1086 	unsigned long com_ring_tail;
1087 
1088 	unsigned long link_ring_head;
1089 	unsigned long link_ring_tail;
1090 };
1091 
1092 #define OCX_MESSAGE_SIZE	SZ_1K
1093 #define OCX_OTHER_SIZE		(50 * ARRAY_SIZE(ocx_com_link_errors))
1094 
1095 /* This handler is threaded */
1096 static irqreturn_t thunderx_ocx_com_isr(int irq, void *irq_id)
1097 {
1098 	struct msix_entry *msix = irq_id;
1099 	struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1100 						msix_ent[msix->entry]);
1101 
1102 	int lane;
1103 	unsigned long head = ring_pos(ocx->com_ring_head,
1104 				      ARRAY_SIZE(ocx->com_err_ctx));
1105 	struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head];
1106 
1107 	ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT);
1108 
1109 	for (lane = 0; lane < OCX_RX_LANES; lane++) {
1110 		ctx->reg_lane_int[lane] =
1111 			readq(ocx->regs + OCX_LNE_INT(lane));
1112 		ctx->reg_lane_stat11[lane] =
1113 			readq(ocx->regs + OCX_LNE_STAT(lane, 11));
1114 
1115 		writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane));
1116 	}
1117 
1118 	writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT);
1119 
1120 	ocx->com_ring_head++;
1121 
1122 	return IRQ_WAKE_THREAD;
1123 }
1124 
1125 static irqreturn_t thunderx_ocx_com_threaded_isr(int irq, void *irq_id)
1126 {
1127 	struct msix_entry *msix = irq_id;
1128 	struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1129 						msix_ent[msix->entry]);
1130 
1131 	irqreturn_t ret = IRQ_NONE;
1132 
1133 	unsigned long tail;
1134 	struct ocx_com_err_ctx *ctx;
1135 	int lane;
1136 	char *msg;
1137 	char *other;
1138 
1139 	msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1140 	other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1141 
1142 	if (!msg || !other)
1143 		goto err_free;
1144 
1145 	while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail,
1146 			ARRAY_SIZE(ocx->com_err_ctx))) {
1147 		tail = ring_pos(ocx->com_ring_tail,
1148 				ARRAY_SIZE(ocx->com_err_ctx));
1149 		ctx = &ocx->com_err_ctx[tail];
1150 
1151 		snprintf(msg, OCX_MESSAGE_SIZE, "%s: OCX_COM_INT: %016llx",
1152 			ocx->edac_dev->ctl_name, ctx->reg_com_int);
1153 
1154 		decode_register(other, OCX_OTHER_SIZE,
1155 				ocx_com_errors, ctx->reg_com_int);
1156 
1157 		strncat(msg, other, OCX_MESSAGE_SIZE);
1158 
1159 		for (lane = 0; lane < OCX_RX_LANES; lane++)
1160 			if (ctx->reg_com_int & BIT(lane)) {
1161 				snprintf(other, OCX_OTHER_SIZE,
1162 					 "\n\tOCX_LNE_INT[%02d]: %016llx OCX_LNE_STAT11[%02d]: %016llx",
1163 					 lane, ctx->reg_lane_int[lane],
1164 					 lane, ctx->reg_lane_stat11[lane]);
1165 
1166 				strncat(msg, other, OCX_MESSAGE_SIZE);
1167 
1168 				decode_register(other, OCX_OTHER_SIZE,
1169 						ocx_lane_errors,
1170 						ctx->reg_lane_int[lane]);
1171 				strncat(msg, other, OCX_MESSAGE_SIZE);
1172 			}
1173 
1174 		if (ctx->reg_com_int & OCX_COM_INT_CE)
1175 			edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1176 
1177 		ocx->com_ring_tail++;
1178 	}
1179 
1180 	ret = IRQ_HANDLED;
1181 
1182 err_free:
1183 	kfree(other);
1184 	kfree(msg);
1185 
1186 	return ret;
1187 }
1188 
1189 static irqreturn_t thunderx_ocx_lnk_isr(int irq, void *irq_id)
1190 {
1191 	struct msix_entry *msix = irq_id;
1192 	struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1193 						msix_ent[msix->entry]);
1194 	unsigned long head = ring_pos(ocx->link_ring_head,
1195 				      ARRAY_SIZE(ocx->link_err_ctx));
1196 	struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head];
1197 
1198 	ctx->link = msix->entry;
1199 	ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1200 
1201 	writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1202 
1203 	ocx->link_ring_head++;
1204 
1205 	return IRQ_WAKE_THREAD;
1206 }
1207 
1208 static irqreturn_t thunderx_ocx_lnk_threaded_isr(int irq, void *irq_id)
1209 {
1210 	struct msix_entry *msix = irq_id;
1211 	struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1212 						msix_ent[msix->entry]);
1213 	irqreturn_t ret = IRQ_NONE;
1214 	unsigned long tail;
1215 	struct ocx_link_err_ctx *ctx;
1216 
1217 	char *msg;
1218 	char *other;
1219 
1220 	msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1221 	other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1222 
1223 	if (!msg || !other)
1224 		goto err_free;
1225 
1226 	while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail,
1227 			ARRAY_SIZE(ocx->link_err_ctx))) {
1228 		tail = ring_pos(ocx->link_ring_head,
1229 				ARRAY_SIZE(ocx->link_err_ctx));
1230 
1231 		ctx = &ocx->link_err_ctx[tail];
1232 
1233 		snprintf(msg, OCX_MESSAGE_SIZE,
1234 			 "%s: OCX_COM_LINK_INT[%d]: %016llx",
1235 			 ocx->edac_dev->ctl_name,
1236 			 ctx->link, ctx->reg_com_link_int);
1237 
1238 		decode_register(other, OCX_OTHER_SIZE,
1239 				ocx_com_link_errors, ctx->reg_com_link_int);
1240 
1241 		strncat(msg, other, OCX_MESSAGE_SIZE);
1242 
1243 		if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE)
1244 			edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);
1245 		else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE)
1246 			edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1247 
1248 		ocx->link_ring_tail++;
1249 	}
1250 
1251 	ret = IRQ_HANDLED;
1252 err_free:
1253 	kfree(other);
1254 	kfree(msg);
1255 
1256 	return ret;
1257 }
1258 
1259 #define OCX_DEBUGFS_ATTR(_name, _reg)	DEBUGFS_REG_ATTR(ocx, _name, _reg)
1260 
1261 OCX_DEBUGFS_ATTR(tlk0_ecc_ctl, OCX_TLKX_ECC_CTL(0));
1262 OCX_DEBUGFS_ATTR(tlk1_ecc_ctl, OCX_TLKX_ECC_CTL(1));
1263 OCX_DEBUGFS_ATTR(tlk2_ecc_ctl, OCX_TLKX_ECC_CTL(2));
1264 
1265 OCX_DEBUGFS_ATTR(rlk0_ecc_ctl, OCX_RLKX_ECC_CTL(0));
1266 OCX_DEBUGFS_ATTR(rlk1_ecc_ctl, OCX_RLKX_ECC_CTL(1));
1267 OCX_DEBUGFS_ATTR(rlk2_ecc_ctl, OCX_RLKX_ECC_CTL(2));
1268 
1269 OCX_DEBUGFS_ATTR(com_link0_int, OCX_COM_LINKX_INT_W1S(0));
1270 OCX_DEBUGFS_ATTR(com_link1_int, OCX_COM_LINKX_INT_W1S(1));
1271 OCX_DEBUGFS_ATTR(com_link2_int, OCX_COM_LINKX_INT_W1S(2));
1272 
1273 OCX_DEBUGFS_ATTR(lne00_badcnt, OCX_LNE_BAD_CNT(0));
1274 OCX_DEBUGFS_ATTR(lne01_badcnt, OCX_LNE_BAD_CNT(1));
1275 OCX_DEBUGFS_ATTR(lne02_badcnt, OCX_LNE_BAD_CNT(2));
1276 OCX_DEBUGFS_ATTR(lne03_badcnt, OCX_LNE_BAD_CNT(3));
1277 OCX_DEBUGFS_ATTR(lne04_badcnt, OCX_LNE_BAD_CNT(4));
1278 OCX_DEBUGFS_ATTR(lne05_badcnt, OCX_LNE_BAD_CNT(5));
1279 OCX_DEBUGFS_ATTR(lne06_badcnt, OCX_LNE_BAD_CNT(6));
1280 OCX_DEBUGFS_ATTR(lne07_badcnt, OCX_LNE_BAD_CNT(7));
1281 
1282 OCX_DEBUGFS_ATTR(lne08_badcnt, OCX_LNE_BAD_CNT(8));
1283 OCX_DEBUGFS_ATTR(lne09_badcnt, OCX_LNE_BAD_CNT(9));
1284 OCX_DEBUGFS_ATTR(lne10_badcnt, OCX_LNE_BAD_CNT(10));
1285 OCX_DEBUGFS_ATTR(lne11_badcnt, OCX_LNE_BAD_CNT(11));
1286 OCX_DEBUGFS_ATTR(lne12_badcnt, OCX_LNE_BAD_CNT(12));
1287 OCX_DEBUGFS_ATTR(lne13_badcnt, OCX_LNE_BAD_CNT(13));
1288 OCX_DEBUGFS_ATTR(lne14_badcnt, OCX_LNE_BAD_CNT(14));
1289 OCX_DEBUGFS_ATTR(lne15_badcnt, OCX_LNE_BAD_CNT(15));
1290 
1291 OCX_DEBUGFS_ATTR(lne16_badcnt, OCX_LNE_BAD_CNT(16));
1292 OCX_DEBUGFS_ATTR(lne17_badcnt, OCX_LNE_BAD_CNT(17));
1293 OCX_DEBUGFS_ATTR(lne18_badcnt, OCX_LNE_BAD_CNT(18));
1294 OCX_DEBUGFS_ATTR(lne19_badcnt, OCX_LNE_BAD_CNT(19));
1295 OCX_DEBUGFS_ATTR(lne20_badcnt, OCX_LNE_BAD_CNT(20));
1296 OCX_DEBUGFS_ATTR(lne21_badcnt, OCX_LNE_BAD_CNT(21));
1297 OCX_DEBUGFS_ATTR(lne22_badcnt, OCX_LNE_BAD_CNT(22));
1298 OCX_DEBUGFS_ATTR(lne23_badcnt, OCX_LNE_BAD_CNT(23));
1299 
1300 OCX_DEBUGFS_ATTR(com_int, OCX_COM_INT_W1S);
1301 
1302 struct debugfs_entry *ocx_dfs_ents[] = {
1303 	&debugfs_tlk0_ecc_ctl,
1304 	&debugfs_tlk1_ecc_ctl,
1305 	&debugfs_tlk2_ecc_ctl,
1306 
1307 	&debugfs_rlk0_ecc_ctl,
1308 	&debugfs_rlk1_ecc_ctl,
1309 	&debugfs_rlk2_ecc_ctl,
1310 
1311 	&debugfs_com_link0_int,
1312 	&debugfs_com_link1_int,
1313 	&debugfs_com_link2_int,
1314 
1315 	&debugfs_lne00_badcnt,
1316 	&debugfs_lne01_badcnt,
1317 	&debugfs_lne02_badcnt,
1318 	&debugfs_lne03_badcnt,
1319 	&debugfs_lne04_badcnt,
1320 	&debugfs_lne05_badcnt,
1321 	&debugfs_lne06_badcnt,
1322 	&debugfs_lne07_badcnt,
1323 	&debugfs_lne08_badcnt,
1324 	&debugfs_lne09_badcnt,
1325 	&debugfs_lne10_badcnt,
1326 	&debugfs_lne11_badcnt,
1327 	&debugfs_lne12_badcnt,
1328 	&debugfs_lne13_badcnt,
1329 	&debugfs_lne14_badcnt,
1330 	&debugfs_lne15_badcnt,
1331 	&debugfs_lne16_badcnt,
1332 	&debugfs_lne17_badcnt,
1333 	&debugfs_lne18_badcnt,
1334 	&debugfs_lne19_badcnt,
1335 	&debugfs_lne20_badcnt,
1336 	&debugfs_lne21_badcnt,
1337 	&debugfs_lne22_badcnt,
1338 	&debugfs_lne23_badcnt,
1339 
1340 	&debugfs_com_int,
1341 };
1342 
1343 static const struct pci_device_id thunderx_ocx_pci_tbl[] = {
1344 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_OCX) },
1345 	{ 0, },
1346 };
1347 
1348 static void thunderx_ocx_clearstats(struct thunderx_ocx *ocx)
1349 {
1350 	int lane, stat, cfg;
1351 
1352 	for (lane = 0; lane < OCX_RX_LANES; lane++) {
1353 		cfg = readq(ocx->regs + OCX_LNE_CFG(lane));
1354 		cfg |= OCX_LNE_CFG_RX_STAT_RDCLR;
1355 		cfg &= ~OCX_LNE_CFG_RX_STAT_ENA;
1356 		writeq(cfg, ocx->regs + OCX_LNE_CFG(lane));
1357 
1358 		for (stat = 0; stat < OCX_RX_LANE_STATS; stat++)
1359 			readq(ocx->regs + OCX_LNE_STAT(lane, stat));
1360 	}
1361 }
1362 
1363 static int thunderx_ocx_probe(struct pci_dev *pdev,
1364 			      const struct pci_device_id *id)
1365 {
1366 	struct thunderx_ocx *ocx;
1367 	struct edac_device_ctl_info *edac_dev;
1368 	char name[32];
1369 	int idx;
1370 	int i;
1371 	int ret;
1372 	u64 reg;
1373 
1374 	ret = pcim_enable_device(pdev);
1375 	if (ret) {
1376 		dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1377 		return ret;
1378 	}
1379 
1380 	ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_ocx");
1381 	if (ret) {
1382 		dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1383 		return ret;
1384 	}
1385 
1386 	idx = edac_device_alloc_index();
1387 	snprintf(name, sizeof(name), "OCX%d", idx);
1388 	edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx),
1389 					      name, 1, "CCPI", 1,
1390 					      0, NULL, 0, idx);
1391 	if (!edac_dev) {
1392 		dev_err(&pdev->dev, "Cannot allocate EDAC device: %d\n", ret);
1393 		return -ENOMEM;
1394 	}
1395 	ocx = edac_dev->pvt_info;
1396 	ocx->edac_dev = edac_dev;
1397 	ocx->com_ring_head = 0;
1398 	ocx->com_ring_tail = 0;
1399 	ocx->link_ring_head = 0;
1400 	ocx->link_ring_tail = 0;
1401 
1402 	ocx->regs = pcim_iomap_table(pdev)[0];
1403 	if (!ocx->regs) {
1404 		dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1405 		ret = -ENODEV;
1406 		goto err_free;
1407 	}
1408 
1409 	ocx->pdev = pdev;
1410 
1411 	for (i = 0; i < OCX_INTS; i++) {
1412 		ocx->msix_ent[i].entry = i;
1413 		ocx->msix_ent[i].vector = 0;
1414 	}
1415 
1416 	ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS);
1417 	if (ret) {
1418 		dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
1419 		goto err_free;
1420 	}
1421 
1422 	for (i = 0; i < OCX_INTS; i++) {
1423 		ret = devm_request_threaded_irq(&pdev->dev,
1424 						ocx->msix_ent[i].vector,
1425 						(i == 3) ?
1426 						 thunderx_ocx_com_isr :
1427 						 thunderx_ocx_lnk_isr,
1428 						(i == 3) ?
1429 						 thunderx_ocx_com_threaded_isr :
1430 						 thunderx_ocx_lnk_threaded_isr,
1431 						0, "[EDAC] ThunderX OCX",
1432 						&ocx->msix_ent[i]);
1433 		if (ret)
1434 			goto err_free;
1435 	}
1436 
1437 	edac_dev->dev = &pdev->dev;
1438 	edac_dev->dev_name = dev_name(&pdev->dev);
1439 	edac_dev->mod_name = "thunderx-ocx";
1440 	edac_dev->ctl_name = "thunderx-ocx";
1441 
1442 	ret = edac_device_add_device(edac_dev);
1443 	if (ret) {
1444 		dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
1445 		goto err_free;
1446 	}
1447 
1448 	if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
1449 		ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
1450 
1451 		ret = thunderx_create_debugfs_nodes(ocx->debugfs,
1452 						    ocx_dfs_ents,
1453 						    ocx,
1454 						    ARRAY_SIZE(ocx_dfs_ents));
1455 		if (ret != ARRAY_SIZE(ocx_dfs_ents)) {
1456 			dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
1457 				 ret, ret >= 0 ? " created" : "");
1458 		}
1459 	}
1460 
1461 	pci_set_drvdata(pdev, edac_dev);
1462 
1463 	thunderx_ocx_clearstats(ocx);
1464 
1465 	for (i = 0; i < OCX_RX_LANES; i++) {
1466 		writeq(OCX_LNE_INT_ENA_ALL,
1467 		       ocx->regs + OCX_LNE_INT_EN(i));
1468 
1469 		reg = readq(ocx->regs + OCX_LNE_INT(i));
1470 		writeq(reg, ocx->regs + OCX_LNE_INT(i));
1471 
1472 	}
1473 
1474 	for (i = 0; i < OCX_LINK_INTS; i++) {
1475 		reg = readq(ocx->regs + OCX_COM_LINKX_INT(i));
1476 		writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i));
1477 
1478 		writeq(OCX_COM_LINKX_INT_ENA_ALL,
1479 		       ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i));
1480 	}
1481 
1482 	reg = readq(ocx->regs + OCX_COM_INT);
1483 	writeq(reg, ocx->regs + OCX_COM_INT);
1484 
1485 	writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S);
1486 
1487 	return 0;
1488 err_free:
1489 	edac_device_free_ctl_info(edac_dev);
1490 
1491 	return ret;
1492 }
1493 
1494 static void thunderx_ocx_remove(struct pci_dev *pdev)
1495 {
1496 	struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
1497 	struct thunderx_ocx *ocx = edac_dev->pvt_info;
1498 	int i;
1499 
1500 	writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C);
1501 
1502 	for (i = 0; i < OCX_INTS; i++) {
1503 		writeq(OCX_COM_LINKX_INT_ENA_ALL,
1504 		       ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i));
1505 	}
1506 
1507 	edac_debugfs_remove_recursive(ocx->debugfs);
1508 
1509 	edac_device_del_device(&pdev->dev);
1510 	edac_device_free_ctl_info(edac_dev);
1511 }
1512 
1513 MODULE_DEVICE_TABLE(pci, thunderx_ocx_pci_tbl);
1514 
1515 static struct pci_driver thunderx_ocx_driver = {
1516 	.name     = "thunderx_ocx_edac",
1517 	.probe    = thunderx_ocx_probe,
1518 	.remove   = thunderx_ocx_remove,
1519 	.id_table = thunderx_ocx_pci_tbl,
1520 };
1521 
1522 /*---------------------- L2C driver ---------------------------------*/
1523 
1524 #define PCI_DEVICE_ID_THUNDER_L2C_TAD 0xa02e
1525 #define PCI_DEVICE_ID_THUNDER_L2C_CBC 0xa02f
1526 #define PCI_DEVICE_ID_THUNDER_L2C_MCI 0xa030
1527 
1528 #define L2C_TAD_INT_W1C		0x40000
1529 #define L2C_TAD_INT_W1S		0x40008
1530 
1531 #define L2C_TAD_INT_ENA_W1C	0x40020
1532 #define L2C_TAD_INT_ENA_W1S	0x40028
1533 
1534 
1535 #define L2C_TAD_INT_L2DDBE	 BIT(1)
1536 #define L2C_TAD_INT_SBFSBE	 BIT(2)
1537 #define L2C_TAD_INT_SBFDBE	 BIT(3)
1538 #define L2C_TAD_INT_FBFSBE	 BIT(4)
1539 #define L2C_TAD_INT_FBFDBE	 BIT(5)
1540 #define L2C_TAD_INT_TAGDBE	 BIT(9)
1541 #define L2C_TAD_INT_RDDISLMC	 BIT(15)
1542 #define L2C_TAD_INT_WRDISLMC	 BIT(16)
1543 #define L2C_TAD_INT_LFBTO	 BIT(17)
1544 #define L2C_TAD_INT_GSYNCTO	 BIT(18)
1545 #define L2C_TAD_INT_RTGSBE	 BIT(32)
1546 #define L2C_TAD_INT_RTGDBE	 BIT(33)
1547 #define L2C_TAD_INT_RDDISOCI	 BIT(34)
1548 #define L2C_TAD_INT_WRDISOCI	 BIT(35)
1549 
1550 #define L2C_TAD_INT_ECC		(L2C_TAD_INT_L2DDBE | \
1551 				 L2C_TAD_INT_SBFSBE | L2C_TAD_INT_SBFDBE | \
1552 				 L2C_TAD_INT_FBFSBE | L2C_TAD_INT_FBFDBE)
1553 
1554 #define L2C_TAD_INT_CE          (L2C_TAD_INT_SBFSBE | \
1555 				 L2C_TAD_INT_FBFSBE)
1556 
1557 #define L2C_TAD_INT_UE          (L2C_TAD_INT_L2DDBE | \
1558 				 L2C_TAD_INT_SBFDBE | \
1559 				 L2C_TAD_INT_FBFDBE | \
1560 				 L2C_TAD_INT_TAGDBE | \
1561 				 L2C_TAD_INT_RTGDBE | \
1562 				 L2C_TAD_INT_WRDISOCI | \
1563 				 L2C_TAD_INT_RDDISOCI | \
1564 				 L2C_TAD_INT_WRDISLMC | \
1565 				 L2C_TAD_INT_RDDISLMC | \
1566 				 L2C_TAD_INT_LFBTO    | \
1567 				 L2C_TAD_INT_GSYNCTO)
1568 
1569 static const struct error_descr l2_tad_errors[] = {
1570 	{
1571 		.type  = ERR_CORRECTED,
1572 		.mask  = L2C_TAD_INT_SBFSBE,
1573 		.descr = "SBF single-bit error",
1574 	},
1575 	{
1576 		.type  = ERR_CORRECTED,
1577 		.mask  = L2C_TAD_INT_FBFSBE,
1578 		.descr = "FBF single-bit error",
1579 	},
1580 	{
1581 		.type  = ERR_UNCORRECTED,
1582 		.mask  = L2C_TAD_INT_L2DDBE,
1583 		.descr = "L2D double-bit error",
1584 	},
1585 	{
1586 		.type  = ERR_UNCORRECTED,
1587 		.mask  = L2C_TAD_INT_SBFDBE,
1588 		.descr = "SBF double-bit error",
1589 	},
1590 	{
1591 		.type  = ERR_UNCORRECTED,
1592 		.mask  = L2C_TAD_INT_FBFDBE,
1593 		.descr = "FBF double-bit error",
1594 	},
1595 	{
1596 		.type  = ERR_UNCORRECTED,
1597 		.mask  = L2C_TAD_INT_TAGDBE,
1598 		.descr = "TAG double-bit error",
1599 	},
1600 	{
1601 		.type  = ERR_UNCORRECTED,
1602 		.mask  = L2C_TAD_INT_RTGDBE,
1603 		.descr = "RTG double-bit error",
1604 	},
1605 	{
1606 		.type  = ERR_UNCORRECTED,
1607 		.mask  = L2C_TAD_INT_WRDISOCI,
1608 		.descr = "Write to a disabled CCPI",
1609 	},
1610 	{
1611 		.type  = ERR_UNCORRECTED,
1612 		.mask  = L2C_TAD_INT_RDDISOCI,
1613 		.descr = "Read from a disabled CCPI",
1614 	},
1615 	{
1616 		.type  = ERR_UNCORRECTED,
1617 		.mask  = L2C_TAD_INT_WRDISLMC,
1618 		.descr = "Write to a disabled LMC",
1619 	},
1620 	{
1621 		.type  = ERR_UNCORRECTED,
1622 		.mask  = L2C_TAD_INT_RDDISLMC,
1623 		.descr = "Read from a disabled LMC",
1624 	},
1625 	{
1626 		.type  = ERR_UNCORRECTED,
1627 		.mask  = L2C_TAD_INT_LFBTO,
1628 		.descr = "LFB entry timeout",
1629 	},
1630 	{
1631 		.type  = ERR_UNCORRECTED,
1632 		.mask  = L2C_TAD_INT_GSYNCTO,
1633 		.descr = "Global sync CCPI timeout",
1634 	},
1635 	{0, 0, NULL},
1636 };
1637 
1638 #define L2C_TAD_INT_TAG		(L2C_TAD_INT_TAGDBE)
1639 
1640 #define L2C_TAD_INT_RTG		(L2C_TAD_INT_RTGDBE)
1641 
1642 #define L2C_TAD_INT_DISLMC	(L2C_TAD_INT_WRDISLMC | L2C_TAD_INT_RDDISLMC)
1643 
1644 #define L2C_TAD_INT_DISOCI	(L2C_TAD_INT_WRDISOCI | L2C_TAD_INT_RDDISOCI)
1645 
1646 #define L2C_TAD_INT_ENA_ALL	(L2C_TAD_INT_ECC | L2C_TAD_INT_TAG | \
1647 				 L2C_TAD_INT_RTG | \
1648 				 L2C_TAD_INT_DISLMC | L2C_TAD_INT_DISOCI | \
1649 				 L2C_TAD_INT_LFBTO)
1650 
1651 #define L2C_TAD_TIMETWO		0x50000
1652 #define L2C_TAD_TIMEOUT		0x50100
1653 #define L2C_TAD_ERR		0x60000
1654 #define L2C_TAD_TQD_ERR		0x60100
1655 #define L2C_TAD_TTG_ERR		0x60200
1656 
1657 
1658 #define L2C_CBC_INT_W1C		0x60000
1659 
1660 #define L2C_CBC_INT_RSDSBE	 BIT(0)
1661 #define L2C_CBC_INT_RSDDBE	 BIT(1)
1662 
1663 #define L2C_CBC_INT_RSD		 (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_RSDDBE)
1664 
1665 #define L2C_CBC_INT_MIBSBE	 BIT(4)
1666 #define L2C_CBC_INT_MIBDBE	 BIT(5)
1667 
1668 #define L2C_CBC_INT_MIB		 (L2C_CBC_INT_MIBSBE | L2C_CBC_INT_MIBDBE)
1669 
1670 #define L2C_CBC_INT_IORDDISOCI	 BIT(6)
1671 #define L2C_CBC_INT_IOWRDISOCI	 BIT(7)
1672 
1673 #define L2C_CBC_INT_IODISOCI	 (L2C_CBC_INT_IORDDISOCI | \
1674 				  L2C_CBC_INT_IOWRDISOCI)
1675 
1676 #define L2C_CBC_INT_CE		 (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_MIBSBE)
1677 #define L2C_CBC_INT_UE		 (L2C_CBC_INT_RSDDBE | L2C_CBC_INT_MIBDBE)
1678 
1679 
1680 static const struct error_descr l2_cbc_errors[] = {
1681 	{
1682 		.type  = ERR_CORRECTED,
1683 		.mask  = L2C_CBC_INT_RSDSBE,
1684 		.descr = "RSD single-bit error",
1685 	},
1686 	{
1687 		.type  = ERR_CORRECTED,
1688 		.mask  = L2C_CBC_INT_MIBSBE,
1689 		.descr = "MIB single-bit error",
1690 	},
1691 	{
1692 		.type  = ERR_UNCORRECTED,
1693 		.mask  = L2C_CBC_INT_RSDDBE,
1694 		.descr = "RSD double-bit error",
1695 	},
1696 	{
1697 		.type  = ERR_UNCORRECTED,
1698 		.mask  = L2C_CBC_INT_MIBDBE,
1699 		.descr = "MIB double-bit error",
1700 	},
1701 	{
1702 		.type  = ERR_UNCORRECTED,
1703 		.mask  = L2C_CBC_INT_IORDDISOCI,
1704 		.descr = "Read from a disabled CCPI",
1705 	},
1706 	{
1707 		.type  = ERR_UNCORRECTED,
1708 		.mask  = L2C_CBC_INT_IOWRDISOCI,
1709 		.descr = "Write to a disabled CCPI",
1710 	},
1711 	{0, 0, NULL},
1712 };
1713 
1714 #define L2C_CBC_INT_W1S		0x60008
1715 #define L2C_CBC_INT_ENA_W1C	0x60020
1716 
1717 #define L2C_CBC_INT_ENA_ALL	 (L2C_CBC_INT_RSD | L2C_CBC_INT_MIB | \
1718 				  L2C_CBC_INT_IODISOCI)
1719 
1720 #define L2C_CBC_INT_ENA_W1S	0x60028
1721 
1722 #define L2C_CBC_IODISOCIERR	0x80008
1723 #define L2C_CBC_IOCERR		0x80010
1724 #define L2C_CBC_RSDERR		0x80018
1725 #define L2C_CBC_MIBERR		0x80020
1726 
1727 
1728 #define L2C_MCI_INT_W1C		0x0
1729 
1730 #define L2C_MCI_INT_VBFSBE	 BIT(0)
1731 #define L2C_MCI_INT_VBFDBE	 BIT(1)
1732 
1733 static const struct error_descr l2_mci_errors[] = {
1734 	{
1735 		.type  = ERR_CORRECTED,
1736 		.mask  = L2C_MCI_INT_VBFSBE,
1737 		.descr = "VBF single-bit error",
1738 	},
1739 	{
1740 		.type  = ERR_UNCORRECTED,
1741 		.mask  = L2C_MCI_INT_VBFDBE,
1742 		.descr = "VBF double-bit error",
1743 	},
1744 	{0, 0, NULL},
1745 };
1746 
1747 #define L2C_MCI_INT_W1S		0x8
1748 #define L2C_MCI_INT_ENA_W1C	0x20
1749 
1750 #define L2C_MCI_INT_ENA_ALL	 (L2C_MCI_INT_VBFSBE | L2C_MCI_INT_VBFDBE)
1751 
1752 #define L2C_MCI_INT_ENA_W1S	0x28
1753 
1754 #define L2C_MCI_ERR		0x10000
1755 
1756 #define L2C_MESSAGE_SIZE	SZ_1K
1757 #define L2C_OTHER_SIZE		(50 * ARRAY_SIZE(l2_tad_errors))
1758 
1759 struct l2c_err_ctx {
1760 	char *reg_ext_name;
1761 	u64  reg_int;
1762 	u64  reg_ext;
1763 };
1764 
1765 struct thunderx_l2c {
1766 	void __iomem *regs;
1767 	struct pci_dev *pdev;
1768 	struct edac_device_ctl_info *edac_dev;
1769 
1770 	struct dentry *debugfs;
1771 
1772 	int index;
1773 
1774 	struct msix_entry msix_ent;
1775 
1776 	struct l2c_err_ctx err_ctx[RING_ENTRIES];
1777 	unsigned long ring_head;
1778 	unsigned long ring_tail;
1779 };
1780 
1781 static irqreturn_t thunderx_l2c_tad_isr(int irq, void *irq_id)
1782 {
1783 	struct msix_entry *msix = irq_id;
1784 	struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c,
1785 						msix_ent);
1786 
1787 	unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx));
1788 	struct l2c_err_ctx *ctx = &tad->err_ctx[head];
1789 
1790 	ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C);
1791 
1792 	if (ctx->reg_int & L2C_TAD_INT_ECC) {
1793 		ctx->reg_ext_name = "TQD_ERR";
1794 		ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR);
1795 	} else if (ctx->reg_int & L2C_TAD_INT_TAG) {
1796 		ctx->reg_ext_name = "TTG_ERR";
1797 		ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR);
1798 	} else if (ctx->reg_int & L2C_TAD_INT_LFBTO) {
1799 		ctx->reg_ext_name = "TIMEOUT";
1800 		ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT);
1801 	} else if (ctx->reg_int & L2C_TAD_INT_DISOCI) {
1802 		ctx->reg_ext_name = "ERR";
1803 		ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);
1804 	}
1805 
1806 	writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C);
1807 
1808 	tad->ring_head++;
1809 
1810 	return IRQ_WAKE_THREAD;
1811 }
1812 
1813 static irqreturn_t thunderx_l2c_cbc_isr(int irq, void *irq_id)
1814 {
1815 	struct msix_entry *msix = irq_id;
1816 	struct thunderx_l2c *cbc = container_of(msix, struct thunderx_l2c,
1817 						msix_ent);
1818 
1819 	unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx));
1820 	struct l2c_err_ctx *ctx = &cbc->err_ctx[head];
1821 
1822 	ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C);
1823 
1824 	if (ctx->reg_int & L2C_CBC_INT_RSD) {
1825 		ctx->reg_ext_name = "RSDERR";
1826 		ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR);
1827 	} else if (ctx->reg_int & L2C_CBC_INT_MIB) {
1828 		ctx->reg_ext_name = "MIBERR";
1829 		ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR);
1830 	} else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) {
1831 		ctx->reg_ext_name = "IODISOCIERR";
1832 		ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR);
1833 	}
1834 
1835 	writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C);
1836 
1837 	cbc->ring_head++;
1838 
1839 	return IRQ_WAKE_THREAD;
1840 }
1841 
1842 static irqreturn_t thunderx_l2c_mci_isr(int irq, void *irq_id)
1843 {
1844 	struct msix_entry *msix = irq_id;
1845 	struct thunderx_l2c *mci = container_of(msix, struct thunderx_l2c,
1846 						msix_ent);
1847 
1848 	unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx));
1849 	struct l2c_err_ctx *ctx = &mci->err_ctx[head];
1850 
1851 	ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C);
1852 	ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR);
1853 
1854 	writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C);
1855 
1856 	ctx->reg_ext_name = "ERR";
1857 
1858 	mci->ring_head++;
1859 
1860 	return IRQ_WAKE_THREAD;
1861 }
1862 
1863 static irqreturn_t thunderx_l2c_threaded_isr(int irq, void *irq_id)
1864 {
1865 	struct msix_entry *msix = irq_id;
1866 	struct thunderx_l2c *l2c = container_of(msix, struct thunderx_l2c,
1867 						msix_ent);
1868 
1869 	unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx));
1870 	struct l2c_err_ctx *ctx = &l2c->err_ctx[tail];
1871 	irqreturn_t ret = IRQ_NONE;
1872 
1873 	u64 mask_ue, mask_ce;
1874 	const struct error_descr *l2_errors;
1875 	char *reg_int_name;
1876 
1877 	char *msg;
1878 	char *other;
1879 
1880 	msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1881 	other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1882 
1883 	if (!msg || !other)
1884 		goto err_free;
1885 
1886 	switch (l2c->pdev->device) {
1887 	case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1888 		reg_int_name = "L2C_TAD_INT";
1889 		mask_ue = L2C_TAD_INT_UE;
1890 		mask_ce = L2C_TAD_INT_CE;
1891 		l2_errors = l2_tad_errors;
1892 		break;
1893 	case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1894 		reg_int_name = "L2C_CBC_INT";
1895 		mask_ue = L2C_CBC_INT_UE;
1896 		mask_ce = L2C_CBC_INT_CE;
1897 		l2_errors = l2_cbc_errors;
1898 		break;
1899 	case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1900 		reg_int_name = "L2C_MCI_INT";
1901 		mask_ue = L2C_MCI_INT_VBFDBE;
1902 		mask_ce = L2C_MCI_INT_VBFSBE;
1903 		l2_errors = l2_mci_errors;
1904 		break;
1905 	default:
1906 		dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n",
1907 			l2c->pdev->device);
1908 		return IRQ_NONE;
1909 	}
1910 
1911 	while (CIRC_CNT(l2c->ring_head, l2c->ring_tail,
1912 			ARRAY_SIZE(l2c->err_ctx))) {
1913 		snprintf(msg, L2C_MESSAGE_SIZE,
1914 			 "%s: %s: %016llx, %s: %016llx",
1915 			 l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int,
1916 			 ctx->reg_ext_name, ctx->reg_ext);
1917 
1918 		decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int);
1919 
1920 		strncat(msg, other, L2C_MESSAGE_SIZE);
1921 
1922 		if (ctx->reg_int & mask_ue)
1923 			edac_device_handle_ue(l2c->edac_dev, 0, 0, msg);
1924 		else if (ctx->reg_int & mask_ce)
1925 			edac_device_handle_ce(l2c->edac_dev, 0, 0, msg);
1926 
1927 		l2c->ring_tail++;
1928 	}
1929 
1930 	return IRQ_HANDLED;
1931 
1932 err_free:
1933 	kfree(other);
1934 	kfree(msg);
1935 
1936 	return ret;
1937 }
1938 
1939 #define L2C_DEBUGFS_ATTR(_name, _reg)	DEBUGFS_REG_ATTR(l2c, _name, _reg)
1940 
1941 L2C_DEBUGFS_ATTR(tad_int, L2C_TAD_INT_W1S);
1942 
1943 struct debugfs_entry *l2c_tad_dfs_ents[] = {
1944 	&debugfs_tad_int,
1945 };
1946 
1947 L2C_DEBUGFS_ATTR(cbc_int, L2C_CBC_INT_W1S);
1948 
1949 struct debugfs_entry *l2c_cbc_dfs_ents[] = {
1950 	&debugfs_cbc_int,
1951 };
1952 
1953 L2C_DEBUGFS_ATTR(mci_int, L2C_MCI_INT_W1S);
1954 
1955 struct debugfs_entry *l2c_mci_dfs_ents[] = {
1956 	&debugfs_mci_int,
1957 };
1958 
1959 static const struct pci_device_id thunderx_l2c_pci_tbl[] = {
1960 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_TAD), },
1961 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_CBC), },
1962 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_MCI), },
1963 	{ 0, },
1964 };
1965 
1966 static int thunderx_l2c_probe(struct pci_dev *pdev,
1967 			      const struct pci_device_id *id)
1968 {
1969 	struct thunderx_l2c *l2c;
1970 	struct edac_device_ctl_info *edac_dev;
1971 	struct debugfs_entry **l2c_devattr;
1972 	size_t dfs_entries;
1973 	irqreturn_t (*thunderx_l2c_isr)(int, void *) = NULL;
1974 	char name[32];
1975 	const char *fmt;
1976 	u64 reg_en_offs, reg_en_mask;
1977 	int idx;
1978 	int ret;
1979 
1980 	ret = pcim_enable_device(pdev);
1981 	if (ret) {
1982 		dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1983 		return ret;
1984 	}
1985 
1986 	ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_l2c");
1987 	if (ret) {
1988 		dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1989 		return ret;
1990 	}
1991 
1992 	switch (pdev->device) {
1993 	case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1994 		thunderx_l2c_isr = thunderx_l2c_tad_isr;
1995 		l2c_devattr = l2c_tad_dfs_ents;
1996 		dfs_entries = ARRAY_SIZE(l2c_tad_dfs_ents);
1997 		fmt = "L2C-TAD%d";
1998 		reg_en_offs = L2C_TAD_INT_ENA_W1S;
1999 		reg_en_mask = L2C_TAD_INT_ENA_ALL;
2000 		break;
2001 	case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2002 		thunderx_l2c_isr = thunderx_l2c_cbc_isr;
2003 		l2c_devattr = l2c_cbc_dfs_ents;
2004 		dfs_entries = ARRAY_SIZE(l2c_cbc_dfs_ents);
2005 		fmt = "L2C-CBC%d";
2006 		reg_en_offs = L2C_CBC_INT_ENA_W1S;
2007 		reg_en_mask = L2C_CBC_INT_ENA_ALL;
2008 		break;
2009 	case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2010 		thunderx_l2c_isr = thunderx_l2c_mci_isr;
2011 		l2c_devattr = l2c_mci_dfs_ents;
2012 		dfs_entries = ARRAY_SIZE(l2c_mci_dfs_ents);
2013 		fmt = "L2C-MCI%d";
2014 		reg_en_offs = L2C_MCI_INT_ENA_W1S;
2015 		reg_en_mask = L2C_MCI_INT_ENA_ALL;
2016 		break;
2017 	default:
2018 		//Should never ever get here
2019 		dev_err(&pdev->dev, "Unsupported PCI device: %04x\n",
2020 			pdev->device);
2021 		return -EINVAL;
2022 	}
2023 
2024 	idx = edac_device_alloc_index();
2025 	snprintf(name, sizeof(name), fmt, idx);
2026 
2027 	edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c),
2028 					      name, 1, "L2C", 1, 0,
2029 					      NULL, 0, idx);
2030 	if (!edac_dev) {
2031 		dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
2032 		return -ENOMEM;
2033 	}
2034 
2035 	l2c = edac_dev->pvt_info;
2036 	l2c->edac_dev = edac_dev;
2037 
2038 	l2c->regs = pcim_iomap_table(pdev)[0];
2039 	if (!l2c->regs) {
2040 		dev_err(&pdev->dev, "Cannot map PCI resources\n");
2041 		ret = -ENODEV;
2042 		goto err_free;
2043 	}
2044 
2045 	l2c->pdev = pdev;
2046 
2047 	l2c->ring_head = 0;
2048 	l2c->ring_tail = 0;
2049 
2050 	l2c->msix_ent.entry = 0;
2051 	l2c->msix_ent.vector = 0;
2052 
2053 	ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1);
2054 	if (ret) {
2055 		dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
2056 		goto err_free;
2057 	}
2058 
2059 	ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector,
2060 					thunderx_l2c_isr,
2061 					thunderx_l2c_threaded_isr,
2062 					0, "[EDAC] ThunderX L2C",
2063 					&l2c->msix_ent);
2064 	if (ret)
2065 		goto err_free;
2066 
2067 	edac_dev->dev = &pdev->dev;
2068 	edac_dev->dev_name = dev_name(&pdev->dev);
2069 	edac_dev->mod_name = "thunderx-l2c";
2070 	edac_dev->ctl_name = "thunderx-l2c";
2071 
2072 	ret = edac_device_add_device(edac_dev);
2073 	if (ret) {
2074 		dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
2075 		goto err_free;
2076 	}
2077 
2078 	if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
2079 		l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
2080 
2081 		ret = thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr,
2082 					      l2c, dfs_entries);
2083 
2084 		if (ret != dfs_entries) {
2085 			dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
2086 				 ret, ret >= 0 ? " created" : "");
2087 		}
2088 	}
2089 
2090 	pci_set_drvdata(pdev, edac_dev);
2091 
2092 	writeq(reg_en_mask, l2c->regs + reg_en_offs);
2093 
2094 	return 0;
2095 
2096 err_free:
2097 	edac_device_free_ctl_info(edac_dev);
2098 
2099 	return ret;
2100 }
2101 
2102 static void thunderx_l2c_remove(struct pci_dev *pdev)
2103 {
2104 	struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
2105 	struct thunderx_l2c *l2c = edac_dev->pvt_info;
2106 
2107 	switch (pdev->device) {
2108 	case PCI_DEVICE_ID_THUNDER_L2C_TAD:
2109 		writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C);
2110 		break;
2111 	case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2112 		writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C);
2113 		break;
2114 	case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2115 		writeq(L2C_MCI_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C);
2116 		break;
2117 	}
2118 
2119 	edac_debugfs_remove_recursive(l2c->debugfs);
2120 
2121 	edac_device_del_device(&pdev->dev);
2122 	edac_device_free_ctl_info(edac_dev);
2123 }
2124 
2125 MODULE_DEVICE_TABLE(pci, thunderx_l2c_pci_tbl);
2126 
2127 static struct pci_driver thunderx_l2c_driver = {
2128 	.name     = "thunderx_l2c_edac",
2129 	.probe    = thunderx_l2c_probe,
2130 	.remove   = thunderx_l2c_remove,
2131 	.id_table = thunderx_l2c_pci_tbl,
2132 };
2133 
2134 static int __init thunderx_edac_init(void)
2135 {
2136 	int rc = 0;
2137 
2138 	rc = pci_register_driver(&thunderx_lmc_driver);
2139 	if (rc)
2140 		return rc;
2141 
2142 	rc = pci_register_driver(&thunderx_ocx_driver);
2143 	if (rc)
2144 		goto err_lmc;
2145 
2146 	rc = pci_register_driver(&thunderx_l2c_driver);
2147 	if (rc)
2148 		goto err_ocx;
2149 
2150 	return rc;
2151 err_ocx:
2152 	pci_unregister_driver(&thunderx_ocx_driver);
2153 err_lmc:
2154 	pci_unregister_driver(&thunderx_lmc_driver);
2155 
2156 	return rc;
2157 }
2158 
2159 static void __exit thunderx_edac_exit(void)
2160 {
2161 	pci_unregister_driver(&thunderx_l2c_driver);
2162 	pci_unregister_driver(&thunderx_ocx_driver);
2163 	pci_unregister_driver(&thunderx_lmc_driver);
2164 
2165 }
2166 
2167 module_init(thunderx_edac_init);
2168 module_exit(thunderx_edac_exit);
2169 
2170 MODULE_LICENSE("GPL v2");
2171 MODULE_AUTHOR("Cavium, Inc.");
2172 MODULE_DESCRIPTION("EDAC Driver for Cavium ThunderX");
2173