8  * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved.
24 #include <asm/page.h>
50 static void decode_register(char *str, size_t size,  in decode_register()  argument
56 	while (descr->type && descr->mask && descr->descr) {  in decode_register()
57 		if (reg & descr->mask) {  in decode_register()
58 			ret = snprintf(str, size, "\n\t%s, %s",  in decode_register()
59 				       descr->type == ERR_CORRECTED ?  in decode_register()
61 				       descr->descr);  in decode_register()
63 			size -= ret;  in decode_register()
71 	return (data >> pos) & ((1 << width) - 1);  in get_bits()
127 		.descr = "Single-bit ECC error",
137 		.descr = "Double-bit ECC error",
142 		.descr = "Non-existent memory write",
207 	struct page *mem;
214 #define ring_pos(pos, size) ((pos) & (size - 1))  argument
233 	struct thunderx_##_type *pdata = file->private_data;		    \
236 	snprintf(buf, count, "0x%016llx", pdata->_field);		    \
245 	struct thunderx_##_type *pdata = file->private_data;		    \
248 	res = kstrtoull_from_user(data, count, 0, &pdata->_field);	    \
262 	struct thunderx_##_type *pdata = file->private_data;		    \
265 	sprintf(buf, "0x%016llx", readq(pdata->regs + _reg));		    \
274 	struct thunderx_##_type *pdata = file->private_data;		    \
281 		writeq(val, pdata->regs + _reg);			    \
296  * - Setup the ECC injection by writing the appropriate parameters:
300  * - Do the actual injection:
307 	struct thunderx_lmc *lmc = file->private_data;  in thunderx_lmc_inject_int_write()
315 		writeq(val, lmc->regs + LMC_INT_W1S);  in thunderx_lmc_inject_int_write()
326 	struct thunderx_lmc *lmc = file->private_data;  in thunderx_lmc_int_read()
328 	u64 lmc_int = readq(lmc->regs + LMC_INT);  in thunderx_lmc_int_read()
344 	addr = (uintptr_t)page_address(lmc->mem);  in inject_ecc_fn()
345 	phys = (uintptr_t)page_to_phys(lmc->mem);  in inject_ecc_fn()
348 	lmc->parity_test &= ~(7ULL << 8);  in inject_ecc_fn()
349 	lmc->parity_test |= (cl_idx << 8);  in inject_ecc_fn()
351 	writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);  in inject_ecc_fn()
352 	writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);  in inject_ecc_fn()
353 	writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);  in inject_ecc_fn()
355 	readq(lmc->regs + LMC_CHAR_MASK0);  in inject_ecc_fn()
356 	readq(lmc->regs + LMC_CHAR_MASK2);  in inject_ecc_fn()
357 	readq(lmc->regs + LMC_ECC_PARITY_TEST);  in inject_ecc_fn()
410 	struct thunderx_lmc *lmc = file->private_data;  in thunderx_lmc_inject_ecc_write()
416 	atomic_set(&lmc->ecc_int, 0);  in thunderx_lmc_inject_ecc_write()
418 	lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);  in thunderx_lmc_inject_ecc_write()
419 	if (!lmc->mem)  in thunderx_lmc_inject_ecc_write()
420 		return -ENOMEM;  in thunderx_lmc_inject_ecc_write()
424 		__free_pages(lmc->mem, 0);  in thunderx_lmc_inject_ecc_write()
425 		return -ENOMEM;  in thunderx_lmc_inject_ecc_write()
428 	addr = page_address(lmc->mem);  in thunderx_lmc_inject_ecc_write()
430 	while (!atomic_read(&lmc->ecc_int) && timeout--) {  in thunderx_lmc_inject_ecc_write()
444 	__free_pages(lmc->mem, 0);  in thunderx_lmc_inject_ecc_write()
478 		return -ENOENT;  in thunderx_create_debugfs_nodes()
481 		ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode,  in thunderx_create_debugfs_nodes()
482 					       parent, data, &attrs[i]->fops);  in thunderx_create_debugfs_nodes()
496 	addr |= lmc->node << 40;  in thunderx_faddr_to_phys()
497 	addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;  in thunderx_faddr_to_phys()
498 	addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;  in thunderx_faddr_to_phys()
499 	addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;  in thunderx_faddr_to_phys()
500 	addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;  in thunderx_faddr_to_phys()
502 	bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;  in thunderx_faddr_to_phys()
504 	if (lmc->xor_bank)  in thunderx_faddr_to_phys()
505 		bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);  in thunderx_faddr_to_phys()
507 	addr |= bank << lmc->bank_lsb;  in thunderx_faddr_to_phys()
509 	xbits = PCI_FUNC(lmc->pdev->devfn);  in thunderx_faddr_to_phys()
511 	if (lmc->l2c_alias)  in thunderx_faddr_to_phys()
512 		xbits ^= get_bits(addr, 20, lmc->xbits) ^  in thunderx_faddr_to_phys()
513 			 get_bits(addr, 12, lmc->xbits);  in thunderx_faddr_to_phys()
531 			if (pdev->dev.numa_node == node)  in thunderx_get_num_lmcs()
548 	struct thunderx_lmc *lmc = mci->pvt_info;  in thunderx_lmc_err_isr()
550 	unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));  in thunderx_lmc_err_isr()
551 	struct lmc_err_ctx *ctx = &lmc->err_ctx[head];  in thunderx_lmc_err_isr()
553 	writeq(0, lmc->regs + LMC_CHAR_MASK0);  in thunderx_lmc_err_isr()
554 	writeq(0, lmc->regs + LMC_CHAR_MASK2);  in thunderx_lmc_err_isr()
555 	writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);  in thunderx_lmc_err_isr()
557 	ctx->reg_int = readq(lmc->regs + LMC_INT);  in thunderx_lmc_err_isr()
558 	ctx->reg_fadr = readq(lmc->regs + LMC_FADR);  in thunderx_lmc_err_isr()
559 	ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);  in thunderx_lmc_err_isr()
560 	ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);  in thunderx_lmc_err_isr()
561 	ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);  in thunderx_lmc_err_isr()
563 	lmc->ring_head++;  in thunderx_lmc_err_isr()
565 	atomic_set(&lmc->ecc_int, 1);  in thunderx_lmc_err_isr()
568 	writeq(ctx->reg_int, lmc->regs + LMC_INT);  in thunderx_lmc_err_isr()
576 	struct thunderx_lmc *lmc = mci->pvt_info;  in thunderx_lmc_threaded_isr()
593 	while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,  in thunderx_lmc_threaded_isr()
594 		ARRAY_SIZE(lmc->err_ctx))) {  in thunderx_lmc_threaded_isr()
595 		tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));  in thunderx_lmc_threaded_isr()
597 		ctx = &lmc->err_ctx[tail];  in thunderx_lmc_threaded_isr()
599 		dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",  in thunderx_lmc_threaded_isr()
600 			ctx->reg_int);  in thunderx_lmc_threaded_isr()
601 		dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",  in thunderx_lmc_threaded_isr()
602 			ctx->reg_fadr);  in thunderx_lmc_threaded_isr()
603 		dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",  in thunderx_lmc_threaded_isr()
604 			ctx->reg_nxm_fadr);  in thunderx_lmc_threaded_isr()
605 		dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",  in thunderx_lmc_threaded_isr()
606 			ctx->reg_scram_fadr);  in thunderx_lmc_threaded_isr()
607 		dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",  in thunderx_lmc_threaded_isr()
608 			ctx->reg_ecc_synd);  in thunderx_lmc_threaded_isr()
612 			 LMC_FADR_FDIMM(ctx->reg_scram_fadr),  in thunderx_lmc_threaded_isr()
613 			 LMC_FADR_FBUNK(ctx->reg_scram_fadr),  in thunderx_lmc_threaded_isr()
614 			 LMC_FADR_FBANK(ctx->reg_scram_fadr),  in thunderx_lmc_threaded_isr()
615 			 LMC_FADR_FROW(ctx->reg_scram_fadr),  in thunderx_lmc_threaded_isr()
616 			 LMC_FADR_FCOL(ctx->reg_scram_fadr));  in thunderx_lmc_threaded_isr()
619 				ctx->reg_int);  in thunderx_lmc_threaded_isr()
621 		phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);  in thunderx_lmc_threaded_isr()
623 		if (ctx->reg_int & LMC_INT_UE)  in thunderx_lmc_threaded_isr()
627 					     0, -1, -1, -1, msg, other);  in thunderx_lmc_threaded_isr()
628 		else if (ctx->reg_int & LMC_INT_CE)  in thunderx_lmc_threaded_isr()
632 					     0, -1, -1, -1, msg, other);  in thunderx_lmc_threaded_isr()
634 		lmc->ring_tail++;  in thunderx_lmc_threaded_isr()
653 	int node = dev_to_node(&pdev->dev);  in pci_dev_to_mc_idx()
654 	int ret = PCI_FUNC(pdev->devfn);  in pci_dev_to_mc_idx()
673 	layer.size = 2;  in thunderx_lmc_probe()
678 		dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);  in thunderx_lmc_probe()
684 		dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);  in thunderx_lmc_probe()
691 		return -ENOMEM;  in thunderx_lmc_probe()
693 	mci->pdev = &pdev->dev;  in thunderx_lmc_probe()
694 	lmc = mci->pvt_info;  in thunderx_lmc_probe()
698 	lmc->regs = pcim_iomap_table(pdev)[0];  in thunderx_lmc_probe()
700 	lmc_control = readq(lmc->regs + LMC_CONTROL);  in thunderx_lmc_probe()
701 	lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);  in thunderx_lmc_probe()
702 	lmc_config = readq(lmc->regs + LMC_CONFIG);  in thunderx_lmc_probe()
705 		mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,  in thunderx_lmc_probe()
709 		mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,  in thunderx_lmc_probe()
714 	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;  in thunderx_lmc_probe()
715 	mci->edac_cap = EDAC_FLAG_SECDED;  in thunderx_lmc_probe()
717 	mci->mod_name = "thunderx-lmc";  in thunderx_lmc_probe()
718 	mci->ctl_name = "thunderx-lmc";  in thunderx_lmc_probe()
719 	mci->dev_name = dev_name(&pdev->dev);  in thunderx_lmc_probe()
720 	mci->scrub_mode = SCRUB_NONE;  in thunderx_lmc_probe()
722 	lmc->pdev = pdev;  in thunderx_lmc_probe()
723 	lmc->msix_ent.entry = 0;  in thunderx_lmc_probe()
725 	lmc->ring_head = 0;  in thunderx_lmc_probe()
726 	lmc->ring_tail = 0;  in thunderx_lmc_probe()
728 	ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);  in thunderx_lmc_probe()
730 		dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);  in thunderx_lmc_probe()
734 	ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,  in thunderx_lmc_probe()
739 		dev_err(&pdev->dev, "Cannot set ISR: %d\n", ret);  in thunderx_lmc_probe()
743 	lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));  in thunderx_lmc_probe()
745 	lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;  in thunderx_lmc_probe()
746 	lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&  in thunderx_lmc_probe()
749 	lmc->pbank_lsb = (lmc_config >> 5) & 0xf;  in thunderx_lmc_probe()
750 	lmc->dimm_lsb  = 28 + lmc->pbank_lsb + lmc->xbits;  in thunderx_lmc_probe()
751 	lmc->rank_lsb = lmc->dimm_lsb;  in thunderx_lmc_probe()
752 	lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;  in thunderx_lmc_probe()
753 	lmc->bank_lsb = 7 + lmc->xbits;  in thunderx_lmc_probe()
754 	lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;  in thunderx_lmc_probe()
756 	lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;  in thunderx_lmc_probe()
758 	lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;  in thunderx_lmc_probe()
760 	l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE);  in thunderx_lmc_probe()
762 		dev_err(&pdev->dev, "Cannot map L2C_CTL\n");  in thunderx_lmc_probe()
763 		ret = -ENOMEM;  in thunderx_lmc_probe()
767 	lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);  in thunderx_lmc_probe()
773 		dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret);  in thunderx_lmc_probe()
777 	lmc_int = readq(lmc->regs + LMC_INT);  in thunderx_lmc_probe()
778 	writeq(lmc_int, lmc->regs + LMC_INT);  in thunderx_lmc_probe()
780 	writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);  in thunderx_lmc_probe()
783 		ret = thunderx_create_debugfs_nodes(mci->debugfs,  in thunderx_lmc_probe()
789 			dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",  in thunderx_lmc_probe()
806 	struct thunderx_lmc *lmc = mci->pvt_info;  in thunderx_lmc_remove()
808 	writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);  in thunderx_lmc_remove()
810 	edac_mc_del_mc(&pdev->dev);  in thunderx_lmc_remove()
823 /*---------------------- OCX driver ---------------------------------*/
903 		.descr = "Replay buffer single-bit error",
908 		.descr = "TX FIFO single-bit error",
913 		.descr = "RX FIFO single-bit error",
933 		.descr = "Replay buffer double-bit error",
938 		.descr = "TX FIFO double-bit error",
943 		.descr = "RX FIFO double-bit error",
1079 						msix_ent[msix->entry]);  in thunderx_ocx_com_isr()
1082 	unsigned long head = ring_pos(ocx->com_ring_head,  in thunderx_ocx_com_isr()
1083 				      ARRAY_SIZE(ocx->com_err_ctx));  in thunderx_ocx_com_isr()
1084 	struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head];  in thunderx_ocx_com_isr()
1086 	ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT);  in thunderx_ocx_com_isr()
1089 		ctx->reg_lane_int[lane] =  in thunderx_ocx_com_isr()
1090 			readq(ocx->regs + OCX_LNE_INT(lane));  in thunderx_ocx_com_isr()
1091 		ctx->reg_lane_stat11[lane] =  in thunderx_ocx_com_isr()
1092 			readq(ocx->regs + OCX_LNE_STAT(lane, 11));  in thunderx_ocx_com_isr()
1094 		writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane));  in thunderx_ocx_com_isr()
1097 	writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT);  in thunderx_ocx_com_isr()
1099 	ocx->com_ring_head++;  in thunderx_ocx_com_isr()
1108 						msix_ent[msix->entry]);  in thunderx_ocx_com_threaded_isr()
1124 	while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail,  in thunderx_ocx_com_threaded_isr()
1125 			ARRAY_SIZE(ocx->com_err_ctx))) {  in thunderx_ocx_com_threaded_isr()
1126 		tail = ring_pos(ocx->com_ring_tail,  in thunderx_ocx_com_threaded_isr()
1127 				ARRAY_SIZE(ocx->com_err_ctx));  in thunderx_ocx_com_threaded_isr()
1128 		ctx = &ocx->com_err_ctx[tail];  in thunderx_ocx_com_threaded_isr()
1131 			ocx->edac_dev->ctl_name, ctx->reg_com_int);  in thunderx_ocx_com_threaded_isr()
1134 				ocx_com_errors, ctx->reg_com_int);  in thunderx_ocx_com_threaded_isr()
1139 			if (ctx->reg_com_int & BIT(lane)) {  in thunderx_ocx_com_threaded_isr()
1142 					 lane, ctx->reg_lane_int[lane],  in thunderx_ocx_com_threaded_isr()
1143 					 lane, ctx->reg_lane_stat11[lane]);  in thunderx_ocx_com_threaded_isr()
1149 						ctx->reg_lane_int[lane]);  in thunderx_ocx_com_threaded_isr()
1153 		if (ctx->reg_com_int & OCX_COM_INT_CE)  in thunderx_ocx_com_threaded_isr()
1154 			edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);  in thunderx_ocx_com_threaded_isr()
1156 		ocx->com_ring_tail++;  in thunderx_ocx_com_threaded_isr()
1172 						msix_ent[msix->entry]);  in thunderx_ocx_lnk_isr()
1173 	unsigned long head = ring_pos(ocx->link_ring_head,  in thunderx_ocx_lnk_isr()
1174 				      ARRAY_SIZE(ocx->link_err_ctx));  in thunderx_ocx_lnk_isr()
1175 	struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head];  in thunderx_ocx_lnk_isr()
1177 	ctx->link = msix->entry;  in thunderx_ocx_lnk_isr()
1178 	ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link));  in thunderx_ocx_lnk_isr()
1180 	writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link));  in thunderx_ocx_lnk_isr()
1182 	ocx->link_ring_head++;  in thunderx_ocx_lnk_isr()
1191 						msix_ent[msix->entry]);  in thunderx_ocx_lnk_threaded_isr()
1205 	while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail,  in thunderx_ocx_lnk_threaded_isr()
1206 			ARRAY_SIZE(ocx->link_err_ctx))) {  in thunderx_ocx_lnk_threaded_isr()
1207 		tail = ring_pos(ocx->link_ring_head,  in thunderx_ocx_lnk_threaded_isr()
1208 				ARRAY_SIZE(ocx->link_err_ctx));  in thunderx_ocx_lnk_threaded_isr()
1210 		ctx = &ocx->link_err_ctx[tail];  in thunderx_ocx_lnk_threaded_isr()
1214 			 ocx->edac_dev->ctl_name,  in thunderx_ocx_lnk_threaded_isr()
1215 			 ctx->link, ctx->reg_com_link_int);  in thunderx_ocx_lnk_threaded_isr()
1218 				ocx_com_link_errors, ctx->reg_com_link_int);  in thunderx_ocx_lnk_threaded_isr()
1222 		if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE)  in thunderx_ocx_lnk_threaded_isr()
1223 			edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);  in thunderx_ocx_lnk_threaded_isr()
1224 		else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE)  in thunderx_ocx_lnk_threaded_isr()
1225 			edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);  in thunderx_ocx_lnk_threaded_isr()
1227 		ocx->link_ring_tail++;  in thunderx_ocx_lnk_threaded_isr()
1332 		cfg = readq(ocx->regs + OCX_LNE_CFG(lane));  in thunderx_ocx_clearstats()
1335 		writeq(cfg, ocx->regs + OCX_LNE_CFG(lane));  in thunderx_ocx_clearstats()
1338 			readq(ocx->regs + OCX_LNE_STAT(lane, stat));  in thunderx_ocx_clearstats()
1355 		dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);  in thunderx_ocx_probe()
1361 		dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);  in thunderx_ocx_probe()
1371 		dev_err(&pdev->dev, "Cannot allocate EDAC device\n");  in thunderx_ocx_probe()
1372 		return -ENOMEM;  in thunderx_ocx_probe()
1374 	ocx = edac_dev->pvt_info;  in thunderx_ocx_probe()
1375 	ocx->edac_dev = edac_dev;  in thunderx_ocx_probe()
1376 	ocx->com_ring_head = 0;  in thunderx_ocx_probe()
1377 	ocx->com_ring_tail = 0;  in thunderx_ocx_probe()
1378 	ocx->link_ring_head = 0;  in thunderx_ocx_probe()
1379 	ocx->link_ring_tail = 0;  in thunderx_ocx_probe()
1381 	ocx->regs = pcim_iomap_table(pdev)[0];  in thunderx_ocx_probe()
1382 	if (!ocx->regs) {  in thunderx_ocx_probe()
1383 		dev_err(&pdev->dev, "Cannot map PCI resources\n");  in thunderx_ocx_probe()
1384 		ret = -ENODEV;  in thunderx_ocx_probe()
1388 	ocx->pdev = pdev;  in thunderx_ocx_probe()
1391 		ocx->msix_ent[i].entry = i;  in thunderx_ocx_probe()
1392 		ocx->msix_ent[i].vector = 0;  in thunderx_ocx_probe()
1395 	ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS);  in thunderx_ocx_probe()
1397 		dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);  in thunderx_ocx_probe()
1402 		ret = devm_request_threaded_irq(&pdev->dev,  in thunderx_ocx_probe()
1403 						ocx->msix_ent[i].vector,  in thunderx_ocx_probe()
1411 						&ocx->msix_ent[i]);  in thunderx_ocx_probe()
1416 	edac_dev->dev = &pdev->dev;  in thunderx_ocx_probe()
1417 	edac_dev->dev_name = dev_name(&pdev->dev);  in thunderx_ocx_probe()
1418 	edac_dev->mod_name = "thunderx-ocx";  in thunderx_ocx_probe()
1419 	edac_dev->ctl_name = "thunderx-ocx";  in thunderx_ocx_probe()
1423 		dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);  in thunderx_ocx_probe()
1428 		ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);  in thunderx_ocx_probe()
1430 		ret = thunderx_create_debugfs_nodes(ocx->debugfs,  in thunderx_ocx_probe()
1435 			dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",  in thunderx_ocx_probe()
1446 		       ocx->regs + OCX_LNE_INT_EN(i));  in thunderx_ocx_probe()
1448 		reg = readq(ocx->regs + OCX_LNE_INT(i));  in thunderx_ocx_probe()
1449 		writeq(reg, ocx->regs + OCX_LNE_INT(i));  in thunderx_ocx_probe()
1454 		reg = readq(ocx->regs + OCX_COM_LINKX_INT(i));  in thunderx_ocx_probe()
1455 		writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i));  in thunderx_ocx_probe()
1458 		       ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i));  in thunderx_ocx_probe()
1461 	reg = readq(ocx->regs + OCX_COM_INT);  in thunderx_ocx_probe()
1462 	writeq(reg, ocx->regs + OCX_COM_INT);  in thunderx_ocx_probe()
1464 	writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S);  in thunderx_ocx_probe()
1476 	struct thunderx_ocx *ocx = edac_dev->pvt_info;  in thunderx_ocx_remove()
1479 	writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C);  in thunderx_ocx_remove()
1483 		       ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i));  in thunderx_ocx_remove()
1486 	edac_debugfs_remove_recursive(ocx->debugfs);  in thunderx_ocx_remove()
1488 	edac_device_del_device(&pdev->dev);  in thunderx_ocx_remove()
1501 /*---------------------- L2C driver ---------------------------------*/
1552 		.descr = "SBF single-bit error",
1557 		.descr = "FBF single-bit error",
1562 		.descr = "L2D double-bit error",
1567 		.descr = "SBF double-bit error",
1572 		.descr = "FBF double-bit error",
1577 		.descr = "TAG double-bit error",
1582 		.descr = "RTG double-bit error",
1663 		.descr = "RSD single-bit error",
1668 		.descr = "MIB single-bit error",
1673 		.descr = "RSD double-bit error",
1678 		.descr = "MIB double-bit error",
1716 		.descr = "VBF single-bit error",
1721 		.descr = "VBF double-bit error",
1763 	struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c,  in thunderx_l2c_tad_isr()  local
1766 	unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx));  in thunderx_l2c_tad_isr()
1767 	struct l2c_err_ctx *ctx = &tad->err_ctx[head];  in thunderx_l2c_tad_isr()
1769 	ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C);  in thunderx_l2c_tad_isr()
1771 	if (ctx->reg_int & L2C_TAD_INT_ECC) {  in thunderx_l2c_tad_isr()
1772 		ctx->reg_ext_name = "TQD_ERR";  in thunderx_l2c_tad_isr()
1773 		ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR);  in thunderx_l2c_tad_isr()
1774 	} else if (ctx->reg_int & L2C_TAD_INT_TAG) {  in thunderx_l2c_tad_isr()
1775 		ctx->reg_ext_name = "TTG_ERR";  in thunderx_l2c_tad_isr()
1776 		ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR);  in thunderx_l2c_tad_isr()
1777 	} else if (ctx->reg_int & L2C_TAD_INT_LFBTO) {  in thunderx_l2c_tad_isr()
1778 		ctx->reg_ext_name = "TIMEOUT";  in thunderx_l2c_tad_isr()
1779 		ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT);  in thunderx_l2c_tad_isr()
1780 	} else if (ctx->reg_int & L2C_TAD_INT_DISOCI) {  in thunderx_l2c_tad_isr()
1781 		ctx->reg_ext_name = "ERR";  in thunderx_l2c_tad_isr()
1782 		ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);  in thunderx_l2c_tad_isr()
1785 	writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C);  in thunderx_l2c_tad_isr()
1787 	tad->ring_head++;  in thunderx_l2c_tad_isr()
1798 	unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx));  in thunderx_l2c_cbc_isr()
1799 	struct l2c_err_ctx *ctx = &cbc->err_ctx[head];  in thunderx_l2c_cbc_isr()
1801 	ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C);  in thunderx_l2c_cbc_isr()
1803 	if (ctx->reg_int & L2C_CBC_INT_RSD) {  in thunderx_l2c_cbc_isr()
1804 		ctx->reg_ext_name = "RSDERR";  in thunderx_l2c_cbc_isr()
1805 		ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR);  in thunderx_l2c_cbc_isr()
1806 	} else if (ctx->reg_int & L2C_CBC_INT_MIB) {  in thunderx_l2c_cbc_isr()
1807 		ctx->reg_ext_name = "MIBERR";  in thunderx_l2c_cbc_isr()
1808 		ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR);  in thunderx_l2c_cbc_isr()
1809 	} else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) {  in thunderx_l2c_cbc_isr()
1810 		ctx->reg_ext_name = "IODISOCIERR";  in thunderx_l2c_cbc_isr()
1811 		ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR);  in thunderx_l2c_cbc_isr()
1814 	writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C);  in thunderx_l2c_cbc_isr()
1816 	cbc->ring_head++;  in thunderx_l2c_cbc_isr()
1827 	unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx));  in thunderx_l2c_mci_isr()
1828 	struct l2c_err_ctx *ctx = &mci->err_ctx[head];  in thunderx_l2c_mci_isr()
1830 	ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C);  in thunderx_l2c_mci_isr()
1831 	ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR);  in thunderx_l2c_mci_isr()
1833 	writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C);  in thunderx_l2c_mci_isr()
1835 	ctx->reg_ext_name = "ERR";  in thunderx_l2c_mci_isr()
1837 	mci->ring_head++;  in thunderx_l2c_mci_isr()
1848 	unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx));  in thunderx_l2c_threaded_isr()
1849 	struct l2c_err_ctx *ctx = &l2c->err_ctx[tail];  in thunderx_l2c_threaded_isr()
1865 	switch (l2c->pdev->device) {  in thunderx_l2c_threaded_isr()
1885 		dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n",  in thunderx_l2c_threaded_isr()
1886 			l2c->pdev->device);  in thunderx_l2c_threaded_isr()
1890 	while (CIRC_CNT(l2c->ring_head, l2c->ring_tail,  in thunderx_l2c_threaded_isr()
1891 			ARRAY_SIZE(l2c->err_ctx))) {  in thunderx_l2c_threaded_isr()
1894 			 l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int,  in thunderx_l2c_threaded_isr()
1895 			 ctx->reg_ext_name, ctx->reg_ext);  in thunderx_l2c_threaded_isr()
1897 		decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int);  in thunderx_l2c_threaded_isr()
1901 		if (ctx->reg_int & mask_ue)  in thunderx_l2c_threaded_isr()
1902 			edac_device_handle_ue(l2c->edac_dev, 0, 0, msg);  in thunderx_l2c_threaded_isr()
1903 		else if (ctx->reg_int & mask_ce)  in thunderx_l2c_threaded_isr()
1904 			edac_device_handle_ce(l2c->edac_dev, 0, 0, msg);  in thunderx_l2c_threaded_isr()
1906 		l2c->ring_tail++;  in thunderx_l2c_threaded_isr()
1961 		dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);  in thunderx_l2c_probe()
1967 		dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);  in thunderx_l2c_probe()
1971 	switch (pdev->device) {  in thunderx_l2c_probe()
1976 		fmt = "L2C-TAD%d";  in thunderx_l2c_probe()
1984 		fmt = "L2C-CBC%d";  in thunderx_l2c_probe()
1992 		fmt = "L2C-MCI%d";  in thunderx_l2c_probe()
1998 		dev_err(&pdev->dev, "Unsupported PCI device: %04x\n",  in thunderx_l2c_probe()
1999 			pdev->device);  in thunderx_l2c_probe()
2000 		return -EINVAL;  in thunderx_l2c_probe()
2010 		dev_err(&pdev->dev, "Cannot allocate EDAC device\n");  in thunderx_l2c_probe()
2011 		return -ENOMEM;  in thunderx_l2c_probe()
2014 	l2c = edac_dev->pvt_info;  in thunderx_l2c_probe()
2015 	l2c->edac_dev = edac_dev;  in thunderx_l2c_probe()
2017 	l2c->regs = pcim_iomap_table(pdev)[0];  in thunderx_l2c_probe()
2018 	if (!l2c->regs) {  in thunderx_l2c_probe()
2019 		dev_err(&pdev->dev, "Cannot map PCI resources\n");  in thunderx_l2c_probe()
2020 		ret = -ENODEV;  in thunderx_l2c_probe()
2024 	l2c->pdev = pdev;  in thunderx_l2c_probe()
2026 	l2c->ring_head = 0;  in thunderx_l2c_probe()
2027 	l2c->ring_tail = 0;  in thunderx_l2c_probe()
2029 	l2c->msix_ent.entry = 0;  in thunderx_l2c_probe()
2030 	l2c->msix_ent.vector = 0;  in thunderx_l2c_probe()
2032 	ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1);  in thunderx_l2c_probe()
2034 		dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);  in thunderx_l2c_probe()
2038 	ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector,  in thunderx_l2c_probe()
2042 					&l2c->msix_ent);  in thunderx_l2c_probe()
2046 	edac_dev->dev = &pdev->dev;  in thunderx_l2c_probe()
2047 	edac_dev->dev_name = dev_name(&pdev->dev);  in thunderx_l2c_probe()
2048 	edac_dev->mod_name = "thunderx-l2c";  in thunderx_l2c_probe()
2049 	edac_dev->ctl_name = "thunderx-l2c";  in thunderx_l2c_probe()
2053 		dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);  in thunderx_l2c_probe()
2058 		l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);  in thunderx_l2c_probe()
2060 		ret = thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr,  in thunderx_l2c_probe()
2064 			dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",  in thunderx_l2c_probe()
2071 	writeq(reg_en_mask, l2c->regs + reg_en_offs);  in thunderx_l2c_probe()
2084 	struct thunderx_l2c *l2c = edac_dev->pvt_info;  in thunderx_l2c_remove()
2086 	switch (pdev->device) {  in thunderx_l2c_remove()
2088 		writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C);  in thunderx_l2c_remove()
2091 		writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C);  in thunderx_l2c_remove()
2094 		writeq(L2C_MCI_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C);  in thunderx_l2c_remove()
2098 	edac_debugfs_remove_recursive(l2c->debugfs);  in thunderx_l2c_remove()
2100 	edac_device_del_device(&pdev->dev);  in thunderx_l2c_remove()
2118 		return -EBUSY;  in thunderx_edac_init()