1 // SPDX-License-Identifier: GPL-2.0
2 /* psycho_common.c: Code common to PSYCHO and derivative PCI controllers.
3  *
4  * Copyright (C) 2008 David S. Miller <davem@davemloft.net>
5  */
6 #include <linux/kernel.h>
7 #include <linux/interrupt.h>
8 #include <linux/numa.h>
9 #include <linux/platform_device.h>
10 
11 #include <asm/upa.h>
12 
13 #include "pci_impl.h"
14 #include "iommu_common.h"
15 #include "psycho_common.h"
16 
17 #define  PSYCHO_STRBUF_CTRL_DENAB	0x0000000000000002ULL
18 #define  PSYCHO_STCERR_WRITE		0x0000000000000002ULL
19 #define  PSYCHO_STCERR_READ		0x0000000000000001ULL
20 #define  PSYCHO_STCTAG_PPN		0x0fffffff00000000ULL
21 #define  PSYCHO_STCTAG_VPN		0x00000000ffffe000ULL
22 #define  PSYCHO_STCTAG_VALID		0x0000000000000002ULL
23 #define  PSYCHO_STCTAG_WRITE		0x0000000000000001ULL
24 #define  PSYCHO_STCLINE_LINDX		0x0000000001e00000ULL
25 #define  PSYCHO_STCLINE_SPTR		0x00000000001f8000ULL
26 #define  PSYCHO_STCLINE_LADDR		0x0000000000007f00ULL
27 #define  PSYCHO_STCLINE_EPTR		0x00000000000000fcULL
28 #define  PSYCHO_STCLINE_VALID		0x0000000000000002ULL
29 #define  PSYCHO_STCLINE_FOFN		0x0000000000000001ULL
30 
31 static DEFINE_SPINLOCK(stc_buf_lock);
32 static unsigned long stc_error_buf[128];
33 static unsigned long stc_tag_buf[16];
34 static unsigned long stc_line_buf[16];
35 
36 static void psycho_check_stc_error(struct pci_pbm_info *pbm)
37 {
38 	unsigned long err_base, tag_base, line_base;
39 	struct strbuf *strbuf = &pbm->stc;
40 	u64 control;
41 	int i;
42 
43 	if (!strbuf->strbuf_control)
44 		return;
45 
46 	err_base = strbuf->strbuf_err_stat;
47 	tag_base = strbuf->strbuf_tag_diag;
48 	line_base = strbuf->strbuf_line_diag;
49 
50 	spin_lock(&stc_buf_lock);
51 
52 	/* This is __REALLY__ dangerous.  When we put the streaming
53 	 * buffer into diagnostic mode to probe it's tags and error
54 	 * status, we _must_ clear all of the line tag valid bits
55 	 * before re-enabling the streaming buffer.  If any dirty data
56 	 * lives in the STC when we do this, we will end up
57 	 * invalidating it before it has a chance to reach main
58 	 * memory.
59 	 */
60 	control = upa_readq(strbuf->strbuf_control);
61 	upa_writeq(control | PSYCHO_STRBUF_CTRL_DENAB, strbuf->strbuf_control);
62 	for (i = 0; i < 128; i++) {
63 		u64 val;
64 
65 		val = upa_readq(err_base + (i * 8UL));
66 		upa_writeq(0UL, err_base + (i * 8UL));
67 		stc_error_buf[i] = val;
68 	}
69 	for (i = 0; i < 16; i++) {
70 		stc_tag_buf[i] = upa_readq(tag_base + (i * 8UL));
71 		stc_line_buf[i] = upa_readq(line_base + (i * 8UL));
72 		upa_writeq(0UL, tag_base + (i * 8UL));
73 		upa_writeq(0UL, line_base + (i * 8UL));
74 	}
75 
76 	/* OK, state is logged, exit diagnostic mode. */
77 	upa_writeq(control, strbuf->strbuf_control);
78 
79 	for (i = 0; i < 16; i++) {
80 		int j, saw_error, first, last;
81 
82 		saw_error = 0;
83 		first = i * 8;
84 		last = first + 8;
85 		for (j = first; j < last; j++) {
86 			u64 errval = stc_error_buf[j];
87 			if (errval != 0) {
88 				saw_error++;
89 				printk(KERN_ERR "%s: STC_ERR(%d)[wr(%d)"
90 				       "rd(%d)]\n",
91 				       pbm->name,
92 				       j,
93 				       (errval & PSYCHO_STCERR_WRITE) ? 1 : 0,
94 				       (errval & PSYCHO_STCERR_READ) ? 1 : 0);
95 			}
96 		}
97 		if (saw_error != 0) {
98 			u64 tagval = stc_tag_buf[i];
99 			u64 lineval = stc_line_buf[i];
100 			printk(KERN_ERR "%s: STC_TAG(%d)[PA(%016llx)VA(%08llx)"
101 			       "V(%d)W(%d)]\n",
102 			       pbm->name,
103 			       i,
104 			       ((tagval & PSYCHO_STCTAG_PPN) >> 19UL),
105 			       (tagval & PSYCHO_STCTAG_VPN),
106 			       ((tagval & PSYCHO_STCTAG_VALID) ? 1 : 0),
107 			       ((tagval & PSYCHO_STCTAG_WRITE) ? 1 : 0));
108 			printk(KERN_ERR "%s: STC_LINE(%d)[LIDX(%llx)SP(%llx)"
109 			       "LADDR(%llx)EP(%llx)V(%d)FOFN(%d)]\n",
110 			       pbm->name,
111 			       i,
112 			       ((lineval & PSYCHO_STCLINE_LINDX) >> 21UL),
113 			       ((lineval & PSYCHO_STCLINE_SPTR) >> 15UL),
114 			       ((lineval & PSYCHO_STCLINE_LADDR) >> 8UL),
115 			       ((lineval & PSYCHO_STCLINE_EPTR) >> 2UL),
116 			       ((lineval & PSYCHO_STCLINE_VALID) ? 1 : 0),
117 			       ((lineval & PSYCHO_STCLINE_FOFN) ? 1 : 0));
118 		}
119 	}
120 
121 	spin_unlock(&stc_buf_lock);
122 }
123 
124 #define PSYCHO_IOMMU_TAG		0xa580UL
125 #define PSYCHO_IOMMU_DATA		0xa600UL
126 
127 static void psycho_record_iommu_tags_and_data(struct pci_pbm_info *pbm,
128 					      u64 *tag, u64 *data)
129 {
130 	int i;
131 
132 	for (i = 0; i < 16; i++) {
133 		unsigned long base = pbm->controller_regs;
134 		unsigned long off = i * 8UL;
135 
136 		tag[i] = upa_readq(base + PSYCHO_IOMMU_TAG+off);
137 		data[i] = upa_readq(base + PSYCHO_IOMMU_DATA+off);
138 
139 		/* Now clear out the entry. */
140 		upa_writeq(0, base + PSYCHO_IOMMU_TAG + off);
141 		upa_writeq(0, base + PSYCHO_IOMMU_DATA + off);
142 	}
143 }
144 
145 #define  PSYCHO_IOMMU_TAG_ERRSTS (0x3UL << 23UL)
146 #define  PSYCHO_IOMMU_TAG_ERR	 (0x1UL << 22UL)
147 #define  PSYCHO_IOMMU_TAG_WRITE	 (0x1UL << 21UL)
148 #define  PSYCHO_IOMMU_TAG_STREAM (0x1UL << 20UL)
149 #define  PSYCHO_IOMMU_TAG_SIZE	 (0x1UL << 19UL)
150 #define  PSYCHO_IOMMU_TAG_VPAGE	 0x7ffffULL
151 #define  PSYCHO_IOMMU_DATA_VALID (1UL << 30UL)
152 #define  PSYCHO_IOMMU_DATA_CACHE (1UL << 28UL)
153 #define  PSYCHO_IOMMU_DATA_PPAGE 0xfffffffULL
154 
155 static void psycho_dump_iommu_tags_and_data(struct pci_pbm_info *pbm,
156 					    u64 *tag, u64 *data)
157 {
158 	int i;
159 
160 	for (i = 0; i < 16; i++) {
161 		u64 tag_val, data_val;
162 		const char *type_str;
163 		tag_val = tag[i];
164 		if (!(tag_val & PSYCHO_IOMMU_TAG_ERR))
165 			continue;
166 
167 		data_val = data[i];
168 		switch((tag_val & PSYCHO_IOMMU_TAG_ERRSTS) >> 23UL) {
169 		case 0:
170 			type_str = "Protection Error";
171 			break;
172 		case 1:
173 			type_str = "Invalid Error";
174 			break;
175 		case 2:
176 			type_str = "TimeOut Error";
177 			break;
178 		case 3:
179 		default:
180 			type_str = "ECC Error";
181 			break;
182 		}
183 
184 		printk(KERN_ERR "%s: IOMMU TAG(%d)[error(%s) wr(%d) "
185 		       "str(%d) sz(%dK) vpg(%08llx)]\n",
186 		       pbm->name, i, type_str,
187 		       ((tag_val & PSYCHO_IOMMU_TAG_WRITE) ? 1 : 0),
188 		       ((tag_val & PSYCHO_IOMMU_TAG_STREAM) ? 1 : 0),
189 		       ((tag_val & PSYCHO_IOMMU_TAG_SIZE) ? 64 : 8),
190 		       (tag_val & PSYCHO_IOMMU_TAG_VPAGE) << IOMMU_PAGE_SHIFT);
191 		printk(KERN_ERR "%s: IOMMU DATA(%d)[valid(%d) cache(%d) "
192 		       "ppg(%016llx)]\n",
193 		       pbm->name, i,
194 		       ((data_val & PSYCHO_IOMMU_DATA_VALID) ? 1 : 0),
195 		       ((data_val & PSYCHO_IOMMU_DATA_CACHE) ? 1 : 0),
196 		       (data_val & PSYCHO_IOMMU_DATA_PPAGE) << IOMMU_PAGE_SHIFT);
197 	}
198 }
199 
200 #define  PSYCHO_IOMMU_CTRL_XLTESTAT	0x0000000006000000UL
201 #define  PSYCHO_IOMMU_CTRL_XLTEERR	0x0000000001000000UL
202 
203 void psycho_check_iommu_error(struct pci_pbm_info *pbm,
204 			      unsigned long afsr,
205 			      unsigned long afar,
206 			      enum psycho_error_type type)
207 {
208 	u64 control, iommu_tag[16], iommu_data[16];
209 	struct iommu *iommu = pbm->iommu;
210 	unsigned long flags;
211 
212 	spin_lock_irqsave(&iommu->lock, flags);
213 	control = upa_readq(iommu->iommu_control);
214 	if (control & PSYCHO_IOMMU_CTRL_XLTEERR) {
215 		const char *type_str;
216 
217 		control &= ~PSYCHO_IOMMU_CTRL_XLTEERR;
218 		upa_writeq(control, iommu->iommu_control);
219 
220 		switch ((control & PSYCHO_IOMMU_CTRL_XLTESTAT) >> 25UL) {
221 		case 0:
222 			type_str = "Protection Error";
223 			break;
224 		case 1:
225 			type_str = "Invalid Error";
226 			break;
227 		case 2:
228 			type_str = "TimeOut Error";
229 			break;
230 		case 3:
231 		default:
232 			type_str = "ECC Error";
233 			break;
234 		}
235 		printk(KERN_ERR "%s: IOMMU Error, type[%s]\n",
236 		       pbm->name, type_str);
237 
238 		/* It is very possible for another DVMA to occur while
239 		 * we do this probe, and corrupt the system further.
240 		 * But we are so screwed at this point that we are
241 		 * likely to crash hard anyways, so get as much
242 		 * diagnostic information to the console as we can.
243 		 */
244 		psycho_record_iommu_tags_and_data(pbm, iommu_tag, iommu_data);
245 		psycho_dump_iommu_tags_and_data(pbm, iommu_tag, iommu_data);
246 	}
247 	psycho_check_stc_error(pbm);
248 	spin_unlock_irqrestore(&iommu->lock, flags);
249 }
250 
251 #define  PSYCHO_PCICTRL_SBH_ERR	 0x0000000800000000UL
252 #define  PSYCHO_PCICTRL_SERR	 0x0000000400000000UL
253 
254 static irqreturn_t psycho_pcierr_intr_other(struct pci_pbm_info *pbm)
255 {
256 	irqreturn_t ret = IRQ_NONE;
257 	u64 csr, csr_error_bits;
258 	u16 stat, *addr;
259 
260 	csr = upa_readq(pbm->pci_csr);
261 	csr_error_bits = csr & (PSYCHO_PCICTRL_SBH_ERR | PSYCHO_PCICTRL_SERR);
262 	if (csr_error_bits) {
263 		/* Clear the errors.  */
264 		upa_writeq(csr, pbm->pci_csr);
265 
266 		/* Log 'em.  */
267 		if (csr_error_bits & PSYCHO_PCICTRL_SBH_ERR)
268 			printk(KERN_ERR "%s: PCI streaming byte hole "
269 			       "error asserted.\n", pbm->name);
270 		if (csr_error_bits & PSYCHO_PCICTRL_SERR)
271 			printk(KERN_ERR "%s: PCI SERR signal asserted.\n",
272 			       pbm->name);
273 		ret = IRQ_HANDLED;
274 	}
275 	addr = psycho_pci_config_mkaddr(pbm, pbm->pci_first_busno,
276 					0, PCI_STATUS);
277 	pci_config_read16(addr, &stat);
278 	if (stat & (PCI_STATUS_PARITY |
279 		    PCI_STATUS_SIG_TARGET_ABORT |
280 		    PCI_STATUS_REC_TARGET_ABORT |
281 		    PCI_STATUS_REC_MASTER_ABORT |
282 		    PCI_STATUS_SIG_SYSTEM_ERROR)) {
283 		printk(KERN_ERR "%s: PCI bus error, PCI_STATUS[%04x]\n",
284 		       pbm->name, stat);
285 		pci_config_write16(addr, 0xffff);
286 		ret = IRQ_HANDLED;
287 	}
288 	return ret;
289 }
290 
291 #define  PSYCHO_PCIAFSR_PMA	0x8000000000000000ULL
292 #define  PSYCHO_PCIAFSR_PTA	0x4000000000000000ULL
293 #define  PSYCHO_PCIAFSR_PRTRY	0x2000000000000000ULL
294 #define  PSYCHO_PCIAFSR_PPERR	0x1000000000000000ULL
295 #define  PSYCHO_PCIAFSR_SMA	0x0800000000000000ULL
296 #define  PSYCHO_PCIAFSR_STA	0x0400000000000000ULL
297 #define  PSYCHO_PCIAFSR_SRTRY	0x0200000000000000ULL
298 #define  PSYCHO_PCIAFSR_SPERR	0x0100000000000000ULL
299 #define  PSYCHO_PCIAFSR_RESV1	0x00ff000000000000ULL
300 #define  PSYCHO_PCIAFSR_BMSK	0x0000ffff00000000ULL
301 #define  PSYCHO_PCIAFSR_BLK	0x0000000080000000ULL
302 #define  PSYCHO_PCIAFSR_RESV2	0x0000000040000000ULL
303 #define  PSYCHO_PCIAFSR_MID	0x000000003e000000ULL
304 #define  PSYCHO_PCIAFSR_RESV3	0x0000000001ffffffULL
305 
306 irqreturn_t psycho_pcierr_intr(int irq, void *dev_id)
307 {
308 	struct pci_pbm_info *pbm = dev_id;
309 	u64 afsr, afar, error_bits;
310 	int reported;
311 
312 	afsr = upa_readq(pbm->pci_afsr);
313 	afar = upa_readq(pbm->pci_afar);
314 	error_bits = afsr &
315 		(PSYCHO_PCIAFSR_PMA | PSYCHO_PCIAFSR_PTA |
316 		 PSYCHO_PCIAFSR_PRTRY | PSYCHO_PCIAFSR_PPERR |
317 		 PSYCHO_PCIAFSR_SMA | PSYCHO_PCIAFSR_STA |
318 		 PSYCHO_PCIAFSR_SRTRY | PSYCHO_PCIAFSR_SPERR);
319 	if (!error_bits)
320 		return psycho_pcierr_intr_other(pbm);
321 	upa_writeq(error_bits, pbm->pci_afsr);
322 	printk(KERN_ERR "%s: PCI Error, primary error type[%s]\n",
323 	       pbm->name,
324 	       (((error_bits & PSYCHO_PCIAFSR_PMA) ?
325 		 "Master Abort" :
326 		 ((error_bits & PSYCHO_PCIAFSR_PTA) ?
327 		  "Target Abort" :
328 		  ((error_bits & PSYCHO_PCIAFSR_PRTRY) ?
329 		   "Excessive Retries" :
330 		   ((error_bits & PSYCHO_PCIAFSR_PPERR) ?
331 		    "Parity Error" : "???"))))));
332 	printk(KERN_ERR "%s: bytemask[%04llx] UPA_MID[%02llx] was_block(%d)\n",
333 	       pbm->name,
334 	       (afsr & PSYCHO_PCIAFSR_BMSK) >> 32UL,
335 	       (afsr & PSYCHO_PCIAFSR_MID) >> 25UL,
336 	       (afsr & PSYCHO_PCIAFSR_BLK) ? 1 : 0);
337 	printk(KERN_ERR "%s: PCI AFAR [%016llx]\n", pbm->name, afar);
338 	printk(KERN_ERR "%s: PCI Secondary errors [", pbm->name);
339 	reported = 0;
340 	if (afsr & PSYCHO_PCIAFSR_SMA) {
341 		reported++;
342 		printk("(Master Abort)");
343 	}
344 	if (afsr & PSYCHO_PCIAFSR_STA) {
345 		reported++;
346 		printk("(Target Abort)");
347 	}
348 	if (afsr & PSYCHO_PCIAFSR_SRTRY) {
349 		reported++;
350 		printk("(Excessive Retries)");
351 	}
352 	if (afsr & PSYCHO_PCIAFSR_SPERR) {
353 		reported++;
354 		printk("(Parity Error)");
355 	}
356 	if (!reported)
357 		printk("(none)");
358 	printk("]\n");
359 
360 	if (error_bits & (PSYCHO_PCIAFSR_PTA | PSYCHO_PCIAFSR_STA)) {
361 		psycho_check_iommu_error(pbm, afsr, afar, PCI_ERR);
362 		pci_scan_for_target_abort(pbm, pbm->pci_bus);
363 	}
364 	if (error_bits & (PSYCHO_PCIAFSR_PMA | PSYCHO_PCIAFSR_SMA))
365 		pci_scan_for_master_abort(pbm, pbm->pci_bus);
366 
367 	if (error_bits & (PSYCHO_PCIAFSR_PPERR | PSYCHO_PCIAFSR_SPERR))
368 		pci_scan_for_parity_error(pbm, pbm->pci_bus);
369 
370 	return IRQ_HANDLED;
371 }
372 
373 static void psycho_iommu_flush(struct pci_pbm_info *pbm)
374 {
375 	int i;
376 
377 	for (i = 0; i < 16; i++) {
378 		unsigned long off = i * 8;
379 
380 		upa_writeq(0, pbm->controller_regs + PSYCHO_IOMMU_TAG + off);
381 		upa_writeq(0, pbm->controller_regs + PSYCHO_IOMMU_DATA + off);
382 	}
383 }
384 
385 #define PSYCHO_IOMMU_CONTROL		0x0200UL
386 #define  PSYCHO_IOMMU_CTRL_TSBSZ	0x0000000000070000UL
387 #define  PSYCHO_IOMMU_TSBSZ_1K      	0x0000000000000000UL
388 #define  PSYCHO_IOMMU_TSBSZ_2K      	0x0000000000010000UL
389 #define  PSYCHO_IOMMU_TSBSZ_4K      	0x0000000000020000UL
390 #define  PSYCHO_IOMMU_TSBSZ_8K      	0x0000000000030000UL
391 #define  PSYCHO_IOMMU_TSBSZ_16K     	0x0000000000040000UL
392 #define  PSYCHO_IOMMU_TSBSZ_32K     	0x0000000000050000UL
393 #define  PSYCHO_IOMMU_TSBSZ_64K     	0x0000000000060000UL
394 #define  PSYCHO_IOMMU_TSBSZ_128K    	0x0000000000070000UL
395 #define  PSYCHO_IOMMU_CTRL_TBWSZ    	0x0000000000000004UL
396 #define  PSYCHO_IOMMU_CTRL_DENAB    	0x0000000000000002UL
397 #define  PSYCHO_IOMMU_CTRL_ENAB     	0x0000000000000001UL
398 #define PSYCHO_IOMMU_FLUSH		0x0210UL
399 #define PSYCHO_IOMMU_TSBBASE		0x0208UL
400 
401 int psycho_iommu_init(struct pci_pbm_info *pbm, int tsbsize,
402 		      u32 dvma_offset, u32 dma_mask,
403 		      unsigned long write_complete_offset)
404 {
405 	struct iommu *iommu = pbm->iommu;
406 	u64 control;
407 	int err;
408 
409 	iommu->iommu_control  = pbm->controller_regs + PSYCHO_IOMMU_CONTROL;
410 	iommu->iommu_tsbbase  = pbm->controller_regs + PSYCHO_IOMMU_TSBBASE;
411 	iommu->iommu_flush    = pbm->controller_regs + PSYCHO_IOMMU_FLUSH;
412 	iommu->iommu_tags     = pbm->controller_regs + PSYCHO_IOMMU_TAG;
413 	iommu->write_complete_reg = (pbm->controller_regs +
414 				     write_complete_offset);
415 
416 	iommu->iommu_ctxflush = 0;
417 
418 	control = upa_readq(iommu->iommu_control);
419 	control |= PSYCHO_IOMMU_CTRL_DENAB;
420 	upa_writeq(control, iommu->iommu_control);
421 
422 	psycho_iommu_flush(pbm);
423 
424 	/* Leave diag mode enabled for full-flushing done in pci_iommu.c */
425 	err = iommu_table_init(iommu, tsbsize * 1024 * 8,
426 			       dvma_offset, dma_mask, pbm->numa_node);
427 	if (err)
428 		return err;
429 
430 	upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase);
431 
432 	control = upa_readq(iommu->iommu_control);
433 	control &= ~(PSYCHO_IOMMU_CTRL_TSBSZ | PSYCHO_IOMMU_CTRL_TBWSZ);
434 	control |= PSYCHO_IOMMU_CTRL_ENAB;
435 
436 	switch (tsbsize) {
437 	case 64:
438 		control |= PSYCHO_IOMMU_TSBSZ_64K;
439 		break;
440 	case 128:
441 		control |= PSYCHO_IOMMU_TSBSZ_128K;
442 		break;
443 	default:
444 		return -EINVAL;
445 	}
446 
447 	upa_writeq(control, iommu->iommu_control);
448 
449 	return 0;
450 
451 }
452 
453 void psycho_pbm_init_common(struct pci_pbm_info *pbm, struct platform_device *op,
454 			    const char *chip_name, int chip_type)
455 {
456 	struct device_node *dp = op->dev.of_node;
457 
458 	pbm->name = dp->full_name;
459 	pbm->numa_node = NUMA_NO_NODE;
460 	pbm->chip_type = chip_type;
461 	pbm->chip_version = of_getintprop_default(dp, "version#", 0);
462 	pbm->chip_revision = of_getintprop_default(dp, "module-revision#", 0);
463 	pbm->op = op;
464 	pbm->pci_ops = &sun4u_pci_ops;
465 	pbm->config_space_reg_bits = 8;
466 	pbm->index = pci_num_pbms++;
467 	pci_get_pbm_props(pbm);
468 	pci_determine_mem_io_space(pbm);
469 
470 	printk(KERN_INFO "%s: %s PCI Bus Module ver[%x:%x]\n",
471 	       pbm->name, chip_name,
472 	       pbm->chip_version, pbm->chip_revision);
473 }
474