1 /* psycho_common.c: Code common to PSYCHO and derivative PCI controllers.
2  *
3  * Copyright (C) 2008 David S. Miller <davem@davemloft.net>
4  */
5 #include <linux/kernel.h>
6 #include <linux/interrupt.h>
7 
8 #include <asm/upa.h>
9 
10 #include "pci_impl.h"
11 #include "iommu_common.h"
12 #include "psycho_common.h"
13 
14 #define  PSYCHO_STRBUF_CTRL_DENAB	0x0000000000000002ULL
15 #define  PSYCHO_STCERR_WRITE		0x0000000000000002ULL
16 #define  PSYCHO_STCERR_READ		0x0000000000000001ULL
17 #define  PSYCHO_STCTAG_PPN		0x0fffffff00000000ULL
18 #define  PSYCHO_STCTAG_VPN		0x00000000ffffe000ULL
19 #define  PSYCHO_STCTAG_VALID		0x0000000000000002ULL
20 #define  PSYCHO_STCTAG_WRITE		0x0000000000000001ULL
21 #define  PSYCHO_STCLINE_LINDX		0x0000000001e00000ULL
22 #define  PSYCHO_STCLINE_SPTR		0x00000000001f8000ULL
23 #define  PSYCHO_STCLINE_LADDR		0x0000000000007f00ULL
24 #define  PSYCHO_STCLINE_EPTR		0x00000000000000fcULL
25 #define  PSYCHO_STCLINE_VALID		0x0000000000000002ULL
26 #define  PSYCHO_STCLINE_FOFN		0x0000000000000001ULL
27 
28 static DEFINE_SPINLOCK(stc_buf_lock);
29 static unsigned long stc_error_buf[128];
30 static unsigned long stc_tag_buf[16];
31 static unsigned long stc_line_buf[16];
32 
33 static void psycho_check_stc_error(struct pci_pbm_info *pbm)
34 {
35 	unsigned long err_base, tag_base, line_base;
36 	struct strbuf *strbuf = &pbm->stc;
37 	u64 control;
38 	int i;
39 
40 	if (!strbuf->strbuf_control)
41 		return;
42 
43 	err_base = strbuf->strbuf_err_stat;
44 	tag_base = strbuf->strbuf_tag_diag;
45 	line_base = strbuf->strbuf_line_diag;
46 
47 	spin_lock(&stc_buf_lock);
48 
49 	/* This is __REALLY__ dangerous.  When we put the streaming
50 	 * buffer into diagnostic mode to probe it's tags and error
51 	 * status, we _must_ clear all of the line tag valid bits
52 	 * before re-enabling the streaming buffer.  If any dirty data
53 	 * lives in the STC when we do this, we will end up
54 	 * invalidating it before it has a chance to reach main
55 	 * memory.
56 	 */
57 	control = upa_readq(strbuf->strbuf_control);
58 	upa_writeq(control | PSYCHO_STRBUF_CTRL_DENAB, strbuf->strbuf_control);
59 	for (i = 0; i < 128; i++) {
60 		u64 val;
61 
62 		val = upa_readq(err_base + (i * 8UL));
63 		upa_writeq(0UL, err_base + (i * 8UL));
64 		stc_error_buf[i] = val;
65 	}
66 	for (i = 0; i < 16; i++) {
67 		stc_tag_buf[i] = upa_readq(tag_base + (i * 8UL));
68 		stc_line_buf[i] = upa_readq(line_base + (i * 8UL));
69 		upa_writeq(0UL, tag_base + (i * 8UL));
70 		upa_writeq(0UL, line_base + (i * 8UL));
71 	}
72 
73 	/* OK, state is logged, exit diagnostic mode. */
74 	upa_writeq(control, strbuf->strbuf_control);
75 
76 	for (i = 0; i < 16; i++) {
77 		int j, saw_error, first, last;
78 
79 		saw_error = 0;
80 		first = i * 8;
81 		last = first + 8;
82 		for (j = first; j < last; j++) {
83 			u64 errval = stc_error_buf[j];
84 			if (errval != 0) {
85 				saw_error++;
86 				printk(KERN_ERR "%s: STC_ERR(%d)[wr(%d)"
87 				       "rd(%d)]\n",
88 				       pbm->name,
89 				       j,
90 				       (errval & PSYCHO_STCERR_WRITE) ? 1 : 0,
91 				       (errval & PSYCHO_STCERR_READ) ? 1 : 0);
92 			}
93 		}
94 		if (saw_error != 0) {
95 			u64 tagval = stc_tag_buf[i];
96 			u64 lineval = stc_line_buf[i];
97 			printk(KERN_ERR "%s: STC_TAG(%d)[PA(%016llx)VA(%08llx)"
98 			       "V(%d)W(%d)]\n",
99 			       pbm->name,
100 			       i,
101 			       ((tagval & PSYCHO_STCTAG_PPN) >> 19UL),
102 			       (tagval & PSYCHO_STCTAG_VPN),
103 			       ((tagval & PSYCHO_STCTAG_VALID) ? 1 : 0),
104 			       ((tagval & PSYCHO_STCTAG_WRITE) ? 1 : 0));
105 			printk(KERN_ERR "%s: STC_LINE(%d)[LIDX(%llx)SP(%llx)"
106 			       "LADDR(%llx)EP(%llx)V(%d)FOFN(%d)]\n",
107 			       pbm->name,
108 			       i,
109 			       ((lineval & PSYCHO_STCLINE_LINDX) >> 21UL),
110 			       ((lineval & PSYCHO_STCLINE_SPTR) >> 15UL),
111 			       ((lineval & PSYCHO_STCLINE_LADDR) >> 8UL),
112 			       ((lineval & PSYCHO_STCLINE_EPTR) >> 2UL),
113 			       ((lineval & PSYCHO_STCLINE_VALID) ? 1 : 0),
114 			       ((lineval & PSYCHO_STCLINE_FOFN) ? 1 : 0));
115 		}
116 	}
117 
118 	spin_unlock(&stc_buf_lock);
119 }
120 
121 #define PSYCHO_IOMMU_TAG		0xa580UL
122 #define PSYCHO_IOMMU_DATA		0xa600UL
123 
124 static void psycho_record_iommu_tags_and_data(struct pci_pbm_info *pbm,
125 					      u64 *tag, u64 *data)
126 {
127 	int i;
128 
129 	for (i = 0; i < 16; i++) {
130 		unsigned long base = pbm->controller_regs;
131 		unsigned long off = i * 8UL;
132 
133 		tag[i] = upa_readq(base + PSYCHO_IOMMU_TAG+off);
134 		data[i] = upa_readq(base + PSYCHO_IOMMU_DATA+off);
135 
136 		/* Now clear out the entry. */
137 		upa_writeq(0, base + PSYCHO_IOMMU_TAG + off);
138 		upa_writeq(0, base + PSYCHO_IOMMU_DATA + off);
139 	}
140 }
141 
142 #define  PSYCHO_IOMMU_TAG_ERRSTS (0x3UL << 23UL)
143 #define  PSYCHO_IOMMU_TAG_ERR	 (0x1UL << 22UL)
144 #define  PSYCHO_IOMMU_TAG_WRITE	 (0x1UL << 21UL)
145 #define  PSYCHO_IOMMU_TAG_STREAM (0x1UL << 20UL)
146 #define  PSYCHO_IOMMU_TAG_SIZE	 (0x1UL << 19UL)
147 #define  PSYCHO_IOMMU_TAG_VPAGE	 0x7ffffULL
148 #define  PSYCHO_IOMMU_DATA_VALID (1UL << 30UL)
149 #define  PSYCHO_IOMMU_DATA_CACHE (1UL << 28UL)
150 #define  PSYCHO_IOMMU_DATA_PPAGE 0xfffffffULL
151 
152 static void psycho_dump_iommu_tags_and_data(struct pci_pbm_info *pbm,
153 					    u64 *tag, u64 *data)
154 {
155 	int i;
156 
157 	for (i = 0; i < 16; i++) {
158 		u64 tag_val, data_val;
159 		const char *type_str;
160 		tag_val = tag[i];
161 		if (!(tag_val & PSYCHO_IOMMU_TAG_ERR))
162 			continue;
163 
164 		data_val = data[i];
165 		switch((tag_val & PSYCHO_IOMMU_TAG_ERRSTS) >> 23UL) {
166 		case 0:
167 			type_str = "Protection Error";
168 			break;
169 		case 1:
170 			type_str = "Invalid Error";
171 			break;
172 		case 2:
173 			type_str = "TimeOut Error";
174 			break;
175 		case 3:
176 		default:
177 			type_str = "ECC Error";
178 			break;
179 		}
180 
181 		printk(KERN_ERR "%s: IOMMU TAG(%d)[error(%s) wr(%d) "
182 		       "str(%d) sz(%dK) vpg(%08llx)]\n",
183 		       pbm->name, i, type_str,
184 		       ((tag_val & PSYCHO_IOMMU_TAG_WRITE) ? 1 : 0),
185 		       ((tag_val & PSYCHO_IOMMU_TAG_STREAM) ? 1 : 0),
186 		       ((tag_val & PSYCHO_IOMMU_TAG_SIZE) ? 64 : 8),
187 		       (tag_val & PSYCHO_IOMMU_TAG_VPAGE) << IOMMU_PAGE_SHIFT);
188 		printk(KERN_ERR "%s: IOMMU DATA(%d)[valid(%d) cache(%d) "
189 		       "ppg(%016llx)]\n",
190 		       pbm->name, i,
191 		       ((data_val & PSYCHO_IOMMU_DATA_VALID) ? 1 : 0),
192 		       ((data_val & PSYCHO_IOMMU_DATA_CACHE) ? 1 : 0),
193 		       (data_val & PSYCHO_IOMMU_DATA_PPAGE) << IOMMU_PAGE_SHIFT);
194 	}
195 }
196 
197 #define  PSYCHO_IOMMU_CTRL_XLTESTAT	0x0000000006000000UL
198 #define  PSYCHO_IOMMU_CTRL_XLTEERR	0x0000000001000000UL
199 
200 void psycho_check_iommu_error(struct pci_pbm_info *pbm,
201 			      unsigned long afsr,
202 			      unsigned long afar,
203 			      enum psycho_error_type type)
204 {
205 	u64 control, iommu_tag[16], iommu_data[16];
206 	struct iommu *iommu = pbm->iommu;
207 	unsigned long flags;
208 
209 	spin_lock_irqsave(&iommu->lock, flags);
210 	control = upa_readq(iommu->iommu_control);
211 	if (control & PSYCHO_IOMMU_CTRL_XLTEERR) {
212 		const char *type_str;
213 
214 		control &= ~PSYCHO_IOMMU_CTRL_XLTEERR;
215 		upa_writeq(control, iommu->iommu_control);
216 
217 		switch ((control & PSYCHO_IOMMU_CTRL_XLTESTAT) >> 25UL) {
218 		case 0:
219 			type_str = "Protection Error";
220 			break;
221 		case 1:
222 			type_str = "Invalid Error";
223 			break;
224 		case 2:
225 			type_str = "TimeOut Error";
226 			break;
227 		case 3:
228 		default:
229 			type_str = "ECC Error";
230 			break;
231 		}
232 		printk(KERN_ERR "%s: IOMMU Error, type[%s]\n",
233 		       pbm->name, type_str);
234 
235 		/* It is very possible for another DVMA to occur while
236 		 * we do this probe, and corrupt the system further.
237 		 * But we are so screwed at this point that we are
238 		 * likely to crash hard anyways, so get as much
239 		 * diagnostic information to the console as we can.
240 		 */
241 		psycho_record_iommu_tags_and_data(pbm, iommu_tag, iommu_data);
242 		psycho_dump_iommu_tags_and_data(pbm, iommu_tag, iommu_data);
243 	}
244 	psycho_check_stc_error(pbm);
245 	spin_unlock_irqrestore(&iommu->lock, flags);
246 }
247 
248 #define  PSYCHO_PCICTRL_SBH_ERR	 0x0000000800000000UL
249 #define  PSYCHO_PCICTRL_SERR	 0x0000000400000000UL
250 
251 static irqreturn_t psycho_pcierr_intr_other(struct pci_pbm_info *pbm)
252 {
253 	irqreturn_t ret = IRQ_NONE;
254 	u64 csr, csr_error_bits;
255 	u16 stat, *addr;
256 
257 	csr = upa_readq(pbm->pci_csr);
258 	csr_error_bits = csr & (PSYCHO_PCICTRL_SBH_ERR | PSYCHO_PCICTRL_SERR);
259 	if (csr_error_bits) {
260 		/* Clear the errors.  */
261 		upa_writeq(csr, pbm->pci_csr);
262 
263 		/* Log 'em.  */
264 		if (csr_error_bits & PSYCHO_PCICTRL_SBH_ERR)
265 			printk(KERN_ERR "%s: PCI streaming byte hole "
266 			       "error asserted.\n", pbm->name);
267 		if (csr_error_bits & PSYCHO_PCICTRL_SERR)
268 			printk(KERN_ERR "%s: PCI SERR signal asserted.\n",
269 			       pbm->name);
270 		ret = IRQ_HANDLED;
271 	}
272 	addr = psycho_pci_config_mkaddr(pbm, pbm->pci_first_busno,
273 					0, PCI_STATUS);
274 	pci_config_read16(addr, &stat);
275 	if (stat & (PCI_STATUS_PARITY |
276 		    PCI_STATUS_SIG_TARGET_ABORT |
277 		    PCI_STATUS_REC_TARGET_ABORT |
278 		    PCI_STATUS_REC_MASTER_ABORT |
279 		    PCI_STATUS_SIG_SYSTEM_ERROR)) {
280 		printk(KERN_ERR "%s: PCI bus error, PCI_STATUS[%04x]\n",
281 		       pbm->name, stat);
282 		pci_config_write16(addr, 0xffff);
283 		ret = IRQ_HANDLED;
284 	}
285 	return ret;
286 }
287 
288 #define  PSYCHO_PCIAFSR_PMA	0x8000000000000000ULL
289 #define  PSYCHO_PCIAFSR_PTA	0x4000000000000000ULL
290 #define  PSYCHO_PCIAFSR_PRTRY	0x2000000000000000ULL
291 #define  PSYCHO_PCIAFSR_PPERR	0x1000000000000000ULL
292 #define  PSYCHO_PCIAFSR_SMA	0x0800000000000000ULL
293 #define  PSYCHO_PCIAFSR_STA	0x0400000000000000ULL
294 #define  PSYCHO_PCIAFSR_SRTRY	0x0200000000000000ULL
295 #define  PSYCHO_PCIAFSR_SPERR	0x0100000000000000ULL
296 #define  PSYCHO_PCIAFSR_RESV1	0x00ff000000000000ULL
297 #define  PSYCHO_PCIAFSR_BMSK	0x0000ffff00000000ULL
298 #define  PSYCHO_PCIAFSR_BLK	0x0000000080000000ULL
299 #define  PSYCHO_PCIAFSR_RESV2	0x0000000040000000ULL
300 #define  PSYCHO_PCIAFSR_MID	0x000000003e000000ULL
301 #define  PSYCHO_PCIAFSR_RESV3	0x0000000001ffffffULL
302 
303 irqreturn_t psycho_pcierr_intr(int irq, void *dev_id)
304 {
305 	struct pci_pbm_info *pbm = dev_id;
306 	u64 afsr, afar, error_bits;
307 	int reported;
308 
309 	afsr = upa_readq(pbm->pci_afsr);
310 	afar = upa_readq(pbm->pci_afar);
311 	error_bits = afsr &
312 		(PSYCHO_PCIAFSR_PMA | PSYCHO_PCIAFSR_PTA |
313 		 PSYCHO_PCIAFSR_PRTRY | PSYCHO_PCIAFSR_PPERR |
314 		 PSYCHO_PCIAFSR_SMA | PSYCHO_PCIAFSR_STA |
315 		 PSYCHO_PCIAFSR_SRTRY | PSYCHO_PCIAFSR_SPERR);
316 	if (!error_bits)
317 		return psycho_pcierr_intr_other(pbm);
318 	upa_writeq(error_bits, pbm->pci_afsr);
319 	printk(KERN_ERR "%s: PCI Error, primary error type[%s]\n",
320 	       pbm->name,
321 	       (((error_bits & PSYCHO_PCIAFSR_PMA) ?
322 		 "Master Abort" :
323 		 ((error_bits & PSYCHO_PCIAFSR_PTA) ?
324 		  "Target Abort" :
325 		  ((error_bits & PSYCHO_PCIAFSR_PRTRY) ?
326 		   "Excessive Retries" :
327 		   ((error_bits & PSYCHO_PCIAFSR_PPERR) ?
328 		    "Parity Error" : "???"))))));
329 	printk(KERN_ERR "%s: bytemask[%04llx] UPA_MID[%02llx] was_block(%d)\n",
330 	       pbm->name,
331 	       (afsr & PSYCHO_PCIAFSR_BMSK) >> 32UL,
332 	       (afsr & PSYCHO_PCIAFSR_MID) >> 25UL,
333 	       (afsr & PSYCHO_PCIAFSR_BLK) ? 1 : 0);
334 	printk(KERN_ERR "%s: PCI AFAR [%016llx]\n", pbm->name, afar);
335 	printk(KERN_ERR "%s: PCI Secondary errors [", pbm->name);
336 	reported = 0;
337 	if (afsr & PSYCHO_PCIAFSR_SMA) {
338 		reported++;
339 		printk("(Master Abort)");
340 	}
341 	if (afsr & PSYCHO_PCIAFSR_STA) {
342 		reported++;
343 		printk("(Target Abort)");
344 	}
345 	if (afsr & PSYCHO_PCIAFSR_SRTRY) {
346 		reported++;
347 		printk("(Excessive Retries)");
348 	}
349 	if (afsr & PSYCHO_PCIAFSR_SPERR) {
350 		reported++;
351 		printk("(Parity Error)");
352 	}
353 	if (!reported)
354 		printk("(none)");
355 	printk("]\n");
356 
357 	if (error_bits & (PSYCHO_PCIAFSR_PTA | PSYCHO_PCIAFSR_STA)) {
358 		psycho_check_iommu_error(pbm, afsr, afar, PCI_ERR);
359 		pci_scan_for_target_abort(pbm, pbm->pci_bus);
360 	}
361 	if (error_bits & (PSYCHO_PCIAFSR_PMA | PSYCHO_PCIAFSR_SMA))
362 		pci_scan_for_master_abort(pbm, pbm->pci_bus);
363 
364 	if (error_bits & (PSYCHO_PCIAFSR_PPERR | PSYCHO_PCIAFSR_SPERR))
365 		pci_scan_for_parity_error(pbm, pbm->pci_bus);
366 
367 	return IRQ_HANDLED;
368 }
369 
370 static void psycho_iommu_flush(struct pci_pbm_info *pbm)
371 {
372 	int i;
373 
374 	for (i = 0; i < 16; i++) {
375 		unsigned long off = i * 8;
376 
377 		upa_writeq(0, pbm->controller_regs + PSYCHO_IOMMU_TAG + off);
378 		upa_writeq(0, pbm->controller_regs + PSYCHO_IOMMU_DATA + off);
379 	}
380 }
381 
382 #define PSYCHO_IOMMU_CONTROL		0x0200UL
383 #define  PSYCHO_IOMMU_CTRL_TSBSZ	0x0000000000070000UL
384 #define  PSYCHO_IOMMU_TSBSZ_1K      	0x0000000000000000UL
385 #define  PSYCHO_IOMMU_TSBSZ_2K      	0x0000000000010000UL
386 #define  PSYCHO_IOMMU_TSBSZ_4K      	0x0000000000020000UL
387 #define  PSYCHO_IOMMU_TSBSZ_8K      	0x0000000000030000UL
388 #define  PSYCHO_IOMMU_TSBSZ_16K     	0x0000000000040000UL
389 #define  PSYCHO_IOMMU_TSBSZ_32K     	0x0000000000050000UL
390 #define  PSYCHO_IOMMU_TSBSZ_64K     	0x0000000000060000UL
391 #define  PSYCHO_IOMMU_TSBSZ_128K    	0x0000000000070000UL
392 #define  PSYCHO_IOMMU_CTRL_TBWSZ    	0x0000000000000004UL
393 #define  PSYCHO_IOMMU_CTRL_DENAB    	0x0000000000000002UL
394 #define  PSYCHO_IOMMU_CTRL_ENAB     	0x0000000000000001UL
395 #define PSYCHO_IOMMU_FLUSH		0x0210UL
396 #define PSYCHO_IOMMU_TSBBASE		0x0208UL
397 
398 int psycho_iommu_init(struct pci_pbm_info *pbm, int tsbsize,
399 		      u32 dvma_offset, u32 dma_mask,
400 		      unsigned long write_complete_offset)
401 {
402 	struct iommu *iommu = pbm->iommu;
403 	u64 control;
404 	int err;
405 
406 	iommu->iommu_control  = pbm->controller_regs + PSYCHO_IOMMU_CONTROL;
407 	iommu->iommu_tsbbase  = pbm->controller_regs + PSYCHO_IOMMU_TSBBASE;
408 	iommu->iommu_flush    = pbm->controller_regs + PSYCHO_IOMMU_FLUSH;
409 	iommu->iommu_tags     = pbm->controller_regs + PSYCHO_IOMMU_TAG;
410 	iommu->write_complete_reg = (pbm->controller_regs +
411 				     write_complete_offset);
412 
413 	iommu->iommu_ctxflush = 0;
414 
415 	control = upa_readq(iommu->iommu_control);
416 	control |= PSYCHO_IOMMU_CTRL_DENAB;
417 	upa_writeq(control, iommu->iommu_control);
418 
419 	psycho_iommu_flush(pbm);
420 
421 	/* Leave diag mode enabled for full-flushing done in pci_iommu.c */
422 	err = iommu_table_init(iommu, tsbsize * 1024 * 8,
423 			       dvma_offset, dma_mask, pbm->numa_node);
424 	if (err)
425 		return err;
426 
427 	upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase);
428 
429 	control = upa_readq(iommu->iommu_control);
430 	control &= ~(PSYCHO_IOMMU_CTRL_TSBSZ | PSYCHO_IOMMU_CTRL_TBWSZ);
431 	control |= PSYCHO_IOMMU_CTRL_ENAB;
432 
433 	switch (tsbsize) {
434 	case 64:
435 		control |= PSYCHO_IOMMU_TSBSZ_64K;
436 		break;
437 	case 128:
438 		control |= PSYCHO_IOMMU_TSBSZ_128K;
439 		break;
440 	default:
441 		return -EINVAL;
442 	}
443 
444 	upa_writeq(control, iommu->iommu_control);
445 
446 	return 0;
447 
448 }
449 
450 void psycho_pbm_init_common(struct pci_pbm_info *pbm, struct platform_device *op,
451 			    const char *chip_name, int chip_type)
452 {
453 	struct device_node *dp = op->dev.of_node;
454 
455 	pbm->name = dp->full_name;
456 	pbm->numa_node = -1;
457 	pbm->chip_type = chip_type;
458 	pbm->chip_version = of_getintprop_default(dp, "version#", 0);
459 	pbm->chip_revision = of_getintprop_default(dp, "module-revision#", 0);
460 	pbm->op = op;
461 	pbm->pci_ops = &sun4u_pci_ops;
462 	pbm->config_space_reg_bits = 8;
463 	pbm->index = pci_num_pbms++;
464 	pci_get_pbm_props(pbm);
465 	pci_determine_mem_io_space(pbm);
466 
467 	printk(KERN_INFO "%s: %s PCI Bus Module ver[%x:%x]\n",
468 	       pbm->name, chip_name,
469 	       pbm->chip_version, pbm->chip_revision);
470 }
471