xref: /openbmc/linux/arch/mips/mm/cerr-sb1.c (revision 1a59d1b8)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2001,2002,2003 Broadcom Corporation
4  */
5 #include <linux/sched.h>
6 #include <asm/mipsregs.h>
7 #include <asm/sibyte/sb1250.h>
8 #include <asm/sibyte/sb1250_regs.h>
9 
10 #if !defined(CONFIG_SIBYTE_BUS_WATCHER) || defined(CONFIG_SIBYTE_BW_TRACE)
11 #include <asm/io.h>
12 #include <asm/sibyte/sb1250_scd.h>
13 #endif
14 
15 /*
16  * We'd like to dump the L2_ECC_TAG register on errors, but errata make
17  * that unsafe... So for now we don't.	(BCM1250/BCM112x erratum SOC-48.)
18  */
19 #undef DUMP_L2_ECC_TAG_ON_ERROR
20 
21 /* SB1 definitions */
22 
23 /* XXX should come from config1 XXX */
24 #define SB1_CACHE_INDEX_MASK   0x1fe0
25 
26 #define CP0_ERRCTL_RECOVERABLE (1 << 31)
27 #define CP0_ERRCTL_DCACHE      (1 << 30)
28 #define CP0_ERRCTL_ICACHE      (1 << 29)
29 #define CP0_ERRCTL_MULTIBUS    (1 << 23)
30 #define CP0_ERRCTL_MC_TLB      (1 << 15)
31 #define CP0_ERRCTL_MC_TIMEOUT  (1 << 14)
32 
33 #define CP0_CERRI_TAG_PARITY   (1 << 29)
34 #define CP0_CERRI_DATA_PARITY  (1 << 28)
35 #define CP0_CERRI_EXTERNAL     (1 << 26)
36 
37 #define CP0_CERRI_IDX_VALID(c) (!((c) & CP0_CERRI_EXTERNAL))
38 #define CP0_CERRI_DATA	       (CP0_CERRI_DATA_PARITY)
39 
40 #define CP0_CERRD_MULTIPLE     (1 << 31)
41 #define CP0_CERRD_TAG_STATE    (1 << 30)
42 #define CP0_CERRD_TAG_ADDRESS  (1 << 29)
43 #define CP0_CERRD_DATA_SBE     (1 << 28)
44 #define CP0_CERRD_DATA_DBE     (1 << 27)
45 #define CP0_CERRD_EXTERNAL     (1 << 26)
46 #define CP0_CERRD_LOAD	       (1 << 25)
47 #define CP0_CERRD_STORE	       (1 << 24)
48 #define CP0_CERRD_FILLWB       (1 << 23)
49 #define CP0_CERRD_COHERENCY    (1 << 22)
50 #define CP0_CERRD_DUPTAG       (1 << 21)
51 
52 #define CP0_CERRD_DPA_VALID(c) (!((c) & CP0_CERRD_EXTERNAL))
53 #define CP0_CERRD_IDX_VALID(c) \
54    (((c) & (CP0_CERRD_LOAD | CP0_CERRD_STORE)) ? (!((c) & CP0_CERRD_EXTERNAL)) : 0)
55 #define CP0_CERRD_CAUSES \
56    (CP0_CERRD_LOAD | CP0_CERRD_STORE | CP0_CERRD_FILLWB | CP0_CERRD_COHERENCY | CP0_CERRD_DUPTAG)
57 #define CP0_CERRD_TYPES \
58    (CP0_CERRD_TAG_STATE | CP0_CERRD_TAG_ADDRESS | CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE | CP0_CERRD_EXTERNAL)
59 #define CP0_CERRD_DATA	       (CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE)
60 
61 static uint32_t extract_ic(unsigned short addr, int data);
62 static uint32_t extract_dc(unsigned short addr, int data);
63 
breakout_errctl(unsigned int val)64 static inline void breakout_errctl(unsigned int val)
65 {
66 	if (val & CP0_ERRCTL_RECOVERABLE)
67 		printk(" recoverable");
68 	if (val & CP0_ERRCTL_DCACHE)
69 		printk(" dcache");
70 	if (val & CP0_ERRCTL_ICACHE)
71 		printk(" icache");
72 	if (val & CP0_ERRCTL_MULTIBUS)
73 		printk(" multiple-buserr");
74 	printk("\n");
75 }
76 
breakout_cerri(unsigned int val)77 static inline void breakout_cerri(unsigned int val)
78 {
79 	if (val & CP0_CERRI_TAG_PARITY)
80 		printk(" tag-parity");
81 	if (val & CP0_CERRI_DATA_PARITY)
82 		printk(" data-parity");
83 	if (val & CP0_CERRI_EXTERNAL)
84 		printk(" external");
85 	printk("\n");
86 }
87 
breakout_cerrd(unsigned int val)88 static inline void breakout_cerrd(unsigned int val)
89 {
90 	switch (val & CP0_CERRD_CAUSES) {
91 	case CP0_CERRD_LOAD:
92 		printk(" load,");
93 		break;
94 	case CP0_CERRD_STORE:
95 		printk(" store,");
96 		break;
97 	case CP0_CERRD_FILLWB:
98 		printk(" fill/wb,");
99 		break;
100 	case CP0_CERRD_COHERENCY:
101 		printk(" coherency,");
102 		break;
103 	case CP0_CERRD_DUPTAG:
104 		printk(" duptags,");
105 		break;
106 	default:
107 		printk(" NO CAUSE,");
108 		break;
109 	}
110 	if (!(val & CP0_CERRD_TYPES))
111 		printk(" NO TYPE");
112 	else {
113 		if (val & CP0_CERRD_MULTIPLE)
114 			printk(" multi-err");
115 		if (val & CP0_CERRD_TAG_STATE)
116 			printk(" tag-state");
117 		if (val & CP0_CERRD_TAG_ADDRESS)
118 			printk(" tag-address");
119 		if (val & CP0_CERRD_DATA_SBE)
120 			printk(" data-SBE");
121 		if (val & CP0_CERRD_DATA_DBE)
122 			printk(" data-DBE");
123 		if (val & CP0_CERRD_EXTERNAL)
124 			printk(" external");
125 	}
126 	printk("\n");
127 }
128 
129 #ifndef CONFIG_SIBYTE_BUS_WATCHER
130 
check_bus_watcher(void)131 static void check_bus_watcher(void)
132 {
133 	uint32_t status, l2_err, memio_err;
134 #ifdef DUMP_L2_ECC_TAG_ON_ERROR
135 	uint64_t l2_tag;
136 #endif
137 
138 	/* Destructive read, clears register and interrupt */
139 	status = csr_in32(IOADDR(A_SCD_BUS_ERR_STATUS));
140 	/* Bit 31 is always on, but there's no #define for that */
141 	if (status & ~(1UL << 31)) {
142 		l2_err = csr_in32(IOADDR(A_BUS_L2_ERRORS));
143 #ifdef DUMP_L2_ECC_TAG_ON_ERROR
144 		l2_tag = in64(IOADDR(A_L2_ECC_TAG));
145 #endif
146 		memio_err = csr_in32(IOADDR(A_BUS_MEM_IO_ERRORS));
147 		printk("Bus watcher error counters: %08x %08x\n", l2_err, memio_err);
148 		printk("\nLast recorded signature:\n");
149 		printk("Request %02x from %d, answered by %d with Dcode %d\n",
150 		       (unsigned int)(G_SCD_BERR_TID(status) & 0x3f),
151 		       (int)(G_SCD_BERR_TID(status) >> 6),
152 		       (int)G_SCD_BERR_RID(status),
153 		       (int)G_SCD_BERR_DCODE(status));
154 #ifdef DUMP_L2_ECC_TAG_ON_ERROR
155 		printk("Last L2 tag w/ bad ECC: %016llx\n", l2_tag);
156 #endif
157 	} else {
158 		printk("Bus watcher indicates no error\n");
159 	}
160 }
161 #else
162 extern void check_bus_watcher(void);
163 #endif
164 
sb1_cache_error(void)165 asmlinkage void sb1_cache_error(void)
166 {
167 	uint32_t errctl, cerr_i, cerr_d, dpalo, dpahi, eepc, res;
168 	unsigned long long cerr_dpa;
169 
170 #ifdef CONFIG_SIBYTE_BW_TRACE
171 	/* Freeze the trace buffer now */
172 	csr_out32(M_SCD_TRACE_CFG_FREEZE, IOADDR(A_SCD_TRACE_CFG));
173 	printk("Trace buffer frozen\n");
174 #endif
175 
176 	printk("Cache error exception on CPU %x:\n",
177 	       (read_c0_prid() >> 25) & 0x7);
178 
179 	__asm__ __volatile__ (
180 	"	.set	push\n\t"
181 	"	.set	mips64\n\t"
182 	"	.set	noat\n\t"
183 	"	mfc0	%0, $26\n\t"
184 	"	mfc0	%1, $27\n\t"
185 	"	mfc0	%2, $27, 1\n\t"
186 	"	dmfc0	$1, $27, 3\n\t"
187 	"	dsrl32	%3, $1, 0 \n\t"
188 	"	sll	%4, $1, 0 \n\t"
189 	"	mfc0	%5, $30\n\t"
190 	"	.set	pop"
191 	: "=r" (errctl), "=r" (cerr_i), "=r" (cerr_d),
192 	  "=r" (dpahi), "=r" (dpalo), "=r" (eepc));
193 
194 	cerr_dpa = (((uint64_t)dpahi) << 32) | dpalo;
195 	printk(" c0_errorepc ==	  %08x\n", eepc);
196 	printk(" c0_errctl   ==	  %08x", errctl);
197 	breakout_errctl(errctl);
198 	if (errctl & CP0_ERRCTL_ICACHE) {
199 		printk(" c0_cerr_i   ==	  %08x", cerr_i);
200 		breakout_cerri(cerr_i);
201 		if (CP0_CERRI_IDX_VALID(cerr_i)) {
202 			/* Check index of EPC, allowing for delay slot */
203 			if (((eepc & SB1_CACHE_INDEX_MASK) != (cerr_i & SB1_CACHE_INDEX_MASK)) &&
204 			    ((eepc & SB1_CACHE_INDEX_MASK) != ((cerr_i & SB1_CACHE_INDEX_MASK) - 4)))
205 				printk(" cerr_i idx doesn't match eepc\n");
206 			else {
207 				res = extract_ic(cerr_i & SB1_CACHE_INDEX_MASK,
208 						 (cerr_i & CP0_CERRI_DATA) != 0);
209 				if (!(res & cerr_i))
210 					printk("...didn't see indicated icache problem\n");
211 			}
212 		}
213 	}
214 	if (errctl & CP0_ERRCTL_DCACHE) {
215 		printk(" c0_cerr_d   ==	  %08x", cerr_d);
216 		breakout_cerrd(cerr_d);
217 		if (CP0_CERRD_DPA_VALID(cerr_d)) {
218 			printk(" c0_cerr_dpa == %010llx\n", cerr_dpa);
219 			if (!CP0_CERRD_IDX_VALID(cerr_d)) {
220 				res = extract_dc(cerr_dpa & SB1_CACHE_INDEX_MASK,
221 						 (cerr_d & CP0_CERRD_DATA) != 0);
222 				if (!(res & cerr_d))
223 					printk("...didn't see indicated dcache problem\n");
224 			} else {
225 				if ((cerr_dpa & SB1_CACHE_INDEX_MASK) != (cerr_d & SB1_CACHE_INDEX_MASK))
226 					printk(" cerr_d idx doesn't match cerr_dpa\n");
227 				else {
228 					res = extract_dc(cerr_d & SB1_CACHE_INDEX_MASK,
229 							 (cerr_d & CP0_CERRD_DATA) != 0);
230 					if (!(res & cerr_d))
231 						printk("...didn't see indicated problem\n");
232 				}
233 			}
234 		}
235 	}
236 
237 	check_bus_watcher();
238 
239 	/*
240 	 * Calling panic() when a fatal cache error occurs scrambles the
241 	 * state of the system (and the cache), making it difficult to
242 	 * investigate after the fact.	However, if you just stall the CPU,
243 	 * the other CPU may keep on running, which is typically very
244 	 * undesirable.
245 	 */
246 #ifdef CONFIG_SB1_CERR_STALL
247 	while (1)
248 		;
249 #else
250 	panic("unhandled cache error");
251 #endif
252 }
253 
254 
255 /* Parity lookup table. */
256 static const uint8_t parity[256] = {
257 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
258 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
259 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
260 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
261 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
262 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
263 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
264 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
265 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
266 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
267 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
268 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
269 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
270 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
271 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
272 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0
273 };
274 
275 /* Masks to select bits for Hamming parity, mask_72_64[i] for bit[i] */
276 static const uint64_t mask_72_64[8] = {
277 	0x0738C808099264FFULL,
278 	0x38C808099264FF07ULL,
279 	0xC808099264FF0738ULL,
280 	0x08099264FF0738C8ULL,
281 	0x099264FF0738C808ULL,
282 	0x9264FF0738C80809ULL,
283 	0x64FF0738C8080992ULL,
284 	0xFF0738C808099264ULL
285 };
286 
287 /* Calculate the parity on a range of bits */
range_parity(uint64_t dword,int max,int min)288 static char range_parity(uint64_t dword, int max, int min)
289 {
290 	char parity = 0;
291 	int i;
292 	dword >>= min;
293 	for (i=max-min; i>=0; i--) {
294 		if (dword & 0x1)
295 			parity = !parity;
296 		dword >>= 1;
297 	}
298 	return parity;
299 }
300 
301 /* Calculate the 4-bit even byte-parity for an instruction */
inst_parity(uint32_t word)302 static unsigned char inst_parity(uint32_t word)
303 {
304 	int i, j;
305 	char parity = 0;
306 	for (j=0; j<4; j++) {
307 		char byte_parity = 0;
308 		for (i=0; i<8; i++) {
309 			if (word & 0x80000000)
310 				byte_parity = !byte_parity;
311 			word <<= 1;
312 		}
313 		parity <<= 1;
314 		parity |= byte_parity;
315 	}
316 	return parity;
317 }
318 
extract_ic(unsigned short addr,int data)319 static uint32_t extract_ic(unsigned short addr, int data)
320 {
321 	unsigned short way;
322 	int valid;
323 	uint32_t taghi, taglolo, taglohi;
324 	unsigned long long taglo, va;
325 	uint64_t tlo_tmp;
326 	uint8_t lru;
327 	int res = 0;
328 
329 	printk("Icache index 0x%04x  ", addr);
330 	for (way = 0; way < 4; way++) {
331 		/* Index-load-tag-I */
332 		__asm__ __volatile__ (
333 		"	.set	push		\n\t"
334 		"	.set	noreorder	\n\t"
335 		"	.set	mips64		\n\t"
336 		"	.set	noat		\n\t"
337 		"	cache	4, 0(%3)	\n\t"
338 		"	mfc0	%0, $29		\n\t"
339 		"	dmfc0	$1, $28		\n\t"
340 		"	dsrl32	%1, $1, 0	\n\t"
341 		"	sll	%2, $1, 0	\n\t"
342 		"	.set	pop"
343 		: "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
344 		: "r" ((way << 13) | addr));
345 
346 		taglo = ((unsigned long long)taglohi << 32) | taglolo;
347 		if (way == 0) {
348 			lru = (taghi >> 14) & 0xff;
349 			printk("[Bank %d Set 0x%02x]  LRU > %d %d %d %d > MRU\n",
350 				    ((addr >> 5) & 0x3), /* bank */
351 				    ((addr >> 7) & 0x3f), /* index */
352 				    (lru & 0x3),
353 				    ((lru >> 2) & 0x3),
354 				    ((lru >> 4) & 0x3),
355 				    ((lru >> 6) & 0x3));
356 		}
357 		va = (taglo & 0xC0000FFFFFFFE000ULL) | addr;
358 		if ((taglo & (1 << 31)) && (((taglo >> 62) & 0x3) == 3))
359 			va |= 0x3FFFF00000000000ULL;
360 		valid = ((taghi >> 29) & 1);
361 		if (valid) {
362 			tlo_tmp = taglo & 0xfff3ff;
363 			if (((taglo >> 10) & 1) ^ range_parity(tlo_tmp, 23, 0)) {
364 				printk("   ** bad parity in VTag0/G/ASID\n");
365 				res |= CP0_CERRI_TAG_PARITY;
366 			}
367 			if (((taglo >> 11) & 1) ^ range_parity(taglo, 63, 24)) {
368 				printk("   ** bad parity in R/VTag1\n");
369 				res |= CP0_CERRI_TAG_PARITY;
370 			}
371 		}
372 		if (valid ^ ((taghi >> 27) & 1)) {
373 			printk("   ** bad parity for valid bit\n");
374 			res |= CP0_CERRI_TAG_PARITY;
375 		}
376 		printk(" %d  [VA %016llx]  [Vld? %d]  raw tags: %08X-%016llX\n",
377 			    way, va, valid, taghi, taglo);
378 
379 		if (data) {
380 			uint32_t datahi, insta, instb;
381 			uint8_t predecode;
382 			int offset;
383 
384 			/* (hit all banks and ways) */
385 			for (offset = 0; offset < 4; offset++) {
386 				/* Index-load-data-I */
387 				__asm__ __volatile__ (
388 				"	.set	push\n\t"
389 				"	.set	noreorder\n\t"
390 				"	.set	mips64\n\t"
391 				"	.set	noat\n\t"
392 				"	cache	6, 0(%3)  \n\t"
393 				"	mfc0	%0, $29, 1\n\t"
394 				"	dmfc0  $1, $28, 1\n\t"
395 				"	dsrl32 %1, $1, 0 \n\t"
396 				"	sll    %2, $1, 0 \n\t"
397 				"	.set	pop	    \n"
398 				: "=r" (datahi), "=r" (insta), "=r" (instb)
399 				: "r" ((way << 13) | addr | (offset << 3)));
400 				predecode = (datahi >> 8) & 0xff;
401 				if (((datahi >> 16) & 1) != (uint32_t)range_parity(predecode, 7, 0)) {
402 					printk("   ** bad parity in predecode\n");
403 					res |= CP0_CERRI_DATA_PARITY;
404 				}
405 				/* XXXKW should/could check predecode bits themselves */
406 				if (((datahi >> 4) & 0xf) ^ inst_parity(insta)) {
407 					printk("   ** bad parity in instruction a\n");
408 					res |= CP0_CERRI_DATA_PARITY;
409 				}
410 				if ((datahi & 0xf) ^ inst_parity(instb)) {
411 					printk("   ** bad parity in instruction b\n");
412 					res |= CP0_CERRI_DATA_PARITY;
413 				}
414 				printk("  %05X-%08X%08X", datahi, insta, instb);
415 			}
416 			printk("\n");
417 		}
418 	}
419 	return res;
420 }
421 
422 /* Compute the ECC for a data doubleword */
dc_ecc(uint64_t dword)423 static uint8_t dc_ecc(uint64_t dword)
424 {
425 	uint64_t t;
426 	uint32_t w;
427 	uint8_t	 p;
428 	int	 i;
429 
430 	p = 0;
431 	for (i = 7; i >= 0; i--)
432 	{
433 		p <<= 1;
434 		t = dword & mask_72_64[i];
435 		w = (uint32_t)(t >> 32);
436 		p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
437 		      ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
438 		w = (uint32_t)(t & 0xFFFFFFFF);
439 		p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
440 		      ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
441 	}
442 	return p;
443 }
444 
445 struct dc_state {
446 	unsigned char val;
447 	char *name;
448 };
449 
450 static struct dc_state dc_states[] = {
451 	{ 0x00, "INVALID" },
452 	{ 0x0f, "COH-SHD" },
453 	{ 0x13, "NCO-E-C" },
454 	{ 0x19, "NCO-E-D" },
455 	{ 0x16, "COH-E-C" },
456 	{ 0x1c, "COH-E-D" },
457 	{ 0xff, "*ERROR*" }
458 };
459 
460 #define DC_TAG_VALID(state) \
461     (((state) == 0x0) || ((state) == 0xf) || ((state) == 0x13) || \
462      ((state) == 0x19) || ((state) == 0x16) || ((state) == 0x1c))
463 
dc_state_str(unsigned char state)464 static char *dc_state_str(unsigned char state)
465 {
466 	struct dc_state *dsc = dc_states;
467 	while (dsc->val != 0xff) {
468 		if (dsc->val == state)
469 			break;
470 		dsc++;
471 	}
472 	return dsc->name;
473 }
474 
extract_dc(unsigned short addr,int data)475 static uint32_t extract_dc(unsigned short addr, int data)
476 {
477 	int valid, way;
478 	unsigned char state;
479 	uint32_t taghi, taglolo, taglohi;
480 	unsigned long long taglo, pa;
481 	uint8_t ecc, lru;
482 	int res = 0;
483 
484 	printk("Dcache index 0x%04x  ", addr);
485 	for (way = 0; way < 4; way++) {
486 		__asm__ __volatile__ (
487 		"	.set	push\n\t"
488 		"	.set	noreorder\n\t"
489 		"	.set	mips64\n\t"
490 		"	.set	noat\n\t"
491 		"	cache	5, 0(%3)\n\t"	/* Index-load-tag-D */
492 		"	mfc0	%0, $29, 2\n\t"
493 		"	dmfc0	$1, $28, 2\n\t"
494 		"	dsrl32	%1, $1, 0\n\t"
495 		"	sll	%2, $1, 0\n\t"
496 		"	.set	pop"
497 		: "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
498 		: "r" ((way << 13) | addr));
499 
500 		taglo = ((unsigned long long)taglohi << 32) | taglolo;
501 		pa = (taglo & 0xFFFFFFE000ULL) | addr;
502 		if (way == 0) {
503 			lru = (taghi >> 14) & 0xff;
504 			printk("[Bank %d Set 0x%02x]  LRU > %d %d %d %d > MRU\n",
505 				    ((addr >> 11) & 0x2) | ((addr >> 5) & 1), /* bank */
506 				    ((addr >> 6) & 0x3f), /* index */
507 				    (lru & 0x3),
508 				    ((lru >> 2) & 0x3),
509 				    ((lru >> 4) & 0x3),
510 				    ((lru >> 6) & 0x3));
511 		}
512 		state = (taghi >> 25) & 0x1f;
513 		valid = DC_TAG_VALID(state);
514 		printk(" %d  [PA %010llx]  [state %s (%02x)]  raw tags: %08X-%016llX\n",
515 			    way, pa, dc_state_str(state), state, taghi, taglo);
516 		if (valid) {
517 			if (((taglo >> 11) & 1) ^ range_parity(taglo, 39, 26)) {
518 				printk("   ** bad parity in PTag1\n");
519 				res |= CP0_CERRD_TAG_ADDRESS;
520 			}
521 			if (((taglo >> 10) & 1) ^ range_parity(taglo, 25, 13)) {
522 				printk("   ** bad parity in PTag0\n");
523 				res |= CP0_CERRD_TAG_ADDRESS;
524 			}
525 		} else {
526 			res |= CP0_CERRD_TAG_STATE;
527 		}
528 
529 		if (data) {
530 			uint32_t datalohi, datalolo, datahi;
531 			unsigned long long datalo;
532 			int offset;
533 			char bad_ecc = 0;
534 
535 			for (offset = 0; offset < 4; offset++) {
536 				/* Index-load-data-D */
537 				__asm__ __volatile__ (
538 				"	.set	push\n\t"
539 				"	.set	noreorder\n\t"
540 				"	.set	mips64\n\t"
541 				"	.set	noat\n\t"
542 				"	cache	7, 0(%3)\n\t" /* Index-load-data-D */
543 				"	mfc0	%0, $29, 3\n\t"
544 				"	dmfc0	$1, $28, 3\n\t"
545 				"	dsrl32	%1, $1, 0 \n\t"
546 				"	sll	%2, $1, 0 \n\t"
547 				"	.set	pop"
548 				: "=r" (datahi), "=r" (datalohi), "=r" (datalolo)
549 				: "r" ((way << 13) | addr | (offset << 3)));
550 				datalo = ((unsigned long long)datalohi << 32) | datalolo;
551 				ecc = dc_ecc(datalo);
552 				if (ecc != datahi) {
553 					int bits;
554 					bad_ecc |= 1 << (3-offset);
555 					ecc ^= datahi;
556 					bits = hweight8(ecc);
557 					res |= (bits == 1) ? CP0_CERRD_DATA_SBE : CP0_CERRD_DATA_DBE;
558 				}
559 				printk("  %02X-%016llX", datahi, datalo);
560 			}
561 			printk("\n");
562 			if (bad_ecc)
563 				printk("  dwords w/ bad ECC: %d %d %d %d\n",
564 				       !!(bad_ecc & 8), !!(bad_ecc & 4),
565 				       !!(bad_ecc & 2), !!(bad_ecc & 1));
566 		}
567 	}
568 	return res;
569 }
570