SB1 cache exception handling.
[linux-2.6] / arch / mips / mm / cerr-sb1.c
1 /*
2  * Copyright (C) 2001,2002,2003 Broadcom Corporation
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
17  */
18 #include <linux/config.h>
19 #include <linux/sched.h>
20 #include <asm/mipsregs.h>
21 #include <asm/sibyte/sb1250.h>
22 #include <asm/sibyte/sb1250_regs.h>
23
24 #if !defined(CONFIG_SIBYTE_BUS_WATCHER) || defined(CONFIG_SIBYTE_BW_TRACE)
25 #include <asm/io.h>
26 #include <asm/sibyte/sb1250_scd.h>
27 #endif
28
29 /*
30  * We'd like to dump the L2_ECC_TAG register on errors, but errata make
31  * that unsafe... So for now we don't.  (BCM1250/BCM112x erratum SOC-48.)
32  */
33 #undef DUMP_L2_ECC_TAG_ON_ERROR
34
35 /* SB1 definitions */
36
37 /* XXX should come from config1 XXX */
38 #define SB1_CACHE_INDEX_MASK   0x1fe0
39
40 #define CP0_ERRCTL_RECOVERABLE (1 << 31)
41 #define CP0_ERRCTL_DCACHE      (1 << 30)
42 #define CP0_ERRCTL_ICACHE      (1 << 29)
43 #define CP0_ERRCTL_MULTIBUS    (1 << 23)
44 #define CP0_ERRCTL_MC_TLB      (1 << 15)
45 #define CP0_ERRCTL_MC_TIMEOUT  (1 << 14)
46
47 #define CP0_CERRI_TAG_PARITY   (1 << 29)
48 #define CP0_CERRI_DATA_PARITY  (1 << 28)
49 #define CP0_CERRI_EXTERNAL     (1 << 26)
50
51 #define CP0_CERRI_IDX_VALID(c) (!((c) & CP0_CERRI_EXTERNAL))
52 #define CP0_CERRI_DATA         (CP0_CERRI_DATA_PARITY)
53
54 #define CP0_CERRD_MULTIPLE     (1 << 31)
55 #define CP0_CERRD_TAG_STATE    (1 << 30)
56 #define CP0_CERRD_TAG_ADDRESS  (1 << 29)
57 #define CP0_CERRD_DATA_SBE     (1 << 28)
58 #define CP0_CERRD_DATA_DBE     (1 << 27)
59 #define CP0_CERRD_EXTERNAL     (1 << 26)
60 #define CP0_CERRD_LOAD         (1 << 25)
61 #define CP0_CERRD_STORE        (1 << 24)
62 #define CP0_CERRD_FILLWB       (1 << 23)
63 #define CP0_CERRD_COHERENCY    (1 << 22)
64 #define CP0_CERRD_DUPTAG       (1 << 21)
65
66 #define CP0_CERRD_DPA_VALID(c) (!((c) & CP0_CERRD_EXTERNAL))
67 #define CP0_CERRD_IDX_VALID(c) \
68    (((c) & (CP0_CERRD_LOAD | CP0_CERRD_STORE)) ? (!((c) & CP0_CERRD_EXTERNAL)) : 0)
69 #define CP0_CERRD_CAUSES \
70    (CP0_CERRD_LOAD | CP0_CERRD_STORE | CP0_CERRD_FILLWB | CP0_CERRD_COHERENCY | CP0_CERRD_DUPTAG)
71 #define CP0_CERRD_TYPES \
72    (CP0_CERRD_TAG_STATE | CP0_CERRD_TAG_ADDRESS | CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE | CP0_CERRD_EXTERNAL)
73 #define CP0_CERRD_DATA         (CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE)
74
75 static uint32_t extract_ic(unsigned short addr, int data);
76 static uint32_t extract_dc(unsigned short addr, int data);
77
78 static inline void breakout_errctl(unsigned int val)
79 {
80         if (val & CP0_ERRCTL_RECOVERABLE)
81                 prom_printf(" recoverable");
82         if (val & CP0_ERRCTL_DCACHE)
83                 prom_printf(" dcache");
84         if (val & CP0_ERRCTL_ICACHE)
85                 prom_printf(" icache");
86         if (val & CP0_ERRCTL_MULTIBUS)
87                 prom_printf(" multiple-buserr");
88         prom_printf("\n");
89 }
90
91 static inline void breakout_cerri(unsigned int val)
92 {
93         if (val & CP0_CERRI_TAG_PARITY)
94                 prom_printf(" tag-parity");
95         if (val & CP0_CERRI_DATA_PARITY)
96                 prom_printf(" data-parity");
97         if (val & CP0_CERRI_EXTERNAL)
98                 prom_printf(" external");
99         prom_printf("\n");
100 }
101
102 static inline void breakout_cerrd(unsigned int val)
103 {
104         switch (val & CP0_CERRD_CAUSES) {
105         case CP0_CERRD_LOAD:
106                 prom_printf(" load,");
107                 break;
108         case CP0_CERRD_STORE:
109                 prom_printf(" store,");
110                 break;
111         case CP0_CERRD_FILLWB:
112                 prom_printf(" fill/wb,");
113                 break;
114         case CP0_CERRD_COHERENCY:
115                 prom_printf(" coherency,");
116                 break;
117         case CP0_CERRD_DUPTAG:
118                 prom_printf(" duptags,");
119                 break;
120         default:
121                 prom_printf(" NO CAUSE,");
122                 break;
123         }
124         if (!(val & CP0_CERRD_TYPES))
125                 prom_printf(" NO TYPE");
126         else {
127                 if (val & CP0_CERRD_MULTIPLE)
128                         prom_printf(" multi-err");
129                 if (val & CP0_CERRD_TAG_STATE)
130                         prom_printf(" tag-state");
131                 if (val & CP0_CERRD_TAG_ADDRESS)
132                         prom_printf(" tag-address");
133                 if (val & CP0_CERRD_DATA_SBE)
134                         prom_printf(" data-SBE");
135                 if (val & CP0_CERRD_DATA_DBE)
136                         prom_printf(" data-DBE");
137                 if (val & CP0_CERRD_EXTERNAL)
138                         prom_printf(" external");
139         }
140         prom_printf("\n");
141 }
142
143 #ifndef CONFIG_SIBYTE_BUS_WATCHER
144
145 static void check_bus_watcher(void)
146 {
147         uint32_t status, l2_err, memio_err;
148 #ifdef DUMP_L2_ECC_TAG_ON_ERROR
149         uint64_t l2_tag;
150 #endif
151
152         /* Destructive read, clears register and interrupt */
153         status = csr_in32(IOADDR(A_SCD_BUS_ERR_STATUS));
154         /* Bit 31 is always on, but there's no #define for that */
155         if (status & ~(1UL << 31)) {
156                 l2_err = csr_in32(IOADDR(A_BUS_L2_ERRORS));
157 #ifdef DUMP_L2_ECC_TAG_ON_ERROR
158                 l2_tag = in64(IO_SPACE_BASE | A_L2_ECC_TAG);
159 #endif
160                 memio_err = csr_in32(IOADDR(A_BUS_MEM_IO_ERRORS));
161                 prom_printf("Bus watcher error counters: %08x %08x\n", l2_err, memio_err);
162                 prom_printf("\nLast recorded signature:\n");
163                 prom_printf("Request %02x from %d, answered by %d with Dcode %d\n",
164                        (unsigned int)(G_SCD_BERR_TID(status) & 0x3f),
165                        (int)(G_SCD_BERR_TID(status) >> 6),
166                        (int)G_SCD_BERR_RID(status),
167                        (int)G_SCD_BERR_DCODE(status));
168 #ifdef DUMP_L2_ECC_TAG_ON_ERROR
169                 prom_printf("Last L2 tag w/ bad ECC: %016llx\n", l2_tag);
170 #endif
171         } else {
172                 prom_printf("Bus watcher indicates no error\n");
173         }
174 }
175 #else
176 extern void check_bus_watcher(void);
177 #endif
178
179 asmlinkage void sb1_cache_error(void)
180 {
181         uint64_t cerr_dpa;
182         uint32_t errctl, cerr_i, cerr_d, dpalo, dpahi, eepc, res;
183
184 #ifdef CONFIG_SIBYTE_BW_TRACE
185         /* Freeze the trace buffer now */
186 #if defined(CONFIG_SIBYTE_BCM1x55) || defined(CONFIG_SIBYTE_BCM1x80)
187         csr_out32(M_BCM1480_SCD_TRACE_CFG_FREEZE, IO_SPACE_BASE | A_SCD_TRACE_CFG);
188 #else
189         csr_out32(M_SCD_TRACE_CFG_FREEZE, IO_SPACE_BASE | A_SCD_TRACE_CFG);
190 #endif
191         prom_printf("Trace buffer frozen\n");
192 #endif
193
194         prom_printf("Cache error exception on CPU %x:\n",
195                     (read_c0_prid() >> 25) & 0x7);
196
197         __asm__ __volatile__ (
198         "       .set    push\n\t"
199         "       .set    mips64\n\t"
200         "       .set    noat\n\t"
201         "       mfc0    %0, $26\n\t"
202         "       mfc0    %1, $27\n\t"
203         "       mfc0    %2, $27, 1\n\t"
204         "       dmfc0   $1, $27, 3\n\t"
205         "       dsrl32  %3, $1, 0 \n\t"
206         "       sll     %4, $1, 0 \n\t"
207         "       mfc0    %5, $30\n\t"
208         "       .set    pop"
209         : "=r" (errctl), "=r" (cerr_i), "=r" (cerr_d),
210           "=r" (dpahi), "=r" (dpalo), "=r" (eepc));
211
212         cerr_dpa = (((uint64_t)dpahi) << 32) | dpalo;
213         prom_printf(" c0_errorepc ==   %08x\n", eepc);
214         prom_printf(" c0_errctl   ==   %08x", errctl);
215         breakout_errctl(errctl);
216         if (errctl & CP0_ERRCTL_ICACHE) {
217                 prom_printf(" c0_cerr_i   ==   %08x", cerr_i);
218                 breakout_cerri(cerr_i);
219                 if (CP0_CERRI_IDX_VALID(cerr_i)) {
220                         /* Check index of EPC, allowing for delay slot */
221                         if (((eepc & SB1_CACHE_INDEX_MASK) != (cerr_i & SB1_CACHE_INDEX_MASK)) &&
222                             ((eepc & SB1_CACHE_INDEX_MASK) != ((cerr_i & SB1_CACHE_INDEX_MASK) - 4)))
223                                 prom_printf(" cerr_i idx doesn't match eepc\n");
224                         else {
225                                 res = extract_ic(cerr_i & SB1_CACHE_INDEX_MASK,
226                                                  (cerr_i & CP0_CERRI_DATA) != 0);
227                                 if (!(res & cerr_i))
228                                         prom_printf("...didn't see indicated icache problem\n");
229                         }
230                 }
231         }
232         if (errctl & CP0_ERRCTL_DCACHE) {
233                 prom_printf(" c0_cerr_d   ==   %08x", cerr_d);
234                 breakout_cerrd(cerr_d);
235                 if (CP0_CERRD_DPA_VALID(cerr_d)) {
236                         prom_printf(" c0_cerr_dpa == %010llx\n", cerr_dpa);
237                         if (!CP0_CERRD_IDX_VALID(cerr_d)) {
238                                 res = extract_dc(cerr_dpa & SB1_CACHE_INDEX_MASK,
239                                                  (cerr_d & CP0_CERRD_DATA) != 0);
240                                 if (!(res & cerr_d))
241                                         prom_printf("...didn't see indicated dcache problem\n");
242                         } else {
243                                 if ((cerr_dpa & SB1_CACHE_INDEX_MASK) != (cerr_d & SB1_CACHE_INDEX_MASK))
244                                         prom_printf(" cerr_d idx doesn't match cerr_dpa\n");
245                                 else {
246                                         res = extract_dc(cerr_d & SB1_CACHE_INDEX_MASK,
247                                                          (cerr_d & CP0_CERRD_DATA) != 0);
248                                         if (!(res & cerr_d))
249                                                 prom_printf("...didn't see indicated problem\n");
250                                 }
251                         }
252                 }
253         }
254
255         check_bus_watcher();
256
257         /*
258          * Calling panic() when a fatal cache error occurs scrambles the
259          * state of the system (and the cache), making it difficult to
260          * investigate after the fact.  However, if you just stall the CPU,
261          * the other CPU may keep on running, which is typically very
262          * undesirable.
263          */
264 #ifdef CONFIG_SB1_CERR_STALL
265         while (1)
266                 ;
267 #else
268         panic("unhandled cache error");
269 #endif
270 }
271
272
273 /* Parity lookup table. */
274 static const uint8_t parity[256] = {
275         0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
276         1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,
277         1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,
278         0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
279         1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,
280         0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
281         0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,
282         1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,1,0,1,1,0,1,0,0,1,1,0,0,1,0,1,1,0
283 };
284
285 /* Masks to select bits for Hamming parity, mask_72_64[i] for bit[i] */
286 static const uint64_t mask_72_64[8] = {
287         0x0738C808099264FFULL,
288         0x38C808099264FF07ULL,
289         0xC808099264FF0738ULL,
290         0x08099264FF0738C8ULL,
291         0x099264FF0738C808ULL,
292         0x9264FF0738C80809ULL,
293         0x64FF0738C8080992ULL,
294         0xFF0738C808099264ULL
295 };
296
297 /* Calculate the parity on a range of bits */
298 static char range_parity(uint64_t dword, int max, int min)
299 {
300         char parity = 0;
301         int i;
302         dword >>= min;
303         for (i=max-min; i>=0; i--) {
304                 if (dword & 0x1)
305                         parity = !parity;
306                 dword >>= 1;
307         }
308         return parity;
309 }
310
311 /* Calculate the 4-bit even byte-parity for an instruction */
312 static unsigned char inst_parity(uint32_t word)
313 {
314         int i, j;
315         char parity = 0;
316         for (j=0; j<4; j++) {
317                 char byte_parity = 0;
318                 for (i=0; i<8; i++) {
319                         if (word & 0x80000000)
320                                 byte_parity = !byte_parity;
321                         word <<= 1;
322                 }
323                 parity <<= 1;
324                 parity |= byte_parity;
325         }
326         return parity;
327 }
328
329 static uint32_t extract_ic(unsigned short addr, int data)
330 {
331         unsigned short way;
332         int valid;
333         uint64_t taglo, va, tlo_tmp;
334         uint32_t taghi, taglolo, taglohi;
335         uint8_t lru;
336         int res = 0;
337
338         prom_printf("Icache index 0x%04x  ", addr);
339         for (way = 0; way < 4; way++) {
340                 /* Index-load-tag-I */
341                 __asm__ __volatile__ (
342                 "       .set    push            \n\t"
343                 "       .set    noreorder       \n\t"
344                 "       .set    mips64          \n\t"
345                 "       .set    noat            \n\t"
346                 "       cache   4, 0(%3)        \n\t"
347                 "       mfc0    %0, $29         \n\t"
348                 "       dmfc0   $1, $28         \n\t"
349                 "       dsrl32  %1, $1, 0       \n\t"
350                 "       sll     %2, $1, 0       \n\t"
351                 "       .set    pop"
352                 : "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
353                 : "r" ((way << 13) | addr));
354
355                 taglo = ((unsigned long long)taglohi << 32) | taglolo;
356                 if (way == 0) {
357                         lru = (taghi >> 14) & 0xff;
358                         prom_printf("[Bank %d Set 0x%02x]  LRU > %d %d %d %d > MRU\n",
359                                     ((addr >> 5) & 0x3), /* bank */
360                                     ((addr >> 7) & 0x3f), /* index */
361                                     (lru & 0x3),
362                                     ((lru >> 2) & 0x3),
363                                     ((lru >> 4) & 0x3),
364                                     ((lru >> 6) & 0x3));
365                 }
366                 va = (taglo & 0xC0000FFFFFFFE000ULL) | addr;
367                 if ((taglo & (1 << 31)) && (((taglo >> 62) & 0x3) == 3))
368                         va |= 0x3FFFF00000000000ULL;
369                 valid = ((taghi >> 29) & 1);
370                 if (valid) {
371                         tlo_tmp = taglo & 0xfff3ff;
372                         if (((taglo >> 10) & 1) ^ range_parity(tlo_tmp, 23, 0)) {
373                                 prom_printf("   ** bad parity in VTag0/G/ASID\n");
374                                 res |= CP0_CERRI_TAG_PARITY;
375                         }
376                         if (((taglo >> 11) & 1) ^ range_parity(taglo, 63, 24)) {
377                                 prom_printf("   ** bad parity in R/VTag1\n");
378                                 res |= CP0_CERRI_TAG_PARITY;
379                         }
380                 }
381                 if (valid ^ ((taghi >> 27) & 1)) {
382                         prom_printf("   ** bad parity for valid bit\n");
383                         res |= CP0_CERRI_TAG_PARITY;
384                 }
385                 prom_printf(" %d  [VA %016llx]  [Vld? %d]  raw tags: %08X-%016llX\n",
386                             way, va, valid, taghi, taglo);
387
388                 if (data) {
389                         uint32_t datahi, insta, instb;
390                         uint8_t predecode;
391                         int offset;
392
393                         /* (hit all banks and ways) */
394                         for (offset = 0; offset < 4; offset++) {
395                                 /* Index-load-data-I */
396                                 __asm__ __volatile__ (
397                                 "       .set    push\n\t"
398                                 "       .set    noreorder\n\t"
399                                 "       .set    mips64\n\t"
400                                 "       .set    noat\n\t"
401                                 "       cache   6, 0(%3)  \n\t"
402                                 "       mfc0    %0, $29, 1\n\t"
403                                 "       dmfc0  $1, $28, 1\n\t"
404                                 "       dsrl32 %1, $1, 0 \n\t"
405                                 "       sll    %2, $1, 0 \n\t"
406                                 "       .set    pop         \n"
407                                 : "=r" (datahi), "=r" (insta), "=r" (instb)
408                                 : "r" ((way << 13) | addr | (offset << 3)));
409                                 predecode = (datahi >> 8) & 0xff;
410                                 if (((datahi >> 16) & 1) != (uint32_t)range_parity(predecode, 7, 0)) {
411                                         prom_printf("   ** bad parity in predecode\n");
412                                         res |= CP0_CERRI_DATA_PARITY;
413                                 }
414                                 /* XXXKW should/could check predecode bits themselves */
415                                 if (((datahi >> 4) & 0xf) ^ inst_parity(insta)) {
416                                         prom_printf("   ** bad parity in instruction a\n");
417                                         res |= CP0_CERRI_DATA_PARITY;
418                                 }
419                                 if ((datahi & 0xf) ^ inst_parity(instb)) {
420                                         prom_printf("   ** bad parity in instruction b\n");
421                                         res |= CP0_CERRI_DATA_PARITY;
422                                 }
423                                 prom_printf("  %05X-%08X%08X", datahi, insta, instb);
424                         }
425                         prom_printf("\n");
426                 }
427         }
428         return res;
429 }
430
431 /* Compute the ECC for a data doubleword */
432 static uint8_t dc_ecc(uint64_t dword)
433 {
434         uint64_t t;
435         uint32_t w;
436         uint8_t  p;
437         int      i;
438
439         p = 0;
440         for (i = 7; i >= 0; i--)
441         {
442                 p <<= 1;
443                 t = dword & mask_72_64[i];
444                 w = (uint32_t)(t >> 32);
445                 p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
446                       ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
447                 w = (uint32_t)(t & 0xFFFFFFFF);
448                 p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
449                       ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
450         }
451         return p;
452 }
453
454 struct dc_state {
455         unsigned char val;
456         char *name;
457 };
458
459 static struct dc_state dc_states[] = {
460         { 0x00, "INVALID" },
461         { 0x0f, "COH-SHD" },
462         { 0x13, "NCO-E-C" },
463         { 0x19, "NCO-E-D" },
464         { 0x16, "COH-E-C" },
465         { 0x1c, "COH-E-D" },
466         { 0xff, "*ERROR*" }
467 };
468
469 #define DC_TAG_VALID(state) \
470     (((state) == 0x0) || ((state) == 0xf) || ((state) == 0x13) || \
471      ((state) == 0x19) || ((state) == 0x16) || ((state) == 0x1c))
472
473 static char *dc_state_str(unsigned char state)
474 {
475         struct dc_state *dsc = dc_states;
476         while (dsc->val != 0xff) {
477                 if (dsc->val == state)
478                         break;
479                 dsc++;
480         }
481         return dsc->name;
482 }
483
484 static uint32_t extract_dc(unsigned short addr, int data)
485 {
486         int valid, way;
487         unsigned char state;
488         uint64_t taglo, pa;
489         uint32_t taghi, taglolo, taglohi;
490         uint8_t ecc, lru;
491         int res = 0;
492
493         prom_printf("Dcache index 0x%04x  ", addr);
494         for (way = 0; way < 4; way++) {
495                 __asm__ __volatile__ (
496                 "       .set    push\n\t"
497                 "       .set    noreorder\n\t"
498                 "       .set    mips64\n\t"
499                 "       .set    noat\n\t"
500                 "       cache   5, 0(%3)\n\t"   /* Index-load-tag-D */
501                 "       mfc0    %0, $29, 2\n\t"
502                 "       dmfc0   $1, $28, 2\n\t"
503                 "       dsrl32  %1, $1, 0\n\t"
504                 "       sll     %2, $1, 0\n\t"
505                 "       .set    pop"
506                 : "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
507                 : "r" ((way << 13) | addr));
508
509                 taglo = ((unsigned long long)taglohi << 32) | taglolo;
510                 pa = (taglo & 0xFFFFFFE000ULL) | addr;
511                 if (way == 0) {
512                         lru = (taghi >> 14) & 0xff;
513                         prom_printf("[Bank %d Set 0x%02x]  LRU > %d %d %d %d > MRU\n",
514                                     ((addr >> 11) & 0x2) | ((addr >> 5) & 1), /* bank */
515                                     ((addr >> 6) & 0x3f), /* index */
516                                     (lru & 0x3),
517                                     ((lru >> 2) & 0x3),
518                                     ((lru >> 4) & 0x3),
519                                     ((lru >> 6) & 0x3));
520                 }
521                 state = (taghi >> 25) & 0x1f;
522                 valid = DC_TAG_VALID(state);
523                 prom_printf(" %d  [PA %010llx]  [state %s (%02x)]  raw tags: %08X-%016llX\n",
524                             way, pa, dc_state_str(state), state, taghi, taglo);
525                 if (valid) {
526                         if (((taglo >> 11) & 1) ^ range_parity(taglo, 39, 26)) {
527                                 prom_printf("   ** bad parity in PTag1\n");
528                                 res |= CP0_CERRD_TAG_ADDRESS;
529                         }
530                         if (((taglo >> 10) & 1) ^ range_parity(taglo, 25, 13)) {
531                                 prom_printf("   ** bad parity in PTag0\n");
532                                 res |= CP0_CERRD_TAG_ADDRESS;
533                         }
534                 } else {
535                         res |= CP0_CERRD_TAG_STATE;
536                 }
537
538                 if (data) {
539                         uint64_t datalo;
540                         uint32_t datalohi, datalolo, datahi;
541                         int offset;
542                         char bad_ecc = 0;
543
544                         for (offset = 0; offset < 4; offset++) {
545                                 /* Index-load-data-D */
546                                 __asm__ __volatile__ (
547                                 "       .set    push\n\t"
548                                 "       .set    noreorder\n\t"
549                                 "       .set    mips64\n\t"
550                                 "       .set    noat\n\t"
551                                 "       cache   7, 0(%3)\n\t" /* Index-load-data-D */
552                                 "       mfc0    %0, $29, 3\n\t"
553                                 "       dmfc0   $1, $28, 3\n\t"
554                                 "       dsrl32  %1, $1, 0 \n\t"
555                                 "       sll     %2, $1, 0 \n\t"
556                                 "       .set    pop"
557                                 : "=r" (datahi), "=r" (datalohi), "=r" (datalolo)
558                                 : "r" ((way << 13) | addr | (offset << 3)));
559                                 datalo = ((unsigned long long)datalohi << 32) | datalolo;
560                                 ecc = dc_ecc(datalo);
561                                 if (ecc != datahi) {
562                                         int bits = 0;
563                                         bad_ecc |= 1 << (3-offset);
564                                         ecc ^= datahi;
565                                         while (ecc) {
566                                                 if (ecc & 1) bits++;
567                                                 ecc >>= 1;
568                                         }
569                                         res |= (bits == 1) ? CP0_CERRD_DATA_SBE : CP0_CERRD_DATA_DBE;
570                                 }
571                                 prom_printf("  %02X-%016llX", datahi, datalo);
572                         }
573                         prom_printf("\n");
574                         if (bad_ecc)
575                                 prom_printf("  dwords w/ bad ECC: %d %d %d %d\n",
576                                             !!(bad_ecc & 8), !!(bad_ecc & 4),
577                                             !!(bad_ecc & 2), !!(bad_ecc & 1));
578                 }
579         }
580         return res;
581 }