1 /* Intel Sandy Bridge -EN/-EP/-EX Memory Controller kernel module 2 * 3 * This driver supports the memory controllers found on the Intel 4 * processor family Sandy Bridge. 5 * 6 * This file may be distributed under the terms of the 7 * GNU General Public License version 2 only. 8 * 9 * Copyright (c) 2011 by: 10 * Mauro Carvalho Chehab 11 */ 12 13 #include <linux/module.h> 14 #include <linux/init.h> 15 #include <linux/pci.h> 16 #include <linux/pci_ids.h> 17 #include <linux/slab.h> 18 #include <linux/delay.h> 19 #include <linux/edac.h> 20 #include <linux/mmzone.h> 21 #include <linux/smp.h> 22 #include <linux/bitmap.h> 23 #include <linux/math64.h> 24 #include <linux/mod_devicetable.h> 25 #include <asm/cpu_device_id.h> 26 #include <asm/intel-family.h> 27 #include <asm/processor.h> 28 #include <asm/mce.h> 29 30 #include "edac_module.h" 31 32 /* Static vars */ 33 static LIST_HEAD(sbridge_edac_list); 34 35 /* 36 * Alter this version for the module when modifications are made 37 */ 38 #define SBRIDGE_REVISION " Ver: 1.1.2 " 39 #define EDAC_MOD_STR "sbridge_edac" 40 41 /* 42 * Debug macros 43 */ 44 #define sbridge_printk(level, fmt, arg...) \ 45 edac_printk(level, "sbridge", fmt, ##arg) 46 47 #define sbridge_mc_printk(mci, level, fmt, arg...) \ 48 edac_mc_chipset_printk(mci, level, "sbridge", fmt, ##arg) 49 50 /* 51 * Get a bit field at register value <v>, from bit <lo> to bit <hi> 52 */ 53 #define GET_BITFIELD(v, lo, hi) \ 54 (((v) & GENMASK_ULL(hi, lo)) >> (lo)) 55 56 /* Devices 12 Function 6, Offsets 0x80 to 0xcc */ 57 static const u32 sbridge_dram_rule[] = { 58 0x80, 0x88, 0x90, 0x98, 0xa0, 59 0xa8, 0xb0, 0xb8, 0xc0, 0xc8, 60 }; 61 62 static const u32 ibridge_dram_rule[] = { 63 0x60, 0x68, 0x70, 0x78, 0x80, 64 0x88, 0x90, 0x98, 0xa0, 0xa8, 65 0xb0, 0xb8, 0xc0, 0xc8, 0xd0, 66 0xd8, 0xe0, 0xe8, 0xf0, 0xf8, 67 }; 68 69 static const u32 knl_dram_rule[] = { 70 0x60, 0x68, 0x70, 0x78, 0x80, /* 0-4 */ 71 0x88, 0x90, 0x98, 0xa0, 0xa8, /* 5-9 */ 72 0xb0, 0xb8, 0xc0, 0xc8, 0xd0, /* 10-14 */ 73 0xd8, 0xe0, 0xe8, 0xf0, 0xf8, /* 15-19 */ 74 0x100, 0x108, 0x110, 0x118, /* 20-23 */ 75 }; 76 77 #define DRAM_RULE_ENABLE(reg) GET_BITFIELD(reg, 0, 0) 78 #define A7MODE(reg) GET_BITFIELD(reg, 26, 26) 79 80 static char *show_dram_attr(u32 attr) 81 { 82 switch (attr) { 83 case 0: 84 return "DRAM"; 85 case 1: 86 return "MMCFG"; 87 case 2: 88 return "NXM"; 89 default: 90 return "unknown"; 91 } 92 } 93 94 static const u32 sbridge_interleave_list[] = { 95 0x84, 0x8c, 0x94, 0x9c, 0xa4, 96 0xac, 0xb4, 0xbc, 0xc4, 0xcc, 97 }; 98 99 static const u32 ibridge_interleave_list[] = { 100 0x64, 0x6c, 0x74, 0x7c, 0x84, 101 0x8c, 0x94, 0x9c, 0xa4, 0xac, 102 0xb4, 0xbc, 0xc4, 0xcc, 0xd4, 103 0xdc, 0xe4, 0xec, 0xf4, 0xfc, 104 }; 105 106 static const u32 knl_interleave_list[] = { 107 0x64, 0x6c, 0x74, 0x7c, 0x84, /* 0-4 */ 108 0x8c, 0x94, 0x9c, 0xa4, 0xac, /* 5-9 */ 109 0xb4, 0xbc, 0xc4, 0xcc, 0xd4, /* 10-14 */ 110 0xdc, 0xe4, 0xec, 0xf4, 0xfc, /* 15-19 */ 111 0x104, 0x10c, 0x114, 0x11c, /* 20-23 */ 112 }; 113 114 struct interleave_pkg { 115 unsigned char start; 116 unsigned char end; 117 }; 118 119 static const struct interleave_pkg sbridge_interleave_pkg[] = { 120 { 0, 2 }, 121 { 3, 5 }, 122 { 8, 10 }, 123 { 11, 13 }, 124 { 16, 18 }, 125 { 19, 21 }, 126 { 24, 26 }, 127 { 27, 29 }, 128 }; 129 130 static const struct interleave_pkg ibridge_interleave_pkg[] = { 131 { 0, 3 }, 132 { 4, 7 }, 133 { 8, 11 }, 134 { 12, 15 }, 135 { 16, 19 }, 136 { 20, 23 }, 137 { 24, 27 }, 138 { 28, 31 }, 139 }; 140 141 static inline int sad_pkg(const struct interleave_pkg *table, u32 reg, 142 int interleave) 143 { 144 return GET_BITFIELD(reg, table[interleave].start, 145 table[interleave].end); 146 } 147 148 /* Devices 12 Function 7 */ 149 150 #define TOLM 0x80 151 #define TOHM 0x84 152 #define HASWELL_TOLM 0xd0 153 #define HASWELL_TOHM_0 0xd4 154 #define HASWELL_TOHM_1 0xd8 155 #define KNL_TOLM 0xd0 156 #define KNL_TOHM_0 0xd4 157 #define KNL_TOHM_1 0xd8 158 159 #define GET_TOLM(reg) ((GET_BITFIELD(reg, 0, 3) << 28) | 0x3ffffff) 160 #define GET_TOHM(reg) ((GET_BITFIELD(reg, 0, 20) << 25) | 0x3ffffff) 161 162 /* Device 13 Function 6 */ 163 164 #define SAD_TARGET 0xf0 165 166 #define SOURCE_ID(reg) GET_BITFIELD(reg, 9, 11) 167 168 #define SOURCE_ID_KNL(reg) GET_BITFIELD(reg, 12, 14) 169 170 #define SAD_CONTROL 0xf4 171 172 /* Device 14 function 0 */ 173 174 static const u32 tad_dram_rule[] = { 175 0x40, 0x44, 0x48, 0x4c, 176 0x50, 0x54, 0x58, 0x5c, 177 0x60, 0x64, 0x68, 0x6c, 178 }; 179 #define MAX_TAD ARRAY_SIZE(tad_dram_rule) 180 181 #define TAD_LIMIT(reg) ((GET_BITFIELD(reg, 12, 31) << 26) | 0x3ffffff) 182 #define TAD_SOCK(reg) GET_BITFIELD(reg, 10, 11) 183 #define TAD_CH(reg) GET_BITFIELD(reg, 8, 9) 184 #define TAD_TGT3(reg) GET_BITFIELD(reg, 6, 7) 185 #define TAD_TGT2(reg) GET_BITFIELD(reg, 4, 5) 186 #define TAD_TGT1(reg) GET_BITFIELD(reg, 2, 3) 187 #define TAD_TGT0(reg) GET_BITFIELD(reg, 0, 1) 188 189 /* Device 15, function 0 */ 190 191 #define MCMTR 0x7c 192 #define KNL_MCMTR 0x624 193 194 #define IS_ECC_ENABLED(mcmtr) GET_BITFIELD(mcmtr, 2, 2) 195 #define IS_LOCKSTEP_ENABLED(mcmtr) GET_BITFIELD(mcmtr, 1, 1) 196 #define IS_CLOSE_PG(mcmtr) GET_BITFIELD(mcmtr, 0, 0) 197 198 /* Device 15, function 1 */ 199 200 #define RASENABLES 0xac 201 #define IS_MIRROR_ENABLED(reg) GET_BITFIELD(reg, 0, 0) 202 203 /* Device 15, functions 2-5 */ 204 205 static const int mtr_regs[] = { 206 0x80, 0x84, 0x88, 207 }; 208 209 static const int knl_mtr_reg = 0xb60; 210 211 #define RANK_DISABLE(mtr) GET_BITFIELD(mtr, 16, 19) 212 #define IS_DIMM_PRESENT(mtr) GET_BITFIELD(mtr, 14, 14) 213 #define RANK_CNT_BITS(mtr) GET_BITFIELD(mtr, 12, 13) 214 #define RANK_WIDTH_BITS(mtr) GET_BITFIELD(mtr, 2, 4) 215 #define COL_WIDTH_BITS(mtr) GET_BITFIELD(mtr, 0, 1) 216 217 static const u32 tad_ch_nilv_offset[] = { 218 0x90, 0x94, 0x98, 0x9c, 219 0xa0, 0xa4, 0xa8, 0xac, 220 0xb0, 0xb4, 0xb8, 0xbc, 221 }; 222 #define CHN_IDX_OFFSET(reg) GET_BITFIELD(reg, 28, 29) 223 #define TAD_OFFSET(reg) (GET_BITFIELD(reg, 6, 25) << 26) 224 225 static const u32 rir_way_limit[] = { 226 0x108, 0x10c, 0x110, 0x114, 0x118, 227 }; 228 #define MAX_RIR_RANGES ARRAY_SIZE(rir_way_limit) 229 230 #define IS_RIR_VALID(reg) GET_BITFIELD(reg, 31, 31) 231 #define RIR_WAY(reg) GET_BITFIELD(reg, 28, 29) 232 233 #define MAX_RIR_WAY 8 234 235 static const u32 rir_offset[MAX_RIR_RANGES][MAX_RIR_WAY] = { 236 { 0x120, 0x124, 0x128, 0x12c, 0x130, 0x134, 0x138, 0x13c }, 237 { 0x140, 0x144, 0x148, 0x14c, 0x150, 0x154, 0x158, 0x15c }, 238 { 0x160, 0x164, 0x168, 0x16c, 0x170, 0x174, 0x178, 0x17c }, 239 { 0x180, 0x184, 0x188, 0x18c, 0x190, 0x194, 0x198, 0x19c }, 240 { 0x1a0, 0x1a4, 0x1a8, 0x1ac, 0x1b0, 0x1b4, 0x1b8, 0x1bc }, 241 }; 242 243 #define RIR_RNK_TGT(type, reg) (((type) == BROADWELL) ? \ 244 GET_BITFIELD(reg, 20, 23) : GET_BITFIELD(reg, 16, 19)) 245 246 #define RIR_OFFSET(type, reg) (((type) == HASWELL || (type) == BROADWELL) ? \ 247 GET_BITFIELD(reg, 2, 15) : GET_BITFIELD(reg, 2, 14)) 248 249 /* Device 16, functions 2-7 */ 250 251 /* 252 * FIXME: Implement the error count reads directly 253 */ 254 255 static const u32 correrrcnt[] = { 256 0x104, 0x108, 0x10c, 0x110, 257 }; 258 259 #define RANK_ODD_OV(reg) GET_BITFIELD(reg, 31, 31) 260 #define RANK_ODD_ERR_CNT(reg) GET_BITFIELD(reg, 16, 30) 261 #define RANK_EVEN_OV(reg) GET_BITFIELD(reg, 15, 15) 262 #define RANK_EVEN_ERR_CNT(reg) GET_BITFIELD(reg, 0, 14) 263 264 static const u32 correrrthrsld[] = { 265 0x11c, 0x120, 0x124, 0x128, 266 }; 267 268 #define RANK_ODD_ERR_THRSLD(reg) GET_BITFIELD(reg, 16, 30) 269 #define RANK_EVEN_ERR_THRSLD(reg) GET_BITFIELD(reg, 0, 14) 270 271 272 /* Device 17, function 0 */ 273 274 #define SB_RANK_CFG_A 0x0328 275 276 #define IB_RANK_CFG_A 0x0320 277 278 /* 279 * sbridge structs 280 */ 281 282 #define NUM_CHANNELS 4 /* Max channels per MC */ 283 #define MAX_DIMMS 3 /* Max DIMMS per channel */ 284 #define KNL_MAX_CHAS 38 /* KNL max num. of Cache Home Agents */ 285 #define KNL_MAX_CHANNELS 6 /* KNL max num. of PCI channels */ 286 #define KNL_MAX_EDCS 8 /* Embedded DRAM controllers */ 287 #define CHANNEL_UNSPECIFIED 0xf /* Intel IA32 SDM 15-14 */ 288 289 enum type { 290 SANDY_BRIDGE, 291 IVY_BRIDGE, 292 HASWELL, 293 BROADWELL, 294 KNIGHTS_LANDING, 295 }; 296 297 enum domain { 298 IMC0 = 0, 299 IMC1, 300 SOCK, 301 }; 302 303 struct sbridge_pvt; 304 struct sbridge_info { 305 enum type type; 306 u32 mcmtr; 307 u32 rankcfgr; 308 u64 (*get_tolm)(struct sbridge_pvt *pvt); 309 u64 (*get_tohm)(struct sbridge_pvt *pvt); 310 u64 (*rir_limit)(u32 reg); 311 u64 (*sad_limit)(u32 reg); 312 u32 (*interleave_mode)(u32 reg); 313 u32 (*dram_attr)(u32 reg); 314 const u32 *dram_rule; 315 const u32 *interleave_list; 316 const struct interleave_pkg *interleave_pkg; 317 u8 max_sad; 318 u8 max_interleave; 319 u8 (*get_node_id)(struct sbridge_pvt *pvt); 320 enum mem_type (*get_memory_type)(struct sbridge_pvt *pvt); 321 enum dev_type (*get_width)(struct sbridge_pvt *pvt, u32 mtr); 322 struct pci_dev *pci_vtd; 323 }; 324 325 struct sbridge_channel { 326 u32 ranks; 327 u32 dimms; 328 }; 329 330 struct pci_id_descr { 331 int dev_id; 332 int optional; 333 enum domain dom; 334 }; 335 336 struct pci_id_table { 337 const struct pci_id_descr *descr; 338 int n_devs_per_imc; 339 int n_devs_per_sock; 340 int n_imcs_per_sock; 341 enum type type; 342 }; 343 344 struct sbridge_dev { 345 struct list_head list; 346 u8 bus, mc; 347 u8 node_id, source_id; 348 struct pci_dev **pdev; 349 enum domain dom; 350 int n_devs; 351 int i_devs; 352 struct mem_ctl_info *mci; 353 }; 354 355 struct knl_pvt { 356 struct pci_dev *pci_cha[KNL_MAX_CHAS]; 357 struct pci_dev *pci_channel[KNL_MAX_CHANNELS]; 358 struct pci_dev *pci_mc0; 359 struct pci_dev *pci_mc1; 360 struct pci_dev *pci_mc0_misc; 361 struct pci_dev *pci_mc1_misc; 362 struct pci_dev *pci_mc_info; /* tolm, tohm */ 363 }; 364 365 struct sbridge_pvt { 366 /* Devices per socket */ 367 struct pci_dev *pci_ddrio; 368 struct pci_dev *pci_sad0, *pci_sad1; 369 struct pci_dev *pci_br0, *pci_br1; 370 /* Devices per memory controller */ 371 struct pci_dev *pci_ha, *pci_ta, *pci_ras; 372 struct pci_dev *pci_tad[NUM_CHANNELS]; 373 374 struct sbridge_dev *sbridge_dev; 375 376 struct sbridge_info info; 377 struct sbridge_channel channel[NUM_CHANNELS]; 378 379 /* Memory type detection */ 380 bool is_mirrored, is_lockstep, is_close_pg; 381 bool is_chan_hash; 382 383 /* Memory description */ 384 u64 tolm, tohm; 385 struct knl_pvt knl; 386 }; 387 388 #define PCI_DESCR(device_id, opt, domain) \ 389 .dev_id = (device_id), \ 390 .optional = opt, \ 391 .dom = domain 392 393 static const struct pci_id_descr pci_dev_descr_sbridge[] = { 394 /* Processor Home Agent */ 395 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0, 0, IMC0) }, 396 397 /* Memory controller */ 398 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA, 0, IMC0) }, 399 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS, 0, IMC0) }, 400 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0, 0, IMC0) }, 401 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1, 0, IMC0) }, 402 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2, 0, IMC0) }, 403 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3, 0, IMC0) }, 404 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO, 1, SOCK) }, 405 406 /* System Address Decoder */ 407 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0, 0, SOCK) }, 408 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1, 0, SOCK) }, 409 410 /* Broadcast Registers */ 411 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_BR, 0, SOCK) }, 412 }; 413 414 #define PCI_ID_TABLE_ENTRY(A, N, M, T) { \ 415 .descr = A, \ 416 .n_devs_per_imc = N, \ 417 .n_devs_per_sock = ARRAY_SIZE(A), \ 418 .n_imcs_per_sock = M, \ 419 .type = T \ 420 } 421 422 static const struct pci_id_table pci_dev_descr_sbridge_table[] = { 423 PCI_ID_TABLE_ENTRY(pci_dev_descr_sbridge, ARRAY_SIZE(pci_dev_descr_sbridge), 1, SANDY_BRIDGE), 424 {0,} /* 0 terminated list. */ 425 }; 426 427 /* This changes depending if 1HA or 2HA: 428 * 1HA: 429 * 0x0eb8 (17.0) is DDRIO0 430 * 2HA: 431 * 0x0ebc (17.4) is DDRIO0 432 */ 433 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0 0x0eb8 434 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0 0x0ebc 435 436 /* pci ids */ 437 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0 0x0ea0 438 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA 0x0ea8 439 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS 0x0e71 440 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0 0x0eaa 441 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1 0x0eab 442 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2 0x0eac 443 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3 0x0ead 444 #define PCI_DEVICE_ID_INTEL_IBRIDGE_SAD 0x0ec8 445 #define PCI_DEVICE_ID_INTEL_IBRIDGE_BR0 0x0ec9 446 #define PCI_DEVICE_ID_INTEL_IBRIDGE_BR1 0x0eca 447 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1 0x0e60 448 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA 0x0e68 449 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS 0x0e79 450 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0 0x0e6a 451 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1 0x0e6b 452 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2 0x0e6c 453 #define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3 0x0e6d 454 455 static const struct pci_id_descr pci_dev_descr_ibridge[] = { 456 /* Processor Home Agent */ 457 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0, 0, IMC0) }, 458 459 /* Memory controller */ 460 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA, 0, IMC0) }, 461 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS, 0, IMC0) }, 462 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0, 0, IMC0) }, 463 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1, 0, IMC0) }, 464 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2, 0, IMC0) }, 465 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3, 0, IMC0) }, 466 467 /* Optional, mode 2HA */ 468 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1, 1, IMC1) }, 469 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA, 1, IMC1) }, 470 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS, 1, IMC1) }, 471 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0, 1, IMC1) }, 472 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1, 1, IMC1) }, 473 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2, 1, IMC1) }, 474 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3, 1, IMC1) }, 475 476 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0, 1, SOCK) }, 477 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0, 1, SOCK) }, 478 479 /* System Address Decoder */ 480 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_SAD, 0, SOCK) }, 481 482 /* Broadcast Registers */ 483 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_BR0, 1, SOCK) }, 484 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_BR1, 0, SOCK) }, 485 486 }; 487 488 static const struct pci_id_table pci_dev_descr_ibridge_table[] = { 489 PCI_ID_TABLE_ENTRY(pci_dev_descr_ibridge, 12, 2, IVY_BRIDGE), 490 {0,} /* 0 terminated list. */ 491 }; 492 493 /* Haswell support */ 494 /* EN processor: 495 * - 1 IMC 496 * - 3 DDR3 channels, 2 DPC per channel 497 * EP processor: 498 * - 1 or 2 IMC 499 * - 4 DDR4 channels, 3 DPC per channel 500 * EP 4S processor: 501 * - 2 IMC 502 * - 4 DDR4 channels, 3 DPC per channel 503 * EX processor: 504 * - 2 IMC 505 * - each IMC interfaces with a SMI 2 channel 506 * - each SMI channel interfaces with a scalable memory buffer 507 * - each scalable memory buffer supports 4 DDR3/DDR4 channels, 3 DPC 508 */ 509 #define HASWELL_DDRCRCLKCONTROLS 0xa10 /* Ditto on Broadwell */ 510 #define HASWELL_HASYSDEFEATURE2 0x84 511 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_VTD_MISC 0x2f28 512 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0 0x2fa0 513 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1 0x2f60 514 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA 0x2fa8 515 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TM 0x2f71 516 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA 0x2f68 517 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TM 0x2f79 518 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0 0x2ffc 519 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1 0x2ffd 520 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0 0x2faa 521 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1 0x2fab 522 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2 0x2fac 523 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3 0x2fad 524 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0 0x2f6a 525 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1 0x2f6b 526 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2 0x2f6c 527 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3 0x2f6d 528 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0 0x2fbd 529 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1 0x2fbf 530 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2 0x2fb9 531 #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3 0x2fbb 532 static const struct pci_id_descr pci_dev_descr_haswell[] = { 533 /* first item must be the HA */ 534 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0, 0, IMC0) }, 535 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1, 1, IMC1) }, 536 537 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA, 0, IMC0) }, 538 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TM, 0, IMC0) }, 539 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0, 0, IMC0) }, 540 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1, 0, IMC0) }, 541 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2, 1, IMC0) }, 542 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3, 1, IMC0) }, 543 544 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA, 1, IMC1) }, 545 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TM, 1, IMC1) }, 546 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0, 1, IMC1) }, 547 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1, 1, IMC1) }, 548 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2, 1, IMC1) }, 549 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3, 1, IMC1) }, 550 551 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0, 0, SOCK) }, 552 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1, 0, SOCK) }, 553 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0, 1, SOCK) }, 554 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1, 1, SOCK) }, 555 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2, 1, SOCK) }, 556 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3, 1, SOCK) }, 557 }; 558 559 static const struct pci_id_table pci_dev_descr_haswell_table[] = { 560 PCI_ID_TABLE_ENTRY(pci_dev_descr_haswell, 13, 2, HASWELL), 561 {0,} /* 0 terminated list. */ 562 }; 563 564 /* Knight's Landing Support */ 565 /* 566 * KNL's memory channels are swizzled between memory controllers. 567 * MC0 is mapped to CH3,4,5 and MC1 is mapped to CH0,1,2 568 */ 569 #define knl_channel_remap(mc, chan) ((mc) ? (chan) : (chan) + 3) 570 571 /* Memory controller, TAD tables, error injection - 2-8-0, 2-9-0 (2 of these) */ 572 #define PCI_DEVICE_ID_INTEL_KNL_IMC_MC 0x7840 573 /* DRAM channel stuff; bank addrs, dimmmtr, etc.. 2-8-2 - 2-9-4 (6 of these) */ 574 #define PCI_DEVICE_ID_INTEL_KNL_IMC_CHAN 0x7843 575 /* kdrwdbu TAD limits/offsets, MCMTR - 2-10-1, 2-11-1 (2 of these) */ 576 #define PCI_DEVICE_ID_INTEL_KNL_IMC_TA 0x7844 577 /* CHA broadcast registers, dram rules - 1-29-0 (1 of these) */ 578 #define PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0 0x782a 579 /* SAD target - 1-29-1 (1 of these) */ 580 #define PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1 0x782b 581 /* Caching / Home Agent */ 582 #define PCI_DEVICE_ID_INTEL_KNL_IMC_CHA 0x782c 583 /* Device with TOLM and TOHM, 0-5-0 (1 of these) */ 584 #define PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM 0x7810 585 586 /* 587 * KNL differs from SB, IB, and Haswell in that it has multiple 588 * instances of the same device with the same device ID, so we handle that 589 * by creating as many copies in the table as we expect to find. 590 * (Like device ID must be grouped together.) 591 */ 592 593 static const struct pci_id_descr pci_dev_descr_knl[] = { 594 [0 ... 1] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_MC, 0, IMC0)}, 595 [2 ... 7] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_CHAN, 0, IMC0) }, 596 [8] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_TA, 0, IMC0) }, 597 [9] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM, 0, IMC0) }, 598 [10] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0, 0, SOCK) }, 599 [11] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1, 0, SOCK) }, 600 [12 ... 49] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_CHA, 0, SOCK) }, 601 }; 602 603 static const struct pci_id_table pci_dev_descr_knl_table[] = { 604 PCI_ID_TABLE_ENTRY(pci_dev_descr_knl, ARRAY_SIZE(pci_dev_descr_knl), 1, KNIGHTS_LANDING), 605 {0,} 606 }; 607 608 /* 609 * Broadwell support 610 * 611 * DE processor: 612 * - 1 IMC 613 * - 2 DDR3 channels, 2 DPC per channel 614 * EP processor: 615 * - 1 or 2 IMC 616 * - 4 DDR4 channels, 3 DPC per channel 617 * EP 4S processor: 618 * - 2 IMC 619 * - 4 DDR4 channels, 3 DPC per channel 620 * EX processor: 621 * - 2 IMC 622 * - each IMC interfaces with a SMI 2 channel 623 * - each SMI channel interfaces with a scalable memory buffer 624 * - each scalable memory buffer supports 4 DDR3/DDR4 channels, 3 DPC 625 */ 626 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_VTD_MISC 0x6f28 627 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0 0x6fa0 628 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1 0x6f60 629 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA 0x6fa8 630 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TM 0x6f71 631 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA 0x6f68 632 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TM 0x6f79 633 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0 0x6ffc 634 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1 0x6ffd 635 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0 0x6faa 636 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1 0x6fab 637 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2 0x6fac 638 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3 0x6fad 639 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0 0x6f6a 640 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1 0x6f6b 641 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2 0x6f6c 642 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3 0x6f6d 643 #define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0 0x6faf 644 645 static const struct pci_id_descr pci_dev_descr_broadwell[] = { 646 /* first item must be the HA */ 647 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0, 0, IMC0) }, 648 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1, 1, IMC1) }, 649 650 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA, 0, IMC0) }, 651 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TM, 0, IMC0) }, 652 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0, 0, IMC0) }, 653 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1, 0, IMC0) }, 654 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2, 1, IMC0) }, 655 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3, 1, IMC0) }, 656 657 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA, 1, IMC1) }, 658 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TM, 1, IMC1) }, 659 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0, 1, IMC1) }, 660 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1, 1, IMC1) }, 661 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2, 1, IMC1) }, 662 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3, 1, IMC1) }, 663 664 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0, 0, SOCK) }, 665 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1, 0, SOCK) }, 666 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0, 1, SOCK) }, 667 }; 668 669 static const struct pci_id_table pci_dev_descr_broadwell_table[] = { 670 PCI_ID_TABLE_ENTRY(pci_dev_descr_broadwell, 10, 2, BROADWELL), 671 {0,} /* 0 terminated list. */ 672 }; 673 674 675 /**************************************************************************** 676 Ancillary status routines 677 ****************************************************************************/ 678 679 static inline int numrank(enum type type, u32 mtr) 680 { 681 int ranks = (1 << RANK_CNT_BITS(mtr)); 682 int max = 4; 683 684 if (type == HASWELL || type == BROADWELL || type == KNIGHTS_LANDING) 685 max = 8; 686 687 if (ranks > max) { 688 edac_dbg(0, "Invalid number of ranks: %d (max = %i) raw value = %x (%04x)\n", 689 ranks, max, (unsigned int)RANK_CNT_BITS(mtr), mtr); 690 return -EINVAL; 691 } 692 693 return ranks; 694 } 695 696 static inline int numrow(u32 mtr) 697 { 698 int rows = (RANK_WIDTH_BITS(mtr) + 12); 699 700 if (rows < 13 || rows > 18) { 701 edac_dbg(0, "Invalid number of rows: %d (should be between 14 and 17) raw value = %x (%04x)\n", 702 rows, (unsigned int)RANK_WIDTH_BITS(mtr), mtr); 703 return -EINVAL; 704 } 705 706 return 1 << rows; 707 } 708 709 static inline int numcol(u32 mtr) 710 { 711 int cols = (COL_WIDTH_BITS(mtr) + 10); 712 713 if (cols > 12) { 714 edac_dbg(0, "Invalid number of cols: %d (max = 4) raw value = %x (%04x)\n", 715 cols, (unsigned int)COL_WIDTH_BITS(mtr), mtr); 716 return -EINVAL; 717 } 718 719 return 1 << cols; 720 } 721 722 static struct sbridge_dev *get_sbridge_dev(u8 bus, enum domain dom, int multi_bus, 723 struct sbridge_dev *prev) 724 { 725 struct sbridge_dev *sbridge_dev; 726 727 /* 728 * If we have devices scattered across several busses that pertain 729 * to the same memory controller, we'll lump them all together. 730 */ 731 if (multi_bus) { 732 return list_first_entry_or_null(&sbridge_edac_list, 733 struct sbridge_dev, list); 734 } 735 736 sbridge_dev = list_entry(prev ? prev->list.next 737 : sbridge_edac_list.next, struct sbridge_dev, list); 738 739 list_for_each_entry_from(sbridge_dev, &sbridge_edac_list, list) { 740 if (sbridge_dev->bus == bus && (dom == SOCK || dom == sbridge_dev->dom)) 741 return sbridge_dev; 742 } 743 744 return NULL; 745 } 746 747 static struct sbridge_dev *alloc_sbridge_dev(u8 bus, enum domain dom, 748 const struct pci_id_table *table) 749 { 750 struct sbridge_dev *sbridge_dev; 751 752 sbridge_dev = kzalloc(sizeof(*sbridge_dev), GFP_KERNEL); 753 if (!sbridge_dev) 754 return NULL; 755 756 sbridge_dev->pdev = kcalloc(table->n_devs_per_imc, 757 sizeof(*sbridge_dev->pdev), 758 GFP_KERNEL); 759 if (!sbridge_dev->pdev) { 760 kfree(sbridge_dev); 761 return NULL; 762 } 763 764 sbridge_dev->bus = bus; 765 sbridge_dev->dom = dom; 766 sbridge_dev->n_devs = table->n_devs_per_imc; 767 list_add_tail(&sbridge_dev->list, &sbridge_edac_list); 768 769 return sbridge_dev; 770 } 771 772 static void free_sbridge_dev(struct sbridge_dev *sbridge_dev) 773 { 774 list_del(&sbridge_dev->list); 775 kfree(sbridge_dev->pdev); 776 kfree(sbridge_dev); 777 } 778 779 static u64 sbridge_get_tolm(struct sbridge_pvt *pvt) 780 { 781 u32 reg; 782 783 /* Address range is 32:28 */ 784 pci_read_config_dword(pvt->pci_sad1, TOLM, ®); 785 return GET_TOLM(reg); 786 } 787 788 static u64 sbridge_get_tohm(struct sbridge_pvt *pvt) 789 { 790 u32 reg; 791 792 pci_read_config_dword(pvt->pci_sad1, TOHM, ®); 793 return GET_TOHM(reg); 794 } 795 796 static u64 ibridge_get_tolm(struct sbridge_pvt *pvt) 797 { 798 u32 reg; 799 800 pci_read_config_dword(pvt->pci_br1, TOLM, ®); 801 802 return GET_TOLM(reg); 803 } 804 805 static u64 ibridge_get_tohm(struct sbridge_pvt *pvt) 806 { 807 u32 reg; 808 809 pci_read_config_dword(pvt->pci_br1, TOHM, ®); 810 811 return GET_TOHM(reg); 812 } 813 814 static u64 rir_limit(u32 reg) 815 { 816 return ((u64)GET_BITFIELD(reg, 1, 10) << 29) | 0x1fffffff; 817 } 818 819 static u64 sad_limit(u32 reg) 820 { 821 return (GET_BITFIELD(reg, 6, 25) << 26) | 0x3ffffff; 822 } 823 824 static u32 interleave_mode(u32 reg) 825 { 826 return GET_BITFIELD(reg, 1, 1); 827 } 828 829 static u32 dram_attr(u32 reg) 830 { 831 return GET_BITFIELD(reg, 2, 3); 832 } 833 834 static u64 knl_sad_limit(u32 reg) 835 { 836 return (GET_BITFIELD(reg, 7, 26) << 26) | 0x3ffffff; 837 } 838 839 static u32 knl_interleave_mode(u32 reg) 840 { 841 return GET_BITFIELD(reg, 1, 2); 842 } 843 844 static const char * const knl_intlv_mode[] = { 845 "[8:6]", "[10:8]", "[14:12]", "[32:30]" 846 }; 847 848 static const char *get_intlv_mode_str(u32 reg, enum type t) 849 { 850 if (t == KNIGHTS_LANDING) 851 return knl_intlv_mode[knl_interleave_mode(reg)]; 852 else 853 return interleave_mode(reg) ? "[8:6]" : "[8:6]XOR[18:16]"; 854 } 855 856 static u32 dram_attr_knl(u32 reg) 857 { 858 return GET_BITFIELD(reg, 3, 4); 859 } 860 861 862 static enum mem_type get_memory_type(struct sbridge_pvt *pvt) 863 { 864 u32 reg; 865 enum mem_type mtype; 866 867 if (pvt->pci_ddrio) { 868 pci_read_config_dword(pvt->pci_ddrio, pvt->info.rankcfgr, 869 ®); 870 if (GET_BITFIELD(reg, 11, 11)) 871 /* FIXME: Can also be LRDIMM */ 872 mtype = MEM_RDDR3; 873 else 874 mtype = MEM_DDR3; 875 } else 876 mtype = MEM_UNKNOWN; 877 878 return mtype; 879 } 880 881 static enum mem_type haswell_get_memory_type(struct sbridge_pvt *pvt) 882 { 883 u32 reg; 884 bool registered = false; 885 enum mem_type mtype = MEM_UNKNOWN; 886 887 if (!pvt->pci_ddrio) 888 goto out; 889 890 pci_read_config_dword(pvt->pci_ddrio, 891 HASWELL_DDRCRCLKCONTROLS, ®); 892 /* Is_Rdimm */ 893 if (GET_BITFIELD(reg, 16, 16)) 894 registered = true; 895 896 pci_read_config_dword(pvt->pci_ta, MCMTR, ®); 897 if (GET_BITFIELD(reg, 14, 14)) { 898 if (registered) 899 mtype = MEM_RDDR4; 900 else 901 mtype = MEM_DDR4; 902 } else { 903 if (registered) 904 mtype = MEM_RDDR3; 905 else 906 mtype = MEM_DDR3; 907 } 908 909 out: 910 return mtype; 911 } 912 913 static enum dev_type knl_get_width(struct sbridge_pvt *pvt, u32 mtr) 914 { 915 /* for KNL value is fixed */ 916 return DEV_X16; 917 } 918 919 static enum dev_type sbridge_get_width(struct sbridge_pvt *pvt, u32 mtr) 920 { 921 /* there's no way to figure out */ 922 return DEV_UNKNOWN; 923 } 924 925 static enum dev_type __ibridge_get_width(u32 mtr) 926 { 927 enum dev_type type; 928 929 switch (mtr) { 930 case 3: 931 type = DEV_UNKNOWN; 932 break; 933 case 2: 934 type = DEV_X16; 935 break; 936 case 1: 937 type = DEV_X8; 938 break; 939 case 0: 940 type = DEV_X4; 941 break; 942 } 943 944 return type; 945 } 946 947 static enum dev_type ibridge_get_width(struct sbridge_pvt *pvt, u32 mtr) 948 { 949 /* 950 * ddr3_width on the documentation but also valid for DDR4 on 951 * Haswell 952 */ 953 return __ibridge_get_width(GET_BITFIELD(mtr, 7, 8)); 954 } 955 956 static enum dev_type broadwell_get_width(struct sbridge_pvt *pvt, u32 mtr) 957 { 958 /* ddr3_width on the documentation but also valid for DDR4 */ 959 return __ibridge_get_width(GET_BITFIELD(mtr, 8, 9)); 960 } 961 962 static enum mem_type knl_get_memory_type(struct sbridge_pvt *pvt) 963 { 964 /* DDR4 RDIMMS and LRDIMMS are supported */ 965 return MEM_RDDR4; 966 } 967 968 static u8 get_node_id(struct sbridge_pvt *pvt) 969 { 970 u32 reg; 971 pci_read_config_dword(pvt->pci_br0, SAD_CONTROL, ®); 972 return GET_BITFIELD(reg, 0, 2); 973 } 974 975 static u8 haswell_get_node_id(struct sbridge_pvt *pvt) 976 { 977 u32 reg; 978 979 pci_read_config_dword(pvt->pci_sad1, SAD_CONTROL, ®); 980 return GET_BITFIELD(reg, 0, 3); 981 } 982 983 static u8 knl_get_node_id(struct sbridge_pvt *pvt) 984 { 985 u32 reg; 986 987 pci_read_config_dword(pvt->pci_sad1, SAD_CONTROL, ®); 988 return GET_BITFIELD(reg, 0, 2); 989 } 990 991 992 static u64 haswell_get_tolm(struct sbridge_pvt *pvt) 993 { 994 u32 reg; 995 996 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOLM, ®); 997 return (GET_BITFIELD(reg, 26, 31) << 26) | 0x3ffffff; 998 } 999 1000 static u64 haswell_get_tohm(struct sbridge_pvt *pvt) 1001 { 1002 u64 rc; 1003 u32 reg; 1004 1005 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_0, ®); 1006 rc = GET_BITFIELD(reg, 26, 31); 1007 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_1, ®); 1008 rc = ((reg << 6) | rc) << 26; 1009 1010 return rc | 0x1ffffff; 1011 } 1012 1013 static u64 knl_get_tolm(struct sbridge_pvt *pvt) 1014 { 1015 u32 reg; 1016 1017 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOLM, ®); 1018 return (GET_BITFIELD(reg, 26, 31) << 26) | 0x3ffffff; 1019 } 1020 1021 static u64 knl_get_tohm(struct sbridge_pvt *pvt) 1022 { 1023 u64 rc; 1024 u32 reg_lo, reg_hi; 1025 1026 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOHM_0, ®_lo); 1027 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOHM_1, ®_hi); 1028 rc = ((u64)reg_hi << 32) | reg_lo; 1029 return rc | 0x3ffffff; 1030 } 1031 1032 1033 static u64 haswell_rir_limit(u32 reg) 1034 { 1035 return (((u64)GET_BITFIELD(reg, 1, 11) + 1) << 29) - 1; 1036 } 1037 1038 static inline u8 sad_pkg_socket(u8 pkg) 1039 { 1040 /* on Ivy Bridge, nodeID is SASS, where A is HA and S is node id */ 1041 return ((pkg >> 3) << 2) | (pkg & 0x3); 1042 } 1043 1044 static inline u8 sad_pkg_ha(u8 pkg) 1045 { 1046 return (pkg >> 2) & 0x1; 1047 } 1048 1049 static int haswell_chan_hash(int idx, u64 addr) 1050 { 1051 int i; 1052 1053 /* 1054 * XOR even bits from 12:26 to bit0 of idx, 1055 * odd bits from 13:27 to bit1 1056 */ 1057 for (i = 12; i < 28; i += 2) 1058 idx ^= (addr >> i) & 3; 1059 1060 return idx; 1061 } 1062 1063 /* Low bits of TAD limit, and some metadata. */ 1064 static const u32 knl_tad_dram_limit_lo[] = { 1065 0x400, 0x500, 0x600, 0x700, 1066 0x800, 0x900, 0xa00, 0xb00, 1067 }; 1068 1069 /* Low bits of TAD offset. */ 1070 static const u32 knl_tad_dram_offset_lo[] = { 1071 0x404, 0x504, 0x604, 0x704, 1072 0x804, 0x904, 0xa04, 0xb04, 1073 }; 1074 1075 /* High 16 bits of TAD limit and offset. */ 1076 static const u32 knl_tad_dram_hi[] = { 1077 0x408, 0x508, 0x608, 0x708, 1078 0x808, 0x908, 0xa08, 0xb08, 1079 }; 1080 1081 /* Number of ways a tad entry is interleaved. */ 1082 static const u32 knl_tad_ways[] = { 1083 8, 6, 4, 3, 2, 1, 1084 }; 1085 1086 /* 1087 * Retrieve the n'th Target Address Decode table entry 1088 * from the memory controller's TAD table. 1089 * 1090 * @pvt: driver private data 1091 * @entry: which entry you want to retrieve 1092 * @mc: which memory controller (0 or 1) 1093 * @offset: output tad range offset 1094 * @limit: output address of first byte above tad range 1095 * @ways: output number of interleave ways 1096 * 1097 * The offset value has curious semantics. It's a sort of running total 1098 * of the sizes of all the memory regions that aren't mapped in this 1099 * tad table. 1100 */ 1101 static int knl_get_tad(const struct sbridge_pvt *pvt, 1102 const int entry, 1103 const int mc, 1104 u64 *offset, 1105 u64 *limit, 1106 int *ways) 1107 { 1108 u32 reg_limit_lo, reg_offset_lo, reg_hi; 1109 struct pci_dev *pci_mc; 1110 int way_id; 1111 1112 switch (mc) { 1113 case 0: 1114 pci_mc = pvt->knl.pci_mc0; 1115 break; 1116 case 1: 1117 pci_mc = pvt->knl.pci_mc1; 1118 break; 1119 default: 1120 WARN_ON(1); 1121 return -EINVAL; 1122 } 1123 1124 pci_read_config_dword(pci_mc, 1125 knl_tad_dram_limit_lo[entry], ®_limit_lo); 1126 pci_read_config_dword(pci_mc, 1127 knl_tad_dram_offset_lo[entry], ®_offset_lo); 1128 pci_read_config_dword(pci_mc, 1129 knl_tad_dram_hi[entry], ®_hi); 1130 1131 /* Is this TAD entry enabled? */ 1132 if (!GET_BITFIELD(reg_limit_lo, 0, 0)) 1133 return -ENODEV; 1134 1135 way_id = GET_BITFIELD(reg_limit_lo, 3, 5); 1136 1137 if (way_id < ARRAY_SIZE(knl_tad_ways)) { 1138 *ways = knl_tad_ways[way_id]; 1139 } else { 1140 *ways = 0; 1141 sbridge_printk(KERN_ERR, 1142 "Unexpected value %d in mc_tad_limit_lo wayness field\n", 1143 way_id); 1144 return -ENODEV; 1145 } 1146 1147 /* 1148 * The least significant 6 bits of base and limit are truncated. 1149 * For limit, we fill the missing bits with 1s. 1150 */ 1151 *offset = ((u64) GET_BITFIELD(reg_offset_lo, 6, 31) << 6) | 1152 ((u64) GET_BITFIELD(reg_hi, 0, 15) << 32); 1153 *limit = ((u64) GET_BITFIELD(reg_limit_lo, 6, 31) << 6) | 63 | 1154 ((u64) GET_BITFIELD(reg_hi, 16, 31) << 32); 1155 1156 return 0; 1157 } 1158 1159 /* Determine which memory controller is responsible for a given channel. */ 1160 static int knl_channel_mc(int channel) 1161 { 1162 WARN_ON(channel < 0 || channel >= 6); 1163 1164 return channel < 3 ? 1 : 0; 1165 } 1166 1167 /* 1168 * Get the Nth entry from EDC_ROUTE_TABLE register. 1169 * (This is the per-tile mapping of logical interleave targets to 1170 * physical EDC modules.) 1171 * 1172 * entry 0: 0:2 1173 * 1: 3:5 1174 * 2: 6:8 1175 * 3: 9:11 1176 * 4: 12:14 1177 * 5: 15:17 1178 * 6: 18:20 1179 * 7: 21:23 1180 * reserved: 24:31 1181 */ 1182 static u32 knl_get_edc_route(int entry, u32 reg) 1183 { 1184 WARN_ON(entry >= KNL_MAX_EDCS); 1185 return GET_BITFIELD(reg, entry*3, (entry*3)+2); 1186 } 1187 1188 /* 1189 * Get the Nth entry from MC_ROUTE_TABLE register. 1190 * (This is the per-tile mapping of logical interleave targets to 1191 * physical DRAM channels modules.) 1192 * 1193 * entry 0: mc 0:2 channel 18:19 1194 * 1: mc 3:5 channel 20:21 1195 * 2: mc 6:8 channel 22:23 1196 * 3: mc 9:11 channel 24:25 1197 * 4: mc 12:14 channel 26:27 1198 * 5: mc 15:17 channel 28:29 1199 * reserved: 30:31 1200 * 1201 * Though we have 3 bits to identify the MC, we should only see 1202 * the values 0 or 1. 1203 */ 1204 1205 static u32 knl_get_mc_route(int entry, u32 reg) 1206 { 1207 int mc, chan; 1208 1209 WARN_ON(entry >= KNL_MAX_CHANNELS); 1210 1211 mc = GET_BITFIELD(reg, entry*3, (entry*3)+2); 1212 chan = GET_BITFIELD(reg, (entry*2) + 18, (entry*2) + 18 + 1); 1213 1214 return knl_channel_remap(mc, chan); 1215 } 1216 1217 /* 1218 * Render the EDC_ROUTE register in human-readable form. 1219 * Output string s should be at least KNL_MAX_EDCS*2 bytes. 1220 */ 1221 static void knl_show_edc_route(u32 reg, char *s) 1222 { 1223 int i; 1224 1225 for (i = 0; i < KNL_MAX_EDCS; i++) { 1226 s[i*2] = knl_get_edc_route(i, reg) + '0'; 1227 s[i*2+1] = '-'; 1228 } 1229 1230 s[KNL_MAX_EDCS*2 - 1] = '\0'; 1231 } 1232 1233 /* 1234 * Render the MC_ROUTE register in human-readable form. 1235 * Output string s should be at least KNL_MAX_CHANNELS*2 bytes. 1236 */ 1237 static void knl_show_mc_route(u32 reg, char *s) 1238 { 1239 int i; 1240 1241 for (i = 0; i < KNL_MAX_CHANNELS; i++) { 1242 s[i*2] = knl_get_mc_route(i, reg) + '0'; 1243 s[i*2+1] = '-'; 1244 } 1245 1246 s[KNL_MAX_CHANNELS*2 - 1] = '\0'; 1247 } 1248 1249 #define KNL_EDC_ROUTE 0xb8 1250 #define KNL_MC_ROUTE 0xb4 1251 1252 /* Is this dram rule backed by regular DRAM in flat mode? */ 1253 #define KNL_EDRAM(reg) GET_BITFIELD(reg, 29, 29) 1254 1255 /* Is this dram rule cached? */ 1256 #define KNL_CACHEABLE(reg) GET_BITFIELD(reg, 28, 28) 1257 1258 /* Is this rule backed by edc ? */ 1259 #define KNL_EDRAM_ONLY(reg) GET_BITFIELD(reg, 29, 29) 1260 1261 /* Is this rule backed by DRAM, cacheable in EDRAM? */ 1262 #define KNL_CACHEABLE(reg) GET_BITFIELD(reg, 28, 28) 1263 1264 /* Is this rule mod3? */ 1265 #define KNL_MOD3(reg) GET_BITFIELD(reg, 27, 27) 1266 1267 /* 1268 * Figure out how big our RAM modules are. 1269 * 1270 * The DIMMMTR register in KNL doesn't tell us the size of the DIMMs, so we 1271 * have to figure this out from the SAD rules, interleave lists, route tables, 1272 * and TAD rules. 1273 * 1274 * SAD rules can have holes in them (e.g. the 3G-4G hole), so we have to 1275 * inspect the TAD rules to figure out how large the SAD regions really are. 1276 * 1277 * When we know the real size of a SAD region and how many ways it's 1278 * interleaved, we know the individual contribution of each channel to 1279 * TAD is size/ways. 1280 * 1281 * Finally, we have to check whether each channel participates in each SAD 1282 * region. 1283 * 1284 * Fortunately, KNL only supports one DIMM per channel, so once we know how 1285 * much memory the channel uses, we know the DIMM is at least that large. 1286 * (The BIOS might possibly choose not to map all available memory, in which 1287 * case we will underreport the size of the DIMM.) 1288 * 1289 * In theory, we could try to determine the EDC sizes as well, but that would 1290 * only work in flat mode, not in cache mode. 1291 * 1292 * @mc_sizes: Output sizes of channels (must have space for KNL_MAX_CHANNELS 1293 * elements) 1294 */ 1295 static int knl_get_dimm_capacity(struct sbridge_pvt *pvt, u64 *mc_sizes) 1296 { 1297 u64 sad_base, sad_size, sad_limit = 0; 1298 u64 tad_base, tad_size, tad_limit, tad_deadspace, tad_livespace; 1299 int sad_rule = 0; 1300 int tad_rule = 0; 1301 int intrlv_ways, tad_ways; 1302 u32 first_pkg, pkg; 1303 int i; 1304 u64 sad_actual_size[2]; /* sad size accounting for holes, per mc */ 1305 u32 dram_rule, interleave_reg; 1306 u32 mc_route_reg[KNL_MAX_CHAS]; 1307 u32 edc_route_reg[KNL_MAX_CHAS]; 1308 int edram_only; 1309 char edc_route_string[KNL_MAX_EDCS*2]; 1310 char mc_route_string[KNL_MAX_CHANNELS*2]; 1311 int cur_reg_start; 1312 int mc; 1313 int channel; 1314 int way; 1315 int participants[KNL_MAX_CHANNELS]; 1316 int participant_count = 0; 1317 1318 for (i = 0; i < KNL_MAX_CHANNELS; i++) 1319 mc_sizes[i] = 0; 1320 1321 /* Read the EDC route table in each CHA. */ 1322 cur_reg_start = 0; 1323 for (i = 0; i < KNL_MAX_CHAS; i++) { 1324 pci_read_config_dword(pvt->knl.pci_cha[i], 1325 KNL_EDC_ROUTE, &edc_route_reg[i]); 1326 1327 if (i > 0 && edc_route_reg[i] != edc_route_reg[i-1]) { 1328 knl_show_edc_route(edc_route_reg[i-1], 1329 edc_route_string); 1330 if (cur_reg_start == i-1) 1331 edac_dbg(0, "edc route table for CHA %d: %s\n", 1332 cur_reg_start, edc_route_string); 1333 else 1334 edac_dbg(0, "edc route table for CHA %d-%d: %s\n", 1335 cur_reg_start, i-1, edc_route_string); 1336 cur_reg_start = i; 1337 } 1338 } 1339 knl_show_edc_route(edc_route_reg[i-1], edc_route_string); 1340 if (cur_reg_start == i-1) 1341 edac_dbg(0, "edc route table for CHA %d: %s\n", 1342 cur_reg_start, edc_route_string); 1343 else 1344 edac_dbg(0, "edc route table for CHA %d-%d: %s\n", 1345 cur_reg_start, i-1, edc_route_string); 1346 1347 /* Read the MC route table in each CHA. */ 1348 cur_reg_start = 0; 1349 for (i = 0; i < KNL_MAX_CHAS; i++) { 1350 pci_read_config_dword(pvt->knl.pci_cha[i], 1351 KNL_MC_ROUTE, &mc_route_reg[i]); 1352 1353 if (i > 0 && mc_route_reg[i] != mc_route_reg[i-1]) { 1354 knl_show_mc_route(mc_route_reg[i-1], mc_route_string); 1355 if (cur_reg_start == i-1) 1356 edac_dbg(0, "mc route table for CHA %d: %s\n", 1357 cur_reg_start, mc_route_string); 1358 else 1359 edac_dbg(0, "mc route table for CHA %d-%d: %s\n", 1360 cur_reg_start, i-1, mc_route_string); 1361 cur_reg_start = i; 1362 } 1363 } 1364 knl_show_mc_route(mc_route_reg[i-1], mc_route_string); 1365 if (cur_reg_start == i-1) 1366 edac_dbg(0, "mc route table for CHA %d: %s\n", 1367 cur_reg_start, mc_route_string); 1368 else 1369 edac_dbg(0, "mc route table for CHA %d-%d: %s\n", 1370 cur_reg_start, i-1, mc_route_string); 1371 1372 /* Process DRAM rules */ 1373 for (sad_rule = 0; sad_rule < pvt->info.max_sad; sad_rule++) { 1374 /* previous limit becomes the new base */ 1375 sad_base = sad_limit; 1376 1377 pci_read_config_dword(pvt->pci_sad0, 1378 pvt->info.dram_rule[sad_rule], &dram_rule); 1379 1380 if (!DRAM_RULE_ENABLE(dram_rule)) 1381 break; 1382 1383 edram_only = KNL_EDRAM_ONLY(dram_rule); 1384 1385 sad_limit = pvt->info.sad_limit(dram_rule)+1; 1386 sad_size = sad_limit - sad_base; 1387 1388 pci_read_config_dword(pvt->pci_sad0, 1389 pvt->info.interleave_list[sad_rule], &interleave_reg); 1390 1391 /* 1392 * Find out how many ways this dram rule is interleaved. 1393 * We stop when we see the first channel again. 1394 */ 1395 first_pkg = sad_pkg(pvt->info.interleave_pkg, 1396 interleave_reg, 0); 1397 for (intrlv_ways = 1; intrlv_ways < 8; intrlv_ways++) { 1398 pkg = sad_pkg(pvt->info.interleave_pkg, 1399 interleave_reg, intrlv_ways); 1400 1401 if ((pkg & 0x8) == 0) { 1402 /* 1403 * 0 bit means memory is non-local, 1404 * which KNL doesn't support 1405 */ 1406 edac_dbg(0, "Unexpected interleave target %d\n", 1407 pkg); 1408 return -1; 1409 } 1410 1411 if (pkg == first_pkg) 1412 break; 1413 } 1414 if (KNL_MOD3(dram_rule)) 1415 intrlv_ways *= 3; 1416 1417 edac_dbg(3, "dram rule %d (base 0x%llx, limit 0x%llx), %d way interleave%s\n", 1418 sad_rule, 1419 sad_base, 1420 sad_limit, 1421 intrlv_ways, 1422 edram_only ? ", EDRAM" : ""); 1423 1424 /* 1425 * Find out how big the SAD region really is by iterating 1426 * over TAD tables (SAD regions may contain holes). 1427 * Each memory controller might have a different TAD table, so 1428 * we have to look at both. 1429 * 1430 * Livespace is the memory that's mapped in this TAD table, 1431 * deadspace is the holes (this could be the MMIO hole, or it 1432 * could be memory that's mapped by the other TAD table but 1433 * not this one). 1434 */ 1435 for (mc = 0; mc < 2; mc++) { 1436 sad_actual_size[mc] = 0; 1437 tad_livespace = 0; 1438 for (tad_rule = 0; 1439 tad_rule < ARRAY_SIZE( 1440 knl_tad_dram_limit_lo); 1441 tad_rule++) { 1442 if (knl_get_tad(pvt, 1443 tad_rule, 1444 mc, 1445 &tad_deadspace, 1446 &tad_limit, 1447 &tad_ways)) 1448 break; 1449 1450 tad_size = (tad_limit+1) - 1451 (tad_livespace + tad_deadspace); 1452 tad_livespace += tad_size; 1453 tad_base = (tad_limit+1) - tad_size; 1454 1455 if (tad_base < sad_base) { 1456 if (tad_limit > sad_base) 1457 edac_dbg(0, "TAD region overlaps lower SAD boundary -- TAD tables may be configured incorrectly.\n"); 1458 } else if (tad_base < sad_limit) { 1459 if (tad_limit+1 > sad_limit) { 1460 edac_dbg(0, "TAD region overlaps upper SAD boundary -- TAD tables may be configured incorrectly.\n"); 1461 } else { 1462 /* TAD region is completely inside SAD region */ 1463 edac_dbg(3, "TAD region %d 0x%llx - 0x%llx (%lld bytes) table%d\n", 1464 tad_rule, tad_base, 1465 tad_limit, tad_size, 1466 mc); 1467 sad_actual_size[mc] += tad_size; 1468 } 1469 } 1470 tad_base = tad_limit+1; 1471 } 1472 } 1473 1474 for (mc = 0; mc < 2; mc++) { 1475 edac_dbg(3, " total TAD DRAM footprint in table%d : 0x%llx (%lld bytes)\n", 1476 mc, sad_actual_size[mc], sad_actual_size[mc]); 1477 } 1478 1479 /* Ignore EDRAM rule */ 1480 if (edram_only) 1481 continue; 1482 1483 /* Figure out which channels participate in interleave. */ 1484 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) 1485 participants[channel] = 0; 1486 1487 /* For each channel, does at least one CHA have 1488 * this channel mapped to the given target? 1489 */ 1490 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) { 1491 for (way = 0; way < intrlv_ways; way++) { 1492 int target; 1493 int cha; 1494 1495 if (KNL_MOD3(dram_rule)) 1496 target = way; 1497 else 1498 target = 0x7 & sad_pkg( 1499 pvt->info.interleave_pkg, interleave_reg, way); 1500 1501 for (cha = 0; cha < KNL_MAX_CHAS; cha++) { 1502 if (knl_get_mc_route(target, 1503 mc_route_reg[cha]) == channel 1504 && !participants[channel]) { 1505 participant_count++; 1506 participants[channel] = 1; 1507 break; 1508 } 1509 } 1510 } 1511 } 1512 1513 if (participant_count != intrlv_ways) 1514 edac_dbg(0, "participant_count (%d) != interleave_ways (%d): DIMM size may be incorrect\n", 1515 participant_count, intrlv_ways); 1516 1517 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) { 1518 mc = knl_channel_mc(channel); 1519 if (participants[channel]) { 1520 edac_dbg(4, "mc channel %d contributes %lld bytes via sad entry %d\n", 1521 channel, 1522 sad_actual_size[mc]/intrlv_ways, 1523 sad_rule); 1524 mc_sizes[channel] += 1525 sad_actual_size[mc]/intrlv_ways; 1526 } 1527 } 1528 } 1529 1530 return 0; 1531 } 1532 1533 static void get_source_id(struct mem_ctl_info *mci) 1534 { 1535 struct sbridge_pvt *pvt = mci->pvt_info; 1536 u32 reg; 1537 1538 if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL || 1539 pvt->info.type == KNIGHTS_LANDING) 1540 pci_read_config_dword(pvt->pci_sad1, SAD_TARGET, ®); 1541 else 1542 pci_read_config_dword(pvt->pci_br0, SAD_TARGET, ®); 1543 1544 if (pvt->info.type == KNIGHTS_LANDING) 1545 pvt->sbridge_dev->source_id = SOURCE_ID_KNL(reg); 1546 else 1547 pvt->sbridge_dev->source_id = SOURCE_ID(reg); 1548 } 1549 1550 static int __populate_dimms(struct mem_ctl_info *mci, 1551 u64 knl_mc_sizes[KNL_MAX_CHANNELS], 1552 enum edac_type mode) 1553 { 1554 struct sbridge_pvt *pvt = mci->pvt_info; 1555 int channels = pvt->info.type == KNIGHTS_LANDING ? KNL_MAX_CHANNELS 1556 : NUM_CHANNELS; 1557 unsigned int i, j, banks, ranks, rows, cols, npages; 1558 struct dimm_info *dimm; 1559 enum mem_type mtype; 1560 u64 size; 1561 1562 mtype = pvt->info.get_memory_type(pvt); 1563 if (mtype == MEM_RDDR3 || mtype == MEM_RDDR4) 1564 edac_dbg(0, "Memory is registered\n"); 1565 else if (mtype == MEM_UNKNOWN) 1566 edac_dbg(0, "Cannot determine memory type\n"); 1567 else 1568 edac_dbg(0, "Memory is unregistered\n"); 1569 1570 if (mtype == MEM_DDR4 || mtype == MEM_RDDR4) 1571 banks = 16; 1572 else 1573 banks = 8; 1574 1575 for (i = 0; i < channels; i++) { 1576 u32 mtr; 1577 1578 int max_dimms_per_channel; 1579 1580 if (pvt->info.type == KNIGHTS_LANDING) { 1581 max_dimms_per_channel = 1; 1582 if (!pvt->knl.pci_channel[i]) 1583 continue; 1584 } else { 1585 max_dimms_per_channel = ARRAY_SIZE(mtr_regs); 1586 if (!pvt->pci_tad[i]) 1587 continue; 1588 } 1589 1590 for (j = 0; j < max_dimms_per_channel; j++) { 1591 dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers, i, j, 0); 1592 if (pvt->info.type == KNIGHTS_LANDING) { 1593 pci_read_config_dword(pvt->knl.pci_channel[i], 1594 knl_mtr_reg, &mtr); 1595 } else { 1596 pci_read_config_dword(pvt->pci_tad[i], 1597 mtr_regs[j], &mtr); 1598 } 1599 edac_dbg(4, "Channel #%d MTR%d = %x\n", i, j, mtr); 1600 if (IS_DIMM_PRESENT(mtr)) { 1601 if (!IS_ECC_ENABLED(pvt->info.mcmtr)) { 1602 sbridge_printk(KERN_ERR, "CPU SrcID #%d, Ha #%d, Channel #%d has DIMMs, but ECC is disabled\n", 1603 pvt->sbridge_dev->source_id, 1604 pvt->sbridge_dev->dom, i); 1605 return -ENODEV; 1606 } 1607 pvt->channel[i].dimms++; 1608 1609 ranks = numrank(pvt->info.type, mtr); 1610 1611 if (pvt->info.type == KNIGHTS_LANDING) { 1612 /* For DDR4, this is fixed. */ 1613 cols = 1 << 10; 1614 rows = knl_mc_sizes[i] / 1615 ((u64) cols * ranks * banks * 8); 1616 } else { 1617 rows = numrow(mtr); 1618 cols = numcol(mtr); 1619 } 1620 1621 size = ((u64)rows * cols * banks * ranks) >> (20 - 3); 1622 npages = MiB_TO_PAGES(size); 1623 1624 edac_dbg(0, "mc#%d: ha %d channel %d, dimm %d, %lld Mb (%d pages) bank: %d, rank: %d, row: %#x, col: %#x\n", 1625 pvt->sbridge_dev->mc, pvt->sbridge_dev->dom, i, j, 1626 size, npages, 1627 banks, ranks, rows, cols); 1628 1629 dimm->nr_pages = npages; 1630 dimm->grain = 32; 1631 dimm->dtype = pvt->info.get_width(pvt, mtr); 1632 dimm->mtype = mtype; 1633 dimm->edac_mode = mode; 1634 snprintf(dimm->label, sizeof(dimm->label), 1635 "CPU_SrcID#%u_Ha#%u_Chan#%u_DIMM#%u", 1636 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom, i, j); 1637 } 1638 } 1639 } 1640 1641 return 0; 1642 } 1643 1644 static int get_dimm_config(struct mem_ctl_info *mci) 1645 { 1646 struct sbridge_pvt *pvt = mci->pvt_info; 1647 u64 knl_mc_sizes[KNL_MAX_CHANNELS]; 1648 enum edac_type mode; 1649 u32 reg; 1650 1651 if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) { 1652 pci_read_config_dword(pvt->pci_ha, HASWELL_HASYSDEFEATURE2, ®); 1653 pvt->is_chan_hash = GET_BITFIELD(reg, 21, 21); 1654 } 1655 pvt->sbridge_dev->node_id = pvt->info.get_node_id(pvt); 1656 edac_dbg(0, "mc#%d: Node ID: %d, source ID: %d\n", 1657 pvt->sbridge_dev->mc, 1658 pvt->sbridge_dev->node_id, 1659 pvt->sbridge_dev->source_id); 1660 1661 /* KNL doesn't support mirroring or lockstep, 1662 * and is always closed page 1663 */ 1664 if (pvt->info.type == KNIGHTS_LANDING) { 1665 mode = EDAC_S4ECD4ED; 1666 pvt->is_mirrored = false; 1667 1668 if (knl_get_dimm_capacity(pvt, knl_mc_sizes) != 0) 1669 return -1; 1670 pci_read_config_dword(pvt->pci_ta, KNL_MCMTR, &pvt->info.mcmtr); 1671 } else { 1672 pci_read_config_dword(pvt->pci_ras, RASENABLES, ®); 1673 if (IS_MIRROR_ENABLED(reg)) { 1674 edac_dbg(0, "Memory mirror is enabled\n"); 1675 pvt->is_mirrored = true; 1676 } else { 1677 edac_dbg(0, "Memory mirror is disabled\n"); 1678 pvt->is_mirrored = false; 1679 } 1680 1681 pci_read_config_dword(pvt->pci_ta, MCMTR, &pvt->info.mcmtr); 1682 if (IS_LOCKSTEP_ENABLED(pvt->info.mcmtr)) { 1683 edac_dbg(0, "Lockstep is enabled\n"); 1684 mode = EDAC_S8ECD8ED; 1685 pvt->is_lockstep = true; 1686 } else { 1687 edac_dbg(0, "Lockstep is disabled\n"); 1688 mode = EDAC_S4ECD4ED; 1689 pvt->is_lockstep = false; 1690 } 1691 if (IS_CLOSE_PG(pvt->info.mcmtr)) { 1692 edac_dbg(0, "address map is on closed page mode\n"); 1693 pvt->is_close_pg = true; 1694 } else { 1695 edac_dbg(0, "address map is on open page mode\n"); 1696 pvt->is_close_pg = false; 1697 } 1698 } 1699 1700 return __populate_dimms(mci, knl_mc_sizes, mode); 1701 } 1702 1703 static void get_memory_layout(const struct mem_ctl_info *mci) 1704 { 1705 struct sbridge_pvt *pvt = mci->pvt_info; 1706 int i, j, k, n_sads, n_tads, sad_interl; 1707 u32 reg; 1708 u64 limit, prv = 0; 1709 u64 tmp_mb; 1710 u32 gb, mb; 1711 u32 rir_way; 1712 1713 /* 1714 * Step 1) Get TOLM/TOHM ranges 1715 */ 1716 1717 pvt->tolm = pvt->info.get_tolm(pvt); 1718 tmp_mb = (1 + pvt->tolm) >> 20; 1719 1720 gb = div_u64_rem(tmp_mb, 1024, &mb); 1721 edac_dbg(0, "TOLM: %u.%03u GB (0x%016Lx)\n", 1722 gb, (mb*1000)/1024, (u64)pvt->tolm); 1723 1724 /* Address range is already 45:25 */ 1725 pvt->tohm = pvt->info.get_tohm(pvt); 1726 tmp_mb = (1 + pvt->tohm) >> 20; 1727 1728 gb = div_u64_rem(tmp_mb, 1024, &mb); 1729 edac_dbg(0, "TOHM: %u.%03u GB (0x%016Lx)\n", 1730 gb, (mb*1000)/1024, (u64)pvt->tohm); 1731 1732 /* 1733 * Step 2) Get SAD range and SAD Interleave list 1734 * TAD registers contain the interleave wayness. However, it 1735 * seems simpler to just discover it indirectly, with the 1736 * algorithm bellow. 1737 */ 1738 prv = 0; 1739 for (n_sads = 0; n_sads < pvt->info.max_sad; n_sads++) { 1740 /* SAD_LIMIT Address range is 45:26 */ 1741 pci_read_config_dword(pvt->pci_sad0, pvt->info.dram_rule[n_sads], 1742 ®); 1743 limit = pvt->info.sad_limit(reg); 1744 1745 if (!DRAM_RULE_ENABLE(reg)) 1746 continue; 1747 1748 if (limit <= prv) 1749 break; 1750 1751 tmp_mb = (limit + 1) >> 20; 1752 gb = div_u64_rem(tmp_mb, 1024, &mb); 1753 edac_dbg(0, "SAD#%d %s up to %u.%03u GB (0x%016Lx) Interleave: %s reg=0x%08x\n", 1754 n_sads, 1755 show_dram_attr(pvt->info.dram_attr(reg)), 1756 gb, (mb*1000)/1024, 1757 ((u64)tmp_mb) << 20L, 1758 get_intlv_mode_str(reg, pvt->info.type), 1759 reg); 1760 prv = limit; 1761 1762 pci_read_config_dword(pvt->pci_sad0, pvt->info.interleave_list[n_sads], 1763 ®); 1764 sad_interl = sad_pkg(pvt->info.interleave_pkg, reg, 0); 1765 for (j = 0; j < 8; j++) { 1766 u32 pkg = sad_pkg(pvt->info.interleave_pkg, reg, j); 1767 if (j > 0 && sad_interl == pkg) 1768 break; 1769 1770 edac_dbg(0, "SAD#%d, interleave #%d: %d\n", 1771 n_sads, j, pkg); 1772 } 1773 } 1774 1775 if (pvt->info.type == KNIGHTS_LANDING) 1776 return; 1777 1778 /* 1779 * Step 3) Get TAD range 1780 */ 1781 prv = 0; 1782 for (n_tads = 0; n_tads < MAX_TAD; n_tads++) { 1783 pci_read_config_dword(pvt->pci_ha, tad_dram_rule[n_tads], ®); 1784 limit = TAD_LIMIT(reg); 1785 if (limit <= prv) 1786 break; 1787 tmp_mb = (limit + 1) >> 20; 1788 1789 gb = div_u64_rem(tmp_mb, 1024, &mb); 1790 edac_dbg(0, "TAD#%d: up to %u.%03u GB (0x%016Lx), socket interleave %d, memory interleave %d, TGT: %d, %d, %d, %d, reg=0x%08x\n", 1791 n_tads, gb, (mb*1000)/1024, 1792 ((u64)tmp_mb) << 20L, 1793 (u32)(1 << TAD_SOCK(reg)), 1794 (u32)TAD_CH(reg) + 1, 1795 (u32)TAD_TGT0(reg), 1796 (u32)TAD_TGT1(reg), 1797 (u32)TAD_TGT2(reg), 1798 (u32)TAD_TGT3(reg), 1799 reg); 1800 prv = limit; 1801 } 1802 1803 /* 1804 * Step 4) Get TAD offsets, per each channel 1805 */ 1806 for (i = 0; i < NUM_CHANNELS; i++) { 1807 if (!pvt->channel[i].dimms) 1808 continue; 1809 for (j = 0; j < n_tads; j++) { 1810 pci_read_config_dword(pvt->pci_tad[i], 1811 tad_ch_nilv_offset[j], 1812 ®); 1813 tmp_mb = TAD_OFFSET(reg) >> 20; 1814 gb = div_u64_rem(tmp_mb, 1024, &mb); 1815 edac_dbg(0, "TAD CH#%d, offset #%d: %u.%03u GB (0x%016Lx), reg=0x%08x\n", 1816 i, j, 1817 gb, (mb*1000)/1024, 1818 ((u64)tmp_mb) << 20L, 1819 reg); 1820 } 1821 } 1822 1823 /* 1824 * Step 6) Get RIR Wayness/Limit, per each channel 1825 */ 1826 for (i = 0; i < NUM_CHANNELS; i++) { 1827 if (!pvt->channel[i].dimms) 1828 continue; 1829 for (j = 0; j < MAX_RIR_RANGES; j++) { 1830 pci_read_config_dword(pvt->pci_tad[i], 1831 rir_way_limit[j], 1832 ®); 1833 1834 if (!IS_RIR_VALID(reg)) 1835 continue; 1836 1837 tmp_mb = pvt->info.rir_limit(reg) >> 20; 1838 rir_way = 1 << RIR_WAY(reg); 1839 gb = div_u64_rem(tmp_mb, 1024, &mb); 1840 edac_dbg(0, "CH#%d RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d, reg=0x%08x\n", 1841 i, j, 1842 gb, (mb*1000)/1024, 1843 ((u64)tmp_mb) << 20L, 1844 rir_way, 1845 reg); 1846 1847 for (k = 0; k < rir_way; k++) { 1848 pci_read_config_dword(pvt->pci_tad[i], 1849 rir_offset[j][k], 1850 ®); 1851 tmp_mb = RIR_OFFSET(pvt->info.type, reg) << 6; 1852 1853 gb = div_u64_rem(tmp_mb, 1024, &mb); 1854 edac_dbg(0, "CH#%d RIR#%d INTL#%d, offset %u.%03u GB (0x%016Lx), tgt: %d, reg=0x%08x\n", 1855 i, j, k, 1856 gb, (mb*1000)/1024, 1857 ((u64)tmp_mb) << 20L, 1858 (u32)RIR_RNK_TGT(pvt->info.type, reg), 1859 reg); 1860 } 1861 } 1862 } 1863 } 1864 1865 static struct mem_ctl_info *get_mci_for_node_id(u8 node_id, u8 ha) 1866 { 1867 struct sbridge_dev *sbridge_dev; 1868 1869 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) { 1870 if (sbridge_dev->node_id == node_id && sbridge_dev->dom == ha) 1871 return sbridge_dev->mci; 1872 } 1873 return NULL; 1874 } 1875 1876 static int get_memory_error_data(struct mem_ctl_info *mci, 1877 u64 addr, 1878 u8 *socket, u8 *ha, 1879 long *channel_mask, 1880 u8 *rank, 1881 char **area_type, char *msg) 1882 { 1883 struct mem_ctl_info *new_mci; 1884 struct sbridge_pvt *pvt = mci->pvt_info; 1885 struct pci_dev *pci_ha; 1886 int n_rir, n_sads, n_tads, sad_way, sck_xch; 1887 int sad_interl, idx, base_ch; 1888 int interleave_mode, shiftup = 0; 1889 unsigned sad_interleave[pvt->info.max_interleave]; 1890 u32 reg, dram_rule; 1891 u8 ch_way, sck_way, pkg, sad_ha = 0; 1892 u32 tad_offset; 1893 u32 rir_way; 1894 u32 mb, gb; 1895 u64 ch_addr, offset, limit = 0, prv = 0; 1896 1897 1898 /* 1899 * Step 0) Check if the address is at special memory ranges 1900 * The check bellow is probably enough to fill all cases where 1901 * the error is not inside a memory, except for the legacy 1902 * range (e. g. VGA addresses). It is unlikely, however, that the 1903 * memory controller would generate an error on that range. 1904 */ 1905 if ((addr > (u64) pvt->tolm) && (addr < (1LL << 32))) { 1906 sprintf(msg, "Error at TOLM area, on addr 0x%08Lx", addr); 1907 return -EINVAL; 1908 } 1909 if (addr >= (u64)pvt->tohm) { 1910 sprintf(msg, "Error at MMIOH area, on addr 0x%016Lx", addr); 1911 return -EINVAL; 1912 } 1913 1914 /* 1915 * Step 1) Get socket 1916 */ 1917 for (n_sads = 0; n_sads < pvt->info.max_sad; n_sads++) { 1918 pci_read_config_dword(pvt->pci_sad0, pvt->info.dram_rule[n_sads], 1919 ®); 1920 1921 if (!DRAM_RULE_ENABLE(reg)) 1922 continue; 1923 1924 limit = pvt->info.sad_limit(reg); 1925 if (limit <= prv) { 1926 sprintf(msg, "Can't discover the memory socket"); 1927 return -EINVAL; 1928 } 1929 if (addr <= limit) 1930 break; 1931 prv = limit; 1932 } 1933 if (n_sads == pvt->info.max_sad) { 1934 sprintf(msg, "Can't discover the memory socket"); 1935 return -EINVAL; 1936 } 1937 dram_rule = reg; 1938 *area_type = show_dram_attr(pvt->info.dram_attr(dram_rule)); 1939 interleave_mode = pvt->info.interleave_mode(dram_rule); 1940 1941 pci_read_config_dword(pvt->pci_sad0, pvt->info.interleave_list[n_sads], 1942 ®); 1943 1944 if (pvt->info.type == SANDY_BRIDGE) { 1945 sad_interl = sad_pkg(pvt->info.interleave_pkg, reg, 0); 1946 for (sad_way = 0; sad_way < 8; sad_way++) { 1947 u32 pkg = sad_pkg(pvt->info.interleave_pkg, reg, sad_way); 1948 if (sad_way > 0 && sad_interl == pkg) 1949 break; 1950 sad_interleave[sad_way] = pkg; 1951 edac_dbg(0, "SAD interleave #%d: %d\n", 1952 sad_way, sad_interleave[sad_way]); 1953 } 1954 edac_dbg(0, "mc#%d: Error detected on SAD#%d: address 0x%016Lx < 0x%016Lx, Interleave [%d:6]%s\n", 1955 pvt->sbridge_dev->mc, 1956 n_sads, 1957 addr, 1958 limit, 1959 sad_way + 7, 1960 !interleave_mode ? "" : "XOR[18:16]"); 1961 if (interleave_mode) 1962 idx = ((addr >> 6) ^ (addr >> 16)) & 7; 1963 else 1964 idx = (addr >> 6) & 7; 1965 switch (sad_way) { 1966 case 1: 1967 idx = 0; 1968 break; 1969 case 2: 1970 idx = idx & 1; 1971 break; 1972 case 4: 1973 idx = idx & 3; 1974 break; 1975 case 8: 1976 break; 1977 default: 1978 sprintf(msg, "Can't discover socket interleave"); 1979 return -EINVAL; 1980 } 1981 *socket = sad_interleave[idx]; 1982 edac_dbg(0, "SAD interleave index: %d (wayness %d) = CPU socket %d\n", 1983 idx, sad_way, *socket); 1984 } else if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) { 1985 int bits, a7mode = A7MODE(dram_rule); 1986 1987 if (a7mode) { 1988 /* A7 mode swaps P9 with P6 */ 1989 bits = GET_BITFIELD(addr, 7, 8) << 1; 1990 bits |= GET_BITFIELD(addr, 9, 9); 1991 } else 1992 bits = GET_BITFIELD(addr, 6, 8); 1993 1994 if (interleave_mode == 0) { 1995 /* interleave mode will XOR {8,7,6} with {18,17,16} */ 1996 idx = GET_BITFIELD(addr, 16, 18); 1997 idx ^= bits; 1998 } else 1999 idx = bits; 2000 2001 pkg = sad_pkg(pvt->info.interleave_pkg, reg, idx); 2002 *socket = sad_pkg_socket(pkg); 2003 sad_ha = sad_pkg_ha(pkg); 2004 2005 if (a7mode) { 2006 /* MCChanShiftUpEnable */ 2007 pci_read_config_dword(pvt->pci_ha, HASWELL_HASYSDEFEATURE2, ®); 2008 shiftup = GET_BITFIELD(reg, 22, 22); 2009 } 2010 2011 edac_dbg(0, "SAD interleave package: %d = CPU socket %d, HA %i, shiftup: %i\n", 2012 idx, *socket, sad_ha, shiftup); 2013 } else { 2014 /* Ivy Bridge's SAD mode doesn't support XOR interleave mode */ 2015 idx = (addr >> 6) & 7; 2016 pkg = sad_pkg(pvt->info.interleave_pkg, reg, idx); 2017 *socket = sad_pkg_socket(pkg); 2018 sad_ha = sad_pkg_ha(pkg); 2019 edac_dbg(0, "SAD interleave package: %d = CPU socket %d, HA %d\n", 2020 idx, *socket, sad_ha); 2021 } 2022 2023 *ha = sad_ha; 2024 2025 /* 2026 * Move to the proper node structure, in order to access the 2027 * right PCI registers 2028 */ 2029 new_mci = get_mci_for_node_id(*socket, sad_ha); 2030 if (!new_mci) { 2031 sprintf(msg, "Struct for socket #%u wasn't initialized", 2032 *socket); 2033 return -EINVAL; 2034 } 2035 mci = new_mci; 2036 pvt = mci->pvt_info; 2037 2038 /* 2039 * Step 2) Get memory channel 2040 */ 2041 prv = 0; 2042 pci_ha = pvt->pci_ha; 2043 for (n_tads = 0; n_tads < MAX_TAD; n_tads++) { 2044 pci_read_config_dword(pci_ha, tad_dram_rule[n_tads], ®); 2045 limit = TAD_LIMIT(reg); 2046 if (limit <= prv) { 2047 sprintf(msg, "Can't discover the memory channel"); 2048 return -EINVAL; 2049 } 2050 if (addr <= limit) 2051 break; 2052 prv = limit; 2053 } 2054 if (n_tads == MAX_TAD) { 2055 sprintf(msg, "Can't discover the memory channel"); 2056 return -EINVAL; 2057 } 2058 2059 ch_way = TAD_CH(reg) + 1; 2060 sck_way = TAD_SOCK(reg); 2061 2062 if (ch_way == 3) 2063 idx = addr >> 6; 2064 else { 2065 idx = (addr >> (6 + sck_way + shiftup)) & 0x3; 2066 if (pvt->is_chan_hash) 2067 idx = haswell_chan_hash(idx, addr); 2068 } 2069 idx = idx % ch_way; 2070 2071 /* 2072 * FIXME: Shouldn't we use CHN_IDX_OFFSET() here, when ch_way == 3 ??? 2073 */ 2074 switch (idx) { 2075 case 0: 2076 base_ch = TAD_TGT0(reg); 2077 break; 2078 case 1: 2079 base_ch = TAD_TGT1(reg); 2080 break; 2081 case 2: 2082 base_ch = TAD_TGT2(reg); 2083 break; 2084 case 3: 2085 base_ch = TAD_TGT3(reg); 2086 break; 2087 default: 2088 sprintf(msg, "Can't discover the TAD target"); 2089 return -EINVAL; 2090 } 2091 *channel_mask = 1 << base_ch; 2092 2093 pci_read_config_dword(pvt->pci_tad[base_ch], tad_ch_nilv_offset[n_tads], &tad_offset); 2094 2095 if (pvt->is_mirrored) { 2096 *channel_mask |= 1 << ((base_ch + 2) % 4); 2097 switch(ch_way) { 2098 case 2: 2099 case 4: 2100 sck_xch = (1 << sck_way) * (ch_way >> 1); 2101 break; 2102 default: 2103 sprintf(msg, "Invalid mirror set. Can't decode addr"); 2104 return -EINVAL; 2105 } 2106 } else 2107 sck_xch = (1 << sck_way) * ch_way; 2108 2109 if (pvt->is_lockstep) 2110 *channel_mask |= 1 << ((base_ch + 1) % 4); 2111 2112 offset = TAD_OFFSET(tad_offset); 2113 2114 edac_dbg(0, "TAD#%d: address 0x%016Lx < 0x%016Lx, socket interleave %d, channel interleave %d (offset 0x%08Lx), index %d, base ch: %d, ch mask: 0x%02lx\n", 2115 n_tads, 2116 addr, 2117 limit, 2118 sck_way, 2119 ch_way, 2120 offset, 2121 idx, 2122 base_ch, 2123 *channel_mask); 2124 2125 /* Calculate channel address */ 2126 /* Remove the TAD offset */ 2127 2128 if (offset > addr) { 2129 sprintf(msg, "Can't calculate ch addr: TAD offset 0x%08Lx is too high for addr 0x%08Lx!", 2130 offset, addr); 2131 return -EINVAL; 2132 } 2133 2134 ch_addr = addr - offset; 2135 ch_addr >>= (6 + shiftup); 2136 ch_addr /= sck_xch; 2137 ch_addr <<= (6 + shiftup); 2138 ch_addr |= addr & ((1 << (6 + shiftup)) - 1); 2139 2140 /* 2141 * Step 3) Decode rank 2142 */ 2143 for (n_rir = 0; n_rir < MAX_RIR_RANGES; n_rir++) { 2144 pci_read_config_dword(pvt->pci_tad[base_ch], rir_way_limit[n_rir], ®); 2145 2146 if (!IS_RIR_VALID(reg)) 2147 continue; 2148 2149 limit = pvt->info.rir_limit(reg); 2150 gb = div_u64_rem(limit >> 20, 1024, &mb); 2151 edac_dbg(0, "RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d\n", 2152 n_rir, 2153 gb, (mb*1000)/1024, 2154 limit, 2155 1 << RIR_WAY(reg)); 2156 if (ch_addr <= limit) 2157 break; 2158 } 2159 if (n_rir == MAX_RIR_RANGES) { 2160 sprintf(msg, "Can't discover the memory rank for ch addr 0x%08Lx", 2161 ch_addr); 2162 return -EINVAL; 2163 } 2164 rir_way = RIR_WAY(reg); 2165 2166 if (pvt->is_close_pg) 2167 idx = (ch_addr >> 6); 2168 else 2169 idx = (ch_addr >> 13); /* FIXME: Datasheet says to shift by 15 */ 2170 idx %= 1 << rir_way; 2171 2172 pci_read_config_dword(pvt->pci_tad[base_ch], rir_offset[n_rir][idx], ®); 2173 *rank = RIR_RNK_TGT(pvt->info.type, reg); 2174 2175 edac_dbg(0, "RIR#%d: channel address 0x%08Lx < 0x%08Lx, RIR interleave %d, index %d\n", 2176 n_rir, 2177 ch_addr, 2178 limit, 2179 rir_way, 2180 idx); 2181 2182 return 0; 2183 } 2184 2185 /**************************************************************************** 2186 Device initialization routines: put/get, init/exit 2187 ****************************************************************************/ 2188 2189 /* 2190 * sbridge_put_all_devices 'put' all the devices that we have 2191 * reserved via 'get' 2192 */ 2193 static void sbridge_put_devices(struct sbridge_dev *sbridge_dev) 2194 { 2195 int i; 2196 2197 edac_dbg(0, "\n"); 2198 for (i = 0; i < sbridge_dev->n_devs; i++) { 2199 struct pci_dev *pdev = sbridge_dev->pdev[i]; 2200 if (!pdev) 2201 continue; 2202 edac_dbg(0, "Removing dev %02x:%02x.%d\n", 2203 pdev->bus->number, 2204 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); 2205 pci_dev_put(pdev); 2206 } 2207 } 2208 2209 static void sbridge_put_all_devices(void) 2210 { 2211 struct sbridge_dev *sbridge_dev, *tmp; 2212 2213 list_for_each_entry_safe(sbridge_dev, tmp, &sbridge_edac_list, list) { 2214 sbridge_put_devices(sbridge_dev); 2215 free_sbridge_dev(sbridge_dev); 2216 } 2217 } 2218 2219 static int sbridge_get_onedevice(struct pci_dev **prev, 2220 u8 *num_mc, 2221 const struct pci_id_table *table, 2222 const unsigned devno, 2223 const int multi_bus) 2224 { 2225 struct sbridge_dev *sbridge_dev = NULL; 2226 const struct pci_id_descr *dev_descr = &table->descr[devno]; 2227 struct pci_dev *pdev = NULL; 2228 u8 bus = 0; 2229 int i = 0; 2230 2231 sbridge_printk(KERN_DEBUG, 2232 "Seeking for: PCI ID %04x:%04x\n", 2233 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2234 2235 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 2236 dev_descr->dev_id, *prev); 2237 2238 if (!pdev) { 2239 if (*prev) { 2240 *prev = pdev; 2241 return 0; 2242 } 2243 2244 if (dev_descr->optional) 2245 return 0; 2246 2247 /* if the HA wasn't found */ 2248 if (devno == 0) 2249 return -ENODEV; 2250 2251 sbridge_printk(KERN_INFO, 2252 "Device not found: %04x:%04x\n", 2253 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2254 2255 /* End of list, leave */ 2256 return -ENODEV; 2257 } 2258 bus = pdev->bus->number; 2259 2260 next_imc: 2261 sbridge_dev = get_sbridge_dev(bus, dev_descr->dom, multi_bus, sbridge_dev); 2262 if (!sbridge_dev) { 2263 2264 if (dev_descr->dom == SOCK) 2265 goto out_imc; 2266 2267 sbridge_dev = alloc_sbridge_dev(bus, dev_descr->dom, table); 2268 if (!sbridge_dev) { 2269 pci_dev_put(pdev); 2270 return -ENOMEM; 2271 } 2272 (*num_mc)++; 2273 } 2274 2275 if (sbridge_dev->pdev[sbridge_dev->i_devs]) { 2276 sbridge_printk(KERN_ERR, 2277 "Duplicated device for %04x:%04x\n", 2278 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2279 pci_dev_put(pdev); 2280 return -ENODEV; 2281 } 2282 2283 sbridge_dev->pdev[sbridge_dev->i_devs++] = pdev; 2284 2285 /* pdev belongs to more than one IMC, do extra gets */ 2286 if (++i > 1) 2287 pci_dev_get(pdev); 2288 2289 if (dev_descr->dom == SOCK && i < table->n_imcs_per_sock) 2290 goto next_imc; 2291 2292 out_imc: 2293 /* Be sure that the device is enabled */ 2294 if (unlikely(pci_enable_device(pdev) < 0)) { 2295 sbridge_printk(KERN_ERR, 2296 "Couldn't enable %04x:%04x\n", 2297 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2298 return -ENODEV; 2299 } 2300 2301 edac_dbg(0, "Detected %04x:%04x\n", 2302 PCI_VENDOR_ID_INTEL, dev_descr->dev_id); 2303 2304 /* 2305 * As stated on drivers/pci/search.c, the reference count for 2306 * @from is always decremented if it is not %NULL. So, as we need 2307 * to get all devices up to null, we need to do a get for the device 2308 */ 2309 pci_dev_get(pdev); 2310 2311 *prev = pdev; 2312 2313 return 0; 2314 } 2315 2316 /* 2317 * sbridge_get_all_devices - Find and perform 'get' operation on the MCH's 2318 * devices we want to reference for this driver. 2319 * @num_mc: pointer to the memory controllers count, to be incremented in case 2320 * of success. 2321 * @table: model specific table 2322 * 2323 * returns 0 in case of success or error code 2324 */ 2325 static int sbridge_get_all_devices(u8 *num_mc, 2326 const struct pci_id_table *table) 2327 { 2328 int i, rc; 2329 struct pci_dev *pdev = NULL; 2330 int allow_dups = 0; 2331 int multi_bus = 0; 2332 2333 if (table->type == KNIGHTS_LANDING) 2334 allow_dups = multi_bus = 1; 2335 while (table && table->descr) { 2336 for (i = 0; i < table->n_devs_per_sock; i++) { 2337 if (!allow_dups || i == 0 || 2338 table->descr[i].dev_id != 2339 table->descr[i-1].dev_id) { 2340 pdev = NULL; 2341 } 2342 do { 2343 rc = sbridge_get_onedevice(&pdev, num_mc, 2344 table, i, multi_bus); 2345 if (rc < 0) { 2346 if (i == 0) { 2347 i = table->n_devs_per_sock; 2348 break; 2349 } 2350 sbridge_put_all_devices(); 2351 return -ENODEV; 2352 } 2353 } while (pdev && !allow_dups); 2354 } 2355 table++; 2356 } 2357 2358 return 0; 2359 } 2360 2361 /* 2362 * Device IDs for {SBRIDGE,IBRIDGE,HASWELL,BROADWELL}_IMC_HA0_TAD0 are in 2363 * the format: XXXa. So we can convert from a device to the corresponding 2364 * channel like this 2365 */ 2366 #define TAD_DEV_TO_CHAN(dev) (((dev) & 0xf) - 0xa) 2367 2368 static int sbridge_mci_bind_devs(struct mem_ctl_info *mci, 2369 struct sbridge_dev *sbridge_dev) 2370 { 2371 struct sbridge_pvt *pvt = mci->pvt_info; 2372 struct pci_dev *pdev; 2373 u8 saw_chan_mask = 0; 2374 int i; 2375 2376 for (i = 0; i < sbridge_dev->n_devs; i++) { 2377 pdev = sbridge_dev->pdev[i]; 2378 if (!pdev) 2379 continue; 2380 2381 switch (pdev->device) { 2382 case PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0: 2383 pvt->pci_sad0 = pdev; 2384 break; 2385 case PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1: 2386 pvt->pci_sad1 = pdev; 2387 break; 2388 case PCI_DEVICE_ID_INTEL_SBRIDGE_BR: 2389 pvt->pci_br0 = pdev; 2390 break; 2391 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0: 2392 pvt->pci_ha = pdev; 2393 break; 2394 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA: 2395 pvt->pci_ta = pdev; 2396 break; 2397 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS: 2398 pvt->pci_ras = pdev; 2399 break; 2400 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0: 2401 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1: 2402 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2: 2403 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3: 2404 { 2405 int id = TAD_DEV_TO_CHAN(pdev->device); 2406 pvt->pci_tad[id] = pdev; 2407 saw_chan_mask |= 1 << id; 2408 } 2409 break; 2410 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO: 2411 pvt->pci_ddrio = pdev; 2412 break; 2413 default: 2414 goto error; 2415 } 2416 2417 edac_dbg(0, "Associated PCI %02x:%02x, bus %d with dev = %p\n", 2418 pdev->vendor, pdev->device, 2419 sbridge_dev->bus, 2420 pdev); 2421 } 2422 2423 /* Check if everything were registered */ 2424 if (!pvt->pci_sad0 || !pvt->pci_sad1 || !pvt->pci_ha || 2425 !pvt->pci_ras || !pvt->pci_ta) 2426 goto enodev; 2427 2428 if (saw_chan_mask != 0x0f) 2429 goto enodev; 2430 return 0; 2431 2432 enodev: 2433 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2434 return -ENODEV; 2435 2436 error: 2437 sbridge_printk(KERN_ERR, "Unexpected device %02x:%02x\n", 2438 PCI_VENDOR_ID_INTEL, pdev->device); 2439 return -EINVAL; 2440 } 2441 2442 static int ibridge_mci_bind_devs(struct mem_ctl_info *mci, 2443 struct sbridge_dev *sbridge_dev) 2444 { 2445 struct sbridge_pvt *pvt = mci->pvt_info; 2446 struct pci_dev *pdev; 2447 u8 saw_chan_mask = 0; 2448 int i; 2449 2450 for (i = 0; i < sbridge_dev->n_devs; i++) { 2451 pdev = sbridge_dev->pdev[i]; 2452 if (!pdev) 2453 continue; 2454 2455 switch (pdev->device) { 2456 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0: 2457 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1: 2458 pvt->pci_ha = pdev; 2459 break; 2460 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA: 2461 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA: 2462 pvt->pci_ta = pdev; 2463 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS: 2464 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS: 2465 pvt->pci_ras = pdev; 2466 break; 2467 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0: 2468 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1: 2469 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2: 2470 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3: 2471 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0: 2472 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1: 2473 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2: 2474 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3: 2475 { 2476 int id = TAD_DEV_TO_CHAN(pdev->device); 2477 pvt->pci_tad[id] = pdev; 2478 saw_chan_mask |= 1 << id; 2479 } 2480 break; 2481 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0: 2482 pvt->pci_ddrio = pdev; 2483 break; 2484 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0: 2485 pvt->pci_ddrio = pdev; 2486 break; 2487 case PCI_DEVICE_ID_INTEL_IBRIDGE_SAD: 2488 pvt->pci_sad0 = pdev; 2489 break; 2490 case PCI_DEVICE_ID_INTEL_IBRIDGE_BR0: 2491 pvt->pci_br0 = pdev; 2492 break; 2493 case PCI_DEVICE_ID_INTEL_IBRIDGE_BR1: 2494 pvt->pci_br1 = pdev; 2495 break; 2496 default: 2497 goto error; 2498 } 2499 2500 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n", 2501 sbridge_dev->bus, 2502 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 2503 pdev); 2504 } 2505 2506 /* Check if everything were registered */ 2507 if (!pvt->pci_sad0 || !pvt->pci_ha || !pvt->pci_br0 || 2508 !pvt->pci_br1 || !pvt->pci_ras || !pvt->pci_ta) 2509 goto enodev; 2510 2511 if (saw_chan_mask != 0x0f && /* -EN/-EX */ 2512 saw_chan_mask != 0x03) /* -EP */ 2513 goto enodev; 2514 return 0; 2515 2516 enodev: 2517 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2518 return -ENODEV; 2519 2520 error: 2521 sbridge_printk(KERN_ERR, 2522 "Unexpected device %02x:%02x\n", PCI_VENDOR_ID_INTEL, 2523 pdev->device); 2524 return -EINVAL; 2525 } 2526 2527 static int haswell_mci_bind_devs(struct mem_ctl_info *mci, 2528 struct sbridge_dev *sbridge_dev) 2529 { 2530 struct sbridge_pvt *pvt = mci->pvt_info; 2531 struct pci_dev *pdev; 2532 u8 saw_chan_mask = 0; 2533 int i; 2534 2535 /* there's only one device per system; not tied to any bus */ 2536 if (pvt->info.pci_vtd == NULL) 2537 /* result will be checked later */ 2538 pvt->info.pci_vtd = pci_get_device(PCI_VENDOR_ID_INTEL, 2539 PCI_DEVICE_ID_INTEL_HASWELL_IMC_VTD_MISC, 2540 NULL); 2541 2542 for (i = 0; i < sbridge_dev->n_devs; i++) { 2543 pdev = sbridge_dev->pdev[i]; 2544 if (!pdev) 2545 continue; 2546 2547 switch (pdev->device) { 2548 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0: 2549 pvt->pci_sad0 = pdev; 2550 break; 2551 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1: 2552 pvt->pci_sad1 = pdev; 2553 break; 2554 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0: 2555 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1: 2556 pvt->pci_ha = pdev; 2557 break; 2558 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA: 2559 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA: 2560 pvt->pci_ta = pdev; 2561 break; 2562 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TM: 2563 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TM: 2564 pvt->pci_ras = pdev; 2565 break; 2566 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0: 2567 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1: 2568 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2: 2569 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3: 2570 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0: 2571 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1: 2572 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2: 2573 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3: 2574 { 2575 int id = TAD_DEV_TO_CHAN(pdev->device); 2576 pvt->pci_tad[id] = pdev; 2577 saw_chan_mask |= 1 << id; 2578 } 2579 break; 2580 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0: 2581 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1: 2582 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2: 2583 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3: 2584 if (!pvt->pci_ddrio) 2585 pvt->pci_ddrio = pdev; 2586 break; 2587 default: 2588 break; 2589 } 2590 2591 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n", 2592 sbridge_dev->bus, 2593 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 2594 pdev); 2595 } 2596 2597 /* Check if everything were registered */ 2598 if (!pvt->pci_sad0 || !pvt->pci_ha || !pvt->pci_sad1 || 2599 !pvt->pci_ras || !pvt->pci_ta || !pvt->info.pci_vtd) 2600 goto enodev; 2601 2602 if (saw_chan_mask != 0x0f && /* -EN/-EX */ 2603 saw_chan_mask != 0x03) /* -EP */ 2604 goto enodev; 2605 return 0; 2606 2607 enodev: 2608 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2609 return -ENODEV; 2610 } 2611 2612 static int broadwell_mci_bind_devs(struct mem_ctl_info *mci, 2613 struct sbridge_dev *sbridge_dev) 2614 { 2615 struct sbridge_pvt *pvt = mci->pvt_info; 2616 struct pci_dev *pdev; 2617 u8 saw_chan_mask = 0; 2618 int i; 2619 2620 /* there's only one device per system; not tied to any bus */ 2621 if (pvt->info.pci_vtd == NULL) 2622 /* result will be checked later */ 2623 pvt->info.pci_vtd = pci_get_device(PCI_VENDOR_ID_INTEL, 2624 PCI_DEVICE_ID_INTEL_BROADWELL_IMC_VTD_MISC, 2625 NULL); 2626 2627 for (i = 0; i < sbridge_dev->n_devs; i++) { 2628 pdev = sbridge_dev->pdev[i]; 2629 if (!pdev) 2630 continue; 2631 2632 switch (pdev->device) { 2633 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0: 2634 pvt->pci_sad0 = pdev; 2635 break; 2636 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1: 2637 pvt->pci_sad1 = pdev; 2638 break; 2639 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0: 2640 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1: 2641 pvt->pci_ha = pdev; 2642 break; 2643 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA: 2644 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA: 2645 pvt->pci_ta = pdev; 2646 break; 2647 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TM: 2648 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TM: 2649 pvt->pci_ras = pdev; 2650 break; 2651 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0: 2652 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1: 2653 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2: 2654 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3: 2655 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0: 2656 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1: 2657 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2: 2658 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3: 2659 { 2660 int id = TAD_DEV_TO_CHAN(pdev->device); 2661 pvt->pci_tad[id] = pdev; 2662 saw_chan_mask |= 1 << id; 2663 } 2664 break; 2665 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0: 2666 pvt->pci_ddrio = pdev; 2667 break; 2668 default: 2669 break; 2670 } 2671 2672 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n", 2673 sbridge_dev->bus, 2674 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), 2675 pdev); 2676 } 2677 2678 /* Check if everything were registered */ 2679 if (!pvt->pci_sad0 || !pvt->pci_ha || !pvt->pci_sad1 || 2680 !pvt->pci_ras || !pvt->pci_ta || !pvt->info.pci_vtd) 2681 goto enodev; 2682 2683 if (saw_chan_mask != 0x0f && /* -EN/-EX */ 2684 saw_chan_mask != 0x03) /* -EP */ 2685 goto enodev; 2686 return 0; 2687 2688 enodev: 2689 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2690 return -ENODEV; 2691 } 2692 2693 static int knl_mci_bind_devs(struct mem_ctl_info *mci, 2694 struct sbridge_dev *sbridge_dev) 2695 { 2696 struct sbridge_pvt *pvt = mci->pvt_info; 2697 struct pci_dev *pdev; 2698 int dev, func; 2699 2700 int i; 2701 int devidx; 2702 2703 for (i = 0; i < sbridge_dev->n_devs; i++) { 2704 pdev = sbridge_dev->pdev[i]; 2705 if (!pdev) 2706 continue; 2707 2708 /* Extract PCI device and function. */ 2709 dev = (pdev->devfn >> 3) & 0x1f; 2710 func = pdev->devfn & 0x7; 2711 2712 switch (pdev->device) { 2713 case PCI_DEVICE_ID_INTEL_KNL_IMC_MC: 2714 if (dev == 8) 2715 pvt->knl.pci_mc0 = pdev; 2716 else if (dev == 9) 2717 pvt->knl.pci_mc1 = pdev; 2718 else { 2719 sbridge_printk(KERN_ERR, 2720 "Memory controller in unexpected place! (dev %d, fn %d)\n", 2721 dev, func); 2722 continue; 2723 } 2724 break; 2725 2726 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0: 2727 pvt->pci_sad0 = pdev; 2728 break; 2729 2730 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1: 2731 pvt->pci_sad1 = pdev; 2732 break; 2733 2734 case PCI_DEVICE_ID_INTEL_KNL_IMC_CHA: 2735 /* There are one of these per tile, and range from 2736 * 1.14.0 to 1.18.5. 2737 */ 2738 devidx = ((dev-14)*8)+func; 2739 2740 if (devidx < 0 || devidx >= KNL_MAX_CHAS) { 2741 sbridge_printk(KERN_ERR, 2742 "Caching and Home Agent in unexpected place! (dev %d, fn %d)\n", 2743 dev, func); 2744 continue; 2745 } 2746 2747 WARN_ON(pvt->knl.pci_cha[devidx] != NULL); 2748 2749 pvt->knl.pci_cha[devidx] = pdev; 2750 break; 2751 2752 case PCI_DEVICE_ID_INTEL_KNL_IMC_CHAN: 2753 devidx = -1; 2754 2755 /* 2756 * MC0 channels 0-2 are device 9 function 2-4, 2757 * MC1 channels 3-5 are device 8 function 2-4. 2758 */ 2759 2760 if (dev == 9) 2761 devidx = func-2; 2762 else if (dev == 8) 2763 devidx = 3 + (func-2); 2764 2765 if (devidx < 0 || devidx >= KNL_MAX_CHANNELS) { 2766 sbridge_printk(KERN_ERR, 2767 "DRAM Channel Registers in unexpected place! (dev %d, fn %d)\n", 2768 dev, func); 2769 continue; 2770 } 2771 2772 WARN_ON(pvt->knl.pci_channel[devidx] != NULL); 2773 pvt->knl.pci_channel[devidx] = pdev; 2774 break; 2775 2776 case PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM: 2777 pvt->knl.pci_mc_info = pdev; 2778 break; 2779 2780 case PCI_DEVICE_ID_INTEL_KNL_IMC_TA: 2781 pvt->pci_ta = pdev; 2782 break; 2783 2784 default: 2785 sbridge_printk(KERN_ERR, "Unexpected device %d\n", 2786 pdev->device); 2787 break; 2788 } 2789 } 2790 2791 if (!pvt->knl.pci_mc0 || !pvt->knl.pci_mc1 || 2792 !pvt->pci_sad0 || !pvt->pci_sad1 || 2793 !pvt->pci_ta) { 2794 goto enodev; 2795 } 2796 2797 for (i = 0; i < KNL_MAX_CHANNELS; i++) { 2798 if (!pvt->knl.pci_channel[i]) { 2799 sbridge_printk(KERN_ERR, "Missing channel %d\n", i); 2800 goto enodev; 2801 } 2802 } 2803 2804 for (i = 0; i < KNL_MAX_CHAS; i++) { 2805 if (!pvt->knl.pci_cha[i]) { 2806 sbridge_printk(KERN_ERR, "Missing CHA %d\n", i); 2807 goto enodev; 2808 } 2809 } 2810 2811 return 0; 2812 2813 enodev: 2814 sbridge_printk(KERN_ERR, "Some needed devices are missing\n"); 2815 return -ENODEV; 2816 } 2817 2818 /**************************************************************************** 2819 Error check routines 2820 ****************************************************************************/ 2821 2822 /* 2823 * While Sandy Bridge has error count registers, SMI BIOS read values from 2824 * and resets the counters. So, they are not reliable for the OS to read 2825 * from them. So, we have no option but to just trust on whatever MCE is 2826 * telling us about the errors. 2827 */ 2828 static void sbridge_mce_output_error(struct mem_ctl_info *mci, 2829 const struct mce *m) 2830 { 2831 struct mem_ctl_info *new_mci; 2832 struct sbridge_pvt *pvt = mci->pvt_info; 2833 enum hw_event_mc_err_type tp_event; 2834 char *type, *optype, msg[256]; 2835 bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0); 2836 bool overflow = GET_BITFIELD(m->status, 62, 62); 2837 bool uncorrected_error = GET_BITFIELD(m->status, 61, 61); 2838 bool recoverable; 2839 u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52); 2840 u32 mscod = GET_BITFIELD(m->status, 16, 31); 2841 u32 errcode = GET_BITFIELD(m->status, 0, 15); 2842 u32 channel = GET_BITFIELD(m->status, 0, 3); 2843 u32 optypenum = GET_BITFIELD(m->status, 4, 6); 2844 long channel_mask, first_channel; 2845 u8 rank, socket, ha; 2846 int rc, dimm; 2847 char *area_type = NULL; 2848 2849 if (pvt->info.type != SANDY_BRIDGE) 2850 recoverable = true; 2851 else 2852 recoverable = GET_BITFIELD(m->status, 56, 56); 2853 2854 if (uncorrected_error) { 2855 if (ripv) { 2856 type = "FATAL"; 2857 tp_event = HW_EVENT_ERR_FATAL; 2858 } else { 2859 type = "NON_FATAL"; 2860 tp_event = HW_EVENT_ERR_UNCORRECTED; 2861 } 2862 } else { 2863 type = "CORRECTED"; 2864 tp_event = HW_EVENT_ERR_CORRECTED; 2865 } 2866 2867 /* 2868 * According with Table 15-9 of the Intel Architecture spec vol 3A, 2869 * memory errors should fit in this mask: 2870 * 000f 0000 1mmm cccc (binary) 2871 * where: 2872 * f = Correction Report Filtering Bit. If 1, subsequent errors 2873 * won't be shown 2874 * mmm = error type 2875 * cccc = channel 2876 * If the mask doesn't match, report an error to the parsing logic 2877 */ 2878 if (! ((errcode & 0xef80) == 0x80)) { 2879 optype = "Can't parse: it is not a mem"; 2880 } else { 2881 switch (optypenum) { 2882 case 0: 2883 optype = "generic undef request error"; 2884 break; 2885 case 1: 2886 optype = "memory read error"; 2887 break; 2888 case 2: 2889 optype = "memory write error"; 2890 break; 2891 case 3: 2892 optype = "addr/cmd error"; 2893 break; 2894 case 4: 2895 optype = "memory scrubbing error"; 2896 break; 2897 default: 2898 optype = "reserved"; 2899 break; 2900 } 2901 } 2902 2903 /* Only decode errors with an valid address (ADDRV) */ 2904 if (!GET_BITFIELD(m->status, 58, 58)) 2905 return; 2906 2907 if (pvt->info.type == KNIGHTS_LANDING) { 2908 if (channel == 14) { 2909 edac_dbg(0, "%s%s err_code:%04x:%04x EDRAM bank %d\n", 2910 overflow ? " OVERFLOW" : "", 2911 (uncorrected_error && recoverable) 2912 ? " recoverable" : "", 2913 mscod, errcode, 2914 m->bank); 2915 } else { 2916 char A = *("A"); 2917 2918 /* 2919 * Reported channel is in range 0-2, so we can't map it 2920 * back to mc. To figure out mc we check machine check 2921 * bank register that reported this error. 2922 * bank15 means mc0 and bank16 means mc1. 2923 */ 2924 channel = knl_channel_remap(m->bank == 16, channel); 2925 channel_mask = 1 << channel; 2926 2927 snprintf(msg, sizeof(msg), 2928 "%s%s err_code:%04x:%04x channel:%d (DIMM_%c)", 2929 overflow ? " OVERFLOW" : "", 2930 (uncorrected_error && recoverable) 2931 ? " recoverable" : " ", 2932 mscod, errcode, channel, A + channel); 2933 edac_mc_handle_error(tp_event, mci, core_err_cnt, 2934 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0, 2935 channel, 0, -1, 2936 optype, msg); 2937 } 2938 return; 2939 } else { 2940 rc = get_memory_error_data(mci, m->addr, &socket, &ha, 2941 &channel_mask, &rank, &area_type, msg); 2942 } 2943 2944 if (rc < 0) 2945 goto err_parsing; 2946 new_mci = get_mci_for_node_id(socket, ha); 2947 if (!new_mci) { 2948 strcpy(msg, "Error: socket got corrupted!"); 2949 goto err_parsing; 2950 } 2951 mci = new_mci; 2952 pvt = mci->pvt_info; 2953 2954 first_channel = find_first_bit(&channel_mask, NUM_CHANNELS); 2955 2956 if (rank < 4) 2957 dimm = 0; 2958 else if (rank < 8) 2959 dimm = 1; 2960 else 2961 dimm = 2; 2962 2963 2964 /* 2965 * FIXME: On some memory configurations (mirror, lockstep), the 2966 * Memory Controller can't point the error to a single DIMM. The 2967 * EDAC core should be handling the channel mask, in order to point 2968 * to the group of dimm's where the error may be happening. 2969 */ 2970 if (!pvt->is_lockstep && !pvt->is_mirrored && !pvt->is_close_pg) 2971 channel = first_channel; 2972 2973 snprintf(msg, sizeof(msg), 2974 "%s%s area:%s err_code:%04x:%04x socket:%d ha:%d channel_mask:%ld rank:%d", 2975 overflow ? " OVERFLOW" : "", 2976 (uncorrected_error && recoverable) ? " recoverable" : "", 2977 area_type, 2978 mscod, errcode, 2979 socket, ha, 2980 channel_mask, 2981 rank); 2982 2983 edac_dbg(0, "%s\n", msg); 2984 2985 /* FIXME: need support for channel mask */ 2986 2987 if (channel == CHANNEL_UNSPECIFIED) 2988 channel = -1; 2989 2990 /* Call the helper to output message */ 2991 edac_mc_handle_error(tp_event, mci, core_err_cnt, 2992 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0, 2993 channel, dimm, -1, 2994 optype, msg); 2995 return; 2996 err_parsing: 2997 edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0, 2998 -1, -1, -1, 2999 msg, ""); 3000 3001 } 3002 3003 /* 3004 * Check that logging is enabled and that this is the right type 3005 * of error for us to handle. 3006 */ 3007 static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val, 3008 void *data) 3009 { 3010 struct mce *mce = (struct mce *)data; 3011 struct mem_ctl_info *mci; 3012 struct sbridge_pvt *pvt; 3013 char *type; 3014 3015 if (edac_get_report_status() == EDAC_REPORTING_DISABLED) 3016 return NOTIFY_DONE; 3017 3018 mci = get_mci_for_node_id(mce->socketid, IMC0); 3019 if (!mci) 3020 return NOTIFY_DONE; 3021 pvt = mci->pvt_info; 3022 3023 /* 3024 * Just let mcelog handle it if the error is 3025 * outside the memory controller. A memory error 3026 * is indicated by bit 7 = 1 and bits = 8-11,13-15 = 0. 3027 * bit 12 has an special meaning. 3028 */ 3029 if ((mce->status & 0xefff) >> 7 != 1) 3030 return NOTIFY_DONE; 3031 3032 if (mce->mcgstatus & MCG_STATUS_MCIP) 3033 type = "Exception"; 3034 else 3035 type = "Event"; 3036 3037 sbridge_mc_printk(mci, KERN_DEBUG, "HANDLING MCE MEMORY ERROR\n"); 3038 3039 sbridge_mc_printk(mci, KERN_DEBUG, "CPU %d: Machine Check %s: %Lx " 3040 "Bank %d: %016Lx\n", mce->extcpu, type, 3041 mce->mcgstatus, mce->bank, mce->status); 3042 sbridge_mc_printk(mci, KERN_DEBUG, "TSC %llx ", mce->tsc); 3043 sbridge_mc_printk(mci, KERN_DEBUG, "ADDR %llx ", mce->addr); 3044 sbridge_mc_printk(mci, KERN_DEBUG, "MISC %llx ", mce->misc); 3045 3046 sbridge_mc_printk(mci, KERN_DEBUG, "PROCESSOR %u:%x TIME %llu SOCKET " 3047 "%u APIC %x\n", mce->cpuvendor, mce->cpuid, 3048 mce->time, mce->socketid, mce->apicid); 3049 3050 sbridge_mce_output_error(mci, mce); 3051 3052 /* Advice mcelog that the error were handled */ 3053 return NOTIFY_STOP; 3054 } 3055 3056 static struct notifier_block sbridge_mce_dec = { 3057 .notifier_call = sbridge_mce_check_error, 3058 .priority = MCE_PRIO_EDAC, 3059 }; 3060 3061 /**************************************************************************** 3062 EDAC register/unregister logic 3063 ****************************************************************************/ 3064 3065 static void sbridge_unregister_mci(struct sbridge_dev *sbridge_dev) 3066 { 3067 struct mem_ctl_info *mci = sbridge_dev->mci; 3068 struct sbridge_pvt *pvt; 3069 3070 if (unlikely(!mci || !mci->pvt_info)) { 3071 edac_dbg(0, "MC: dev = %p\n", &sbridge_dev->pdev[0]->dev); 3072 3073 sbridge_printk(KERN_ERR, "Couldn't find mci handler\n"); 3074 return; 3075 } 3076 3077 pvt = mci->pvt_info; 3078 3079 edac_dbg(0, "MC: mci = %p, dev = %p\n", 3080 mci, &sbridge_dev->pdev[0]->dev); 3081 3082 /* Remove MC sysfs nodes */ 3083 edac_mc_del_mc(mci->pdev); 3084 3085 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name); 3086 kfree(mci->ctl_name); 3087 edac_mc_free(mci); 3088 sbridge_dev->mci = NULL; 3089 } 3090 3091 static int sbridge_register_mci(struct sbridge_dev *sbridge_dev, enum type type) 3092 { 3093 struct mem_ctl_info *mci; 3094 struct edac_mc_layer layers[2]; 3095 struct sbridge_pvt *pvt; 3096 struct pci_dev *pdev = sbridge_dev->pdev[0]; 3097 int rc; 3098 3099 /* allocate a new MC control structure */ 3100 layers[0].type = EDAC_MC_LAYER_CHANNEL; 3101 layers[0].size = type == KNIGHTS_LANDING ? 3102 KNL_MAX_CHANNELS : NUM_CHANNELS; 3103 layers[0].is_virt_csrow = false; 3104 layers[1].type = EDAC_MC_LAYER_SLOT; 3105 layers[1].size = type == KNIGHTS_LANDING ? 1 : MAX_DIMMS; 3106 layers[1].is_virt_csrow = true; 3107 mci = edac_mc_alloc(sbridge_dev->mc, ARRAY_SIZE(layers), layers, 3108 sizeof(*pvt)); 3109 3110 if (unlikely(!mci)) 3111 return -ENOMEM; 3112 3113 edac_dbg(0, "MC: mci = %p, dev = %p\n", 3114 mci, &pdev->dev); 3115 3116 pvt = mci->pvt_info; 3117 memset(pvt, 0, sizeof(*pvt)); 3118 3119 /* Associate sbridge_dev and mci for future usage */ 3120 pvt->sbridge_dev = sbridge_dev; 3121 sbridge_dev->mci = mci; 3122 3123 mci->mtype_cap = type == KNIGHTS_LANDING ? 3124 MEM_FLAG_DDR4 : MEM_FLAG_DDR3; 3125 mci->edac_ctl_cap = EDAC_FLAG_NONE; 3126 mci->edac_cap = EDAC_FLAG_NONE; 3127 mci->mod_name = "sb_edac.c"; 3128 mci->mod_ver = SBRIDGE_REVISION; 3129 mci->dev_name = pci_name(pdev); 3130 mci->ctl_page_to_phys = NULL; 3131 3132 pvt->info.type = type; 3133 switch (type) { 3134 case IVY_BRIDGE: 3135 pvt->info.rankcfgr = IB_RANK_CFG_A; 3136 pvt->info.get_tolm = ibridge_get_tolm; 3137 pvt->info.get_tohm = ibridge_get_tohm; 3138 pvt->info.dram_rule = ibridge_dram_rule; 3139 pvt->info.get_memory_type = get_memory_type; 3140 pvt->info.get_node_id = get_node_id; 3141 pvt->info.rir_limit = rir_limit; 3142 pvt->info.sad_limit = sad_limit; 3143 pvt->info.interleave_mode = interleave_mode; 3144 pvt->info.dram_attr = dram_attr; 3145 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule); 3146 pvt->info.interleave_list = ibridge_interleave_list; 3147 pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list); 3148 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3149 pvt->info.get_width = ibridge_get_width; 3150 3151 /* Store pci devices at mci for faster access */ 3152 rc = ibridge_mci_bind_devs(mci, sbridge_dev); 3153 if (unlikely(rc < 0)) 3154 goto fail0; 3155 get_source_id(mci); 3156 mci->ctl_name = kasprintf(GFP_KERNEL, "Ivy Bridge SrcID#%d_Ha#%d", 3157 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3158 break; 3159 case SANDY_BRIDGE: 3160 pvt->info.rankcfgr = SB_RANK_CFG_A; 3161 pvt->info.get_tolm = sbridge_get_tolm; 3162 pvt->info.get_tohm = sbridge_get_tohm; 3163 pvt->info.dram_rule = sbridge_dram_rule; 3164 pvt->info.get_memory_type = get_memory_type; 3165 pvt->info.get_node_id = get_node_id; 3166 pvt->info.rir_limit = rir_limit; 3167 pvt->info.sad_limit = sad_limit; 3168 pvt->info.interleave_mode = interleave_mode; 3169 pvt->info.dram_attr = dram_attr; 3170 pvt->info.max_sad = ARRAY_SIZE(sbridge_dram_rule); 3171 pvt->info.interleave_list = sbridge_interleave_list; 3172 pvt->info.max_interleave = ARRAY_SIZE(sbridge_interleave_list); 3173 pvt->info.interleave_pkg = sbridge_interleave_pkg; 3174 pvt->info.get_width = sbridge_get_width; 3175 3176 /* Store pci devices at mci for faster access */ 3177 rc = sbridge_mci_bind_devs(mci, sbridge_dev); 3178 if (unlikely(rc < 0)) 3179 goto fail0; 3180 get_source_id(mci); 3181 mci->ctl_name = kasprintf(GFP_KERNEL, "Sandy Bridge SrcID#%d_Ha#%d", 3182 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3183 break; 3184 case HASWELL: 3185 /* rankcfgr isn't used */ 3186 pvt->info.get_tolm = haswell_get_tolm; 3187 pvt->info.get_tohm = haswell_get_tohm; 3188 pvt->info.dram_rule = ibridge_dram_rule; 3189 pvt->info.get_memory_type = haswell_get_memory_type; 3190 pvt->info.get_node_id = haswell_get_node_id; 3191 pvt->info.rir_limit = haswell_rir_limit; 3192 pvt->info.sad_limit = sad_limit; 3193 pvt->info.interleave_mode = interleave_mode; 3194 pvt->info.dram_attr = dram_attr; 3195 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule); 3196 pvt->info.interleave_list = ibridge_interleave_list; 3197 pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list); 3198 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3199 pvt->info.get_width = ibridge_get_width; 3200 3201 /* Store pci devices at mci for faster access */ 3202 rc = haswell_mci_bind_devs(mci, sbridge_dev); 3203 if (unlikely(rc < 0)) 3204 goto fail0; 3205 get_source_id(mci); 3206 mci->ctl_name = kasprintf(GFP_KERNEL, "Haswell SrcID#%d_Ha#%d", 3207 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3208 break; 3209 case BROADWELL: 3210 /* rankcfgr isn't used */ 3211 pvt->info.get_tolm = haswell_get_tolm; 3212 pvt->info.get_tohm = haswell_get_tohm; 3213 pvt->info.dram_rule = ibridge_dram_rule; 3214 pvt->info.get_memory_type = haswell_get_memory_type; 3215 pvt->info.get_node_id = haswell_get_node_id; 3216 pvt->info.rir_limit = haswell_rir_limit; 3217 pvt->info.sad_limit = sad_limit; 3218 pvt->info.interleave_mode = interleave_mode; 3219 pvt->info.dram_attr = dram_attr; 3220 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule); 3221 pvt->info.interleave_list = ibridge_interleave_list; 3222 pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list); 3223 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3224 pvt->info.get_width = broadwell_get_width; 3225 3226 /* Store pci devices at mci for faster access */ 3227 rc = broadwell_mci_bind_devs(mci, sbridge_dev); 3228 if (unlikely(rc < 0)) 3229 goto fail0; 3230 get_source_id(mci); 3231 mci->ctl_name = kasprintf(GFP_KERNEL, "Broadwell SrcID#%d_Ha#%d", 3232 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3233 break; 3234 case KNIGHTS_LANDING: 3235 /* pvt->info.rankcfgr == ??? */ 3236 pvt->info.get_tolm = knl_get_tolm; 3237 pvt->info.get_tohm = knl_get_tohm; 3238 pvt->info.dram_rule = knl_dram_rule; 3239 pvt->info.get_memory_type = knl_get_memory_type; 3240 pvt->info.get_node_id = knl_get_node_id; 3241 pvt->info.rir_limit = NULL; 3242 pvt->info.sad_limit = knl_sad_limit; 3243 pvt->info.interleave_mode = knl_interleave_mode; 3244 pvt->info.dram_attr = dram_attr_knl; 3245 pvt->info.max_sad = ARRAY_SIZE(knl_dram_rule); 3246 pvt->info.interleave_list = knl_interleave_list; 3247 pvt->info.max_interleave = ARRAY_SIZE(knl_interleave_list); 3248 pvt->info.interleave_pkg = ibridge_interleave_pkg; 3249 pvt->info.get_width = knl_get_width; 3250 3251 rc = knl_mci_bind_devs(mci, sbridge_dev); 3252 if (unlikely(rc < 0)) 3253 goto fail0; 3254 get_source_id(mci); 3255 mci->ctl_name = kasprintf(GFP_KERNEL, "Knights Landing SrcID#%d_Ha#%d", 3256 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom); 3257 break; 3258 } 3259 3260 /* Get dimm basic config and the memory layout */ 3261 rc = get_dimm_config(mci); 3262 if (rc < 0) { 3263 edac_dbg(0, "MC: failed to get_dimm_config()\n"); 3264 goto fail; 3265 } 3266 get_memory_layout(mci); 3267 3268 /* record ptr to the generic device */ 3269 mci->pdev = &pdev->dev; 3270 3271 /* add this new MC control structure to EDAC's list of MCs */ 3272 if (unlikely(edac_mc_add_mc(mci))) { 3273 edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); 3274 rc = -EINVAL; 3275 goto fail; 3276 } 3277 3278 return 0; 3279 3280 fail: 3281 kfree(mci->ctl_name); 3282 fail0: 3283 edac_mc_free(mci); 3284 sbridge_dev->mci = NULL; 3285 return rc; 3286 } 3287 3288 #define ICPU(model, table) \ 3289 { X86_VENDOR_INTEL, 6, model, 0, (unsigned long)&table } 3290 3291 static const struct x86_cpu_id sbridge_cpuids[] = { 3292 ICPU(INTEL_FAM6_SANDYBRIDGE_X, pci_dev_descr_sbridge_table), 3293 ICPU(INTEL_FAM6_IVYBRIDGE_X, pci_dev_descr_ibridge_table), 3294 ICPU(INTEL_FAM6_HASWELL_X, pci_dev_descr_haswell_table), 3295 ICPU(INTEL_FAM6_BROADWELL_X, pci_dev_descr_broadwell_table), 3296 ICPU(INTEL_FAM6_BROADWELL_XEON_D, pci_dev_descr_broadwell_table), 3297 ICPU(INTEL_FAM6_XEON_PHI_KNL, pci_dev_descr_knl_table), 3298 ICPU(INTEL_FAM6_XEON_PHI_KNM, pci_dev_descr_knl_table), 3299 { } 3300 }; 3301 MODULE_DEVICE_TABLE(x86cpu, sbridge_cpuids); 3302 3303 /* 3304 * sbridge_probe Get all devices and register memory controllers 3305 * present. 3306 * return: 3307 * 0 for FOUND a device 3308 * < 0 for error code 3309 */ 3310 3311 static int sbridge_probe(const struct x86_cpu_id *id) 3312 { 3313 int rc = -ENODEV; 3314 u8 mc, num_mc = 0; 3315 struct sbridge_dev *sbridge_dev; 3316 struct pci_id_table *ptable = (struct pci_id_table *)id->driver_data; 3317 3318 /* get the pci devices we want to reserve for our use */ 3319 rc = sbridge_get_all_devices(&num_mc, ptable); 3320 3321 if (unlikely(rc < 0)) { 3322 edac_dbg(0, "couldn't get all devices\n"); 3323 goto fail0; 3324 } 3325 3326 mc = 0; 3327 3328 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) { 3329 edac_dbg(0, "Registering MC#%d (%d of %d)\n", 3330 mc, mc + 1, num_mc); 3331 3332 sbridge_dev->mc = mc++; 3333 rc = sbridge_register_mci(sbridge_dev, ptable->type); 3334 if (unlikely(rc < 0)) 3335 goto fail1; 3336 } 3337 3338 sbridge_printk(KERN_INFO, "%s\n", SBRIDGE_REVISION); 3339 3340 return 0; 3341 3342 fail1: 3343 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) 3344 sbridge_unregister_mci(sbridge_dev); 3345 3346 sbridge_put_all_devices(); 3347 fail0: 3348 return rc; 3349 } 3350 3351 /* 3352 * sbridge_remove cleanup 3353 * 3354 */ 3355 static void sbridge_remove(void) 3356 { 3357 struct sbridge_dev *sbridge_dev; 3358 3359 edac_dbg(0, "\n"); 3360 3361 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) 3362 sbridge_unregister_mci(sbridge_dev); 3363 3364 /* Release PCI resources */ 3365 sbridge_put_all_devices(); 3366 } 3367 3368 /* 3369 * sbridge_init Module entry function 3370 * Try to initialize this module for its devices 3371 */ 3372 static int __init sbridge_init(void) 3373 { 3374 const struct x86_cpu_id *id; 3375 int rc; 3376 3377 edac_dbg(2, "\n"); 3378 3379 id = x86_match_cpu(sbridge_cpuids); 3380 if (!id) 3381 return -ENODEV; 3382 3383 /* Ensure that the OPSTATE is set correctly for POLL or NMI */ 3384 opstate_init(); 3385 3386 rc = sbridge_probe(id); 3387 3388 if (rc >= 0) { 3389 mce_register_decode_chain(&sbridge_mce_dec); 3390 if (edac_get_report_status() == EDAC_REPORTING_DISABLED) 3391 sbridge_printk(KERN_WARNING, "Loading driver, error reporting disabled.\n"); 3392 return 0; 3393 } 3394 3395 sbridge_printk(KERN_ERR, "Failed to register device with error %d.\n", 3396 rc); 3397 3398 return rc; 3399 } 3400 3401 /* 3402 * sbridge_exit() Module exit function 3403 * Unregister the driver 3404 */ 3405 static void __exit sbridge_exit(void) 3406 { 3407 edac_dbg(2, "\n"); 3408 sbridge_remove(); 3409 mce_unregister_decode_chain(&sbridge_mce_dec); 3410 } 3411 3412 module_init(sbridge_init); 3413 module_exit(sbridge_exit); 3414 3415 module_param(edac_op_state, int, 0444); 3416 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 3417 3418 MODULE_LICENSE("GPL"); 3419 MODULE_AUTHOR("Mauro Carvalho Chehab"); 3420 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)"); 3421 MODULE_DESCRIPTION("MC Driver for Intel Sandy Bridge and Ivy Bridge memory controllers - " 3422 SBRIDGE_REVISION); 3423