1 /* This only handles 32bit MTRR on 32bit hosts. This is strictly wrong 2 because MTRRs can span upto 40 bits (36bits on most modern x86) */ 3 #include <linux/init.h> 4 #include <linux/slab.h> 5 #include <linux/mm.h> 6 #include <linux/module.h> 7 #include <asm/io.h> 8 #include <asm/mtrr.h> 9 #include <asm/msr.h> 10 #include <asm/system.h> 11 #include <asm/cpufeature.h> 12 #include <asm/processor-flags.h> 13 #include <asm/tlbflush.h> 14 #include "mtrr.h" 15 16 struct mtrr_state { 17 struct mtrr_var_range var_ranges[MAX_VAR_RANGES]; 18 mtrr_type fixed_ranges[NUM_FIXED_RANGES]; 19 unsigned char enabled; 20 unsigned char have_fixed; 21 mtrr_type def_type; 22 }; 23 24 struct fixed_range_block { 25 int base_msr; /* start address of an MTRR block */ 26 int ranges; /* number of MTRRs in this block */ 27 }; 28 29 static struct fixed_range_block fixed_range_blocks[] = { 30 { MTRRfix64K_00000_MSR, 1 }, /* one 64k MTRR */ 31 { MTRRfix16K_80000_MSR, 2 }, /* two 16k MTRRs */ 32 { MTRRfix4K_C0000_MSR, 8 }, /* eight 4k MTRRs */ 33 {} 34 }; 35 36 static unsigned long smp_changes_mask; 37 static struct mtrr_state mtrr_state = {}; 38 39 #undef MODULE_PARAM_PREFIX 40 #define MODULE_PARAM_PREFIX "mtrr." 41 42 static int mtrr_show; 43 module_param_named(show, mtrr_show, bool, 0); 44 45 /* Get the MSR pair relating to a var range */ 46 static void 47 get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr) 48 { 49 rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); 50 rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); 51 } 52 53 static void 54 get_fixed_ranges(mtrr_type * frs) 55 { 56 unsigned int *p = (unsigned int *) frs; 57 int i; 58 59 rdmsr(MTRRfix64K_00000_MSR, p[0], p[1]); 60 61 for (i = 0; i < 2; i++) 62 rdmsr(MTRRfix16K_80000_MSR + i, p[2 + i * 2], p[3 + i * 2]); 63 for (i = 0; i < 8; i++) 64 rdmsr(MTRRfix4K_C0000_MSR + i, p[6 + i * 2], p[7 + i * 2]); 65 } 66 67 void mtrr_save_fixed_ranges(void *info) 68 { 69 if (cpu_has_mtrr) 70 get_fixed_ranges(mtrr_state.fixed_ranges); 71 } 72 73 static void print_fixed(unsigned base, unsigned step, const mtrr_type*types) 74 { 75 unsigned i; 76 77 for (i = 0; i < 8; ++i, ++types, base += step) 78 printk(KERN_INFO "MTRR %05X-%05X %s\n", 79 base, base + step - 1, mtrr_attrib_to_str(*types)); 80 } 81 82 /* Grab all of the MTRR state for this CPU into *state */ 83 void __init get_mtrr_state(void) 84 { 85 unsigned int i; 86 struct mtrr_var_range *vrs; 87 unsigned lo, dummy; 88 89 vrs = mtrr_state.var_ranges; 90 91 rdmsr(MTRRcap_MSR, lo, dummy); 92 mtrr_state.have_fixed = (lo >> 8) & 1; 93 94 for (i = 0; i < num_var_ranges; i++) 95 get_mtrr_var_range(i, &vrs[i]); 96 if (mtrr_state.have_fixed) 97 get_fixed_ranges(mtrr_state.fixed_ranges); 98 99 rdmsr(MTRRdefType_MSR, lo, dummy); 100 mtrr_state.def_type = (lo & 0xff); 101 mtrr_state.enabled = (lo & 0xc00) >> 10; 102 103 if (mtrr_show) { 104 int high_width; 105 106 printk(KERN_INFO "MTRR default type: %s\n", mtrr_attrib_to_str(mtrr_state.def_type)); 107 if (mtrr_state.have_fixed) { 108 printk(KERN_INFO "MTRR fixed ranges %sabled:\n", 109 mtrr_state.enabled & 1 ? "en" : "dis"); 110 print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0); 111 for (i = 0; i < 2; ++i) 112 print_fixed(0x80000 + i * 0x20000, 0x04000, mtrr_state.fixed_ranges + (i + 1) * 8); 113 for (i = 0; i < 8; ++i) 114 print_fixed(0xC0000 + i * 0x08000, 0x01000, mtrr_state.fixed_ranges + (i + 3) * 8); 115 } 116 printk(KERN_INFO "MTRR variable ranges %sabled:\n", 117 mtrr_state.enabled & 2 ? "en" : "dis"); 118 high_width = ((size_or_mask ? ffs(size_or_mask) - 1 : 32) - (32 - PAGE_SHIFT) + 3) / 4; 119 for (i = 0; i < num_var_ranges; ++i) { 120 if (mtrr_state.var_ranges[i].mask_lo & (1 << 11)) 121 printk(KERN_INFO "MTRR %u base %0*X%05X000 mask %0*X%05X000 %s\n", 122 i, 123 high_width, 124 mtrr_state.var_ranges[i].base_hi, 125 mtrr_state.var_ranges[i].base_lo >> 12, 126 high_width, 127 mtrr_state.var_ranges[i].mask_hi, 128 mtrr_state.var_ranges[i].mask_lo >> 12, 129 mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff)); 130 else 131 printk(KERN_INFO "MTRR %u disabled\n", i); 132 } 133 } 134 } 135 136 /* Some BIOS's are fucked and don't set all MTRRs the same! */ 137 void __init mtrr_state_warn(void) 138 { 139 unsigned long mask = smp_changes_mask; 140 141 if (!mask) 142 return; 143 if (mask & MTRR_CHANGE_MASK_FIXED) 144 printk(KERN_WARNING "mtrr: your CPUs had inconsistent fixed MTRR settings\n"); 145 if (mask & MTRR_CHANGE_MASK_VARIABLE) 146 printk(KERN_WARNING "mtrr: your CPUs had inconsistent variable MTRR settings\n"); 147 if (mask & MTRR_CHANGE_MASK_DEFTYPE) 148 printk(KERN_WARNING "mtrr: your CPUs had inconsistent MTRRdefType settings\n"); 149 printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n"); 150 printk(KERN_INFO "mtrr: corrected configuration.\n"); 151 } 152 153 /* Doesn't attempt to pass an error out to MTRR users 154 because it's quite complicated in some cases and probably not 155 worth it because the best error handling is to ignore it. */ 156 void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b) 157 { 158 if (wrmsr_safe(msr, a, b) < 0) 159 printk(KERN_ERR 160 "MTRR: CPU %u: Writing MSR %x to %x:%x failed\n", 161 smp_processor_id(), msr, a, b); 162 } 163 164 /** 165 * Enable and allow read/write of extended fixed-range MTRR bits on K8 CPUs 166 * see AMD publication no. 24593, chapter 3.2.1 for more information 167 */ 168 static inline void k8_enable_fixed_iorrs(void) 169 { 170 unsigned lo, hi; 171 172 rdmsr(MSR_K8_SYSCFG, lo, hi); 173 mtrr_wrmsr(MSR_K8_SYSCFG, lo 174 | K8_MTRRFIXRANGE_DRAM_ENABLE 175 | K8_MTRRFIXRANGE_DRAM_MODIFY, hi); 176 } 177 178 /** 179 * set_fixed_range - checks & updates a fixed-range MTRR if it differs from the value it should have 180 * @msr: MSR address of the MTTR which should be checked and updated 181 * @changed: pointer which indicates whether the MTRR needed to be changed 182 * @msrwords: pointer to the MSR values which the MSR should have 183 * 184 * If K8 extentions are wanted, update the K8 SYSCFG MSR also. 185 * See AMD publication no. 24593, chapter 7.8.1, page 233 for more information. 186 */ 187 static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords) 188 { 189 unsigned lo, hi; 190 191 rdmsr(msr, lo, hi); 192 193 if (lo != msrwords[0] || hi != msrwords[1]) { 194 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && 195 boot_cpu_data.x86 == 15 && 196 ((msrwords[0] | msrwords[1]) & K8_MTRR_RDMEM_WRMEM_MASK)) 197 k8_enable_fixed_iorrs(); 198 mtrr_wrmsr(msr, msrwords[0], msrwords[1]); 199 *changed = true; 200 } 201 } 202 203 /** 204 * generic_get_free_region - Get a free MTRR. 205 * @base: The starting (base) address of the region. 206 * @size: The size (in bytes) of the region. 207 * @replace_reg: mtrr index to be replaced; set to invalid value if none. 208 * 209 * Returns: The index of the region on success, else negative on error. 210 */ 211 int generic_get_free_region(unsigned long base, unsigned long size, int replace_reg) 212 { 213 int i, max; 214 mtrr_type ltype; 215 unsigned long lbase, lsize; 216 217 max = num_var_ranges; 218 if (replace_reg >= 0 && replace_reg < max) 219 return replace_reg; 220 for (i = 0; i < max; ++i) { 221 mtrr_if->get(i, &lbase, &lsize, <ype); 222 if (lsize == 0) 223 return i; 224 } 225 return -ENOSPC; 226 } 227 228 static void generic_get_mtrr(unsigned int reg, unsigned long *base, 229 unsigned long *size, mtrr_type *type) 230 { 231 unsigned int mask_lo, mask_hi, base_lo, base_hi; 232 233 rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi); 234 if ((mask_lo & 0x800) == 0) { 235 /* Invalid (i.e. free) range */ 236 *base = 0; 237 *size = 0; 238 *type = 0; 239 return; 240 } 241 242 rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi); 243 244 /* Work out the shifted address mask. */ 245 mask_lo = size_or_mask | mask_hi << (32 - PAGE_SHIFT) 246 | mask_lo >> PAGE_SHIFT; 247 248 /* This works correctly if size is a power of two, i.e. a 249 contiguous range. */ 250 *size = -mask_lo; 251 *base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT; 252 *type = base_lo & 0xff; 253 } 254 255 /** 256 * set_fixed_ranges - checks & updates the fixed-range MTRRs if they differ from the saved set 257 * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges() 258 */ 259 static int set_fixed_ranges(mtrr_type * frs) 260 { 261 unsigned long long *saved = (unsigned long long *) frs; 262 bool changed = false; 263 int block=-1, range; 264 265 while (fixed_range_blocks[++block].ranges) 266 for (range=0; range < fixed_range_blocks[block].ranges; range++) 267 set_fixed_range(fixed_range_blocks[block].base_msr + range, 268 &changed, (unsigned int *) saved++); 269 270 return changed; 271 } 272 273 /* Set the MSR pair relating to a var range. Returns TRUE if 274 changes are made */ 275 static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr) 276 { 277 unsigned int lo, hi; 278 bool changed = false; 279 280 rdmsr(MTRRphysBase_MSR(index), lo, hi); 281 if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL) 282 || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) != 283 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) { 284 mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); 285 changed = true; 286 } 287 288 rdmsr(MTRRphysMask_MSR(index), lo, hi); 289 290 if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL) 291 || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) != 292 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) { 293 mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); 294 changed = true; 295 } 296 return changed; 297 } 298 299 static u32 deftype_lo, deftype_hi; 300 301 /** 302 * set_mtrr_state - Set the MTRR state for this CPU. 303 * 304 * NOTE: The CPU must already be in a safe state for MTRR changes. 305 * RETURNS: 0 if no changes made, else a mask indicating what was changed. 306 */ 307 static unsigned long set_mtrr_state(void) 308 { 309 unsigned int i; 310 unsigned long change_mask = 0; 311 312 for (i = 0; i < num_var_ranges; i++) 313 if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i])) 314 change_mask |= MTRR_CHANGE_MASK_VARIABLE; 315 316 if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges)) 317 change_mask |= MTRR_CHANGE_MASK_FIXED; 318 319 /* Set_mtrr_restore restores the old value of MTRRdefType, 320 so to set it we fiddle with the saved value */ 321 if ((deftype_lo & 0xff) != mtrr_state.def_type 322 || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) { 323 deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type | (mtrr_state.enabled << 10); 324 change_mask |= MTRR_CHANGE_MASK_DEFTYPE; 325 } 326 327 return change_mask; 328 } 329 330 331 static unsigned long cr4 = 0; 332 static DEFINE_SPINLOCK(set_atomicity_lock); 333 334 /* 335 * Since we are disabling the cache don't allow any interrupts - they 336 * would run extremely slow and would only increase the pain. The caller must 337 * ensure that local interrupts are disabled and are reenabled after post_set() 338 * has been called. 339 */ 340 341 static void prepare_set(void) __acquires(set_atomicity_lock) 342 { 343 unsigned long cr0; 344 345 /* Note that this is not ideal, since the cache is only flushed/disabled 346 for this CPU while the MTRRs are changed, but changing this requires 347 more invasive changes to the way the kernel boots */ 348 349 spin_lock(&set_atomicity_lock); 350 351 /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */ 352 cr0 = read_cr0() | X86_CR0_CD; 353 write_cr0(cr0); 354 wbinvd(); 355 356 /* Save value of CR4 and clear Page Global Enable (bit 7) */ 357 if ( cpu_has_pge ) { 358 cr4 = read_cr4(); 359 write_cr4(cr4 & ~X86_CR4_PGE); 360 } 361 362 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ 363 __flush_tlb(); 364 365 /* Save MTRR state */ 366 rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi); 367 368 /* Disable MTRRs, and set the default type to uncached */ 369 mtrr_wrmsr(MTRRdefType_MSR, deftype_lo & ~0xcff, deftype_hi); 370 } 371 372 static void post_set(void) __releases(set_atomicity_lock) 373 { 374 /* Flush TLBs (no need to flush caches - they are disabled) */ 375 __flush_tlb(); 376 377 /* Intel (P6) standard MTRRs */ 378 mtrr_wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi); 379 380 /* Enable caches */ 381 write_cr0(read_cr0() & 0xbfffffff); 382 383 /* Restore value of CR4 */ 384 if ( cpu_has_pge ) 385 write_cr4(cr4); 386 spin_unlock(&set_atomicity_lock); 387 } 388 389 static void generic_set_all(void) 390 { 391 unsigned long mask, count; 392 unsigned long flags; 393 394 local_irq_save(flags); 395 prepare_set(); 396 397 /* Actually set the state */ 398 mask = set_mtrr_state(); 399 400 post_set(); 401 local_irq_restore(flags); 402 403 /* Use the atomic bitops to update the global mask */ 404 for (count = 0; count < sizeof mask * 8; ++count) { 405 if (mask & 0x01) 406 set_bit(count, &smp_changes_mask); 407 mask >>= 1; 408 } 409 410 } 411 412 static void generic_set_mtrr(unsigned int reg, unsigned long base, 413 unsigned long size, mtrr_type type) 414 /* [SUMMARY] Set variable MTRR register on the local CPU. 415 <reg> The register to set. 416 <base> The base address of the region. 417 <size> The size of the region. If this is 0 the region is disabled. 418 <type> The type of the region. 419 [RETURNS] Nothing. 420 */ 421 { 422 unsigned long flags; 423 struct mtrr_var_range *vr; 424 425 vr = &mtrr_state.var_ranges[reg]; 426 427 local_irq_save(flags); 428 prepare_set(); 429 430 if (size == 0) { 431 /* The invalid bit is kept in the mask, so we simply clear the 432 relevant mask register to disable a range. */ 433 mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0); 434 memset(vr, 0, sizeof(struct mtrr_var_range)); 435 } else { 436 vr->base_lo = base << PAGE_SHIFT | type; 437 vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT); 438 vr->mask_lo = -size << PAGE_SHIFT | 0x800; 439 vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT); 440 441 mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi); 442 mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi); 443 } 444 445 post_set(); 446 local_irq_restore(flags); 447 } 448 449 int generic_validate_add_page(unsigned long base, unsigned long size, unsigned int type) 450 { 451 unsigned long lbase, last; 452 453 /* For Intel PPro stepping <= 7, must be 4 MiB aligned 454 and not touch 0x70000000->0x7003FFFF */ 455 if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 && 456 boot_cpu_data.x86_model == 1 && 457 boot_cpu_data.x86_mask <= 7) { 458 if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) { 459 printk(KERN_WARNING "mtrr: base(0x%lx000) is not 4 MiB aligned\n", base); 460 return -EINVAL; 461 } 462 if (!(base + size < 0x70000 || base > 0x7003F) && 463 (type == MTRR_TYPE_WRCOMB 464 || type == MTRR_TYPE_WRBACK)) { 465 printk(KERN_WARNING "mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n"); 466 return -EINVAL; 467 } 468 } 469 470 /* Check upper bits of base and last are equal and lower bits are 0 471 for base and 1 for last */ 472 last = base + size - 1; 473 for (lbase = base; !(lbase & 1) && (last & 1); 474 lbase = lbase >> 1, last = last >> 1) ; 475 if (lbase != last) { 476 printk(KERN_WARNING "mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", 477 base, size); 478 return -EINVAL; 479 } 480 return 0; 481 } 482 483 484 static int generic_have_wrcomb(void) 485 { 486 unsigned long config, dummy; 487 rdmsr(MTRRcap_MSR, config, dummy); 488 return (config & (1 << 10)); 489 } 490 491 int positive_have_wrcomb(void) 492 { 493 return 1; 494 } 495 496 /* generic structure... 497 */ 498 struct mtrr_ops generic_mtrr_ops = { 499 .use_intel_if = 1, 500 .set_all = generic_set_all, 501 .get = generic_get_mtrr, 502 .get_free_region = generic_get_free_region, 503 .set = generic_set_mtrr, 504 .validate_add_page = generic_validate_add_page, 505 .have_wrcomb = generic_have_wrcomb, 506 }; 507