1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * This only handles 32bit MTRR on 32bit hosts. This is strictly wrong 4 * because MTRRs can span up to 40 bits (36bits on most modern x86) 5 */ 6 7 #include <linux/export.h> 8 #include <linux/init.h> 9 #include <linux/io.h> 10 #include <linux/mm.h> 11 #include <linux/cc_platform.h> 12 #include <asm/processor-flags.h> 13 #include <asm/cacheinfo.h> 14 #include <asm/cpufeature.h> 15 #include <asm/hypervisor.h> 16 #include <asm/mshyperv.h> 17 #include <asm/tlbflush.h> 18 #include <asm/mtrr.h> 19 #include <asm/msr.h> 20 #include <asm/memtype.h> 21 22 #include "mtrr.h" 23 24 struct fixed_range_block { 25 int base_msr; /* start address of an MTRR block */ 26 int ranges; /* number of MTRRs in this block */ 27 }; 28 29 static struct fixed_range_block fixed_range_blocks[] = { 30 { MSR_MTRRfix64K_00000, 1 }, /* one 64k MTRR */ 31 { MSR_MTRRfix16K_80000, 2 }, /* two 16k MTRRs */ 32 { MSR_MTRRfix4K_C0000, 8 }, /* eight 4k MTRRs */ 33 {} 34 }; 35 36 struct cache_map { 37 u64 start; 38 u64 end; 39 u64 flags; 40 u64 type:8; 41 u64 fixed:1; 42 }; 43 44 /* 45 * CACHE_MAP_MAX is the maximum number of memory ranges in cache_map, where 46 * no 2 adjacent ranges have the same cache mode (those would be merged). 47 * The number is based on the worst case: 48 * - no two adjacent fixed MTRRs share the same cache mode 49 * - one variable MTRR is spanning a huge area with mode WB 50 * - 255 variable MTRRs with mode UC all overlap with the WB MTRR, creating 2 51 * additional ranges each (result like "ababababa...aba" with a = WB, b = UC), 52 * accounting for MTRR_MAX_VAR_RANGES * 2 - 1 range entries 53 * - a TOP_MEM2 area (even with overlapping an UC MTRR can't add 2 range entries 54 * to the possible maximum, as it always starts at 4GB, thus it can't be in 55 * the middle of that MTRR, unless that MTRR starts at 0, which would remove 56 * the initial "a" from the "abababa" pattern above) 57 * The map won't contain ranges with no matching MTRR (those fall back to the 58 * default cache mode). 59 */ 60 #define CACHE_MAP_MAX (MTRR_NUM_FIXED_RANGES + MTRR_MAX_VAR_RANGES * 2) 61 62 static struct cache_map init_cache_map[CACHE_MAP_MAX] __initdata; 63 static struct cache_map *cache_map __refdata = init_cache_map; 64 static unsigned int cache_map_size = CACHE_MAP_MAX; 65 static unsigned int cache_map_n; 66 static unsigned int cache_map_fixed; 67 68 static unsigned long smp_changes_mask; 69 static int mtrr_state_set; 70 u64 mtrr_tom2; 71 72 struct mtrr_state_type mtrr_state; 73 EXPORT_SYMBOL_GPL(mtrr_state); 74 75 /* Reserved bits in the high portion of the MTRRphysBaseN MSR. */ 76 u32 phys_hi_rsvd; 77 78 /* 79 * BIOS is expected to clear MtrrFixDramModEn bit, see for example 80 * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD 81 * Opteron Processors" (26094 Rev. 3.30 February 2006), section 82 * "13.2.1.2 SYSCFG Register": "The MtrrFixDramModEn bit should be set 83 * to 1 during BIOS initialization of the fixed MTRRs, then cleared to 84 * 0 for operation." 85 */ 86 static inline void k8_check_syscfg_dram_mod_en(void) 87 { 88 u32 lo, hi; 89 90 if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && 91 (boot_cpu_data.x86 >= 0x0f))) 92 return; 93 94 rdmsr(MSR_AMD64_SYSCFG, lo, hi); 95 if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) { 96 pr_err(FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]" 97 " not cleared by BIOS, clearing this bit\n", 98 smp_processor_id()); 99 lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY; 100 mtrr_wrmsr(MSR_AMD64_SYSCFG, lo, hi); 101 } 102 } 103 104 /* Get the size of contiguous MTRR range */ 105 static u64 get_mtrr_size(u64 mask) 106 { 107 u64 size; 108 109 mask |= (u64)phys_hi_rsvd << 32; 110 size = -mask; 111 112 return size; 113 } 114 115 static u8 get_var_mtrr_state(unsigned int reg, u64 *start, u64 *size) 116 { 117 struct mtrr_var_range *mtrr = mtrr_state.var_ranges + reg; 118 119 if (!(mtrr->mask_lo & MTRR_PHYSMASK_V)) 120 return MTRR_TYPE_INVALID; 121 122 *start = (((u64)mtrr->base_hi) << 32) + (mtrr->base_lo & PAGE_MASK); 123 *size = get_mtrr_size((((u64)mtrr->mask_hi) << 32) + 124 (mtrr->mask_lo & PAGE_MASK)); 125 126 return mtrr->base_lo & MTRR_PHYSBASE_TYPE; 127 } 128 129 static u8 get_effective_type(u8 type1, u8 type2) 130 { 131 if (type1 == MTRR_TYPE_UNCACHABLE || type2 == MTRR_TYPE_UNCACHABLE) 132 return MTRR_TYPE_UNCACHABLE; 133 134 if ((type1 == MTRR_TYPE_WRBACK && type2 == MTRR_TYPE_WRTHROUGH) || 135 (type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK)) 136 return MTRR_TYPE_WRTHROUGH; 137 138 if (type1 != type2) 139 return MTRR_TYPE_UNCACHABLE; 140 141 return type1; 142 } 143 144 /* 145 * Check and return the effective type for MTRR-MTRR type overlap. 146 * Returns true if the effective type is UNCACHEABLE, else returns false 147 */ 148 static bool check_type_overlap(u8 *prev, u8 *curr) 149 { 150 *prev = *curr = get_effective_type(*curr, *prev); 151 152 return *prev == MTRR_TYPE_UNCACHABLE; 153 } 154 155 /** 156 * mtrr_type_lookup_fixed - look up memory type in MTRR fixed entries 157 * 158 * Return the MTRR fixed memory type of 'start'. 159 * 160 * MTRR fixed entries are divided into the following ways: 161 * 0x00000 - 0x7FFFF : This range is divided into eight 64KB sub-ranges 162 * 0x80000 - 0xBFFFF : This range is divided into sixteen 16KB sub-ranges 163 * 0xC0000 - 0xFFFFF : This range is divided into sixty-four 4KB sub-ranges 164 * 165 * Return Values: 166 * MTRR_TYPE_(type) - Matched memory type 167 * MTRR_TYPE_INVALID - Unmatched 168 */ 169 static u8 mtrr_type_lookup_fixed(u64 start, u64 end) 170 { 171 int idx; 172 173 if (start >= 0x100000) 174 return MTRR_TYPE_INVALID; 175 176 /* 0x0 - 0x7FFFF */ 177 if (start < 0x80000) { 178 idx = 0; 179 idx += (start >> 16); 180 return mtrr_state.fixed_ranges[idx]; 181 /* 0x80000 - 0xBFFFF */ 182 } else if (start < 0xC0000) { 183 idx = 1 * 8; 184 idx += ((start - 0x80000) >> 14); 185 return mtrr_state.fixed_ranges[idx]; 186 } 187 188 /* 0xC0000 - 0xFFFFF */ 189 idx = 3 * 8; 190 idx += ((start - 0xC0000) >> 12); 191 return mtrr_state.fixed_ranges[idx]; 192 } 193 194 /** 195 * mtrr_type_lookup_variable - look up memory type in MTRR variable entries 196 * 197 * Return Value: 198 * MTRR_TYPE_(type) - Matched memory type or default memory type (unmatched) 199 * 200 * Output Arguments: 201 * repeat - Set to 1 when [start:end] spanned across MTRR range and type 202 * returned corresponds only to [start:*partial_end]. Caller has 203 * to lookup again for [*partial_end:end]. 204 * 205 * uniform - Set to 1 when an MTRR covers the region uniformly, i.e. the 206 * region is fully covered by a single MTRR entry or the default 207 * type. 208 */ 209 static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end, 210 int *repeat, u8 *uniform) 211 { 212 int i; 213 u64 base, mask; 214 u8 prev_match, curr_match; 215 216 *repeat = 0; 217 *uniform = 1; 218 219 prev_match = MTRR_TYPE_INVALID; 220 for (i = 0; i < num_var_ranges; ++i) { 221 unsigned short start_state, end_state, inclusive; 222 223 if (!(mtrr_state.var_ranges[i].mask_lo & MTRR_PHYSMASK_V)) 224 continue; 225 226 base = (((u64)mtrr_state.var_ranges[i].base_hi) << 32) + 227 (mtrr_state.var_ranges[i].base_lo & PAGE_MASK); 228 mask = (((u64)mtrr_state.var_ranges[i].mask_hi) << 32) + 229 (mtrr_state.var_ranges[i].mask_lo & PAGE_MASK); 230 231 start_state = ((start & mask) == (base & mask)); 232 end_state = ((end & mask) == (base & mask)); 233 inclusive = ((start < base) && (end > base)); 234 235 if ((start_state != end_state) || inclusive) { 236 /* 237 * We have start:end spanning across an MTRR. 238 * We split the region into either 239 * 240 * - start_state:1 241 * (start:mtrr_end)(mtrr_end:end) 242 * - end_state:1 243 * (start:mtrr_start)(mtrr_start:end) 244 * - inclusive:1 245 * (start:mtrr_start)(mtrr_start:mtrr_end)(mtrr_end:end) 246 * 247 * depending on kind of overlap. 248 * 249 * Return the type of the first region and a pointer 250 * to the start of next region so that caller will be 251 * advised to lookup again after having adjusted start 252 * and end. 253 * 254 * Note: This way we handle overlaps with multiple 255 * entries and the default type properly. 256 */ 257 if (start_state) 258 *partial_end = base + get_mtrr_size(mask); 259 else 260 *partial_end = base; 261 262 if (unlikely(*partial_end <= start)) { 263 WARN_ON(1); 264 *partial_end = start + PAGE_SIZE; 265 } 266 267 end = *partial_end - 1; /* end is inclusive */ 268 *repeat = 1; 269 *uniform = 0; 270 } 271 272 if ((start & mask) != (base & mask)) 273 continue; 274 275 curr_match = mtrr_state.var_ranges[i].base_lo & MTRR_PHYSBASE_TYPE; 276 if (prev_match == MTRR_TYPE_INVALID) { 277 prev_match = curr_match; 278 continue; 279 } 280 281 *uniform = 0; 282 if (check_type_overlap(&prev_match, &curr_match)) 283 return curr_match; 284 } 285 286 if (prev_match != MTRR_TYPE_INVALID) 287 return prev_match; 288 289 return mtrr_state.def_type; 290 } 291 292 static void rm_map_entry_at(int idx) 293 { 294 cache_map_n--; 295 if (cache_map_n > idx) { 296 memmove(cache_map + idx, cache_map + idx + 1, 297 sizeof(*cache_map) * (cache_map_n - idx)); 298 } 299 } 300 301 /* 302 * Add an entry into cache_map at a specific index. Merges adjacent entries if 303 * appropriate. Return the number of merges for correcting the scan index 304 * (this is needed as merging will reduce the number of entries, which will 305 * result in skipping entries in future iterations if the scan index isn't 306 * corrected). 307 * Note that the corrected index can never go below -1 (resulting in being 0 in 308 * the next scan iteration), as "2" is returned only if the current index is 309 * larger than zero. 310 */ 311 static int add_map_entry_at(u64 start, u64 end, u8 type, int idx) 312 { 313 bool merge_prev = false, merge_next = false; 314 315 if (start >= end) 316 return 0; 317 318 if (idx > 0) { 319 struct cache_map *prev = cache_map + idx - 1; 320 321 if (!prev->fixed && start == prev->end && type == prev->type) 322 merge_prev = true; 323 } 324 325 if (idx < cache_map_n) { 326 struct cache_map *next = cache_map + idx; 327 328 if (!next->fixed && end == next->start && type == next->type) 329 merge_next = true; 330 } 331 332 if (merge_prev && merge_next) { 333 cache_map[idx - 1].end = cache_map[idx].end; 334 rm_map_entry_at(idx); 335 return 2; 336 } 337 if (merge_prev) { 338 cache_map[idx - 1].end = end; 339 return 1; 340 } 341 if (merge_next) { 342 cache_map[idx].start = start; 343 return 1; 344 } 345 346 /* Sanity check: the array should NEVER be too small! */ 347 if (cache_map_n == cache_map_size) { 348 WARN(1, "MTRR cache mode memory map exhausted!\n"); 349 cache_map_n = cache_map_fixed; 350 return 0; 351 } 352 353 if (cache_map_n > idx) { 354 memmove(cache_map + idx + 1, cache_map + idx, 355 sizeof(*cache_map) * (cache_map_n - idx)); 356 } 357 358 cache_map[idx].start = start; 359 cache_map[idx].end = end; 360 cache_map[idx].type = type; 361 cache_map[idx].fixed = 0; 362 cache_map_n++; 363 364 return 0; 365 } 366 367 /* Clear a part of an entry. Return 1 if start of entry is still valid. */ 368 static int clr_map_range_at(u64 start, u64 end, int idx) 369 { 370 int ret = start != cache_map[idx].start; 371 u64 tmp; 372 373 if (start == cache_map[idx].start && end == cache_map[idx].end) { 374 rm_map_entry_at(idx); 375 } else if (start == cache_map[idx].start) { 376 cache_map[idx].start = end; 377 } else if (end == cache_map[idx].end) { 378 cache_map[idx].end = start; 379 } else { 380 tmp = cache_map[idx].end; 381 cache_map[idx].end = start; 382 add_map_entry_at(end, tmp, cache_map[idx].type, idx + 1); 383 } 384 385 return ret; 386 } 387 388 /* 389 * Add MTRR to the map. The current map is scanned and each part of the MTRR 390 * either overlapping with an existing entry or with a hole in the map is 391 * handled separately. 392 */ 393 static void add_map_entry(u64 start, u64 end, u8 type) 394 { 395 u8 new_type, old_type; 396 u64 tmp; 397 int i; 398 399 for (i = 0; i < cache_map_n && start < end; i++) { 400 if (start >= cache_map[i].end) 401 continue; 402 403 if (start < cache_map[i].start) { 404 /* Region start has no overlap. */ 405 tmp = min(end, cache_map[i].start); 406 i -= add_map_entry_at(start, tmp, type, i); 407 start = tmp; 408 continue; 409 } 410 411 new_type = get_effective_type(type, cache_map[i].type); 412 old_type = cache_map[i].type; 413 414 if (cache_map[i].fixed || new_type == old_type) { 415 /* Cut off start of new entry. */ 416 start = cache_map[i].end; 417 continue; 418 } 419 420 /* Handle only overlapping part of region. */ 421 tmp = min(end, cache_map[i].end); 422 i += clr_map_range_at(start, tmp, i); 423 i -= add_map_entry_at(start, tmp, new_type, i); 424 start = tmp; 425 } 426 427 /* Add rest of region after last map entry (rest might be empty). */ 428 add_map_entry_at(start, end, type, i); 429 } 430 431 /* Add variable MTRRs to cache map. */ 432 static void map_add_var(void) 433 { 434 u64 start, size; 435 unsigned int i; 436 u8 type; 437 438 /* 439 * Add AMD TOP_MEM2 area. Can't be added in mtrr_build_map(), as it 440 * needs to be added again when rebuilding the map due to potentially 441 * having moved as a result of variable MTRRs for memory below 4GB. 442 */ 443 if (mtrr_tom2) { 444 add_map_entry(BIT_ULL(32), mtrr_tom2, MTRR_TYPE_WRBACK); 445 cache_map[cache_map_n - 1].fixed = 1; 446 } 447 448 for (i = 0; i < num_var_ranges; i++) { 449 type = get_var_mtrr_state(i, &start, &size); 450 if (type != MTRR_TYPE_INVALID) 451 add_map_entry(start, start + size, type); 452 } 453 } 454 455 /* 456 * Rebuild map by replacing variable entries. Needs to be called when MTRR 457 * registers are being changed after boot, as such changes could include 458 * removals of registers, which are complicated to handle without rebuild of 459 * the map. 460 */ 461 void generic_rebuild_map(void) 462 { 463 if (mtrr_if != &generic_mtrr_ops) 464 return; 465 466 cache_map_n = cache_map_fixed; 467 468 map_add_var(); 469 } 470 471 static unsigned int __init get_cache_map_size(void) 472 { 473 return cache_map_fixed + 2 * num_var_ranges + (mtrr_tom2 != 0); 474 } 475 476 /* Build the cache_map containing the cache modes per memory range. */ 477 void __init mtrr_build_map(void) 478 { 479 u64 start, end, size; 480 unsigned int i; 481 u8 type; 482 483 /* Add fixed MTRRs, optimize for adjacent entries with same type. */ 484 if (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED) { 485 /* 486 * Start with 64k size fixed entries, preset 1st one (hence the 487 * loop below is starting with index 1). 488 */ 489 start = 0; 490 end = size = 0x10000; 491 type = mtrr_state.fixed_ranges[0]; 492 493 for (i = 1; i < MTRR_NUM_FIXED_RANGES; i++) { 494 /* 8 64k entries, then 16 16k ones, rest 4k. */ 495 if (i == 8 || i == 24) 496 size >>= 2; 497 498 if (mtrr_state.fixed_ranges[i] != type) { 499 add_map_entry(start, end, type); 500 start = end; 501 type = mtrr_state.fixed_ranges[i]; 502 } 503 end += size; 504 } 505 add_map_entry(start, end, type); 506 } 507 508 /* Mark fixed, they take precedence. */ 509 for (i = 0; i < cache_map_n; i++) 510 cache_map[i].fixed = 1; 511 cache_map_fixed = cache_map_n; 512 513 map_add_var(); 514 515 pr_info("MTRR map: %u entries (%u fixed + %u variable; max %u), built from %u variable MTRRs\n", 516 cache_map_n, cache_map_fixed, cache_map_n - cache_map_fixed, 517 get_cache_map_size(), num_var_ranges + (mtrr_tom2 != 0)); 518 } 519 520 /* Copy the cache_map from __initdata memory to dynamically allocated one. */ 521 void __init mtrr_copy_map(void) 522 { 523 unsigned int new_size = get_cache_map_size(); 524 525 if (!mtrr_state.enabled || !new_size) { 526 cache_map = NULL; 527 return; 528 } 529 530 mutex_lock(&mtrr_mutex); 531 532 cache_map = kcalloc(new_size, sizeof(*cache_map), GFP_KERNEL); 533 if (cache_map) { 534 memmove(cache_map, init_cache_map, 535 cache_map_n * sizeof(*cache_map)); 536 cache_map_size = new_size; 537 } else { 538 mtrr_state.enabled = 0; 539 pr_err("MTRRs disabled due to allocation failure for lookup map.\n"); 540 } 541 542 mutex_unlock(&mtrr_mutex); 543 } 544 545 /** 546 * mtrr_overwrite_state - set static MTRR state 547 * 548 * Used to set MTRR state via different means (e.g. with data obtained from 549 * a hypervisor). 550 * Is allowed only for special cases when running virtualized. Must be called 551 * from the x86_init.hyper.init_platform() hook. It can be called only once. 552 * The MTRR state can't be changed afterwards. To ensure that, X86_FEATURE_MTRR 553 * is cleared. 554 */ 555 void mtrr_overwrite_state(struct mtrr_var_range *var, unsigned int num_var, 556 mtrr_type def_type) 557 { 558 unsigned int i; 559 560 /* Only allowed to be called once before mtrr_bp_init(). */ 561 if (WARN_ON_ONCE(mtrr_state_set)) 562 return; 563 564 /* Only allowed when running virtualized. */ 565 if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) 566 return; 567 568 /* 569 * Only allowed for special virtualization cases: 570 * - when running as Hyper-V, SEV-SNP guest using vTOM 571 * - when running as Xen PV guest 572 * - when running as SEV-SNP or TDX guest to avoid unnecessary 573 * VMM communication/Virtualization exceptions (#VC, #VE) 574 */ 575 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP) && 576 !hv_is_isolation_supported() && 577 !cpu_feature_enabled(X86_FEATURE_XENPV) && 578 !cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) 579 return; 580 581 /* Disable MTRR in order to disable MTRR modifications. */ 582 setup_clear_cpu_cap(X86_FEATURE_MTRR); 583 584 if (var) { 585 if (num_var > MTRR_MAX_VAR_RANGES) { 586 pr_warn("Trying to overwrite MTRR state with %u variable entries\n", 587 num_var); 588 num_var = MTRR_MAX_VAR_RANGES; 589 } 590 for (i = 0; i < num_var; i++) 591 mtrr_state.var_ranges[i] = var[i]; 592 num_var_ranges = num_var; 593 } 594 595 mtrr_state.def_type = def_type; 596 mtrr_state.enabled |= MTRR_STATE_MTRR_ENABLED; 597 598 mtrr_state_set = 1; 599 } 600 601 /** 602 * mtrr_type_lookup - look up memory type in MTRR 603 * 604 * Return Values: 605 * MTRR_TYPE_(type) - The effective MTRR type for the region 606 * MTRR_TYPE_INVALID - MTRR is disabled 607 * 608 * Output Argument: 609 * uniform - Set to 1 when an MTRR covers the region uniformly, i.e. the 610 * region is fully covered by a single MTRR entry or the default 611 * type. 612 */ 613 u8 mtrr_type_lookup(u64 start, u64 end, u8 *uniform) 614 { 615 u8 type, prev_type, is_uniform = 1, dummy; 616 int repeat; 617 u64 partial_end; 618 619 /* Make end inclusive instead of exclusive */ 620 end--; 621 622 if (!mtrr_state_set) 623 return MTRR_TYPE_INVALID; 624 625 if (!(mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED)) 626 return MTRR_TYPE_INVALID; 627 628 /* 629 * Look up the fixed ranges first, which take priority over 630 * the variable ranges. 631 */ 632 if ((start < 0x100000) && 633 (mtrr_state.have_fixed) && 634 (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) { 635 is_uniform = 0; 636 type = mtrr_type_lookup_fixed(start, end); 637 goto out; 638 } 639 640 /* 641 * Look up the variable ranges. Look of multiple ranges matching 642 * this address and pick type as per MTRR precedence. 643 */ 644 type = mtrr_type_lookup_variable(start, end, &partial_end, 645 &repeat, &is_uniform); 646 647 /* 648 * Common path is with repeat = 0. 649 * However, we can have cases where [start:end] spans across some 650 * MTRR ranges and/or the default type. Do repeated lookups for 651 * that case here. 652 */ 653 while (repeat) { 654 prev_type = type; 655 start = partial_end; 656 is_uniform = 0; 657 type = mtrr_type_lookup_variable(start, end, &partial_end, 658 &repeat, &dummy); 659 660 if (check_type_overlap(&prev_type, &type)) 661 goto out; 662 } 663 664 if (mtrr_tom2 && (start >= (1ULL<<32)) && (end < mtrr_tom2)) 665 type = MTRR_TYPE_WRBACK; 666 667 out: 668 *uniform = is_uniform; 669 return type; 670 } 671 672 /* Get the MSR pair relating to a var range */ 673 static void 674 get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr) 675 { 676 rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); 677 rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); 678 } 679 680 /* Fill the MSR pair relating to a var range */ 681 void fill_mtrr_var_range(unsigned int index, 682 u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi) 683 { 684 struct mtrr_var_range *vr; 685 686 vr = mtrr_state.var_ranges; 687 688 vr[index].base_lo = base_lo; 689 vr[index].base_hi = base_hi; 690 vr[index].mask_lo = mask_lo; 691 vr[index].mask_hi = mask_hi; 692 } 693 694 static void get_fixed_ranges(mtrr_type *frs) 695 { 696 unsigned int *p = (unsigned int *)frs; 697 int i; 698 699 k8_check_syscfg_dram_mod_en(); 700 701 rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]); 702 703 for (i = 0; i < 2; i++) 704 rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]); 705 for (i = 0; i < 8; i++) 706 rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]); 707 } 708 709 void mtrr_save_fixed_ranges(void *info) 710 { 711 if (boot_cpu_has(X86_FEATURE_MTRR)) 712 get_fixed_ranges(mtrr_state.fixed_ranges); 713 } 714 715 static unsigned __initdata last_fixed_start; 716 static unsigned __initdata last_fixed_end; 717 static mtrr_type __initdata last_fixed_type; 718 719 static void __init print_fixed_last(void) 720 { 721 if (!last_fixed_end) 722 return; 723 724 pr_debug(" %05X-%05X %s\n", last_fixed_start, 725 last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type)); 726 727 last_fixed_end = 0; 728 } 729 730 static void __init update_fixed_last(unsigned base, unsigned end, 731 mtrr_type type) 732 { 733 last_fixed_start = base; 734 last_fixed_end = end; 735 last_fixed_type = type; 736 } 737 738 static void __init 739 print_fixed(unsigned base, unsigned step, const mtrr_type *types) 740 { 741 unsigned i; 742 743 for (i = 0; i < 8; ++i, ++types, base += step) { 744 if (last_fixed_end == 0) { 745 update_fixed_last(base, base + step, *types); 746 continue; 747 } 748 if (last_fixed_end == base && last_fixed_type == *types) { 749 last_fixed_end = base + step; 750 continue; 751 } 752 /* new segments: gap or different type */ 753 print_fixed_last(); 754 update_fixed_last(base, base + step, *types); 755 } 756 } 757 758 static void __init print_mtrr_state(void) 759 { 760 unsigned int i; 761 int high_width; 762 763 pr_debug("MTRR default type: %s\n", 764 mtrr_attrib_to_str(mtrr_state.def_type)); 765 if (mtrr_state.have_fixed) { 766 pr_debug("MTRR fixed ranges %sabled:\n", 767 ((mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED) && 768 (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) ? 769 "en" : "dis"); 770 print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0); 771 for (i = 0; i < 2; ++i) 772 print_fixed(0x80000 + i * 0x20000, 0x04000, 773 mtrr_state.fixed_ranges + (i + 1) * 8); 774 for (i = 0; i < 8; ++i) 775 print_fixed(0xC0000 + i * 0x08000, 0x01000, 776 mtrr_state.fixed_ranges + (i + 3) * 8); 777 778 /* tail */ 779 print_fixed_last(); 780 } 781 pr_debug("MTRR variable ranges %sabled:\n", 782 mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED ? "en" : "dis"); 783 high_width = (boot_cpu_data.x86_phys_bits - (32 - PAGE_SHIFT) + 3) / 4; 784 785 for (i = 0; i < num_var_ranges; ++i) { 786 if (mtrr_state.var_ranges[i].mask_lo & MTRR_PHYSMASK_V) 787 pr_debug(" %u base %0*X%05X000 mask %0*X%05X000 %s\n", 788 i, 789 high_width, 790 mtrr_state.var_ranges[i].base_hi, 791 mtrr_state.var_ranges[i].base_lo >> 12, 792 high_width, 793 mtrr_state.var_ranges[i].mask_hi, 794 mtrr_state.var_ranges[i].mask_lo >> 12, 795 mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 796 MTRR_PHYSBASE_TYPE)); 797 else 798 pr_debug(" %u disabled\n", i); 799 } 800 if (mtrr_tom2) 801 pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20); 802 } 803 804 /* Grab all of the MTRR state for this CPU into *state */ 805 bool __init get_mtrr_state(void) 806 { 807 struct mtrr_var_range *vrs; 808 unsigned lo, dummy; 809 unsigned int i; 810 811 vrs = mtrr_state.var_ranges; 812 813 rdmsr(MSR_MTRRcap, lo, dummy); 814 mtrr_state.have_fixed = lo & MTRR_CAP_FIX; 815 816 for (i = 0; i < num_var_ranges; i++) 817 get_mtrr_var_range(i, &vrs[i]); 818 if (mtrr_state.have_fixed) 819 get_fixed_ranges(mtrr_state.fixed_ranges); 820 821 rdmsr(MSR_MTRRdefType, lo, dummy); 822 mtrr_state.def_type = lo & MTRR_DEF_TYPE_TYPE; 823 mtrr_state.enabled = (lo & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT; 824 825 if (amd_special_default_mtrr()) { 826 unsigned low, high; 827 828 /* TOP_MEM2 */ 829 rdmsr(MSR_K8_TOP_MEM2, low, high); 830 mtrr_tom2 = high; 831 mtrr_tom2 <<= 32; 832 mtrr_tom2 |= low; 833 mtrr_tom2 &= 0xffffff800000ULL; 834 } 835 836 print_mtrr_state(); 837 838 mtrr_state_set = 1; 839 840 return !!(mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED); 841 } 842 843 /* Some BIOS's are messed up and don't set all MTRRs the same! */ 844 void __init mtrr_state_warn(void) 845 { 846 unsigned long mask = smp_changes_mask; 847 848 if (!mask) 849 return; 850 if (mask & MTRR_CHANGE_MASK_FIXED) 851 pr_warn("mtrr: your CPUs had inconsistent fixed MTRR settings\n"); 852 if (mask & MTRR_CHANGE_MASK_VARIABLE) 853 pr_warn("mtrr: your CPUs had inconsistent variable MTRR settings\n"); 854 if (mask & MTRR_CHANGE_MASK_DEFTYPE) 855 pr_warn("mtrr: your CPUs had inconsistent MTRRdefType settings\n"); 856 857 pr_info("mtrr: probably your BIOS does not setup all CPUs.\n"); 858 pr_info("mtrr: corrected configuration.\n"); 859 } 860 861 /* 862 * Doesn't attempt to pass an error out to MTRR users 863 * because it's quite complicated in some cases and probably not 864 * worth it because the best error handling is to ignore it. 865 */ 866 void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b) 867 { 868 if (wrmsr_safe(msr, a, b) < 0) { 869 pr_err("MTRR: CPU %u: Writing MSR %x to %x:%x failed\n", 870 smp_processor_id(), msr, a, b); 871 } 872 } 873 874 /** 875 * set_fixed_range - checks & updates a fixed-range MTRR if it 876 * differs from the value it should have 877 * @msr: MSR address of the MTTR which should be checked and updated 878 * @changed: pointer which indicates whether the MTRR needed to be changed 879 * @msrwords: pointer to the MSR values which the MSR should have 880 */ 881 static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords) 882 { 883 unsigned lo, hi; 884 885 rdmsr(msr, lo, hi); 886 887 if (lo != msrwords[0] || hi != msrwords[1]) { 888 mtrr_wrmsr(msr, msrwords[0], msrwords[1]); 889 *changed = true; 890 } 891 } 892 893 /** 894 * generic_get_free_region - Get a free MTRR. 895 * @base: The starting (base) address of the region. 896 * @size: The size (in bytes) of the region. 897 * @replace_reg: mtrr index to be replaced; set to invalid value if none. 898 * 899 * Returns: The index of the region on success, else negative on error. 900 */ 901 int 902 generic_get_free_region(unsigned long base, unsigned long size, int replace_reg) 903 { 904 unsigned long lbase, lsize; 905 mtrr_type ltype; 906 int i, max; 907 908 max = num_var_ranges; 909 if (replace_reg >= 0 && replace_reg < max) 910 return replace_reg; 911 912 for (i = 0; i < max; ++i) { 913 mtrr_if->get(i, &lbase, &lsize, <ype); 914 if (lsize == 0) 915 return i; 916 } 917 918 return -ENOSPC; 919 } 920 921 static void generic_get_mtrr(unsigned int reg, unsigned long *base, 922 unsigned long *size, mtrr_type *type) 923 { 924 u32 mask_lo, mask_hi, base_lo, base_hi; 925 unsigned int hi; 926 u64 tmp, mask; 927 928 /* 929 * get_mtrr doesn't need to update mtrr_state, also it could be called 930 * from any cpu, so try to print it out directly. 931 */ 932 get_cpu(); 933 934 rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi); 935 936 if (!(mask_lo & MTRR_PHYSMASK_V)) { 937 /* Invalid (i.e. free) range */ 938 *base = 0; 939 *size = 0; 940 *type = 0; 941 goto out_put_cpu; 942 } 943 944 rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi); 945 946 /* Work out the shifted address mask: */ 947 tmp = (u64)mask_hi << 32 | (mask_lo & PAGE_MASK); 948 mask = (u64)phys_hi_rsvd << 32 | tmp; 949 950 /* Expand tmp with high bits to all 1s: */ 951 hi = fls64(tmp); 952 if (hi > 0) { 953 tmp |= ~((1ULL<<(hi - 1)) - 1); 954 955 if (tmp != mask) { 956 pr_warn("mtrr: your BIOS has configured an incorrect mask, fixing it.\n"); 957 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); 958 mask = tmp; 959 } 960 } 961 962 /* 963 * This works correctly if size is a power of two, i.e. a 964 * contiguous range: 965 */ 966 *size = -mask >> PAGE_SHIFT; 967 *base = (u64)base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT; 968 *type = base_lo & MTRR_PHYSBASE_TYPE; 969 970 out_put_cpu: 971 put_cpu(); 972 } 973 974 /** 975 * set_fixed_ranges - checks & updates the fixed-range MTRRs if they 976 * differ from the saved set 977 * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges() 978 */ 979 static int set_fixed_ranges(mtrr_type *frs) 980 { 981 unsigned long long *saved = (unsigned long long *)frs; 982 bool changed = false; 983 int block = -1, range; 984 985 k8_check_syscfg_dram_mod_en(); 986 987 while (fixed_range_blocks[++block].ranges) { 988 for (range = 0; range < fixed_range_blocks[block].ranges; range++) 989 set_fixed_range(fixed_range_blocks[block].base_msr + range, 990 &changed, (unsigned int *)saved++); 991 } 992 993 return changed; 994 } 995 996 /* 997 * Set the MSR pair relating to a var range. 998 * Returns true if changes are made. 999 */ 1000 static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr) 1001 { 1002 unsigned int lo, hi; 1003 bool changed = false; 1004 1005 rdmsr(MTRRphysBase_MSR(index), lo, hi); 1006 if ((vr->base_lo & ~MTRR_PHYSBASE_RSVD) != (lo & ~MTRR_PHYSBASE_RSVD) 1007 || (vr->base_hi & ~phys_hi_rsvd) != (hi & ~phys_hi_rsvd)) { 1008 1009 mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); 1010 changed = true; 1011 } 1012 1013 rdmsr(MTRRphysMask_MSR(index), lo, hi); 1014 1015 if ((vr->mask_lo & ~MTRR_PHYSMASK_RSVD) != (lo & ~MTRR_PHYSMASK_RSVD) 1016 || (vr->mask_hi & ~phys_hi_rsvd) != (hi & ~phys_hi_rsvd)) { 1017 mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); 1018 changed = true; 1019 } 1020 return changed; 1021 } 1022 1023 static u32 deftype_lo, deftype_hi; 1024 1025 /** 1026 * set_mtrr_state - Set the MTRR state for this CPU. 1027 * 1028 * NOTE: The CPU must already be in a safe state for MTRR changes, including 1029 * measures that only a single CPU can be active in set_mtrr_state() in 1030 * order to not be subject to races for usage of deftype_lo. This is 1031 * accomplished by taking cache_disable_lock. 1032 * RETURNS: 0 if no changes made, else a mask indicating what was changed. 1033 */ 1034 static unsigned long set_mtrr_state(void) 1035 { 1036 unsigned long change_mask = 0; 1037 unsigned int i; 1038 1039 for (i = 0; i < num_var_ranges; i++) { 1040 if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i])) 1041 change_mask |= MTRR_CHANGE_MASK_VARIABLE; 1042 } 1043 1044 if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges)) 1045 change_mask |= MTRR_CHANGE_MASK_FIXED; 1046 1047 /* 1048 * Set_mtrr_restore restores the old value of MTRRdefType, 1049 * so to set it we fiddle with the saved value: 1050 */ 1051 if ((deftype_lo & MTRR_DEF_TYPE_TYPE) != mtrr_state.def_type || 1052 ((deftype_lo & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT) != mtrr_state.enabled) { 1053 1054 deftype_lo = (deftype_lo & MTRR_DEF_TYPE_DISABLE) | 1055 mtrr_state.def_type | 1056 (mtrr_state.enabled << MTRR_STATE_SHIFT); 1057 change_mask |= MTRR_CHANGE_MASK_DEFTYPE; 1058 } 1059 1060 return change_mask; 1061 } 1062 1063 void mtrr_disable(void) 1064 { 1065 /* Save MTRR state */ 1066 rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi); 1067 1068 /* Disable MTRRs, and set the default type to uncached */ 1069 mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & MTRR_DEF_TYPE_DISABLE, deftype_hi); 1070 } 1071 1072 void mtrr_enable(void) 1073 { 1074 /* Intel (P6) standard MTRRs */ 1075 mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi); 1076 } 1077 1078 void mtrr_generic_set_state(void) 1079 { 1080 unsigned long mask, count; 1081 1082 /* Actually set the state */ 1083 mask = set_mtrr_state(); 1084 1085 /* Use the atomic bitops to update the global mask */ 1086 for (count = 0; count < sizeof(mask) * 8; ++count) { 1087 if (mask & 0x01) 1088 set_bit(count, &smp_changes_mask); 1089 mask >>= 1; 1090 } 1091 } 1092 1093 /** 1094 * generic_set_mtrr - set variable MTRR register on the local CPU. 1095 * 1096 * @reg: The register to set. 1097 * @base: The base address of the region. 1098 * @size: The size of the region. If this is 0 the region is disabled. 1099 * @type: The type of the region. 1100 * 1101 * Returns nothing. 1102 */ 1103 static void generic_set_mtrr(unsigned int reg, unsigned long base, 1104 unsigned long size, mtrr_type type) 1105 { 1106 unsigned long flags; 1107 struct mtrr_var_range *vr; 1108 1109 vr = &mtrr_state.var_ranges[reg]; 1110 1111 local_irq_save(flags); 1112 cache_disable(); 1113 1114 if (size == 0) { 1115 /* 1116 * The invalid bit is kept in the mask, so we simply 1117 * clear the relevant mask register to disable a range. 1118 */ 1119 mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0); 1120 memset(vr, 0, sizeof(struct mtrr_var_range)); 1121 } else { 1122 vr->base_lo = base << PAGE_SHIFT | type; 1123 vr->base_hi = (base >> (32 - PAGE_SHIFT)) & ~phys_hi_rsvd; 1124 vr->mask_lo = -size << PAGE_SHIFT | MTRR_PHYSMASK_V; 1125 vr->mask_hi = (-size >> (32 - PAGE_SHIFT)) & ~phys_hi_rsvd; 1126 1127 mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi); 1128 mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi); 1129 } 1130 1131 cache_enable(); 1132 local_irq_restore(flags); 1133 } 1134 1135 int generic_validate_add_page(unsigned long base, unsigned long size, 1136 unsigned int type) 1137 { 1138 unsigned long lbase, last; 1139 1140 /* 1141 * For Intel PPro stepping <= 7 1142 * must be 4 MiB aligned and not touch 0x70000000 -> 0x7003FFFF 1143 */ 1144 if (mtrr_if == &generic_mtrr_ops && boot_cpu_data.x86 == 6 && 1145 boot_cpu_data.x86_model == 1 && 1146 boot_cpu_data.x86_stepping <= 7) { 1147 if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) { 1148 pr_warn("mtrr: base(0x%lx000) is not 4 MiB aligned\n", base); 1149 return -EINVAL; 1150 } 1151 if (!(base + size < 0x70000 || base > 0x7003F) && 1152 (type == MTRR_TYPE_WRCOMB 1153 || type == MTRR_TYPE_WRBACK)) { 1154 pr_warn("mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n"); 1155 return -EINVAL; 1156 } 1157 } 1158 1159 /* 1160 * Check upper bits of base and last are equal and lower bits are 0 1161 * for base and 1 for last 1162 */ 1163 last = base + size - 1; 1164 for (lbase = base; !(lbase & 1) && (last & 1); 1165 lbase = lbase >> 1, last = last >> 1) 1166 ; 1167 if (lbase != last) { 1168 pr_warn("mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", base, size); 1169 return -EINVAL; 1170 } 1171 return 0; 1172 } 1173 1174 static int generic_have_wrcomb(void) 1175 { 1176 unsigned long config, dummy; 1177 rdmsr(MSR_MTRRcap, config, dummy); 1178 return config & MTRR_CAP_WC; 1179 } 1180 1181 int positive_have_wrcomb(void) 1182 { 1183 return 1; 1184 } 1185 1186 /* 1187 * Generic structure... 1188 */ 1189 const struct mtrr_ops generic_mtrr_ops = { 1190 .get = generic_get_mtrr, 1191 .get_free_region = generic_get_free_region, 1192 .set = generic_set_mtrr, 1193 .validate_add_page = generic_validate_add_page, 1194 .have_wrcomb = generic_have_wrcomb, 1195 }; 1196