1 #include <linux/bootmem.h> 2 #include <linux/linkage.h> 3 #include <linux/bitops.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/percpu.h> 7 #include <linux/string.h> 8 #include <linux/delay.h> 9 #include <linux/sched.h> 10 #include <linux/init.h> 11 #include <linux/kgdb.h> 12 #include <linux/smp.h> 13 #include <linux/io.h> 14 15 #include <asm/stackprotector.h> 16 #include <asm/perf_event.h> 17 #include <asm/mmu_context.h> 18 #include <asm/hypervisor.h> 19 #include <asm/processor.h> 20 #include <asm/sections.h> 21 #include <linux/topology.h> 22 #include <linux/cpumask.h> 23 #include <asm/pgtable.h> 24 #include <asm/atomic.h> 25 #include <asm/proto.h> 26 #include <asm/setup.h> 27 #include <asm/apic.h> 28 #include <asm/desc.h> 29 #include <asm/i387.h> 30 #include <asm/mtrr.h> 31 #include <linux/numa.h> 32 #include <asm/asm.h> 33 #include <asm/cpu.h> 34 #include <asm/mce.h> 35 #include <asm/msr.h> 36 #include <asm/pat.h> 37 38 #ifdef CONFIG_X86_LOCAL_APIC 39 #include <asm/uv/uv.h> 40 #endif 41 42 #include "cpu.h" 43 44 /* all of these masks are initialized in setup_cpu_local_masks() */ 45 cpumask_var_t cpu_initialized_mask; 46 cpumask_var_t cpu_callout_mask; 47 cpumask_var_t cpu_callin_mask; 48 49 /* representing cpus for which sibling maps can be computed */ 50 cpumask_var_t cpu_sibling_setup_mask; 51 52 /* correctly size the local cpu masks */ 53 void __init setup_cpu_local_masks(void) 54 { 55 alloc_bootmem_cpumask_var(&cpu_initialized_mask); 56 alloc_bootmem_cpumask_var(&cpu_callin_mask); 57 alloc_bootmem_cpumask_var(&cpu_callout_mask); 58 alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask); 59 } 60 61 static void __cpuinit default_init(struct cpuinfo_x86 *c) 62 { 63 #ifdef CONFIG_X86_64 64 cpu_detect_cache_sizes(c); 65 #else 66 /* Not much we can do here... */ 67 /* Check if at least it has cpuid */ 68 if (c->cpuid_level == -1) { 69 /* No cpuid. It must be an ancient CPU */ 70 if (c->x86 == 4) 71 strcpy(c->x86_model_id, "486"); 72 else if (c->x86 == 3) 73 strcpy(c->x86_model_id, "386"); 74 } 75 #endif 76 } 77 78 static const struct cpu_dev __cpuinitconst default_cpu = { 79 .c_init = default_init, 80 .c_vendor = "Unknown", 81 .c_x86_vendor = X86_VENDOR_UNKNOWN, 82 }; 83 84 static const struct cpu_dev *this_cpu __cpuinitdata = &default_cpu; 85 86 DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = { 87 #ifdef CONFIG_X86_64 88 /* 89 * We need valid kernel segments for data and code in long mode too 90 * IRET will check the segment types kkeil 2000/10/28 91 * Also sysret mandates a special GDT layout 92 * 93 * TLS descriptors are currently at a different place compared to i386. 94 * Hopefully nobody expects them at a fixed place (Wine?) 95 */ 96 [GDT_ENTRY_KERNEL32_CS] = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff), 97 [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff), 98 [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc093, 0, 0xfffff), 99 [GDT_ENTRY_DEFAULT_USER32_CS] = GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff), 100 [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff), 101 [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff), 102 #else 103 [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xc09a, 0, 0xfffff), 104 [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff), 105 [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff), 106 [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff), 107 /* 108 * Segments used for calling PnP BIOS have byte granularity. 109 * They code segments and data segments have fixed 64k limits, 110 * the transfer segment sizes are set at run time. 111 */ 112 /* 32-bit code */ 113 [GDT_ENTRY_PNPBIOS_CS32] = GDT_ENTRY_INIT(0x409a, 0, 0xffff), 114 /* 16-bit code */ 115 [GDT_ENTRY_PNPBIOS_CS16] = GDT_ENTRY_INIT(0x009a, 0, 0xffff), 116 /* 16-bit data */ 117 [GDT_ENTRY_PNPBIOS_DS] = GDT_ENTRY_INIT(0x0092, 0, 0xffff), 118 /* 16-bit data */ 119 [GDT_ENTRY_PNPBIOS_TS1] = GDT_ENTRY_INIT(0x0092, 0, 0), 120 /* 16-bit data */ 121 [GDT_ENTRY_PNPBIOS_TS2] = GDT_ENTRY_INIT(0x0092, 0, 0), 122 /* 123 * The APM segments have byte granularity and their bases 124 * are set at run time. All have 64k limits. 125 */ 126 /* 32-bit code */ 127 [GDT_ENTRY_APMBIOS_BASE] = GDT_ENTRY_INIT(0x409a, 0, 0xffff), 128 /* 16-bit code */ 129 [GDT_ENTRY_APMBIOS_BASE+1] = GDT_ENTRY_INIT(0x009a, 0, 0xffff), 130 /* data */ 131 [GDT_ENTRY_APMBIOS_BASE+2] = GDT_ENTRY_INIT(0x4092, 0, 0xffff), 132 133 [GDT_ENTRY_ESPFIX_SS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff), 134 [GDT_ENTRY_PERCPU] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff), 135 GDT_STACK_CANARY_INIT 136 #endif 137 } }; 138 EXPORT_PER_CPU_SYMBOL_GPL(gdt_page); 139 140 static int __init x86_xsave_setup(char *s) 141 { 142 setup_clear_cpu_cap(X86_FEATURE_XSAVE); 143 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT); 144 return 1; 145 } 146 __setup("noxsave", x86_xsave_setup); 147 148 static int __init x86_xsaveopt_setup(char *s) 149 { 150 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT); 151 return 1; 152 } 153 __setup("noxsaveopt", x86_xsaveopt_setup); 154 155 #ifdef CONFIG_X86_32 156 static int cachesize_override __cpuinitdata = -1; 157 static int disable_x86_serial_nr __cpuinitdata = 1; 158 159 static int __init cachesize_setup(char *str) 160 { 161 get_option(&str, &cachesize_override); 162 return 1; 163 } 164 __setup("cachesize=", cachesize_setup); 165 166 static int __init x86_fxsr_setup(char *s) 167 { 168 setup_clear_cpu_cap(X86_FEATURE_FXSR); 169 setup_clear_cpu_cap(X86_FEATURE_XMM); 170 return 1; 171 } 172 __setup("nofxsr", x86_fxsr_setup); 173 174 static int __init x86_sep_setup(char *s) 175 { 176 setup_clear_cpu_cap(X86_FEATURE_SEP); 177 return 1; 178 } 179 __setup("nosep", x86_sep_setup); 180 181 /* Standard macro to see if a specific flag is changeable */ 182 static inline int flag_is_changeable_p(u32 flag) 183 { 184 u32 f1, f2; 185 186 /* 187 * Cyrix and IDT cpus allow disabling of CPUID 188 * so the code below may return different results 189 * when it is executed before and after enabling 190 * the CPUID. Add "volatile" to not allow gcc to 191 * optimize the subsequent calls to this function. 192 */ 193 asm volatile ("pushfl \n\t" 194 "pushfl \n\t" 195 "popl %0 \n\t" 196 "movl %0, %1 \n\t" 197 "xorl %2, %0 \n\t" 198 "pushl %0 \n\t" 199 "popfl \n\t" 200 "pushfl \n\t" 201 "popl %0 \n\t" 202 "popfl \n\t" 203 204 : "=&r" (f1), "=&r" (f2) 205 : "ir" (flag)); 206 207 return ((f1^f2) & flag) != 0; 208 } 209 210 /* Probe for the CPUID instruction */ 211 static int __cpuinit have_cpuid_p(void) 212 { 213 return flag_is_changeable_p(X86_EFLAGS_ID); 214 } 215 216 static void __cpuinit squash_the_stupid_serial_number(struct cpuinfo_x86 *c) 217 { 218 unsigned long lo, hi; 219 220 if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr) 221 return; 222 223 /* Disable processor serial number: */ 224 225 rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi); 226 lo |= 0x200000; 227 wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi); 228 229 printk(KERN_NOTICE "CPU serial number disabled.\n"); 230 clear_cpu_cap(c, X86_FEATURE_PN); 231 232 /* Disabling the serial number may affect the cpuid level */ 233 c->cpuid_level = cpuid_eax(0); 234 } 235 236 static int __init x86_serial_nr_setup(char *s) 237 { 238 disable_x86_serial_nr = 0; 239 return 1; 240 } 241 __setup("serialnumber", x86_serial_nr_setup); 242 #else 243 static inline int flag_is_changeable_p(u32 flag) 244 { 245 return 1; 246 } 247 /* Probe for the CPUID instruction */ 248 static inline int have_cpuid_p(void) 249 { 250 return 1; 251 } 252 static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c) 253 { 254 } 255 #endif 256 257 static int disable_smep __cpuinitdata; 258 static __init int setup_disable_smep(char *arg) 259 { 260 disable_smep = 1; 261 return 1; 262 } 263 __setup("nosmep", setup_disable_smep); 264 265 static __cpuinit void setup_smep(struct cpuinfo_x86 *c) 266 { 267 if (cpu_has(c, X86_FEATURE_SMEP)) { 268 if (unlikely(disable_smep)) { 269 setup_clear_cpu_cap(X86_FEATURE_SMEP); 270 clear_in_cr4(X86_CR4_SMEP); 271 } else 272 set_in_cr4(X86_CR4_SMEP); 273 } 274 } 275 276 /* 277 * Some CPU features depend on higher CPUID levels, which may not always 278 * be available due to CPUID level capping or broken virtualization 279 * software. Add those features to this table to auto-disable them. 280 */ 281 struct cpuid_dependent_feature { 282 u32 feature; 283 u32 level; 284 }; 285 286 static const struct cpuid_dependent_feature __cpuinitconst 287 cpuid_dependent_features[] = { 288 { X86_FEATURE_MWAIT, 0x00000005 }, 289 { X86_FEATURE_DCA, 0x00000009 }, 290 { X86_FEATURE_XSAVE, 0x0000000d }, 291 { 0, 0 } 292 }; 293 294 static void __cpuinit filter_cpuid_features(struct cpuinfo_x86 *c, bool warn) 295 { 296 const struct cpuid_dependent_feature *df; 297 298 for (df = cpuid_dependent_features; df->feature; df++) { 299 300 if (!cpu_has(c, df->feature)) 301 continue; 302 /* 303 * Note: cpuid_level is set to -1 if unavailable, but 304 * extended_extended_level is set to 0 if unavailable 305 * and the legitimate extended levels are all negative 306 * when signed; hence the weird messing around with 307 * signs here... 308 */ 309 if (!((s32)df->level < 0 ? 310 (u32)df->level > (u32)c->extended_cpuid_level : 311 (s32)df->level > (s32)c->cpuid_level)) 312 continue; 313 314 clear_cpu_cap(c, df->feature); 315 if (!warn) 316 continue; 317 318 printk(KERN_WARNING 319 "CPU: CPU feature %s disabled, no CPUID level 0x%x\n", 320 x86_cap_flags[df->feature], df->level); 321 } 322 } 323 324 /* 325 * Naming convention should be: <Name> [(<Codename>)] 326 * This table only is used unless init_<vendor>() below doesn't set it; 327 * in particular, if CPUID levels 0x80000002..4 are supported, this 328 * isn't used 329 */ 330 331 /* Look up CPU names by table lookup. */ 332 static const char *__cpuinit table_lookup_model(struct cpuinfo_x86 *c) 333 { 334 const struct cpu_model_info *info; 335 336 if (c->x86_model >= 16) 337 return NULL; /* Range check */ 338 339 if (!this_cpu) 340 return NULL; 341 342 info = this_cpu->c_models; 343 344 while (info && info->family) { 345 if (info->family == c->x86) 346 return info->model_names[c->x86_model]; 347 info++; 348 } 349 return NULL; /* Not found */ 350 } 351 352 __u32 cpu_caps_cleared[NCAPINTS] __cpuinitdata; 353 __u32 cpu_caps_set[NCAPINTS] __cpuinitdata; 354 355 void load_percpu_segment(int cpu) 356 { 357 #ifdef CONFIG_X86_32 358 loadsegment(fs, __KERNEL_PERCPU); 359 #else 360 loadsegment(gs, 0); 361 wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu)); 362 #endif 363 load_stack_canary_segment(); 364 } 365 366 /* 367 * Current gdt points %fs at the "master" per-cpu area: after this, 368 * it's on the real one. 369 */ 370 void switch_to_new_gdt(int cpu) 371 { 372 struct desc_ptr gdt_descr; 373 374 gdt_descr.address = (long)get_cpu_gdt_table(cpu); 375 gdt_descr.size = GDT_SIZE - 1; 376 load_gdt(&gdt_descr); 377 /* Reload the per-cpu base */ 378 379 load_percpu_segment(cpu); 380 } 381 382 static const struct cpu_dev *__cpuinitdata cpu_devs[X86_VENDOR_NUM] = {}; 383 384 static void __cpuinit get_model_name(struct cpuinfo_x86 *c) 385 { 386 unsigned int *v; 387 char *p, *q; 388 389 if (c->extended_cpuid_level < 0x80000004) 390 return; 391 392 v = (unsigned int *)c->x86_model_id; 393 cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]); 394 cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]); 395 cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]); 396 c->x86_model_id[48] = 0; 397 398 /* 399 * Intel chips right-justify this string for some dumb reason; 400 * undo that brain damage: 401 */ 402 p = q = &c->x86_model_id[0]; 403 while (*p == ' ') 404 p++; 405 if (p != q) { 406 while (*p) 407 *q++ = *p++; 408 while (q <= &c->x86_model_id[48]) 409 *q++ = '\0'; /* Zero-pad the rest */ 410 } 411 } 412 413 void __cpuinit cpu_detect_cache_sizes(struct cpuinfo_x86 *c) 414 { 415 unsigned int n, dummy, ebx, ecx, edx, l2size; 416 417 n = c->extended_cpuid_level; 418 419 if (n >= 0x80000005) { 420 cpuid(0x80000005, &dummy, &ebx, &ecx, &edx); 421 c->x86_cache_size = (ecx>>24) + (edx>>24); 422 #ifdef CONFIG_X86_64 423 /* On K8 L1 TLB is inclusive, so don't count it */ 424 c->x86_tlbsize = 0; 425 #endif 426 } 427 428 if (n < 0x80000006) /* Some chips just has a large L1. */ 429 return; 430 431 cpuid(0x80000006, &dummy, &ebx, &ecx, &edx); 432 l2size = ecx >> 16; 433 434 #ifdef CONFIG_X86_64 435 c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff); 436 #else 437 /* do processor-specific cache resizing */ 438 if (this_cpu->c_size_cache) 439 l2size = this_cpu->c_size_cache(c, l2size); 440 441 /* Allow user to override all this if necessary. */ 442 if (cachesize_override != -1) 443 l2size = cachesize_override; 444 445 if (l2size == 0) 446 return; /* Again, no L2 cache is possible */ 447 #endif 448 449 c->x86_cache_size = l2size; 450 } 451 452 void __cpuinit detect_ht(struct cpuinfo_x86 *c) 453 { 454 #ifdef CONFIG_X86_HT 455 u32 eax, ebx, ecx, edx; 456 int index_msb, core_bits; 457 static bool printed; 458 459 if (!cpu_has(c, X86_FEATURE_HT)) 460 return; 461 462 if (cpu_has(c, X86_FEATURE_CMP_LEGACY)) 463 goto out; 464 465 if (cpu_has(c, X86_FEATURE_XTOPOLOGY)) 466 return; 467 468 cpuid(1, &eax, &ebx, &ecx, &edx); 469 470 smp_num_siblings = (ebx & 0xff0000) >> 16; 471 472 if (smp_num_siblings == 1) { 473 printk_once(KERN_INFO "CPU0: Hyper-Threading is disabled\n"); 474 goto out; 475 } 476 477 if (smp_num_siblings <= 1) 478 goto out; 479 480 index_msb = get_count_order(smp_num_siblings); 481 c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb); 482 483 smp_num_siblings = smp_num_siblings / c->x86_max_cores; 484 485 index_msb = get_count_order(smp_num_siblings); 486 487 core_bits = get_count_order(c->x86_max_cores); 488 489 c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) & 490 ((1 << core_bits) - 1); 491 492 out: 493 if (!printed && (c->x86_max_cores * smp_num_siblings) > 1) { 494 printk(KERN_INFO "CPU: Physical Processor ID: %d\n", 495 c->phys_proc_id); 496 printk(KERN_INFO "CPU: Processor Core ID: %d\n", 497 c->cpu_core_id); 498 printed = 1; 499 } 500 #endif 501 } 502 503 static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c) 504 { 505 char *v = c->x86_vendor_id; 506 int i; 507 508 for (i = 0; i < X86_VENDOR_NUM; i++) { 509 if (!cpu_devs[i]) 510 break; 511 512 if (!strcmp(v, cpu_devs[i]->c_ident[0]) || 513 (cpu_devs[i]->c_ident[1] && 514 !strcmp(v, cpu_devs[i]->c_ident[1]))) { 515 516 this_cpu = cpu_devs[i]; 517 c->x86_vendor = this_cpu->c_x86_vendor; 518 return; 519 } 520 } 521 522 printk_once(KERN_ERR 523 "CPU: vendor_id '%s' unknown, using generic init.\n" \ 524 "CPU: Your system may be unstable.\n", v); 525 526 c->x86_vendor = X86_VENDOR_UNKNOWN; 527 this_cpu = &default_cpu; 528 } 529 530 void __cpuinit cpu_detect(struct cpuinfo_x86 *c) 531 { 532 /* Get vendor name */ 533 cpuid(0x00000000, (unsigned int *)&c->cpuid_level, 534 (unsigned int *)&c->x86_vendor_id[0], 535 (unsigned int *)&c->x86_vendor_id[8], 536 (unsigned int *)&c->x86_vendor_id[4]); 537 538 c->x86 = 4; 539 /* Intel-defined flags: level 0x00000001 */ 540 if (c->cpuid_level >= 0x00000001) { 541 u32 junk, tfms, cap0, misc; 542 543 cpuid(0x00000001, &tfms, &misc, &junk, &cap0); 544 c->x86 = (tfms >> 8) & 0xf; 545 c->x86_model = (tfms >> 4) & 0xf; 546 c->x86_mask = tfms & 0xf; 547 548 if (c->x86 == 0xf) 549 c->x86 += (tfms >> 20) & 0xff; 550 if (c->x86 >= 0x6) 551 c->x86_model += ((tfms >> 16) & 0xf) << 4; 552 553 if (cap0 & (1<<19)) { 554 c->x86_clflush_size = ((misc >> 8) & 0xff) * 8; 555 c->x86_cache_alignment = c->x86_clflush_size; 556 } 557 } 558 } 559 560 void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c) 561 { 562 u32 tfms, xlvl; 563 u32 ebx; 564 565 /* Intel-defined flags: level 0x00000001 */ 566 if (c->cpuid_level >= 0x00000001) { 567 u32 capability, excap; 568 569 cpuid(0x00000001, &tfms, &ebx, &excap, &capability); 570 c->x86_capability[0] = capability; 571 c->x86_capability[4] = excap; 572 } 573 574 /* Additional Intel-defined flags: level 0x00000007 */ 575 if (c->cpuid_level >= 0x00000007) { 576 u32 eax, ebx, ecx, edx; 577 578 cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx); 579 580 c->x86_capability[9] = ebx; 581 } 582 583 /* AMD-defined flags: level 0x80000001 */ 584 xlvl = cpuid_eax(0x80000000); 585 c->extended_cpuid_level = xlvl; 586 587 if ((xlvl & 0xffff0000) == 0x80000000) { 588 if (xlvl >= 0x80000001) { 589 c->x86_capability[1] = cpuid_edx(0x80000001); 590 c->x86_capability[6] = cpuid_ecx(0x80000001); 591 } 592 } 593 594 if (c->extended_cpuid_level >= 0x80000008) { 595 u32 eax = cpuid_eax(0x80000008); 596 597 c->x86_virt_bits = (eax >> 8) & 0xff; 598 c->x86_phys_bits = eax & 0xff; 599 } 600 #ifdef CONFIG_X86_32 601 else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36)) 602 c->x86_phys_bits = 36; 603 #endif 604 605 if (c->extended_cpuid_level >= 0x80000007) 606 c->x86_power = cpuid_edx(0x80000007); 607 608 init_scattered_cpuid_features(c); 609 } 610 611 static void __cpuinit identify_cpu_without_cpuid(struct cpuinfo_x86 *c) 612 { 613 #ifdef CONFIG_X86_32 614 int i; 615 616 /* 617 * First of all, decide if this is a 486 or higher 618 * It's a 486 if we can modify the AC flag 619 */ 620 if (flag_is_changeable_p(X86_EFLAGS_AC)) 621 c->x86 = 4; 622 else 623 c->x86 = 3; 624 625 for (i = 0; i < X86_VENDOR_NUM; i++) 626 if (cpu_devs[i] && cpu_devs[i]->c_identify) { 627 c->x86_vendor_id[0] = 0; 628 cpu_devs[i]->c_identify(c); 629 if (c->x86_vendor_id[0]) { 630 get_cpu_vendor(c); 631 break; 632 } 633 } 634 #endif 635 } 636 637 /* 638 * Do minimum CPU detection early. 639 * Fields really needed: vendor, cpuid_level, family, model, mask, 640 * cache alignment. 641 * The others are not touched to avoid unwanted side effects. 642 * 643 * WARNING: this function is only called on the BP. Don't add code here 644 * that is supposed to run on all CPUs. 645 */ 646 static void __init early_identify_cpu(struct cpuinfo_x86 *c) 647 { 648 #ifdef CONFIG_X86_64 649 c->x86_clflush_size = 64; 650 c->x86_phys_bits = 36; 651 c->x86_virt_bits = 48; 652 #else 653 c->x86_clflush_size = 32; 654 c->x86_phys_bits = 32; 655 c->x86_virt_bits = 32; 656 #endif 657 c->x86_cache_alignment = c->x86_clflush_size; 658 659 memset(&c->x86_capability, 0, sizeof c->x86_capability); 660 c->extended_cpuid_level = 0; 661 662 if (!have_cpuid_p()) 663 identify_cpu_without_cpuid(c); 664 665 /* cyrix could have cpuid enabled via c_identify()*/ 666 if (!have_cpuid_p()) 667 return; 668 669 cpu_detect(c); 670 671 get_cpu_vendor(c); 672 673 get_cpu_cap(c); 674 675 if (this_cpu->c_early_init) 676 this_cpu->c_early_init(c); 677 678 #ifdef CONFIG_SMP 679 c->cpu_index = 0; 680 #endif 681 filter_cpuid_features(c, false); 682 683 setup_smep(c); 684 } 685 686 void __init early_cpu_init(void) 687 { 688 const struct cpu_dev *const *cdev; 689 int count = 0; 690 691 #ifdef CONFIG_PROCESSOR_SELECT 692 printk(KERN_INFO "KERNEL supported cpus:\n"); 693 #endif 694 695 for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) { 696 const struct cpu_dev *cpudev = *cdev; 697 698 if (count >= X86_VENDOR_NUM) 699 break; 700 cpu_devs[count] = cpudev; 701 count++; 702 703 #ifdef CONFIG_PROCESSOR_SELECT 704 { 705 unsigned int j; 706 707 for (j = 0; j < 2; j++) { 708 if (!cpudev->c_ident[j]) 709 continue; 710 printk(KERN_INFO " %s %s\n", cpudev->c_vendor, 711 cpudev->c_ident[j]); 712 } 713 } 714 #endif 715 } 716 early_identify_cpu(&boot_cpu_data); 717 } 718 719 /* 720 * The NOPL instruction is supposed to exist on all CPUs of family >= 6; 721 * unfortunately, that's not true in practice because of early VIA 722 * chips and (more importantly) broken virtualizers that are not easy 723 * to detect. In the latter case it doesn't even *fail* reliably, so 724 * probing for it doesn't even work. Disable it completely on 32-bit 725 * unless we can find a reliable way to detect all the broken cases. 726 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has(). 727 */ 728 static void __cpuinit detect_nopl(struct cpuinfo_x86 *c) 729 { 730 #ifdef CONFIG_X86_32 731 clear_cpu_cap(c, X86_FEATURE_NOPL); 732 #else 733 set_cpu_cap(c, X86_FEATURE_NOPL); 734 #endif 735 } 736 737 static void __cpuinit generic_identify(struct cpuinfo_x86 *c) 738 { 739 c->extended_cpuid_level = 0; 740 741 if (!have_cpuid_p()) 742 identify_cpu_without_cpuid(c); 743 744 /* cyrix could have cpuid enabled via c_identify()*/ 745 if (!have_cpuid_p()) 746 return; 747 748 cpu_detect(c); 749 750 get_cpu_vendor(c); 751 752 get_cpu_cap(c); 753 754 if (c->cpuid_level >= 0x00000001) { 755 c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF; 756 #ifdef CONFIG_X86_32 757 # ifdef CONFIG_X86_HT 758 c->apicid = apic->phys_pkg_id(c->initial_apicid, 0); 759 # else 760 c->apicid = c->initial_apicid; 761 # endif 762 #endif 763 764 #ifdef CONFIG_X86_HT 765 c->phys_proc_id = c->initial_apicid; 766 #endif 767 } 768 769 setup_smep(c); 770 771 get_model_name(c); /* Default name */ 772 773 detect_nopl(c); 774 } 775 776 /* 777 * This does the hard work of actually picking apart the CPU stuff... 778 */ 779 static void __cpuinit identify_cpu(struct cpuinfo_x86 *c) 780 { 781 int i; 782 783 c->loops_per_jiffy = loops_per_jiffy; 784 c->x86_cache_size = -1; 785 c->x86_vendor = X86_VENDOR_UNKNOWN; 786 c->x86_model = c->x86_mask = 0; /* So far unknown... */ 787 c->x86_vendor_id[0] = '\0'; /* Unset */ 788 c->x86_model_id[0] = '\0'; /* Unset */ 789 c->x86_max_cores = 1; 790 c->x86_coreid_bits = 0; 791 #ifdef CONFIG_X86_64 792 c->x86_clflush_size = 64; 793 c->x86_phys_bits = 36; 794 c->x86_virt_bits = 48; 795 #else 796 c->cpuid_level = -1; /* CPUID not detected */ 797 c->x86_clflush_size = 32; 798 c->x86_phys_bits = 32; 799 c->x86_virt_bits = 32; 800 #endif 801 c->x86_cache_alignment = c->x86_clflush_size; 802 memset(&c->x86_capability, 0, sizeof c->x86_capability); 803 804 generic_identify(c); 805 806 if (this_cpu->c_identify) 807 this_cpu->c_identify(c); 808 809 /* Clear/Set all flags overriden by options, after probe */ 810 for (i = 0; i < NCAPINTS; i++) { 811 c->x86_capability[i] &= ~cpu_caps_cleared[i]; 812 c->x86_capability[i] |= cpu_caps_set[i]; 813 } 814 815 #ifdef CONFIG_X86_64 816 c->apicid = apic->phys_pkg_id(c->initial_apicid, 0); 817 #endif 818 819 /* 820 * Vendor-specific initialization. In this section we 821 * canonicalize the feature flags, meaning if there are 822 * features a certain CPU supports which CPUID doesn't 823 * tell us, CPUID claiming incorrect flags, or other bugs, 824 * we handle them here. 825 * 826 * At the end of this section, c->x86_capability better 827 * indicate the features this CPU genuinely supports! 828 */ 829 if (this_cpu->c_init) 830 this_cpu->c_init(c); 831 832 /* Disable the PN if appropriate */ 833 squash_the_stupid_serial_number(c); 834 835 /* 836 * The vendor-specific functions might have changed features. 837 * Now we do "generic changes." 838 */ 839 840 /* Filter out anything that depends on CPUID levels we don't have */ 841 filter_cpuid_features(c, true); 842 843 /* If the model name is still unset, do table lookup. */ 844 if (!c->x86_model_id[0]) { 845 const char *p; 846 p = table_lookup_model(c); 847 if (p) 848 strcpy(c->x86_model_id, p); 849 else 850 /* Last resort... */ 851 sprintf(c->x86_model_id, "%02x/%02x", 852 c->x86, c->x86_model); 853 } 854 855 #ifdef CONFIG_X86_64 856 detect_ht(c); 857 #endif 858 859 init_hypervisor(c); 860 861 /* 862 * Clear/Set all flags overriden by options, need do it 863 * before following smp all cpus cap AND. 864 */ 865 for (i = 0; i < NCAPINTS; i++) { 866 c->x86_capability[i] &= ~cpu_caps_cleared[i]; 867 c->x86_capability[i] |= cpu_caps_set[i]; 868 } 869 870 /* 871 * On SMP, boot_cpu_data holds the common feature set between 872 * all CPUs; so make sure that we indicate which features are 873 * common between the CPUs. The first time this routine gets 874 * executed, c == &boot_cpu_data. 875 */ 876 if (c != &boot_cpu_data) { 877 /* AND the already accumulated flags with these */ 878 for (i = 0; i < NCAPINTS; i++) 879 boot_cpu_data.x86_capability[i] &= c->x86_capability[i]; 880 } 881 882 /* Init Machine Check Exception if available. */ 883 mcheck_cpu_init(c); 884 885 select_idle_routine(c); 886 887 #ifdef CONFIG_NUMA 888 numa_add_cpu(smp_processor_id()); 889 #endif 890 } 891 892 #ifdef CONFIG_X86_64 893 static void vgetcpu_set_mode(void) 894 { 895 if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP)) 896 vgetcpu_mode = VGETCPU_RDTSCP; 897 else 898 vgetcpu_mode = VGETCPU_LSL; 899 } 900 #endif 901 902 void __init identify_boot_cpu(void) 903 { 904 identify_cpu(&boot_cpu_data); 905 init_amd_e400_c1e_mask(); 906 #ifdef CONFIG_X86_32 907 sysenter_setup(); 908 enable_sep_cpu(); 909 #else 910 vgetcpu_set_mode(); 911 #endif 912 } 913 914 void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c) 915 { 916 BUG_ON(c == &boot_cpu_data); 917 identify_cpu(c); 918 #ifdef CONFIG_X86_32 919 enable_sep_cpu(); 920 #endif 921 mtrr_ap_init(); 922 } 923 924 struct msr_range { 925 unsigned min; 926 unsigned max; 927 }; 928 929 static const struct msr_range msr_range_array[] __cpuinitconst = { 930 { 0x00000000, 0x00000418}, 931 { 0xc0000000, 0xc000040b}, 932 { 0xc0010000, 0xc0010142}, 933 { 0xc0011000, 0xc001103b}, 934 }; 935 936 static void __cpuinit print_cpu_msr(void) 937 { 938 unsigned index_min, index_max; 939 unsigned index; 940 u64 val; 941 int i; 942 943 for (i = 0; i < ARRAY_SIZE(msr_range_array); i++) { 944 index_min = msr_range_array[i].min; 945 index_max = msr_range_array[i].max; 946 947 for (index = index_min; index < index_max; index++) { 948 if (rdmsrl_amd_safe(index, &val)) 949 continue; 950 printk(KERN_INFO " MSR%08x: %016llx\n", index, val); 951 } 952 } 953 } 954 955 static int show_msr __cpuinitdata; 956 957 static __init int setup_show_msr(char *arg) 958 { 959 int num; 960 961 get_option(&arg, &num); 962 963 if (num > 0) 964 show_msr = num; 965 return 1; 966 } 967 __setup("show_msr=", setup_show_msr); 968 969 static __init int setup_noclflush(char *arg) 970 { 971 setup_clear_cpu_cap(X86_FEATURE_CLFLSH); 972 return 1; 973 } 974 __setup("noclflush", setup_noclflush); 975 976 void __cpuinit print_cpu_info(struct cpuinfo_x86 *c) 977 { 978 const char *vendor = NULL; 979 980 if (c->x86_vendor < X86_VENDOR_NUM) { 981 vendor = this_cpu->c_vendor; 982 } else { 983 if (c->cpuid_level >= 0) 984 vendor = c->x86_vendor_id; 985 } 986 987 if (vendor && !strstr(c->x86_model_id, vendor)) 988 printk(KERN_CONT "%s ", vendor); 989 990 if (c->x86_model_id[0]) 991 printk(KERN_CONT "%s", c->x86_model_id); 992 else 993 printk(KERN_CONT "%d86", c->x86); 994 995 if (c->x86_mask || c->cpuid_level >= 0) 996 printk(KERN_CONT " stepping %02x\n", c->x86_mask); 997 else 998 printk(KERN_CONT "\n"); 999 1000 #ifdef CONFIG_SMP 1001 if (c->cpu_index < show_msr) 1002 print_cpu_msr(); 1003 #else 1004 if (show_msr) 1005 print_cpu_msr(); 1006 #endif 1007 } 1008 1009 static __init int setup_disablecpuid(char *arg) 1010 { 1011 int bit; 1012 1013 if (get_option(&arg, &bit) && bit < NCAPINTS*32) 1014 setup_clear_cpu_cap(bit); 1015 else 1016 return 0; 1017 1018 return 1; 1019 } 1020 __setup("clearcpuid=", setup_disablecpuid); 1021 1022 #ifdef CONFIG_X86_64 1023 struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table }; 1024 1025 DEFINE_PER_CPU_FIRST(union irq_stack_union, 1026 irq_stack_union) __aligned(PAGE_SIZE); 1027 1028 /* 1029 * The following four percpu variables are hot. Align current_task to 1030 * cacheline size such that all four fall in the same cacheline. 1031 */ 1032 DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned = 1033 &init_task; 1034 EXPORT_PER_CPU_SYMBOL(current_task); 1035 1036 DEFINE_PER_CPU(unsigned long, kernel_stack) = 1037 (unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE; 1038 EXPORT_PER_CPU_SYMBOL(kernel_stack); 1039 1040 DEFINE_PER_CPU(char *, irq_stack_ptr) = 1041 init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64; 1042 1043 DEFINE_PER_CPU(unsigned int, irq_count) = -1; 1044 1045 /* 1046 * Special IST stacks which the CPU switches to when it calls 1047 * an IST-marked descriptor entry. Up to 7 stacks (hardware 1048 * limit), all of them are 4K, except the debug stack which 1049 * is 8K. 1050 */ 1051 static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = { 1052 [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STKSZ, 1053 [DEBUG_STACK - 1] = DEBUG_STKSZ 1054 }; 1055 1056 static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks 1057 [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); 1058 1059 /* May not be marked __init: used by software suspend */ 1060 void syscall_init(void) 1061 { 1062 /* 1063 * LSTAR and STAR live in a bit strange symbiosis. 1064 * They both write to the same internal register. STAR allows to 1065 * set CS/DS but only a 32bit target. LSTAR sets the 64bit rip. 1066 */ 1067 wrmsrl(MSR_STAR, ((u64)__USER32_CS)<<48 | ((u64)__KERNEL_CS)<<32); 1068 wrmsrl(MSR_LSTAR, system_call); 1069 wrmsrl(MSR_CSTAR, ignore_sysret); 1070 1071 #ifdef CONFIG_IA32_EMULATION 1072 syscall32_cpu_init(); 1073 #endif 1074 1075 /* Flags to clear on syscall */ 1076 wrmsrl(MSR_SYSCALL_MASK, 1077 X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|X86_EFLAGS_IOPL); 1078 } 1079 1080 unsigned long kernel_eflags; 1081 1082 /* 1083 * Copies of the original ist values from the tss are only accessed during 1084 * debugging, no special alignment required. 1085 */ 1086 DEFINE_PER_CPU(struct orig_ist, orig_ist); 1087 1088 #else /* CONFIG_X86_64 */ 1089 1090 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task; 1091 EXPORT_PER_CPU_SYMBOL(current_task); 1092 1093 #ifdef CONFIG_CC_STACKPROTECTOR 1094 DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary); 1095 #endif 1096 1097 /* Make sure %fs and %gs are initialized properly in idle threads */ 1098 struct pt_regs * __cpuinit idle_regs(struct pt_regs *regs) 1099 { 1100 memset(regs, 0, sizeof(struct pt_regs)); 1101 regs->fs = __KERNEL_PERCPU; 1102 regs->gs = __KERNEL_STACK_CANARY; 1103 1104 return regs; 1105 } 1106 #endif /* CONFIG_X86_64 */ 1107 1108 /* 1109 * Clear all 6 debug registers: 1110 */ 1111 static void clear_all_debug_regs(void) 1112 { 1113 int i; 1114 1115 for (i = 0; i < 8; i++) { 1116 /* Ignore db4, db5 */ 1117 if ((i == 4) || (i == 5)) 1118 continue; 1119 1120 set_debugreg(0, i); 1121 } 1122 } 1123 1124 #ifdef CONFIG_KGDB 1125 /* 1126 * Restore debug regs if using kgdbwait and you have a kernel debugger 1127 * connection established. 1128 */ 1129 static void dbg_restore_debug_regs(void) 1130 { 1131 if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break)) 1132 arch_kgdb_ops.correct_hw_break(); 1133 } 1134 #else /* ! CONFIG_KGDB */ 1135 #define dbg_restore_debug_regs() 1136 #endif /* ! CONFIG_KGDB */ 1137 1138 /* 1139 * cpu_init() initializes state that is per-CPU. Some data is already 1140 * initialized (naturally) in the bootstrap process, such as the GDT 1141 * and IDT. We reload them nevertheless, this function acts as a 1142 * 'CPU state barrier', nothing should get across. 1143 * A lot of state is already set up in PDA init for 64 bit 1144 */ 1145 #ifdef CONFIG_X86_64 1146 1147 void __cpuinit cpu_init(void) 1148 { 1149 struct orig_ist *oist; 1150 struct task_struct *me; 1151 struct tss_struct *t; 1152 unsigned long v; 1153 int cpu; 1154 int i; 1155 1156 cpu = stack_smp_processor_id(); 1157 t = &per_cpu(init_tss, cpu); 1158 oist = &per_cpu(orig_ist, cpu); 1159 1160 #ifdef CONFIG_NUMA 1161 if (cpu != 0 && percpu_read(numa_node) == 0 && 1162 early_cpu_to_node(cpu) != NUMA_NO_NODE) 1163 set_numa_node(early_cpu_to_node(cpu)); 1164 #endif 1165 1166 me = current; 1167 1168 if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask)) 1169 panic("CPU#%d already initialized!\n", cpu); 1170 1171 pr_debug("Initializing CPU#%d\n", cpu); 1172 1173 clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE); 1174 1175 /* 1176 * Initialize the per-CPU GDT with the boot GDT, 1177 * and set up the GDT descriptor: 1178 */ 1179 1180 switch_to_new_gdt(cpu); 1181 loadsegment(fs, 0); 1182 1183 load_idt((const struct desc_ptr *)&idt_descr); 1184 1185 memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8); 1186 syscall_init(); 1187 1188 wrmsrl(MSR_FS_BASE, 0); 1189 wrmsrl(MSR_KERNEL_GS_BASE, 0); 1190 barrier(); 1191 1192 x86_configure_nx(); 1193 if (cpu != 0) 1194 enable_x2apic(); 1195 1196 /* 1197 * set up and load the per-CPU TSS 1198 */ 1199 if (!oist->ist[0]) { 1200 char *estacks = per_cpu(exception_stacks, cpu); 1201 1202 for (v = 0; v < N_EXCEPTION_STACKS; v++) { 1203 estacks += exception_stack_sizes[v]; 1204 oist->ist[v] = t->x86_tss.ist[v] = 1205 (unsigned long)estacks; 1206 } 1207 } 1208 1209 t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap); 1210 1211 /* 1212 * <= is required because the CPU will access up to 1213 * 8 bits beyond the end of the IO permission bitmap. 1214 */ 1215 for (i = 0; i <= IO_BITMAP_LONGS; i++) 1216 t->io_bitmap[i] = ~0UL; 1217 1218 atomic_inc(&init_mm.mm_count); 1219 me->active_mm = &init_mm; 1220 BUG_ON(me->mm); 1221 enter_lazy_tlb(&init_mm, me); 1222 1223 load_sp0(t, ¤t->thread); 1224 set_tss_desc(cpu, t); 1225 load_TR_desc(); 1226 load_LDT(&init_mm.context); 1227 1228 clear_all_debug_regs(); 1229 dbg_restore_debug_regs(); 1230 1231 fpu_init(); 1232 xsave_init(); 1233 1234 raw_local_save_flags(kernel_eflags); 1235 1236 if (is_uv_system()) 1237 uv_cpu_init(); 1238 } 1239 1240 #else 1241 1242 void __cpuinit cpu_init(void) 1243 { 1244 int cpu = smp_processor_id(); 1245 struct task_struct *curr = current; 1246 struct tss_struct *t = &per_cpu(init_tss, cpu); 1247 struct thread_struct *thread = &curr->thread; 1248 1249 if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask)) { 1250 printk(KERN_WARNING "CPU#%d already initialized!\n", cpu); 1251 for (;;) 1252 local_irq_enable(); 1253 } 1254 1255 printk(KERN_INFO "Initializing CPU#%d\n", cpu); 1256 1257 if (cpu_has_vme || cpu_has_tsc || cpu_has_de) 1258 clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE); 1259 1260 load_idt(&idt_descr); 1261 switch_to_new_gdt(cpu); 1262 1263 /* 1264 * Set up and load the per-CPU TSS and LDT 1265 */ 1266 atomic_inc(&init_mm.mm_count); 1267 curr->active_mm = &init_mm; 1268 BUG_ON(curr->mm); 1269 enter_lazy_tlb(&init_mm, curr); 1270 1271 load_sp0(t, thread); 1272 set_tss_desc(cpu, t); 1273 load_TR_desc(); 1274 load_LDT(&init_mm.context); 1275 1276 t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap); 1277 1278 #ifdef CONFIG_DOUBLEFAULT 1279 /* Set up doublefault TSS pointer in the GDT */ 1280 __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss); 1281 #endif 1282 1283 clear_all_debug_regs(); 1284 dbg_restore_debug_regs(); 1285 1286 fpu_init(); 1287 xsave_init(); 1288 } 1289 #endif 1290