1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Local APIC handling, local APIC timers 4 * 5 * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com> 6 * 7 * Fixes 8 * Maciej W. Rozycki : Bits for genuine 82489DX APICs; 9 * thanks to Eric Gilmore 10 * and Rolf G. Tews 11 * for testing these extensively. 12 * Maciej W. Rozycki : Various updates and fixes. 13 * Mikael Pettersson : Power Management for UP-APIC. 14 * Pavel Machek and 15 * Mikael Pettersson : PM converted to driver model. 16 */ 17 18 #include <linux/perf_event.h> 19 #include <linux/kernel_stat.h> 20 #include <linux/mc146818rtc.h> 21 #include <linux/acpi_pmtmr.h> 22 #include <linux/clockchips.h> 23 #include <linux/interrupt.h> 24 #include <linux/memblock.h> 25 #include <linux/ftrace.h> 26 #include <linux/ioport.h> 27 #include <linux/export.h> 28 #include <linux/syscore_ops.h> 29 #include <linux/delay.h> 30 #include <linux/timex.h> 31 #include <linux/i8253.h> 32 #include <linux/dmar.h> 33 #include <linux/init.h> 34 #include <linux/cpu.h> 35 #include <linux/dmi.h> 36 #include <linux/smp.h> 37 #include <linux/mm.h> 38 39 #include <asm/trace/irq_vectors.h> 40 #include <asm/irq_remapping.h> 41 #include <asm/pc-conf-reg.h> 42 #include <asm/perf_event.h> 43 #include <asm/x86_init.h> 44 #include <linux/atomic.h> 45 #include <asm/barrier.h> 46 #include <asm/mpspec.h> 47 #include <asm/i8259.h> 48 #include <asm/proto.h> 49 #include <asm/traps.h> 50 #include <asm/apic.h> 51 #include <asm/acpi.h> 52 #include <asm/io_apic.h> 53 #include <asm/desc.h> 54 #include <asm/hpet.h> 55 #include <asm/mtrr.h> 56 #include <asm/time.h> 57 #include <asm/smp.h> 58 #include <asm/mce.h> 59 #include <asm/tsc.h> 60 #include <asm/hypervisor.h> 61 #include <asm/cpu_device_id.h> 62 #include <asm/intel-family.h> 63 #include <asm/irq_regs.h> 64 #include <asm/cpu.h> 65 66 #include "local.h" 67 68 unsigned int num_processors; 69 70 unsigned disabled_cpus; 71 72 /* Processor that is doing the boot up */ 73 unsigned int boot_cpu_physical_apicid __ro_after_init = -1U; 74 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid); 75 76 u8 boot_cpu_apic_version __ro_after_init; 77 78 /* 79 * Bitmask of physically existing CPUs: 80 */ 81 physid_mask_t phys_cpu_present_map; 82 83 /* 84 * Processor to be disabled specified by kernel parameter 85 * disable_cpu_apicid=<int>, mostly used for the kdump 2nd kernel to 86 * avoid undefined behaviour caused by sending INIT from AP to BSP. 87 */ 88 static unsigned int disabled_cpu_apicid __ro_after_init = BAD_APICID; 89 90 /* 91 * This variable controls which CPUs receive external NMIs. By default, 92 * external NMIs are delivered only to the BSP. 93 */ 94 static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP; 95 96 /* 97 * Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID 98 */ 99 static bool virt_ext_dest_id __ro_after_init; 100 101 /* For parallel bootup. */ 102 unsigned long apic_mmio_base __ro_after_init; 103 104 static inline bool apic_accessible(void) 105 { 106 return x2apic_mode || apic_mmio_base; 107 } 108 109 /* 110 * Map cpu index to physical APIC ID 111 */ 112 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID); 113 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid, U32_MAX); 114 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid); 115 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid); 116 117 #ifdef CONFIG_X86_32 118 /* Local APIC was disabled by the BIOS and enabled by the kernel */ 119 static int enabled_via_apicbase __ro_after_init; 120 121 /* 122 * Handle interrupt mode configuration register (IMCR). 123 * This register controls whether the interrupt signals 124 * that reach the BSP come from the master PIC or from the 125 * local APIC. Before entering Symmetric I/O Mode, either 126 * the BIOS or the operating system must switch out of 127 * PIC Mode by changing the IMCR. 128 */ 129 static inline void imcr_pic_to_apic(void) 130 { 131 /* NMI and 8259 INTR go through APIC */ 132 pc_conf_set(PC_CONF_MPS_IMCR, 0x01); 133 } 134 135 static inline void imcr_apic_to_pic(void) 136 { 137 /* NMI and 8259 INTR go directly to BSP */ 138 pc_conf_set(PC_CONF_MPS_IMCR, 0x00); 139 } 140 #endif 141 142 /* 143 * Knob to control our willingness to enable the local APIC. 144 * 145 * +1=force-enable 146 */ 147 static int force_enable_local_apic __initdata; 148 149 /* 150 * APIC command line parameters 151 */ 152 static int __init parse_lapic(char *arg) 153 { 154 if (IS_ENABLED(CONFIG_X86_32) && !arg) 155 force_enable_local_apic = 1; 156 else if (arg && !strncmp(arg, "notscdeadline", 13)) 157 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); 158 return 0; 159 } 160 early_param("lapic", parse_lapic); 161 162 #ifdef CONFIG_X86_64 163 static int apic_calibrate_pmtmr __initdata; 164 static __init int setup_apicpmtimer(char *s) 165 { 166 apic_calibrate_pmtmr = 1; 167 notsc_setup(NULL); 168 return 1; 169 } 170 __setup("apicpmtimer", setup_apicpmtimer); 171 #endif 172 173 static unsigned long mp_lapic_addr __ro_after_init; 174 bool apic_is_disabled __ro_after_init; 175 /* Disable local APIC timer from the kernel commandline or via dmi quirk */ 176 static int disable_apic_timer __initdata; 177 /* Local APIC timer works in C2 */ 178 int local_apic_timer_c2_ok __ro_after_init; 179 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok); 180 181 /* 182 * Debug level, exported for io_apic.c 183 */ 184 int apic_verbosity __ro_after_init; 185 186 int pic_mode __ro_after_init; 187 188 /* Have we found an MP table */ 189 int smp_found_config __ro_after_init; 190 191 static struct resource lapic_resource = { 192 .name = "Local APIC", 193 .flags = IORESOURCE_MEM | IORESOURCE_BUSY, 194 }; 195 196 unsigned int lapic_timer_period = 0; 197 198 static void apic_pm_activate(void); 199 200 /* 201 * Get the LAPIC version 202 */ 203 static inline int lapic_get_version(void) 204 { 205 return GET_APIC_VERSION(apic_read(APIC_LVR)); 206 } 207 208 /* 209 * Check, if the APIC is integrated or a separate chip 210 */ 211 static inline int lapic_is_integrated(void) 212 { 213 return APIC_INTEGRATED(lapic_get_version()); 214 } 215 216 /* 217 * Check, whether this is a modern or a first generation APIC 218 */ 219 static int modern_apic(void) 220 { 221 /* AMD systems use old APIC versions, so check the CPU */ 222 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && 223 boot_cpu_data.x86 >= 0xf) 224 return 1; 225 226 /* Hygon systems use modern APIC */ 227 if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) 228 return 1; 229 230 return lapic_get_version() >= 0x14; 231 } 232 233 /* 234 * right after this call apic become NOOP driven 235 * so apic->write/read doesn't do anything 236 */ 237 static void __init apic_disable(void) 238 { 239 apic_install_driver(&apic_noop); 240 } 241 242 void native_apic_icr_write(u32 low, u32 id) 243 { 244 unsigned long flags; 245 246 local_irq_save(flags); 247 apic_write(APIC_ICR2, SET_XAPIC_DEST_FIELD(id)); 248 apic_write(APIC_ICR, low); 249 local_irq_restore(flags); 250 } 251 252 u64 native_apic_icr_read(void) 253 { 254 u32 icr1, icr2; 255 256 icr2 = apic_read(APIC_ICR2); 257 icr1 = apic_read(APIC_ICR); 258 259 return icr1 | ((u64)icr2 << 32); 260 } 261 262 #ifdef CONFIG_X86_32 263 /** 264 * get_physical_broadcast - Get number of physical broadcast IDs 265 */ 266 int get_physical_broadcast(void) 267 { 268 return modern_apic() ? 0xff : 0xf; 269 } 270 #endif 271 272 /** 273 * lapic_get_maxlvt - get the maximum number of local vector table entries 274 */ 275 int lapic_get_maxlvt(void) 276 { 277 /* 278 * - we always have APIC integrated on 64bit mode 279 * - 82489DXs do not report # of LVT entries 280 */ 281 return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2; 282 } 283 284 /* 285 * Local APIC timer 286 */ 287 288 /* Clock divisor */ 289 #define APIC_DIVISOR 16 290 #define TSC_DIVISOR 8 291 292 /* i82489DX specific */ 293 #define I82489DX_BASE_DIVIDER (((0x2) << 18)) 294 295 /* 296 * This function sets up the local APIC timer, with a timeout of 297 * 'clocks' APIC bus clock. During calibration we actually call 298 * this function twice on the boot CPU, once with a bogus timeout 299 * value, second time for real. The other (noncalibrating) CPUs 300 * call this function only once, with the real, calibrated value. 301 * 302 * We do reads before writes even if unnecessary, to get around the 303 * P5 APIC double write bug. 304 */ 305 static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen) 306 { 307 unsigned int lvtt_value, tmp_value; 308 309 lvtt_value = LOCAL_TIMER_VECTOR; 310 if (!oneshot) 311 lvtt_value |= APIC_LVT_TIMER_PERIODIC; 312 else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 313 lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE; 314 315 /* 316 * The i82489DX APIC uses bit 18 and 19 for the base divider. This 317 * overlaps with bit 18 on integrated APICs, but is not documented 318 * in the SDM. No problem though. i82489DX equipped systems do not 319 * have TSC deadline timer. 320 */ 321 if (!lapic_is_integrated()) 322 lvtt_value |= I82489DX_BASE_DIVIDER; 323 324 if (!irqen) 325 lvtt_value |= APIC_LVT_MASKED; 326 327 apic_write(APIC_LVTT, lvtt_value); 328 329 if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) { 330 /* 331 * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode, 332 * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized. 333 * According to Intel, MFENCE can do the serialization here. 334 */ 335 asm volatile("mfence" : : : "memory"); 336 return; 337 } 338 339 /* 340 * Divide PICLK by 16 341 */ 342 tmp_value = apic_read(APIC_TDCR); 343 apic_write(APIC_TDCR, 344 (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) | 345 APIC_TDR_DIV_16); 346 347 if (!oneshot) 348 apic_write(APIC_TMICT, clocks / APIC_DIVISOR); 349 } 350 351 /* 352 * Setup extended LVT, AMD specific 353 * 354 * Software should use the LVT offsets the BIOS provides. The offsets 355 * are determined by the subsystems using it like those for MCE 356 * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts 357 * are supported. Beginning with family 10h at least 4 offsets are 358 * available. 359 * 360 * Since the offsets must be consistent for all cores, we keep track 361 * of the LVT offsets in software and reserve the offset for the same 362 * vector also to be used on other cores. An offset is freed by 363 * setting the entry to APIC_EILVT_MASKED. 364 * 365 * If the BIOS is right, there should be no conflicts. Otherwise a 366 * "[Firmware Bug]: ..." error message is generated. However, if 367 * software does not properly determines the offsets, it is not 368 * necessarily a BIOS bug. 369 */ 370 371 static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX]; 372 373 static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new) 374 { 375 return (old & APIC_EILVT_MASKED) 376 || (new == APIC_EILVT_MASKED) 377 || ((new & ~APIC_EILVT_MASKED) == old); 378 } 379 380 static unsigned int reserve_eilvt_offset(int offset, unsigned int new) 381 { 382 unsigned int rsvd, vector; 383 384 if (offset >= APIC_EILVT_NR_MAX) 385 return ~0; 386 387 rsvd = atomic_read(&eilvt_offsets[offset]); 388 do { 389 vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */ 390 if (vector && !eilvt_entry_is_changeable(vector, new)) 391 /* may not change if vectors are different */ 392 return rsvd; 393 } while (!atomic_try_cmpxchg(&eilvt_offsets[offset], &rsvd, new)); 394 395 rsvd = new & ~APIC_EILVT_MASKED; 396 if (rsvd && rsvd != vector) 397 pr_info("LVT offset %d assigned for vector 0x%02x\n", 398 offset, rsvd); 399 400 return new; 401 } 402 403 /* 404 * If mask=1, the LVT entry does not generate interrupts while mask=0 405 * enables the vector. See also the BKDGs. Must be called with 406 * preemption disabled. 407 */ 408 409 int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask) 410 { 411 unsigned long reg = APIC_EILVTn(offset); 412 unsigned int new, old, reserved; 413 414 new = (mask << 16) | (msg_type << 8) | vector; 415 old = apic_read(reg); 416 reserved = reserve_eilvt_offset(offset, new); 417 418 if (reserved != new) { 419 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for " 420 "vector 0x%x, but the register is already in use for " 421 "vector 0x%x on another cpu\n", 422 smp_processor_id(), reg, offset, new, reserved); 423 return -EINVAL; 424 } 425 426 if (!eilvt_entry_is_changeable(old, new)) { 427 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for " 428 "vector 0x%x, but the register is already in use for " 429 "vector 0x%x on this cpu\n", 430 smp_processor_id(), reg, offset, new, old); 431 return -EBUSY; 432 } 433 434 apic_write(reg, new); 435 436 return 0; 437 } 438 EXPORT_SYMBOL_GPL(setup_APIC_eilvt); 439 440 /* 441 * Program the next event, relative to now 442 */ 443 static int lapic_next_event(unsigned long delta, 444 struct clock_event_device *evt) 445 { 446 apic_write(APIC_TMICT, delta); 447 return 0; 448 } 449 450 static int lapic_next_deadline(unsigned long delta, 451 struct clock_event_device *evt) 452 { 453 u64 tsc; 454 455 /* This MSR is special and need a special fence: */ 456 weak_wrmsr_fence(); 457 458 tsc = rdtsc(); 459 wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR)); 460 return 0; 461 } 462 463 static int lapic_timer_shutdown(struct clock_event_device *evt) 464 { 465 unsigned int v; 466 467 /* Lapic used as dummy for broadcast ? */ 468 if (evt->features & CLOCK_EVT_FEAT_DUMMY) 469 return 0; 470 471 v = apic_read(APIC_LVTT); 472 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); 473 apic_write(APIC_LVTT, v); 474 apic_write(APIC_TMICT, 0); 475 return 0; 476 } 477 478 static inline int 479 lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot) 480 { 481 /* Lapic used as dummy for broadcast ? */ 482 if (evt->features & CLOCK_EVT_FEAT_DUMMY) 483 return 0; 484 485 __setup_APIC_LVTT(lapic_timer_period, oneshot, 1); 486 return 0; 487 } 488 489 static int lapic_timer_set_periodic(struct clock_event_device *evt) 490 { 491 return lapic_timer_set_periodic_oneshot(evt, false); 492 } 493 494 static int lapic_timer_set_oneshot(struct clock_event_device *evt) 495 { 496 return lapic_timer_set_periodic_oneshot(evt, true); 497 } 498 499 /* 500 * Local APIC timer broadcast function 501 */ 502 static void lapic_timer_broadcast(const struct cpumask *mask) 503 { 504 #ifdef CONFIG_SMP 505 __apic_send_IPI_mask(mask, LOCAL_TIMER_VECTOR); 506 #endif 507 } 508 509 510 /* 511 * The local apic timer can be used for any function which is CPU local. 512 */ 513 static struct clock_event_device lapic_clockevent = { 514 .name = "lapic", 515 .features = CLOCK_EVT_FEAT_PERIODIC | 516 CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP 517 | CLOCK_EVT_FEAT_DUMMY, 518 .shift = 32, 519 .set_state_shutdown = lapic_timer_shutdown, 520 .set_state_periodic = lapic_timer_set_periodic, 521 .set_state_oneshot = lapic_timer_set_oneshot, 522 .set_state_oneshot_stopped = lapic_timer_shutdown, 523 .set_next_event = lapic_next_event, 524 .broadcast = lapic_timer_broadcast, 525 .rating = 100, 526 .irq = -1, 527 }; 528 static DEFINE_PER_CPU(struct clock_event_device, lapic_events); 529 530 static const struct x86_cpu_id deadline_match[] __initconst = { 531 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */ 532 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x4, 0x4), 0x0f), /* EX */ 533 534 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_X, 0x0b000020), 535 536 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x2, 0x2), 0x00000011), 537 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x3, 0x3), 0x0700000e), 538 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x4, 0x4), 0x0f00000c), 539 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x5, 0x5), 0x0e000003), 540 541 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x3, 0x3), 0x01000136), 542 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x4, 0x4), 0x02000014), 543 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x5, 0xf), 0), 544 545 X86_MATCH_INTEL_FAM6_MODEL( HASWELL, 0x22), 546 X86_MATCH_INTEL_FAM6_MODEL( HASWELL_L, 0x20), 547 X86_MATCH_INTEL_FAM6_MODEL( HASWELL_G, 0x17), 548 549 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL, 0x25), 550 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_G, 0x17), 551 552 X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE_L, 0xb2), 553 X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE, 0xb2), 554 555 X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE_L, 0x52), 556 X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE, 0x52), 557 558 {}, 559 }; 560 561 static __init bool apic_validate_deadline_timer(void) 562 { 563 const struct x86_cpu_id *m; 564 u32 rev; 565 566 if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 567 return false; 568 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 569 return true; 570 571 m = x86_match_cpu(deadline_match); 572 if (!m) 573 return true; 574 575 rev = (u32)m->driver_data; 576 577 if (boot_cpu_data.microcode >= rev) 578 return true; 579 580 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); 581 pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; " 582 "please update microcode to version: 0x%x (or later)\n", rev); 583 return false; 584 } 585 586 /* 587 * Setup the local APIC timer for this CPU. Copy the initialized values 588 * of the boot CPU and register the clock event in the framework. 589 */ 590 static void setup_APIC_timer(void) 591 { 592 struct clock_event_device *levt = this_cpu_ptr(&lapic_events); 593 594 if (this_cpu_has(X86_FEATURE_ARAT)) { 595 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP; 596 /* Make LAPIC timer preferable over percpu HPET */ 597 lapic_clockevent.rating = 150; 598 } 599 600 memcpy(levt, &lapic_clockevent, sizeof(*levt)); 601 levt->cpumask = cpumask_of(smp_processor_id()); 602 603 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) { 604 levt->name = "lapic-deadline"; 605 levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC | 606 CLOCK_EVT_FEAT_DUMMY); 607 levt->set_next_event = lapic_next_deadline; 608 clockevents_config_and_register(levt, 609 tsc_khz * (1000 / TSC_DIVISOR), 610 0xF, ~0UL); 611 } else 612 clockevents_register_device(levt); 613 } 614 615 /* 616 * Install the updated TSC frequency from recalibration at the TSC 617 * deadline clockevent devices. 618 */ 619 static void __lapic_update_tsc_freq(void *info) 620 { 621 struct clock_event_device *levt = this_cpu_ptr(&lapic_events); 622 623 if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 624 return; 625 626 clockevents_update_freq(levt, tsc_khz * (1000 / TSC_DIVISOR)); 627 } 628 629 void lapic_update_tsc_freq(void) 630 { 631 /* 632 * The clockevent device's ->mult and ->shift can both be 633 * changed. In order to avoid races, schedule the frequency 634 * update code on each CPU. 635 */ 636 on_each_cpu(__lapic_update_tsc_freq, NULL, 0); 637 } 638 639 /* 640 * In this functions we calibrate APIC bus clocks to the external timer. 641 * 642 * We want to do the calibration only once since we want to have local timer 643 * irqs synchronous. CPUs connected by the same APIC bus have the very same bus 644 * frequency. 645 * 646 * This was previously done by reading the PIT/HPET and waiting for a wrap 647 * around to find out, that a tick has elapsed. I have a box, where the PIT 648 * readout is broken, so it never gets out of the wait loop again. This was 649 * also reported by others. 650 * 651 * Monitoring the jiffies value is inaccurate and the clockevents 652 * infrastructure allows us to do a simple substitution of the interrupt 653 * handler. 654 * 655 * The calibration routine also uses the pm_timer when possible, as the PIT 656 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes 657 * back to normal later in the boot process). 658 */ 659 660 #define LAPIC_CAL_LOOPS (HZ/10) 661 662 static __initdata int lapic_cal_loops = -1; 663 static __initdata long lapic_cal_t1, lapic_cal_t2; 664 static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2; 665 static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2; 666 static __initdata unsigned long lapic_cal_j1, lapic_cal_j2; 667 668 /* 669 * Temporary interrupt handler and polled calibration function. 670 */ 671 static void __init lapic_cal_handler(struct clock_event_device *dev) 672 { 673 unsigned long long tsc = 0; 674 long tapic = apic_read(APIC_TMCCT); 675 unsigned long pm = acpi_pm_read_early(); 676 677 if (boot_cpu_has(X86_FEATURE_TSC)) 678 tsc = rdtsc(); 679 680 switch (lapic_cal_loops++) { 681 case 0: 682 lapic_cal_t1 = tapic; 683 lapic_cal_tsc1 = tsc; 684 lapic_cal_pm1 = pm; 685 lapic_cal_j1 = jiffies; 686 break; 687 688 case LAPIC_CAL_LOOPS: 689 lapic_cal_t2 = tapic; 690 lapic_cal_tsc2 = tsc; 691 if (pm < lapic_cal_pm1) 692 pm += ACPI_PM_OVRRUN; 693 lapic_cal_pm2 = pm; 694 lapic_cal_j2 = jiffies; 695 break; 696 } 697 } 698 699 static int __init 700 calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc) 701 { 702 const long pm_100ms = PMTMR_TICKS_PER_SEC / 10; 703 const long pm_thresh = pm_100ms / 100; 704 unsigned long mult; 705 u64 res; 706 707 #ifndef CONFIG_X86_PM_TIMER 708 return -1; 709 #endif 710 711 apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm); 712 713 /* Check, if the PM timer is available */ 714 if (!deltapm) 715 return -1; 716 717 mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22); 718 719 if (deltapm > (pm_100ms - pm_thresh) && 720 deltapm < (pm_100ms + pm_thresh)) { 721 apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n"); 722 return 0; 723 } 724 725 res = (((u64)deltapm) * mult) >> 22; 726 do_div(res, 1000000); 727 pr_warn("APIC calibration not consistent " 728 "with PM-Timer: %ldms instead of 100ms\n", (long)res); 729 730 /* Correct the lapic counter value */ 731 res = (((u64)(*delta)) * pm_100ms); 732 do_div(res, deltapm); 733 pr_info("APIC delta adjusted to PM-Timer: " 734 "%lu (%ld)\n", (unsigned long)res, *delta); 735 *delta = (long)res; 736 737 /* Correct the tsc counter value */ 738 if (boot_cpu_has(X86_FEATURE_TSC)) { 739 res = (((u64)(*deltatsc)) * pm_100ms); 740 do_div(res, deltapm); 741 apic_printk(APIC_VERBOSE, "TSC delta adjusted to " 742 "PM-Timer: %lu (%ld)\n", 743 (unsigned long)res, *deltatsc); 744 *deltatsc = (long)res; 745 } 746 747 return 0; 748 } 749 750 static int __init lapic_init_clockevent(void) 751 { 752 if (!lapic_timer_period) 753 return -1; 754 755 /* Calculate the scaled math multiplication factor */ 756 lapic_clockevent.mult = div_sc(lapic_timer_period/APIC_DIVISOR, 757 TICK_NSEC, lapic_clockevent.shift); 758 lapic_clockevent.max_delta_ns = 759 clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent); 760 lapic_clockevent.max_delta_ticks = 0x7FFFFFFF; 761 lapic_clockevent.min_delta_ns = 762 clockevent_delta2ns(0xF, &lapic_clockevent); 763 lapic_clockevent.min_delta_ticks = 0xF; 764 765 return 0; 766 } 767 768 bool __init apic_needs_pit(void) 769 { 770 /* 771 * If the frequencies are not known, PIT is required for both TSC 772 * and apic timer calibration. 773 */ 774 if (!tsc_khz || !cpu_khz) 775 return true; 776 777 /* Is there an APIC at all or is it disabled? */ 778 if (!boot_cpu_has(X86_FEATURE_APIC) || apic_is_disabled) 779 return true; 780 781 /* 782 * If interrupt delivery mode is legacy PIC or virtual wire without 783 * configuration, the local APIC timer wont be set up. Make sure 784 * that the PIT is initialized. 785 */ 786 if (apic_intr_mode == APIC_PIC || 787 apic_intr_mode == APIC_VIRTUAL_WIRE_NO_CONFIG) 788 return true; 789 790 /* Virt guests may lack ARAT, but still have DEADLINE */ 791 if (!boot_cpu_has(X86_FEATURE_ARAT)) 792 return true; 793 794 /* Deadline timer is based on TSC so no further PIT action required */ 795 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 796 return false; 797 798 /* APIC timer disabled? */ 799 if (disable_apic_timer) 800 return true; 801 /* 802 * The APIC timer frequency is known already, no PIT calibration 803 * required. If unknown, let the PIT be initialized. 804 */ 805 return lapic_timer_period == 0; 806 } 807 808 static int __init calibrate_APIC_clock(void) 809 { 810 struct clock_event_device *levt = this_cpu_ptr(&lapic_events); 811 u64 tsc_perj = 0, tsc_start = 0; 812 unsigned long jif_start; 813 unsigned long deltaj; 814 long delta, deltatsc; 815 int pm_referenced = 0; 816 817 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 818 return 0; 819 820 /* 821 * Check if lapic timer has already been calibrated by platform 822 * specific routine, such as tsc calibration code. If so just fill 823 * in the clockevent structure and return. 824 */ 825 if (!lapic_init_clockevent()) { 826 apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n", 827 lapic_timer_period); 828 /* 829 * Direct calibration methods must have an always running 830 * local APIC timer, no need for broadcast timer. 831 */ 832 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY; 833 return 0; 834 } 835 836 apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n" 837 "calibrating APIC timer ...\n"); 838 839 /* 840 * There are platforms w/o global clockevent devices. Instead of 841 * making the calibration conditional on that, use a polling based 842 * approach everywhere. 843 */ 844 local_irq_disable(); 845 846 /* 847 * Setup the APIC counter to maximum. There is no way the lapic 848 * can underflow in the 100ms detection time frame 849 */ 850 __setup_APIC_LVTT(0xffffffff, 0, 0); 851 852 /* 853 * Methods to terminate the calibration loop: 854 * 1) Global clockevent if available (jiffies) 855 * 2) TSC if available and frequency is known 856 */ 857 jif_start = READ_ONCE(jiffies); 858 859 if (tsc_khz) { 860 tsc_start = rdtsc(); 861 tsc_perj = div_u64((u64)tsc_khz * 1000, HZ); 862 } 863 864 /* 865 * Enable interrupts so the tick can fire, if a global 866 * clockevent device is available 867 */ 868 local_irq_enable(); 869 870 while (lapic_cal_loops <= LAPIC_CAL_LOOPS) { 871 /* Wait for a tick to elapse */ 872 while (1) { 873 if (tsc_khz) { 874 u64 tsc_now = rdtsc(); 875 if ((tsc_now - tsc_start) >= tsc_perj) { 876 tsc_start += tsc_perj; 877 break; 878 } 879 } else { 880 unsigned long jif_now = READ_ONCE(jiffies); 881 882 if (time_after(jif_now, jif_start)) { 883 jif_start = jif_now; 884 break; 885 } 886 } 887 cpu_relax(); 888 } 889 890 /* Invoke the calibration routine */ 891 local_irq_disable(); 892 lapic_cal_handler(NULL); 893 local_irq_enable(); 894 } 895 896 local_irq_disable(); 897 898 /* Build delta t1-t2 as apic timer counts down */ 899 delta = lapic_cal_t1 - lapic_cal_t2; 900 apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta); 901 902 deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1); 903 904 /* we trust the PM based calibration if possible */ 905 pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1, 906 &delta, &deltatsc); 907 908 lapic_timer_period = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS; 909 lapic_init_clockevent(); 910 911 apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta); 912 apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult); 913 apic_printk(APIC_VERBOSE, "..... calibration result: %u\n", 914 lapic_timer_period); 915 916 if (boot_cpu_has(X86_FEATURE_TSC)) { 917 apic_printk(APIC_VERBOSE, "..... CPU clock speed is " 918 "%ld.%04ld MHz.\n", 919 (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ), 920 (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ)); 921 } 922 923 apic_printk(APIC_VERBOSE, "..... host bus clock speed is " 924 "%u.%04u MHz.\n", 925 lapic_timer_period / (1000000 / HZ), 926 lapic_timer_period % (1000000 / HZ)); 927 928 /* 929 * Do a sanity check on the APIC calibration result 930 */ 931 if (lapic_timer_period < (1000000 / HZ)) { 932 local_irq_enable(); 933 pr_warn("APIC frequency too slow, disabling apic timer\n"); 934 return -1; 935 } 936 937 levt->features &= ~CLOCK_EVT_FEAT_DUMMY; 938 939 /* 940 * PM timer calibration failed or not turned on so lets try APIC 941 * timer based calibration, if a global clockevent device is 942 * available. 943 */ 944 if (!pm_referenced && global_clock_event) { 945 apic_printk(APIC_VERBOSE, "... verify APIC timer\n"); 946 947 /* 948 * Setup the apic timer manually 949 */ 950 levt->event_handler = lapic_cal_handler; 951 lapic_timer_set_periodic(levt); 952 lapic_cal_loops = -1; 953 954 /* Let the interrupts run */ 955 local_irq_enable(); 956 957 while (lapic_cal_loops <= LAPIC_CAL_LOOPS) 958 cpu_relax(); 959 960 /* Stop the lapic timer */ 961 local_irq_disable(); 962 lapic_timer_shutdown(levt); 963 964 /* Jiffies delta */ 965 deltaj = lapic_cal_j2 - lapic_cal_j1; 966 apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj); 967 968 /* Check, if the jiffies result is consistent */ 969 if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2) 970 apic_printk(APIC_VERBOSE, "... jiffies result ok\n"); 971 else 972 levt->features |= CLOCK_EVT_FEAT_DUMMY; 973 } 974 local_irq_enable(); 975 976 if (levt->features & CLOCK_EVT_FEAT_DUMMY) { 977 pr_warn("APIC timer disabled due to verification failure\n"); 978 return -1; 979 } 980 981 return 0; 982 } 983 984 /* 985 * Setup the boot APIC 986 * 987 * Calibrate and verify the result. 988 */ 989 void __init setup_boot_APIC_clock(void) 990 { 991 /* 992 * The local apic timer can be disabled via the kernel 993 * commandline or from the CPU detection code. Register the lapic 994 * timer as a dummy clock event source on SMP systems, so the 995 * broadcast mechanism is used. On UP systems simply ignore it. 996 */ 997 if (disable_apic_timer) { 998 pr_info("Disabling APIC timer\n"); 999 /* No broadcast on UP ! */ 1000 if (num_possible_cpus() > 1) { 1001 lapic_clockevent.mult = 1; 1002 setup_APIC_timer(); 1003 } 1004 return; 1005 } 1006 1007 if (calibrate_APIC_clock()) { 1008 /* No broadcast on UP ! */ 1009 if (num_possible_cpus() > 1) 1010 setup_APIC_timer(); 1011 return; 1012 } 1013 1014 /* 1015 * If nmi_watchdog is set to IO_APIC, we need the 1016 * PIT/HPET going. Otherwise register lapic as a dummy 1017 * device. 1018 */ 1019 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY; 1020 1021 /* Setup the lapic or request the broadcast */ 1022 setup_APIC_timer(); 1023 amd_e400_c1e_apic_setup(); 1024 } 1025 1026 void setup_secondary_APIC_clock(void) 1027 { 1028 setup_APIC_timer(); 1029 amd_e400_c1e_apic_setup(); 1030 } 1031 1032 /* 1033 * The guts of the apic timer interrupt 1034 */ 1035 static void local_apic_timer_interrupt(void) 1036 { 1037 struct clock_event_device *evt = this_cpu_ptr(&lapic_events); 1038 1039 /* 1040 * Normally we should not be here till LAPIC has been initialized but 1041 * in some cases like kdump, its possible that there is a pending LAPIC 1042 * timer interrupt from previous kernel's context and is delivered in 1043 * new kernel the moment interrupts are enabled. 1044 * 1045 * Interrupts are enabled early and LAPIC is setup much later, hence 1046 * its possible that when we get here evt->event_handler is NULL. 1047 * Check for event_handler being NULL and discard the interrupt as 1048 * spurious. 1049 */ 1050 if (!evt->event_handler) { 1051 pr_warn("Spurious LAPIC timer interrupt on cpu %d\n", 1052 smp_processor_id()); 1053 /* Switch it off */ 1054 lapic_timer_shutdown(evt); 1055 return; 1056 } 1057 1058 /* 1059 * the NMI deadlock-detector uses this. 1060 */ 1061 inc_irq_stat(apic_timer_irqs); 1062 1063 evt->event_handler(evt); 1064 } 1065 1066 /* 1067 * Local APIC timer interrupt. This is the most natural way for doing 1068 * local interrupts, but local timer interrupts can be emulated by 1069 * broadcast interrupts too. [in case the hw doesn't support APIC timers] 1070 * 1071 * [ if a single-CPU system runs an SMP kernel then we call the local 1072 * interrupt as well. Thus we cannot inline the local irq ... ] 1073 */ 1074 DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt) 1075 { 1076 struct pt_regs *old_regs = set_irq_regs(regs); 1077 1078 apic_eoi(); 1079 trace_local_timer_entry(LOCAL_TIMER_VECTOR); 1080 local_apic_timer_interrupt(); 1081 trace_local_timer_exit(LOCAL_TIMER_VECTOR); 1082 1083 set_irq_regs(old_regs); 1084 } 1085 1086 /* 1087 * Local APIC start and shutdown 1088 */ 1089 1090 /** 1091 * clear_local_APIC - shutdown the local APIC 1092 * 1093 * This is called, when a CPU is disabled and before rebooting, so the state of 1094 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS 1095 * leftovers during boot. 1096 */ 1097 void clear_local_APIC(void) 1098 { 1099 int maxlvt; 1100 u32 v; 1101 1102 if (!apic_accessible()) 1103 return; 1104 1105 maxlvt = lapic_get_maxlvt(); 1106 /* 1107 * Masking an LVT entry can trigger a local APIC error 1108 * if the vector is zero. Mask LVTERR first to prevent this. 1109 */ 1110 if (maxlvt >= 3) { 1111 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */ 1112 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED); 1113 } 1114 /* 1115 * Careful: we have to set masks only first to deassert 1116 * any level-triggered sources. 1117 */ 1118 v = apic_read(APIC_LVTT); 1119 apic_write(APIC_LVTT, v | APIC_LVT_MASKED); 1120 v = apic_read(APIC_LVT0); 1121 apic_write(APIC_LVT0, v | APIC_LVT_MASKED); 1122 v = apic_read(APIC_LVT1); 1123 apic_write(APIC_LVT1, v | APIC_LVT_MASKED); 1124 if (maxlvt >= 4) { 1125 v = apic_read(APIC_LVTPC); 1126 apic_write(APIC_LVTPC, v | APIC_LVT_MASKED); 1127 } 1128 1129 /* lets not touch this if we didn't frob it */ 1130 #ifdef CONFIG_X86_THERMAL_VECTOR 1131 if (maxlvt >= 5) { 1132 v = apic_read(APIC_LVTTHMR); 1133 apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED); 1134 } 1135 #endif 1136 #ifdef CONFIG_X86_MCE_INTEL 1137 if (maxlvt >= 6) { 1138 v = apic_read(APIC_LVTCMCI); 1139 if (!(v & APIC_LVT_MASKED)) 1140 apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED); 1141 } 1142 #endif 1143 1144 /* 1145 * Clean APIC state for other OSs: 1146 */ 1147 apic_write(APIC_LVTT, APIC_LVT_MASKED); 1148 apic_write(APIC_LVT0, APIC_LVT_MASKED); 1149 apic_write(APIC_LVT1, APIC_LVT_MASKED); 1150 if (maxlvt >= 3) 1151 apic_write(APIC_LVTERR, APIC_LVT_MASKED); 1152 if (maxlvt >= 4) 1153 apic_write(APIC_LVTPC, APIC_LVT_MASKED); 1154 1155 /* Integrated APIC (!82489DX) ? */ 1156 if (lapic_is_integrated()) { 1157 if (maxlvt > 3) 1158 /* Clear ESR due to Pentium errata 3AP and 11AP */ 1159 apic_write(APIC_ESR, 0); 1160 apic_read(APIC_ESR); 1161 } 1162 } 1163 1164 /** 1165 * apic_soft_disable - Clears and software disables the local APIC on hotplug 1166 * 1167 * Contrary to disable_local_APIC() this does not touch the enable bit in 1168 * MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC 1169 * bus would require a hardware reset as the APIC would lose track of bus 1170 * arbitration. On systems with FSB delivery APICBASE could be disabled, 1171 * but it has to be guaranteed that no interrupt is sent to the APIC while 1172 * in that state and it's not clear from the SDM whether it still responds 1173 * to INIT/SIPI messages. Stay on the safe side and use software disable. 1174 */ 1175 void apic_soft_disable(void) 1176 { 1177 u32 value; 1178 1179 clear_local_APIC(); 1180 1181 /* Soft disable APIC (implies clearing of registers for 82489DX!). */ 1182 value = apic_read(APIC_SPIV); 1183 value &= ~APIC_SPIV_APIC_ENABLED; 1184 apic_write(APIC_SPIV, value); 1185 } 1186 1187 /** 1188 * disable_local_APIC - clear and disable the local APIC 1189 */ 1190 void disable_local_APIC(void) 1191 { 1192 if (!apic_accessible()) 1193 return; 1194 1195 apic_soft_disable(); 1196 1197 #ifdef CONFIG_X86_32 1198 /* 1199 * When LAPIC was disabled by the BIOS and enabled by the kernel, 1200 * restore the disabled state. 1201 */ 1202 if (enabled_via_apicbase) { 1203 unsigned int l, h; 1204 1205 rdmsr(MSR_IA32_APICBASE, l, h); 1206 l &= ~MSR_IA32_APICBASE_ENABLE; 1207 wrmsr(MSR_IA32_APICBASE, l, h); 1208 } 1209 #endif 1210 } 1211 1212 /* 1213 * If Linux enabled the LAPIC against the BIOS default disable it down before 1214 * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and 1215 * not power-off. Additionally clear all LVT entries before disable_local_APIC 1216 * for the case where Linux didn't enable the LAPIC. 1217 */ 1218 void lapic_shutdown(void) 1219 { 1220 unsigned long flags; 1221 1222 if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config()) 1223 return; 1224 1225 local_irq_save(flags); 1226 1227 #ifdef CONFIG_X86_32 1228 if (!enabled_via_apicbase) 1229 clear_local_APIC(); 1230 else 1231 #endif 1232 disable_local_APIC(); 1233 1234 1235 local_irq_restore(flags); 1236 } 1237 1238 /** 1239 * sync_Arb_IDs - synchronize APIC bus arbitration IDs 1240 */ 1241 void __init sync_Arb_IDs(void) 1242 { 1243 /* 1244 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not 1245 * needed on AMD. 1246 */ 1247 if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 1248 return; 1249 1250 /* 1251 * Wait for idle. 1252 */ 1253 apic_wait_icr_idle(); 1254 1255 apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n"); 1256 apic_write(APIC_ICR, APIC_DEST_ALLINC | 1257 APIC_INT_LEVELTRIG | APIC_DM_INIT); 1258 } 1259 1260 enum apic_intr_mode_id apic_intr_mode __ro_after_init; 1261 1262 static int __init __apic_intr_mode_select(void) 1263 { 1264 /* Check kernel option */ 1265 if (apic_is_disabled) { 1266 pr_info("APIC disabled via kernel command line\n"); 1267 return APIC_PIC; 1268 } 1269 1270 /* Check BIOS */ 1271 #ifdef CONFIG_X86_64 1272 /* On 64-bit, the APIC must be integrated, Check local APIC only */ 1273 if (!boot_cpu_has(X86_FEATURE_APIC)) { 1274 apic_is_disabled = true; 1275 pr_info("APIC disabled by BIOS\n"); 1276 return APIC_PIC; 1277 } 1278 #else 1279 /* On 32-bit, the APIC may be integrated APIC or 82489DX */ 1280 1281 /* Neither 82489DX nor integrated APIC ? */ 1282 if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) { 1283 apic_is_disabled = true; 1284 return APIC_PIC; 1285 } 1286 1287 /* If the BIOS pretends there is an integrated APIC ? */ 1288 if (!boot_cpu_has(X86_FEATURE_APIC) && 1289 APIC_INTEGRATED(boot_cpu_apic_version)) { 1290 apic_is_disabled = true; 1291 pr_err(FW_BUG "Local APIC not detected, force emulation\n"); 1292 return APIC_PIC; 1293 } 1294 #endif 1295 1296 /* Check MP table or ACPI MADT configuration */ 1297 if (!smp_found_config) { 1298 disable_ioapic_support(); 1299 if (!acpi_lapic) { 1300 pr_info("APIC: ACPI MADT or MP tables are not detected\n"); 1301 return APIC_VIRTUAL_WIRE_NO_CONFIG; 1302 } 1303 return APIC_VIRTUAL_WIRE; 1304 } 1305 1306 #ifdef CONFIG_SMP 1307 /* If SMP should be disabled, then really disable it! */ 1308 if (!setup_max_cpus) { 1309 pr_info("APIC: SMP mode deactivated\n"); 1310 return APIC_SYMMETRIC_IO_NO_ROUTING; 1311 } 1312 #endif 1313 1314 return APIC_SYMMETRIC_IO; 1315 } 1316 1317 /* Select the interrupt delivery mode for the BSP */ 1318 void __init apic_intr_mode_select(void) 1319 { 1320 apic_intr_mode = __apic_intr_mode_select(); 1321 } 1322 1323 /* 1324 * An initial setup of the virtual wire mode. 1325 */ 1326 void __init init_bsp_APIC(void) 1327 { 1328 unsigned int value; 1329 1330 /* 1331 * Don't do the setup now if we have a SMP BIOS as the 1332 * through-I/O-APIC virtual wire mode might be active. 1333 */ 1334 if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC)) 1335 return; 1336 1337 /* 1338 * Do not trust the local APIC being empty at bootup. 1339 */ 1340 clear_local_APIC(); 1341 1342 /* 1343 * Enable APIC. 1344 */ 1345 value = apic_read(APIC_SPIV); 1346 value &= ~APIC_VECTOR_MASK; 1347 value |= APIC_SPIV_APIC_ENABLED; 1348 1349 #ifdef CONFIG_X86_32 1350 /* This bit is reserved on P4/Xeon and should be cleared */ 1351 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && 1352 (boot_cpu_data.x86 == 15)) 1353 value &= ~APIC_SPIV_FOCUS_DISABLED; 1354 else 1355 #endif 1356 value |= APIC_SPIV_FOCUS_DISABLED; 1357 value |= SPURIOUS_APIC_VECTOR; 1358 apic_write(APIC_SPIV, value); 1359 1360 /* 1361 * Set up the virtual wire mode. 1362 */ 1363 apic_write(APIC_LVT0, APIC_DM_EXTINT); 1364 value = APIC_DM_NMI; 1365 if (!lapic_is_integrated()) /* 82489DX */ 1366 value |= APIC_LVT_LEVEL_TRIGGER; 1367 if (apic_extnmi == APIC_EXTNMI_NONE) 1368 value |= APIC_LVT_MASKED; 1369 apic_write(APIC_LVT1, value); 1370 } 1371 1372 static void __init apic_bsp_setup(bool upmode); 1373 1374 /* Init the interrupt delivery mode for the BSP */ 1375 void __init apic_intr_mode_init(void) 1376 { 1377 bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT); 1378 1379 switch (apic_intr_mode) { 1380 case APIC_PIC: 1381 pr_info("APIC: Keep in PIC mode(8259)\n"); 1382 return; 1383 case APIC_VIRTUAL_WIRE: 1384 pr_info("APIC: Switch to virtual wire mode setup\n"); 1385 break; 1386 case APIC_VIRTUAL_WIRE_NO_CONFIG: 1387 pr_info("APIC: Switch to virtual wire mode setup with no configuration\n"); 1388 upmode = true; 1389 break; 1390 case APIC_SYMMETRIC_IO: 1391 pr_info("APIC: Switch to symmetric I/O mode setup\n"); 1392 break; 1393 case APIC_SYMMETRIC_IO_NO_ROUTING: 1394 pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n"); 1395 break; 1396 } 1397 1398 x86_64_probe_apic(); 1399 1400 x86_32_install_bigsmp(); 1401 1402 if (x86_platform.apic_post_init) 1403 x86_platform.apic_post_init(); 1404 1405 apic_bsp_setup(upmode); 1406 } 1407 1408 static void lapic_setup_esr(void) 1409 { 1410 unsigned int oldvalue, value, maxlvt; 1411 1412 if (!lapic_is_integrated()) { 1413 pr_info("No ESR for 82489DX.\n"); 1414 return; 1415 } 1416 1417 if (apic->disable_esr) { 1418 /* 1419 * Something untraceable is creating bad interrupts on 1420 * secondary quads ... for the moment, just leave the 1421 * ESR disabled - we can't do anything useful with the 1422 * errors anyway - mbligh 1423 */ 1424 pr_info("Leaving ESR disabled.\n"); 1425 return; 1426 } 1427 1428 maxlvt = lapic_get_maxlvt(); 1429 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ 1430 apic_write(APIC_ESR, 0); 1431 oldvalue = apic_read(APIC_ESR); 1432 1433 /* enables sending errors */ 1434 value = ERROR_APIC_VECTOR; 1435 apic_write(APIC_LVTERR, value); 1436 1437 /* 1438 * spec says clear errors after enabling vector. 1439 */ 1440 if (maxlvt > 3) 1441 apic_write(APIC_ESR, 0); 1442 value = apic_read(APIC_ESR); 1443 if (value != oldvalue) 1444 apic_printk(APIC_VERBOSE, "ESR value before enabling " 1445 "vector: 0x%08x after: 0x%08x\n", 1446 oldvalue, value); 1447 } 1448 1449 #define APIC_IR_REGS APIC_ISR_NR 1450 #define APIC_IR_BITS (APIC_IR_REGS * 32) 1451 #define APIC_IR_MAPSIZE (APIC_IR_BITS / BITS_PER_LONG) 1452 1453 union apic_ir { 1454 unsigned long map[APIC_IR_MAPSIZE]; 1455 u32 regs[APIC_IR_REGS]; 1456 }; 1457 1458 static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr) 1459 { 1460 int i, bit; 1461 1462 /* Read the IRRs */ 1463 for (i = 0; i < APIC_IR_REGS; i++) 1464 irr->regs[i] = apic_read(APIC_IRR + i * 0x10); 1465 1466 /* Read the ISRs */ 1467 for (i = 0; i < APIC_IR_REGS; i++) 1468 isr->regs[i] = apic_read(APIC_ISR + i * 0x10); 1469 1470 /* 1471 * If the ISR map is not empty. ACK the APIC and run another round 1472 * to verify whether a pending IRR has been unblocked and turned 1473 * into a ISR. 1474 */ 1475 if (!bitmap_empty(isr->map, APIC_IR_BITS)) { 1476 /* 1477 * There can be multiple ISR bits set when a high priority 1478 * interrupt preempted a lower priority one. Issue an ACK 1479 * per set bit. 1480 */ 1481 for_each_set_bit(bit, isr->map, APIC_IR_BITS) 1482 apic_eoi(); 1483 return true; 1484 } 1485 1486 return !bitmap_empty(irr->map, APIC_IR_BITS); 1487 } 1488 1489 /* 1490 * After a crash, we no longer service the interrupts and a pending 1491 * interrupt from previous kernel might still have ISR bit set. 1492 * 1493 * Most probably by now the CPU has serviced that pending interrupt and it 1494 * might not have done the apic_eoi() because it thought, interrupt 1495 * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear 1496 * the ISR bit and cpu thinks it has already serviced the interrupt. Hence 1497 * a vector might get locked. It was noticed for timer irq (vector 1498 * 0x31). Issue an extra EOI to clear ISR. 1499 * 1500 * If there are pending IRR bits they turn into ISR bits after a higher 1501 * priority ISR bit has been acked. 1502 */ 1503 static void apic_pending_intr_clear(void) 1504 { 1505 union apic_ir irr, isr; 1506 unsigned int i; 1507 1508 /* 512 loops are way oversized and give the APIC a chance to obey. */ 1509 for (i = 0; i < 512; i++) { 1510 if (!apic_check_and_ack(&irr, &isr)) 1511 return; 1512 } 1513 /* Dump the IRR/ISR content if that failed */ 1514 pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr.map, isr.map); 1515 } 1516 1517 /** 1518 * setup_local_APIC - setup the local APIC 1519 * 1520 * Used to setup local APIC while initializing BSP or bringing up APs. 1521 * Always called with preemption disabled. 1522 */ 1523 static void setup_local_APIC(void) 1524 { 1525 int cpu = smp_processor_id(); 1526 unsigned int value; 1527 1528 if (apic_is_disabled) { 1529 disable_ioapic_support(); 1530 return; 1531 } 1532 1533 /* 1534 * If this comes from kexec/kcrash the APIC might be enabled in 1535 * SPIV. Soft disable it before doing further initialization. 1536 */ 1537 value = apic_read(APIC_SPIV); 1538 value &= ~APIC_SPIV_APIC_ENABLED; 1539 apic_write(APIC_SPIV, value); 1540 1541 #ifdef CONFIG_X86_32 1542 /* Pound the ESR really hard over the head with a big hammer - mbligh */ 1543 if (lapic_is_integrated() && apic->disable_esr) { 1544 apic_write(APIC_ESR, 0); 1545 apic_write(APIC_ESR, 0); 1546 apic_write(APIC_ESR, 0); 1547 apic_write(APIC_ESR, 0); 1548 } 1549 #endif 1550 /* Validate that the APIC is registered if required */ 1551 BUG_ON(apic->apic_id_registered && !apic->apic_id_registered()); 1552 1553 /* 1554 * Intel recommends to set DFR, LDR and TPR before enabling 1555 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel 1556 * document number 292116). 1557 * 1558 * Except for APICs which operate in physical destination mode. 1559 */ 1560 if (apic->init_apic_ldr) 1561 apic->init_apic_ldr(); 1562 1563 /* 1564 * Set Task Priority to 'accept all except vectors 0-31'. An APIC 1565 * vector in the 16-31 range could be delivered if TPR == 0, but we 1566 * would think it's an exception and terrible things will happen. We 1567 * never change this later on. 1568 */ 1569 value = apic_read(APIC_TASKPRI); 1570 value &= ~APIC_TPRI_MASK; 1571 value |= 0x10; 1572 apic_write(APIC_TASKPRI, value); 1573 1574 /* Clear eventually stale ISR/IRR bits */ 1575 apic_pending_intr_clear(); 1576 1577 /* 1578 * Now that we are all set up, enable the APIC 1579 */ 1580 value = apic_read(APIC_SPIV); 1581 value &= ~APIC_VECTOR_MASK; 1582 /* 1583 * Enable APIC 1584 */ 1585 value |= APIC_SPIV_APIC_ENABLED; 1586 1587 #ifdef CONFIG_X86_32 1588 /* 1589 * Some unknown Intel IO/APIC (or APIC) errata is biting us with 1590 * certain networking cards. If high frequency interrupts are 1591 * happening on a particular IOAPIC pin, plus the IOAPIC routing 1592 * entry is masked/unmasked at a high rate as well then sooner or 1593 * later IOAPIC line gets 'stuck', no more interrupts are received 1594 * from the device. If focus CPU is disabled then the hang goes 1595 * away, oh well :-( 1596 * 1597 * [ This bug can be reproduced easily with a level-triggered 1598 * PCI Ne2000 networking cards and PII/PIII processors, dual 1599 * BX chipset. ] 1600 */ 1601 /* 1602 * Actually disabling the focus CPU check just makes the hang less 1603 * frequent as it makes the interrupt distribution model be more 1604 * like LRU than MRU (the short-term load is more even across CPUs). 1605 */ 1606 1607 /* 1608 * - enable focus processor (bit==0) 1609 * - 64bit mode always use processor focus 1610 * so no need to set it 1611 */ 1612 value &= ~APIC_SPIV_FOCUS_DISABLED; 1613 #endif 1614 1615 /* 1616 * Set spurious IRQ vector 1617 */ 1618 value |= SPURIOUS_APIC_VECTOR; 1619 apic_write(APIC_SPIV, value); 1620 1621 perf_events_lapic_init(); 1622 1623 /* 1624 * Set up LVT0, LVT1: 1625 * 1626 * set up through-local-APIC on the boot CPU's LINT0. This is not 1627 * strictly necessary in pure symmetric-IO mode, but sometimes 1628 * we delegate interrupts to the 8259A. 1629 */ 1630 /* 1631 * TODO: set up through-local-APIC from through-I/O-APIC? --macro 1632 */ 1633 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED; 1634 if (!cpu && (pic_mode || !value || ioapic_is_disabled)) { 1635 value = APIC_DM_EXTINT; 1636 apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu); 1637 } else { 1638 value = APIC_DM_EXTINT | APIC_LVT_MASKED; 1639 apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu); 1640 } 1641 apic_write(APIC_LVT0, value); 1642 1643 /* 1644 * Only the BSP sees the LINT1 NMI signal by default. This can be 1645 * modified by apic_extnmi= boot option. 1646 */ 1647 if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) || 1648 apic_extnmi == APIC_EXTNMI_ALL) 1649 value = APIC_DM_NMI; 1650 else 1651 value = APIC_DM_NMI | APIC_LVT_MASKED; 1652 1653 /* Is 82489DX ? */ 1654 if (!lapic_is_integrated()) 1655 value |= APIC_LVT_LEVEL_TRIGGER; 1656 apic_write(APIC_LVT1, value); 1657 1658 #ifdef CONFIG_X86_MCE_INTEL 1659 /* Recheck CMCI information after local APIC is up on CPU #0 */ 1660 if (!cpu) 1661 cmci_recheck(); 1662 #endif 1663 } 1664 1665 static void end_local_APIC_setup(void) 1666 { 1667 lapic_setup_esr(); 1668 1669 #ifdef CONFIG_X86_32 1670 { 1671 unsigned int value; 1672 /* Disable the local apic timer */ 1673 value = apic_read(APIC_LVTT); 1674 value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); 1675 apic_write(APIC_LVTT, value); 1676 } 1677 #endif 1678 1679 apic_pm_activate(); 1680 } 1681 1682 /* 1683 * APIC setup function for application processors. Called from smpboot.c 1684 */ 1685 void apic_ap_setup(void) 1686 { 1687 setup_local_APIC(); 1688 end_local_APIC_setup(); 1689 } 1690 1691 static __init void cpu_set_boot_apic(void); 1692 1693 static __init void apic_read_boot_cpu_id(bool x2apic) 1694 { 1695 /* 1696 * This can be invoked from check_x2apic() before the APIC has been 1697 * selected. But that code knows for sure that the BIOS enabled 1698 * X2APIC. 1699 */ 1700 if (x2apic) { 1701 boot_cpu_physical_apicid = native_apic_msr_read(APIC_ID); 1702 boot_cpu_apic_version = GET_APIC_VERSION(native_apic_msr_read(APIC_LVR)); 1703 } else { 1704 boot_cpu_physical_apicid = read_apic_id(); 1705 boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR)); 1706 } 1707 cpu_set_boot_apic(); 1708 } 1709 1710 #ifdef CONFIG_X86_X2APIC 1711 int x2apic_mode; 1712 EXPORT_SYMBOL_GPL(x2apic_mode); 1713 1714 enum { 1715 X2APIC_OFF, 1716 X2APIC_DISABLED, 1717 /* All states below here have X2APIC enabled */ 1718 X2APIC_ON, 1719 X2APIC_ON_LOCKED 1720 }; 1721 static int x2apic_state; 1722 1723 static bool x2apic_hw_locked(void) 1724 { 1725 u64 ia32_cap; 1726 u64 msr; 1727 1728 ia32_cap = x86_read_arch_cap_msr(); 1729 if (ia32_cap & ARCH_CAP_XAPIC_DISABLE) { 1730 rdmsrl(MSR_IA32_XAPIC_DISABLE_STATUS, msr); 1731 return (msr & LEGACY_XAPIC_DISABLED); 1732 } 1733 return false; 1734 } 1735 1736 static void __x2apic_disable(void) 1737 { 1738 u64 msr; 1739 1740 if (!boot_cpu_has(X86_FEATURE_APIC)) 1741 return; 1742 1743 rdmsrl(MSR_IA32_APICBASE, msr); 1744 if (!(msr & X2APIC_ENABLE)) 1745 return; 1746 /* Disable xapic and x2apic first and then reenable xapic mode */ 1747 wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE)); 1748 wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE); 1749 printk_once(KERN_INFO "x2apic disabled\n"); 1750 } 1751 1752 static void __x2apic_enable(void) 1753 { 1754 u64 msr; 1755 1756 rdmsrl(MSR_IA32_APICBASE, msr); 1757 if (msr & X2APIC_ENABLE) 1758 return; 1759 wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE); 1760 printk_once(KERN_INFO "x2apic enabled\n"); 1761 } 1762 1763 static int __init setup_nox2apic(char *str) 1764 { 1765 if (x2apic_enabled()) { 1766 int apicid = native_apic_msr_read(APIC_ID); 1767 1768 if (apicid >= 255) { 1769 pr_warn("Apicid: %08x, cannot enforce nox2apic\n", 1770 apicid); 1771 return 0; 1772 } 1773 if (x2apic_hw_locked()) { 1774 pr_warn("APIC locked in x2apic mode, can't disable\n"); 1775 return 0; 1776 } 1777 pr_warn("x2apic already enabled.\n"); 1778 __x2apic_disable(); 1779 } 1780 setup_clear_cpu_cap(X86_FEATURE_X2APIC); 1781 x2apic_state = X2APIC_DISABLED; 1782 x2apic_mode = 0; 1783 return 0; 1784 } 1785 early_param("nox2apic", setup_nox2apic); 1786 1787 /* Called from cpu_init() to enable x2apic on (secondary) cpus */ 1788 void x2apic_setup(void) 1789 { 1790 /* 1791 * Try to make the AP's APIC state match that of the BSP, but if the 1792 * BSP is unlocked and the AP is locked then there is a state mismatch. 1793 * Warn about the mismatch in case a GP fault occurs due to a locked AP 1794 * trying to be turned off. 1795 */ 1796 if (x2apic_state != X2APIC_ON_LOCKED && x2apic_hw_locked()) 1797 pr_warn("x2apic lock mismatch between BSP and AP.\n"); 1798 /* 1799 * If x2apic is not in ON or LOCKED state, disable it if already enabled 1800 * from BIOS. 1801 */ 1802 if (x2apic_state < X2APIC_ON) { 1803 __x2apic_disable(); 1804 return; 1805 } 1806 __x2apic_enable(); 1807 } 1808 1809 static __init void apic_set_fixmap(void); 1810 1811 static __init void x2apic_disable(void) 1812 { 1813 u32 x2apic_id, state = x2apic_state; 1814 1815 x2apic_mode = 0; 1816 x2apic_state = X2APIC_DISABLED; 1817 1818 if (state != X2APIC_ON) 1819 return; 1820 1821 x2apic_id = read_apic_id(); 1822 if (x2apic_id >= 255) 1823 panic("Cannot disable x2apic, id: %08x\n", x2apic_id); 1824 1825 if (x2apic_hw_locked()) { 1826 pr_warn("Cannot disable locked x2apic, id: %08x\n", x2apic_id); 1827 return; 1828 } 1829 1830 __x2apic_disable(); 1831 apic_set_fixmap(); 1832 } 1833 1834 static __init void x2apic_enable(void) 1835 { 1836 if (x2apic_state != X2APIC_OFF) 1837 return; 1838 1839 x2apic_mode = 1; 1840 x2apic_state = X2APIC_ON; 1841 __x2apic_enable(); 1842 } 1843 1844 static __init void try_to_enable_x2apic(int remap_mode) 1845 { 1846 if (x2apic_state == X2APIC_DISABLED) 1847 return; 1848 1849 if (remap_mode != IRQ_REMAP_X2APIC_MODE) { 1850 u32 apic_limit = 255; 1851 1852 /* 1853 * Using X2APIC without IR is not architecturally supported 1854 * on bare metal but may be supported in guests. 1855 */ 1856 if (!x86_init.hyper.x2apic_available()) { 1857 pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n"); 1858 x2apic_disable(); 1859 return; 1860 } 1861 1862 /* 1863 * If the hypervisor supports extended destination ID in 1864 * MSI, that increases the maximum APIC ID that can be 1865 * used for non-remapped IRQ domains. 1866 */ 1867 if (x86_init.hyper.msi_ext_dest_id()) { 1868 virt_ext_dest_id = 1; 1869 apic_limit = 32767; 1870 } 1871 1872 /* 1873 * Without IR, all CPUs can be addressed by IOAPIC/MSI only 1874 * in physical mode, and CPUs with an APIC ID that cannot 1875 * be addressed must not be brought online. 1876 */ 1877 x2apic_set_max_apicid(apic_limit); 1878 x2apic_phys = 1; 1879 } 1880 x2apic_enable(); 1881 } 1882 1883 void __init check_x2apic(void) 1884 { 1885 if (x2apic_enabled()) { 1886 pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n"); 1887 x2apic_mode = 1; 1888 if (x2apic_hw_locked()) 1889 x2apic_state = X2APIC_ON_LOCKED; 1890 else 1891 x2apic_state = X2APIC_ON; 1892 apic_read_boot_cpu_id(true); 1893 } else if (!boot_cpu_has(X86_FEATURE_X2APIC)) { 1894 x2apic_state = X2APIC_DISABLED; 1895 } 1896 } 1897 #else /* CONFIG_X86_X2APIC */ 1898 void __init check_x2apic(void) 1899 { 1900 if (!apic_is_x2apic_enabled()) 1901 return; 1902 /* 1903 * Checkme: Can we simply turn off x2APIC here instead of disabling the APIC? 1904 */ 1905 pr_err("Kernel does not support x2APIC, please recompile with CONFIG_X86_X2APIC.\n"); 1906 pr_err("Disabling APIC, expect reduced performance and functionality.\n"); 1907 1908 apic_is_disabled = true; 1909 setup_clear_cpu_cap(X86_FEATURE_APIC); 1910 } 1911 1912 static inline void try_to_enable_x2apic(int remap_mode) { } 1913 static inline void __x2apic_enable(void) { } 1914 #endif /* !CONFIG_X86_X2APIC */ 1915 1916 void __init enable_IR_x2apic(void) 1917 { 1918 unsigned long flags; 1919 int ret, ir_stat; 1920 1921 if (ioapic_is_disabled) { 1922 pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n"); 1923 return; 1924 } 1925 1926 ir_stat = irq_remapping_prepare(); 1927 if (ir_stat < 0 && !x2apic_supported()) 1928 return; 1929 1930 ret = save_ioapic_entries(); 1931 if (ret) { 1932 pr_info("Saving IO-APIC state failed: %d\n", ret); 1933 return; 1934 } 1935 1936 local_irq_save(flags); 1937 legacy_pic->mask_all(); 1938 mask_ioapic_entries(); 1939 1940 /* If irq_remapping_prepare() succeeded, try to enable it */ 1941 if (ir_stat >= 0) 1942 ir_stat = irq_remapping_enable(); 1943 /* ir_stat contains the remap mode or an error code */ 1944 try_to_enable_x2apic(ir_stat); 1945 1946 if (ir_stat < 0) 1947 restore_ioapic_entries(); 1948 legacy_pic->restore_mask(); 1949 local_irq_restore(flags); 1950 } 1951 1952 #ifdef CONFIG_X86_64 1953 /* 1954 * Detect and enable local APICs on non-SMP boards. 1955 * Original code written by Keir Fraser. 1956 * On AMD64 we trust the BIOS - if it says no APIC it is likely 1957 * not correctly set up (usually the APIC timer won't work etc.) 1958 */ 1959 static bool __init detect_init_APIC(void) 1960 { 1961 if (!boot_cpu_has(X86_FEATURE_APIC)) { 1962 pr_info("No local APIC present\n"); 1963 return false; 1964 } 1965 1966 register_lapic_address(APIC_DEFAULT_PHYS_BASE); 1967 return true; 1968 } 1969 #else 1970 1971 static bool __init apic_verify(unsigned long addr) 1972 { 1973 u32 features, h, l; 1974 1975 /* 1976 * The APIC feature bit should now be enabled 1977 * in `cpuid' 1978 */ 1979 features = cpuid_edx(1); 1980 if (!(features & (1 << X86_FEATURE_APIC))) { 1981 pr_warn("Could not enable APIC!\n"); 1982 return false; 1983 } 1984 set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC); 1985 1986 /* The BIOS may have set up the APIC at some other address */ 1987 if (boot_cpu_data.x86 >= 6) { 1988 rdmsr(MSR_IA32_APICBASE, l, h); 1989 if (l & MSR_IA32_APICBASE_ENABLE) 1990 addr = l & MSR_IA32_APICBASE_BASE; 1991 } 1992 1993 register_lapic_address(addr); 1994 pr_info("Found and enabled local APIC!\n"); 1995 return true; 1996 } 1997 1998 bool __init apic_force_enable(unsigned long addr) 1999 { 2000 u32 h, l; 2001 2002 if (apic_is_disabled) 2003 return false; 2004 2005 /* 2006 * Some BIOSes disable the local APIC in the APIC_BASE 2007 * MSR. This can only be done in software for Intel P6 or later 2008 * and AMD K7 (Model > 1) or later. 2009 */ 2010 if (boot_cpu_data.x86 >= 6) { 2011 rdmsr(MSR_IA32_APICBASE, l, h); 2012 if (!(l & MSR_IA32_APICBASE_ENABLE)) { 2013 pr_info("Local APIC disabled by BIOS -- reenabling.\n"); 2014 l &= ~MSR_IA32_APICBASE_BASE; 2015 l |= MSR_IA32_APICBASE_ENABLE | addr; 2016 wrmsr(MSR_IA32_APICBASE, l, h); 2017 enabled_via_apicbase = 1; 2018 } 2019 } 2020 return apic_verify(addr); 2021 } 2022 2023 /* 2024 * Detect and initialize APIC 2025 */ 2026 static bool __init detect_init_APIC(void) 2027 { 2028 /* Disabled by kernel option? */ 2029 if (apic_is_disabled) 2030 return false; 2031 2032 switch (boot_cpu_data.x86_vendor) { 2033 case X86_VENDOR_AMD: 2034 if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) || 2035 (boot_cpu_data.x86 >= 15)) 2036 break; 2037 goto no_apic; 2038 case X86_VENDOR_HYGON: 2039 break; 2040 case X86_VENDOR_INTEL: 2041 if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 || 2042 (boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC))) 2043 break; 2044 goto no_apic; 2045 default: 2046 goto no_apic; 2047 } 2048 2049 if (!boot_cpu_has(X86_FEATURE_APIC)) { 2050 /* 2051 * Over-ride BIOS and try to enable the local APIC only if 2052 * "lapic" specified. 2053 */ 2054 if (!force_enable_local_apic) { 2055 pr_info("Local APIC disabled by BIOS -- " 2056 "you can enable it with \"lapic\"\n"); 2057 return false; 2058 } 2059 if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE)) 2060 return false; 2061 } else { 2062 if (!apic_verify(APIC_DEFAULT_PHYS_BASE)) 2063 return false; 2064 } 2065 2066 apic_pm_activate(); 2067 2068 return true; 2069 2070 no_apic: 2071 pr_info("No local APIC present or hardware disabled\n"); 2072 return false; 2073 } 2074 #endif 2075 2076 /** 2077 * init_apic_mappings - initialize APIC mappings 2078 */ 2079 void __init init_apic_mappings(void) 2080 { 2081 if (apic_validate_deadline_timer()) 2082 pr_info("TSC deadline timer available\n"); 2083 2084 if (x2apic_mode) 2085 return; 2086 2087 if (!smp_found_config) { 2088 if (!detect_init_APIC()) { 2089 pr_info("APIC: disable apic facility\n"); 2090 apic_disable(); 2091 } 2092 num_processors = 1; 2093 } 2094 } 2095 2096 static __init void apic_set_fixmap(void) 2097 { 2098 set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr); 2099 apic_mmio_base = APIC_BASE; 2100 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", 2101 apic_mmio_base, mp_lapic_addr); 2102 apic_read_boot_cpu_id(false); 2103 } 2104 2105 void __init register_lapic_address(unsigned long address) 2106 { 2107 /* This should only happen once */ 2108 WARN_ON_ONCE(mp_lapic_addr); 2109 mp_lapic_addr = address; 2110 2111 if (!x2apic_mode) 2112 apic_set_fixmap(); 2113 } 2114 2115 /* 2116 * Local APIC interrupts 2117 */ 2118 2119 /* 2120 * Common handling code for spurious_interrupt and spurious_vector entry 2121 * points below. No point in allowing the compiler to inline it twice. 2122 */ 2123 static noinline void handle_spurious_interrupt(u8 vector) 2124 { 2125 u32 v; 2126 2127 trace_spurious_apic_entry(vector); 2128 2129 inc_irq_stat(irq_spurious_count); 2130 2131 /* 2132 * If this is a spurious interrupt then do not acknowledge 2133 */ 2134 if (vector == SPURIOUS_APIC_VECTOR) { 2135 /* See SDM vol 3 */ 2136 pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n", 2137 smp_processor_id()); 2138 goto out; 2139 } 2140 2141 /* 2142 * If it is a vectored one, verify it's set in the ISR. If set, 2143 * acknowledge it. 2144 */ 2145 v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1)); 2146 if (v & (1 << (vector & 0x1f))) { 2147 pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n", 2148 vector, smp_processor_id()); 2149 apic_eoi(); 2150 } else { 2151 pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n", 2152 vector, smp_processor_id()); 2153 } 2154 out: 2155 trace_spurious_apic_exit(vector); 2156 } 2157 2158 /** 2159 * spurious_interrupt - Catch all for interrupts raised on unused vectors 2160 * @regs: Pointer to pt_regs on stack 2161 * @vector: The vector number 2162 * 2163 * This is invoked from ASM entry code to catch all interrupts which 2164 * trigger on an entry which is routed to the common_spurious idtentry 2165 * point. 2166 */ 2167 DEFINE_IDTENTRY_IRQ(spurious_interrupt) 2168 { 2169 handle_spurious_interrupt(vector); 2170 } 2171 2172 DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt) 2173 { 2174 handle_spurious_interrupt(SPURIOUS_APIC_VECTOR); 2175 } 2176 2177 /* 2178 * This interrupt should never happen with our APIC/SMP architecture 2179 */ 2180 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt) 2181 { 2182 static const char * const error_interrupt_reason[] = { 2183 "Send CS error", /* APIC Error Bit 0 */ 2184 "Receive CS error", /* APIC Error Bit 1 */ 2185 "Send accept error", /* APIC Error Bit 2 */ 2186 "Receive accept error", /* APIC Error Bit 3 */ 2187 "Redirectable IPI", /* APIC Error Bit 4 */ 2188 "Send illegal vector", /* APIC Error Bit 5 */ 2189 "Received illegal vector", /* APIC Error Bit 6 */ 2190 "Illegal register address", /* APIC Error Bit 7 */ 2191 }; 2192 u32 v, i = 0; 2193 2194 trace_error_apic_entry(ERROR_APIC_VECTOR); 2195 2196 /* First tickle the hardware, only then report what went on. -- REW */ 2197 if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */ 2198 apic_write(APIC_ESR, 0); 2199 v = apic_read(APIC_ESR); 2200 apic_eoi(); 2201 atomic_inc(&irq_err_count); 2202 2203 apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x", 2204 smp_processor_id(), v); 2205 2206 v &= 0xff; 2207 while (v) { 2208 if (v & 0x1) 2209 apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]); 2210 i++; 2211 v >>= 1; 2212 } 2213 2214 apic_printk(APIC_DEBUG, KERN_CONT "\n"); 2215 2216 trace_error_apic_exit(ERROR_APIC_VECTOR); 2217 } 2218 2219 /** 2220 * connect_bsp_APIC - attach the APIC to the interrupt system 2221 */ 2222 static void __init connect_bsp_APIC(void) 2223 { 2224 #ifdef CONFIG_X86_32 2225 if (pic_mode) { 2226 /* 2227 * Do not trust the local APIC being empty at bootup. 2228 */ 2229 clear_local_APIC(); 2230 /* 2231 * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's 2232 * local APIC to INT and NMI lines. 2233 */ 2234 apic_printk(APIC_VERBOSE, "leaving PIC mode, " 2235 "enabling APIC mode.\n"); 2236 imcr_pic_to_apic(); 2237 } 2238 #endif 2239 } 2240 2241 /** 2242 * disconnect_bsp_APIC - detach the APIC from the interrupt system 2243 * @virt_wire_setup: indicates, whether virtual wire mode is selected 2244 * 2245 * Virtual wire mode is necessary to deliver legacy interrupts even when the 2246 * APIC is disabled. 2247 */ 2248 void disconnect_bsp_APIC(int virt_wire_setup) 2249 { 2250 unsigned int value; 2251 2252 #ifdef CONFIG_X86_32 2253 if (pic_mode) { 2254 /* 2255 * Put the board back into PIC mode (has an effect only on 2256 * certain older boards). Note that APIC interrupts, including 2257 * IPIs, won't work beyond this point! The only exception are 2258 * INIT IPIs. 2259 */ 2260 apic_printk(APIC_VERBOSE, "disabling APIC mode, " 2261 "entering PIC mode.\n"); 2262 imcr_apic_to_pic(); 2263 return; 2264 } 2265 #endif 2266 2267 /* Go back to Virtual Wire compatibility mode */ 2268 2269 /* For the spurious interrupt use vector F, and enable it */ 2270 value = apic_read(APIC_SPIV); 2271 value &= ~APIC_VECTOR_MASK; 2272 value |= APIC_SPIV_APIC_ENABLED; 2273 value |= 0xf; 2274 apic_write(APIC_SPIV, value); 2275 2276 if (!virt_wire_setup) { 2277 /* 2278 * For LVT0 make it edge triggered, active high, 2279 * external and enabled 2280 */ 2281 value = apic_read(APIC_LVT0); 2282 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | 2283 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | 2284 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED); 2285 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; 2286 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT); 2287 apic_write(APIC_LVT0, value); 2288 } else { 2289 /* Disable LVT0 */ 2290 apic_write(APIC_LVT0, APIC_LVT_MASKED); 2291 } 2292 2293 /* 2294 * For LVT1 make it edge triggered, active high, 2295 * nmi and enabled 2296 */ 2297 value = apic_read(APIC_LVT1); 2298 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | 2299 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | 2300 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED); 2301 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; 2302 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI); 2303 apic_write(APIC_LVT1, value); 2304 } 2305 2306 /* 2307 * The number of allocated logical CPU IDs. Since logical CPU IDs are allocated 2308 * contiguously, it equals to current allocated max logical CPU ID plus 1. 2309 * All allocated CPU IDs should be in the [0, nr_logical_cpuids) range, 2310 * so the maximum of nr_logical_cpuids is nr_cpu_ids. 2311 * 2312 * NOTE: Reserve 0 for BSP. 2313 */ 2314 static int nr_logical_cpuids = 1; 2315 2316 /* 2317 * Used to store mapping between logical CPU IDs and APIC IDs. 2318 */ 2319 int cpuid_to_apicid[] = { 2320 [0 ... NR_CPUS - 1] = -1, 2321 }; 2322 2323 bool arch_match_cpu_phys_id(int cpu, u64 phys_id) 2324 { 2325 return phys_id == cpuid_to_apicid[cpu]; 2326 } 2327 2328 #ifdef CONFIG_SMP 2329 static void cpu_mark_primary_thread(unsigned int cpu, unsigned int apicid) 2330 { 2331 /* Isolate the SMT bit(s) in the APICID and check for 0 */ 2332 u32 mask = (1U << (fls(smp_num_siblings) - 1)) - 1; 2333 2334 if (smp_num_siblings == 1 || !(apicid & mask)) 2335 cpumask_set_cpu(cpu, &__cpu_primary_thread_mask); 2336 } 2337 2338 /* 2339 * Due to the utter mess of CPUID evaluation smp_num_siblings is not valid 2340 * during early boot. Initialize the primary thread mask before SMP 2341 * bringup. 2342 */ 2343 static int __init smp_init_primary_thread_mask(void) 2344 { 2345 unsigned int cpu; 2346 2347 for (cpu = 0; cpu < nr_logical_cpuids; cpu++) 2348 cpu_mark_primary_thread(cpu, cpuid_to_apicid[cpu]); 2349 return 0; 2350 } 2351 early_initcall(smp_init_primary_thread_mask); 2352 #else 2353 static inline void cpu_mark_primary_thread(unsigned int cpu, unsigned int apicid) { } 2354 #endif 2355 2356 /* 2357 * Should use this API to allocate logical CPU IDs to keep nr_logical_cpuids 2358 * and cpuid_to_apicid[] synchronized. 2359 */ 2360 static int allocate_logical_cpuid(int apicid) 2361 { 2362 int i; 2363 2364 /* 2365 * cpuid <-> apicid mapping is persistent, so when a cpu is up, 2366 * check if the kernel has allocated a cpuid for it. 2367 */ 2368 for (i = 0; i < nr_logical_cpuids; i++) { 2369 if (cpuid_to_apicid[i] == apicid) 2370 return i; 2371 } 2372 2373 /* Allocate a new cpuid. */ 2374 if (nr_logical_cpuids >= nr_cpu_ids) { 2375 WARN_ONCE(1, "APIC: NR_CPUS/possible_cpus limit of %u reached. " 2376 "Processor %d/0x%x and the rest are ignored.\n", 2377 nr_cpu_ids, nr_logical_cpuids, apicid); 2378 return -EINVAL; 2379 } 2380 2381 cpuid_to_apicid[nr_logical_cpuids] = apicid; 2382 return nr_logical_cpuids++; 2383 } 2384 2385 static void cpu_update_apic(int cpu, int apicid) 2386 { 2387 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64) 2388 early_per_cpu(x86_cpu_to_apicid, cpu) = apicid; 2389 #endif 2390 set_cpu_possible(cpu, true); 2391 physid_set(apicid, phys_cpu_present_map); 2392 set_cpu_present(cpu, true); 2393 num_processors++; 2394 2395 if (system_state != SYSTEM_BOOTING) 2396 cpu_mark_primary_thread(cpu, apicid); 2397 } 2398 2399 static __init void cpu_set_boot_apic(void) 2400 { 2401 cpuid_to_apicid[0] = boot_cpu_physical_apicid; 2402 cpu_update_apic(0, boot_cpu_physical_apicid); 2403 x86_32_probe_bigsmp_early(); 2404 } 2405 2406 int generic_processor_info(int apicid) 2407 { 2408 int cpu, max = nr_cpu_ids; 2409 2410 /* The boot CPU must be set before MADT/MPTABLE parsing happens */ 2411 if (cpuid_to_apicid[0] == BAD_APICID) 2412 panic("Boot CPU APIC not registered yet\n"); 2413 2414 if (apicid == boot_cpu_physical_apicid) 2415 return 0; 2416 2417 if (disabled_cpu_apicid == apicid) { 2418 int thiscpu = num_processors + disabled_cpus; 2419 2420 pr_warn("APIC: Disabling requested cpu. Processor %d/0x%x ignored.\n", 2421 thiscpu, apicid); 2422 2423 disabled_cpus++; 2424 return -ENODEV; 2425 } 2426 2427 if (num_processors >= nr_cpu_ids) { 2428 int thiscpu = max + disabled_cpus; 2429 2430 pr_warn("APIC: NR_CPUS/possible_cpus limit of %i reached. " 2431 "Processor %d/0x%x ignored.\n", max, thiscpu, apicid); 2432 2433 disabled_cpus++; 2434 return -EINVAL; 2435 } 2436 2437 cpu = allocate_logical_cpuid(apicid); 2438 if (cpu < 0) { 2439 disabled_cpus++; 2440 return -EINVAL; 2441 } 2442 2443 cpu_update_apic(cpu, apicid); 2444 return cpu; 2445 } 2446 2447 2448 void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg, 2449 bool dmar) 2450 { 2451 memset(msg, 0, sizeof(*msg)); 2452 2453 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW; 2454 msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical; 2455 msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF; 2456 2457 msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED; 2458 msg->arch_data.vector = cfg->vector; 2459 2460 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH; 2461 /* 2462 * Only the IOMMU itself can use the trick of putting destination 2463 * APIC ID into the high bits of the address. Anything else would 2464 * just be writing to memory if it tried that, and needs IR to 2465 * address APICs which can't be addressed in the normal 32-bit 2466 * address range at 0xFFExxxxx. That is typically just 8 bits, but 2467 * some hypervisors allow the extended destination ID field in bits 2468 * 5-11 to be used, giving support for 15 bits of APIC IDs in total. 2469 */ 2470 if (dmar) 2471 msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8; 2472 else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000) 2473 msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8; 2474 else 2475 WARN_ON_ONCE(cfg->dest_apicid > 0xFF); 2476 } 2477 2478 u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid) 2479 { 2480 u32 dest = msg->arch_addr_lo.destid_0_7; 2481 2482 if (extid) 2483 dest |= msg->arch_addr_hi.destid_8_31 << 8; 2484 return dest; 2485 } 2486 EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid); 2487 2488 static void __init apic_bsp_up_setup(void) 2489 { 2490 #ifdef CONFIG_X86_64 2491 apic_write(APIC_ID, apic->set_apic_id(boot_cpu_physical_apicid)); 2492 #endif 2493 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map); 2494 } 2495 2496 /** 2497 * apic_bsp_setup - Setup function for local apic and io-apic 2498 * @upmode: Force UP mode (for APIC_init_uniprocessor) 2499 */ 2500 static void __init apic_bsp_setup(bool upmode) 2501 { 2502 connect_bsp_APIC(); 2503 if (upmode) 2504 apic_bsp_up_setup(); 2505 setup_local_APIC(); 2506 2507 enable_IO_APIC(); 2508 end_local_APIC_setup(); 2509 irq_remap_enable_fault_handling(); 2510 setup_IO_APIC(); 2511 lapic_update_legacy_vectors(); 2512 } 2513 2514 #ifdef CONFIG_UP_LATE_INIT 2515 void __init up_late_init(void) 2516 { 2517 if (apic_intr_mode == APIC_PIC) 2518 return; 2519 2520 /* Setup local timer */ 2521 x86_init.timers.setup_percpu_clockev(); 2522 } 2523 #endif 2524 2525 /* 2526 * Power management 2527 */ 2528 #ifdef CONFIG_PM 2529 2530 static struct { 2531 /* 2532 * 'active' is true if the local APIC was enabled by us and 2533 * not the BIOS; this signifies that we are also responsible 2534 * for disabling it before entering apm/acpi suspend 2535 */ 2536 int active; 2537 /* r/w apic fields */ 2538 unsigned int apic_id; 2539 unsigned int apic_taskpri; 2540 unsigned int apic_ldr; 2541 unsigned int apic_dfr; 2542 unsigned int apic_spiv; 2543 unsigned int apic_lvtt; 2544 unsigned int apic_lvtpc; 2545 unsigned int apic_lvt0; 2546 unsigned int apic_lvt1; 2547 unsigned int apic_lvterr; 2548 unsigned int apic_tmict; 2549 unsigned int apic_tdcr; 2550 unsigned int apic_thmr; 2551 unsigned int apic_cmci; 2552 } apic_pm_state; 2553 2554 static int lapic_suspend(void) 2555 { 2556 unsigned long flags; 2557 int maxlvt; 2558 2559 if (!apic_pm_state.active) 2560 return 0; 2561 2562 maxlvt = lapic_get_maxlvt(); 2563 2564 apic_pm_state.apic_id = apic_read(APIC_ID); 2565 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI); 2566 apic_pm_state.apic_ldr = apic_read(APIC_LDR); 2567 apic_pm_state.apic_dfr = apic_read(APIC_DFR); 2568 apic_pm_state.apic_spiv = apic_read(APIC_SPIV); 2569 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT); 2570 if (maxlvt >= 4) 2571 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC); 2572 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0); 2573 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1); 2574 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR); 2575 apic_pm_state.apic_tmict = apic_read(APIC_TMICT); 2576 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR); 2577 #ifdef CONFIG_X86_THERMAL_VECTOR 2578 if (maxlvt >= 5) 2579 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR); 2580 #endif 2581 #ifdef CONFIG_X86_MCE_INTEL 2582 if (maxlvt >= 6) 2583 apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI); 2584 #endif 2585 2586 local_irq_save(flags); 2587 2588 /* 2589 * Mask IOAPIC before disabling the local APIC to prevent stale IRR 2590 * entries on some implementations. 2591 */ 2592 mask_ioapic_entries(); 2593 2594 disable_local_APIC(); 2595 2596 irq_remapping_disable(); 2597 2598 local_irq_restore(flags); 2599 return 0; 2600 } 2601 2602 static void lapic_resume(void) 2603 { 2604 unsigned int l, h; 2605 unsigned long flags; 2606 int maxlvt; 2607 2608 if (!apic_pm_state.active) 2609 return; 2610 2611 local_irq_save(flags); 2612 2613 /* 2614 * IO-APIC and PIC have their own resume routines. 2615 * We just mask them here to make sure the interrupt 2616 * subsystem is completely quiet while we enable x2apic 2617 * and interrupt-remapping. 2618 */ 2619 mask_ioapic_entries(); 2620 legacy_pic->mask_all(); 2621 2622 if (x2apic_mode) { 2623 __x2apic_enable(); 2624 } else { 2625 /* 2626 * Make sure the APICBASE points to the right address 2627 * 2628 * FIXME! This will be wrong if we ever support suspend on 2629 * SMP! We'll need to do this as part of the CPU restore! 2630 */ 2631 if (boot_cpu_data.x86 >= 6) { 2632 rdmsr(MSR_IA32_APICBASE, l, h); 2633 l &= ~MSR_IA32_APICBASE_BASE; 2634 l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; 2635 wrmsr(MSR_IA32_APICBASE, l, h); 2636 } 2637 } 2638 2639 maxlvt = lapic_get_maxlvt(); 2640 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED); 2641 apic_write(APIC_ID, apic_pm_state.apic_id); 2642 apic_write(APIC_DFR, apic_pm_state.apic_dfr); 2643 apic_write(APIC_LDR, apic_pm_state.apic_ldr); 2644 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri); 2645 apic_write(APIC_SPIV, apic_pm_state.apic_spiv); 2646 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0); 2647 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1); 2648 #ifdef CONFIG_X86_THERMAL_VECTOR 2649 if (maxlvt >= 5) 2650 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr); 2651 #endif 2652 #ifdef CONFIG_X86_MCE_INTEL 2653 if (maxlvt >= 6) 2654 apic_write(APIC_LVTCMCI, apic_pm_state.apic_cmci); 2655 #endif 2656 if (maxlvt >= 4) 2657 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc); 2658 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt); 2659 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr); 2660 apic_write(APIC_TMICT, apic_pm_state.apic_tmict); 2661 apic_write(APIC_ESR, 0); 2662 apic_read(APIC_ESR); 2663 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr); 2664 apic_write(APIC_ESR, 0); 2665 apic_read(APIC_ESR); 2666 2667 irq_remapping_reenable(x2apic_mode); 2668 2669 local_irq_restore(flags); 2670 } 2671 2672 /* 2673 * This device has no shutdown method - fully functioning local APICs 2674 * are needed on every CPU up until machine_halt/restart/poweroff. 2675 */ 2676 2677 static struct syscore_ops lapic_syscore_ops = { 2678 .resume = lapic_resume, 2679 .suspend = lapic_suspend, 2680 }; 2681 2682 static void apic_pm_activate(void) 2683 { 2684 apic_pm_state.active = 1; 2685 } 2686 2687 static int __init init_lapic_sysfs(void) 2688 { 2689 /* XXX: remove suspend/resume procs if !apic_pm_state.active? */ 2690 if (boot_cpu_has(X86_FEATURE_APIC)) 2691 register_syscore_ops(&lapic_syscore_ops); 2692 2693 return 0; 2694 } 2695 2696 /* local apic needs to resume before other devices access its registers. */ 2697 core_initcall(init_lapic_sysfs); 2698 2699 #else /* CONFIG_PM */ 2700 2701 static void apic_pm_activate(void) { } 2702 2703 #endif /* CONFIG_PM */ 2704 2705 #ifdef CONFIG_X86_64 2706 2707 static int multi_checked; 2708 static int multi; 2709 2710 static int set_multi(const struct dmi_system_id *d) 2711 { 2712 if (multi) 2713 return 0; 2714 pr_info("APIC: %s detected, Multi Chassis\n", d->ident); 2715 multi = 1; 2716 return 0; 2717 } 2718 2719 static const struct dmi_system_id multi_dmi_table[] = { 2720 { 2721 .callback = set_multi, 2722 .ident = "IBM System Summit2", 2723 .matches = { 2724 DMI_MATCH(DMI_SYS_VENDOR, "IBM"), 2725 DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"), 2726 }, 2727 }, 2728 {} 2729 }; 2730 2731 static void dmi_check_multi(void) 2732 { 2733 if (multi_checked) 2734 return; 2735 2736 dmi_check_system(multi_dmi_table); 2737 multi_checked = 1; 2738 } 2739 2740 /* 2741 * apic_is_clustered_box() -- Check if we can expect good TSC 2742 * 2743 * Thus far, the major user of this is IBM's Summit2 series: 2744 * Clustered boxes may have unsynced TSC problems if they are 2745 * multi-chassis. 2746 * Use DMI to check them 2747 */ 2748 int apic_is_clustered_box(void) 2749 { 2750 dmi_check_multi(); 2751 return multi; 2752 } 2753 #endif 2754 2755 /* 2756 * APIC command line parameters 2757 */ 2758 static int __init setup_disableapic(char *arg) 2759 { 2760 apic_is_disabled = true; 2761 setup_clear_cpu_cap(X86_FEATURE_APIC); 2762 return 0; 2763 } 2764 early_param("disableapic", setup_disableapic); 2765 2766 /* same as disableapic, for compatibility */ 2767 static int __init setup_nolapic(char *arg) 2768 { 2769 return setup_disableapic(arg); 2770 } 2771 early_param("nolapic", setup_nolapic); 2772 2773 static int __init parse_lapic_timer_c2_ok(char *arg) 2774 { 2775 local_apic_timer_c2_ok = 1; 2776 return 0; 2777 } 2778 early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok); 2779 2780 static int __init parse_disable_apic_timer(char *arg) 2781 { 2782 disable_apic_timer = 1; 2783 return 0; 2784 } 2785 early_param("noapictimer", parse_disable_apic_timer); 2786 2787 static int __init parse_nolapic_timer(char *arg) 2788 { 2789 disable_apic_timer = 1; 2790 return 0; 2791 } 2792 early_param("nolapic_timer", parse_nolapic_timer); 2793 2794 static int __init apic_set_verbosity(char *arg) 2795 { 2796 if (!arg) { 2797 if (IS_ENABLED(CONFIG_X86_32)) 2798 return -EINVAL; 2799 2800 ioapic_is_disabled = false; 2801 return 0; 2802 } 2803 2804 if (strcmp("debug", arg) == 0) 2805 apic_verbosity = APIC_DEBUG; 2806 else if (strcmp("verbose", arg) == 0) 2807 apic_verbosity = APIC_VERBOSE; 2808 #ifdef CONFIG_X86_64 2809 else { 2810 pr_warn("APIC Verbosity level %s not recognised" 2811 " use apic=verbose or apic=debug\n", arg); 2812 return -EINVAL; 2813 } 2814 #endif 2815 2816 return 0; 2817 } 2818 early_param("apic", apic_set_verbosity); 2819 2820 static int __init lapic_insert_resource(void) 2821 { 2822 if (!apic_mmio_base) 2823 return -1; 2824 2825 /* Put local APIC into the resource map. */ 2826 lapic_resource.start = apic_mmio_base; 2827 lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1; 2828 insert_resource(&iomem_resource, &lapic_resource); 2829 2830 return 0; 2831 } 2832 2833 /* 2834 * need call insert after e820__reserve_resources() 2835 * that is using request_resource 2836 */ 2837 late_initcall(lapic_insert_resource); 2838 2839 static int __init apic_set_disabled_cpu_apicid(char *arg) 2840 { 2841 if (!arg || !get_option(&arg, &disabled_cpu_apicid)) 2842 return -EINVAL; 2843 2844 return 0; 2845 } 2846 early_param("disable_cpu_apicid", apic_set_disabled_cpu_apicid); 2847 2848 static int __init apic_set_extnmi(char *arg) 2849 { 2850 if (!arg) 2851 return -EINVAL; 2852 2853 if (!strncmp("all", arg, 3)) 2854 apic_extnmi = APIC_EXTNMI_ALL; 2855 else if (!strncmp("none", arg, 4)) 2856 apic_extnmi = APIC_EXTNMI_NONE; 2857 else if (!strncmp("bsp", arg, 3)) 2858 apic_extnmi = APIC_EXTNMI_BSP; 2859 else { 2860 pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg); 2861 return -EINVAL; 2862 } 2863 2864 return 0; 2865 } 2866 early_param("apic_extnmi", apic_set_extnmi); 2867