1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Local APIC handling, local APIC timers 4 * 5 * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com> 6 * 7 * Fixes 8 * Maciej W. Rozycki : Bits for genuine 82489DX APICs; 9 * thanks to Eric Gilmore 10 * and Rolf G. Tews 11 * for testing these extensively. 12 * Maciej W. Rozycki : Various updates and fixes. 13 * Mikael Pettersson : Power Management for UP-APIC. 14 * Pavel Machek and 15 * Mikael Pettersson : PM converted to driver model. 16 */ 17 18 #include <linux/perf_event.h> 19 #include <linux/kernel_stat.h> 20 #include <linux/mc146818rtc.h> 21 #include <linux/acpi_pmtmr.h> 22 #include <linux/clockchips.h> 23 #include <linux/interrupt.h> 24 #include <linux/memblock.h> 25 #include <linux/ftrace.h> 26 #include <linux/ioport.h> 27 #include <linux/export.h> 28 #include <linux/syscore_ops.h> 29 #include <linux/delay.h> 30 #include <linux/timex.h> 31 #include <linux/i8253.h> 32 #include <linux/dmar.h> 33 #include <linux/init.h> 34 #include <linux/cpu.h> 35 #include <linux/dmi.h> 36 #include <linux/smp.h> 37 #include <linux/mm.h> 38 39 #include <asm/trace/irq_vectors.h> 40 #include <asm/irq_remapping.h> 41 #include <asm/perf_event.h> 42 #include <asm/x86_init.h> 43 #include <linux/atomic.h> 44 #include <asm/barrier.h> 45 #include <asm/mpspec.h> 46 #include <asm/i8259.h> 47 #include <asm/proto.h> 48 #include <asm/traps.h> 49 #include <asm/apic.h> 50 #include <asm/acpi.h> 51 #include <asm/io_apic.h> 52 #include <asm/desc.h> 53 #include <asm/hpet.h> 54 #include <asm/mtrr.h> 55 #include <asm/time.h> 56 #include <asm/smp.h> 57 #include <asm/mce.h> 58 #include <asm/tsc.h> 59 #include <asm/hypervisor.h> 60 #include <asm/cpu_device_id.h> 61 #include <asm/intel-family.h> 62 #include <asm/irq_regs.h> 63 64 unsigned int num_processors; 65 66 unsigned disabled_cpus; 67 68 /* Processor that is doing the boot up */ 69 unsigned int boot_cpu_physical_apicid __ro_after_init = -1U; 70 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid); 71 72 u8 boot_cpu_apic_version __ro_after_init; 73 74 /* 75 * The highest APIC ID seen during enumeration. 76 */ 77 static unsigned int max_physical_apicid; 78 79 /* 80 * Bitmask of physically existing CPUs: 81 */ 82 physid_mask_t phys_cpu_present_map; 83 84 /* 85 * Processor to be disabled specified by kernel parameter 86 * disable_cpu_apicid=<int>, mostly used for the kdump 2nd kernel to 87 * avoid undefined behaviour caused by sending INIT from AP to BSP. 88 */ 89 static unsigned int disabled_cpu_apicid __ro_after_init = BAD_APICID; 90 91 /* 92 * This variable controls which CPUs receive external NMIs. By default, 93 * external NMIs are delivered only to the BSP. 94 */ 95 static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP; 96 97 /* 98 * Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID 99 */ 100 static bool virt_ext_dest_id __ro_after_init; 101 102 /* 103 * Map cpu index to physical APIC ID 104 */ 105 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID); 106 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID); 107 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid, U32_MAX); 108 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid); 109 EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid); 110 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid); 111 112 #ifdef CONFIG_X86_32 113 114 /* 115 * On x86_32, the mapping between cpu and logical apicid may vary 116 * depending on apic in use. The following early percpu variable is 117 * used for the mapping. This is where the behaviors of x86_64 and 32 118 * actually diverge. Let's keep it ugly for now. 119 */ 120 DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID); 121 122 /* Local APIC was disabled by the BIOS and enabled by the kernel */ 123 static int enabled_via_apicbase __ro_after_init; 124 125 /* 126 * Handle interrupt mode configuration register (IMCR). 127 * This register controls whether the interrupt signals 128 * that reach the BSP come from the master PIC or from the 129 * local APIC. Before entering Symmetric I/O Mode, either 130 * the BIOS or the operating system must switch out of 131 * PIC Mode by changing the IMCR. 132 */ 133 static inline void imcr_pic_to_apic(void) 134 { 135 /* select IMCR register */ 136 outb(0x70, 0x22); 137 /* NMI and 8259 INTR go through APIC */ 138 outb(0x01, 0x23); 139 } 140 141 static inline void imcr_apic_to_pic(void) 142 { 143 /* select IMCR register */ 144 outb(0x70, 0x22); 145 /* NMI and 8259 INTR go directly to BSP */ 146 outb(0x00, 0x23); 147 } 148 #endif 149 150 /* 151 * Knob to control our willingness to enable the local APIC. 152 * 153 * +1=force-enable 154 */ 155 static int force_enable_local_apic __initdata; 156 157 /* 158 * APIC command line parameters 159 */ 160 static int __init parse_lapic(char *arg) 161 { 162 if (IS_ENABLED(CONFIG_X86_32) && !arg) 163 force_enable_local_apic = 1; 164 else if (arg && !strncmp(arg, "notscdeadline", 13)) 165 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); 166 return 0; 167 } 168 early_param("lapic", parse_lapic); 169 170 #ifdef CONFIG_X86_64 171 static int apic_calibrate_pmtmr __initdata; 172 static __init int setup_apicpmtimer(char *s) 173 { 174 apic_calibrate_pmtmr = 1; 175 notsc_setup(NULL); 176 return 0; 177 } 178 __setup("apicpmtimer", setup_apicpmtimer); 179 #endif 180 181 unsigned long mp_lapic_addr __ro_after_init; 182 int disable_apic __ro_after_init; 183 /* Disable local APIC timer from the kernel commandline or via dmi quirk */ 184 static int disable_apic_timer __initdata; 185 /* Local APIC timer works in C2 */ 186 int local_apic_timer_c2_ok __ro_after_init; 187 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok); 188 189 /* 190 * Debug level, exported for io_apic.c 191 */ 192 int apic_verbosity __ro_after_init; 193 194 int pic_mode __ro_after_init; 195 196 /* Have we found an MP table */ 197 int smp_found_config __ro_after_init; 198 199 static struct resource lapic_resource = { 200 .name = "Local APIC", 201 .flags = IORESOURCE_MEM | IORESOURCE_BUSY, 202 }; 203 204 unsigned int lapic_timer_period = 0; 205 206 static void apic_pm_activate(void); 207 208 static unsigned long apic_phys __ro_after_init; 209 210 /* 211 * Get the LAPIC version 212 */ 213 static inline int lapic_get_version(void) 214 { 215 return GET_APIC_VERSION(apic_read(APIC_LVR)); 216 } 217 218 /* 219 * Check, if the APIC is integrated or a separate chip 220 */ 221 static inline int lapic_is_integrated(void) 222 { 223 return APIC_INTEGRATED(lapic_get_version()); 224 } 225 226 /* 227 * Check, whether this is a modern or a first generation APIC 228 */ 229 static int modern_apic(void) 230 { 231 /* AMD systems use old APIC versions, so check the CPU */ 232 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && 233 boot_cpu_data.x86 >= 0xf) 234 return 1; 235 236 /* Hygon systems use modern APIC */ 237 if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) 238 return 1; 239 240 return lapic_get_version() >= 0x14; 241 } 242 243 /* 244 * right after this call apic become NOOP driven 245 * so apic->write/read doesn't do anything 246 */ 247 static void __init apic_disable(void) 248 { 249 pr_info("APIC: switched to apic NOOP\n"); 250 apic = &apic_noop; 251 } 252 253 void native_apic_wait_icr_idle(void) 254 { 255 while (apic_read(APIC_ICR) & APIC_ICR_BUSY) 256 cpu_relax(); 257 } 258 259 u32 native_safe_apic_wait_icr_idle(void) 260 { 261 u32 send_status; 262 int timeout; 263 264 timeout = 0; 265 do { 266 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY; 267 if (!send_status) 268 break; 269 inc_irq_stat(icr_read_retry_count); 270 udelay(100); 271 } while (timeout++ < 1000); 272 273 return send_status; 274 } 275 276 void native_apic_icr_write(u32 low, u32 id) 277 { 278 unsigned long flags; 279 280 local_irq_save(flags); 281 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id)); 282 apic_write(APIC_ICR, low); 283 local_irq_restore(flags); 284 } 285 286 u64 native_apic_icr_read(void) 287 { 288 u32 icr1, icr2; 289 290 icr2 = apic_read(APIC_ICR2); 291 icr1 = apic_read(APIC_ICR); 292 293 return icr1 | ((u64)icr2 << 32); 294 } 295 296 #ifdef CONFIG_X86_32 297 /** 298 * get_physical_broadcast - Get number of physical broadcast IDs 299 */ 300 int get_physical_broadcast(void) 301 { 302 return modern_apic() ? 0xff : 0xf; 303 } 304 #endif 305 306 /** 307 * lapic_get_maxlvt - get the maximum number of local vector table entries 308 */ 309 int lapic_get_maxlvt(void) 310 { 311 /* 312 * - we always have APIC integrated on 64bit mode 313 * - 82489DXs do not report # of LVT entries 314 */ 315 return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2; 316 } 317 318 /* 319 * Local APIC timer 320 */ 321 322 /* Clock divisor */ 323 #define APIC_DIVISOR 16 324 #define TSC_DIVISOR 8 325 326 /* 327 * This function sets up the local APIC timer, with a timeout of 328 * 'clocks' APIC bus clock. During calibration we actually call 329 * this function twice on the boot CPU, once with a bogus timeout 330 * value, second time for real. The other (noncalibrating) CPUs 331 * call this function only once, with the real, calibrated value. 332 * 333 * We do reads before writes even if unnecessary, to get around the 334 * P5 APIC double write bug. 335 */ 336 static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen) 337 { 338 unsigned int lvtt_value, tmp_value; 339 340 lvtt_value = LOCAL_TIMER_VECTOR; 341 if (!oneshot) 342 lvtt_value |= APIC_LVT_TIMER_PERIODIC; 343 else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 344 lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE; 345 346 if (!lapic_is_integrated()) 347 lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV); 348 349 if (!irqen) 350 lvtt_value |= APIC_LVT_MASKED; 351 352 apic_write(APIC_LVTT, lvtt_value); 353 354 if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) { 355 /* 356 * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode, 357 * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized. 358 * According to Intel, MFENCE can do the serialization here. 359 */ 360 asm volatile("mfence" : : : "memory"); 361 return; 362 } 363 364 /* 365 * Divide PICLK by 16 366 */ 367 tmp_value = apic_read(APIC_TDCR); 368 apic_write(APIC_TDCR, 369 (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) | 370 APIC_TDR_DIV_16); 371 372 if (!oneshot) 373 apic_write(APIC_TMICT, clocks / APIC_DIVISOR); 374 } 375 376 /* 377 * Setup extended LVT, AMD specific 378 * 379 * Software should use the LVT offsets the BIOS provides. The offsets 380 * are determined by the subsystems using it like those for MCE 381 * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts 382 * are supported. Beginning with family 10h at least 4 offsets are 383 * available. 384 * 385 * Since the offsets must be consistent for all cores, we keep track 386 * of the LVT offsets in software and reserve the offset for the same 387 * vector also to be used on other cores. An offset is freed by 388 * setting the entry to APIC_EILVT_MASKED. 389 * 390 * If the BIOS is right, there should be no conflicts. Otherwise a 391 * "[Firmware Bug]: ..." error message is generated. However, if 392 * software does not properly determines the offsets, it is not 393 * necessarily a BIOS bug. 394 */ 395 396 static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX]; 397 398 static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new) 399 { 400 return (old & APIC_EILVT_MASKED) 401 || (new == APIC_EILVT_MASKED) 402 || ((new & ~APIC_EILVT_MASKED) == old); 403 } 404 405 static unsigned int reserve_eilvt_offset(int offset, unsigned int new) 406 { 407 unsigned int rsvd, vector; 408 409 if (offset >= APIC_EILVT_NR_MAX) 410 return ~0; 411 412 rsvd = atomic_read(&eilvt_offsets[offset]); 413 do { 414 vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */ 415 if (vector && !eilvt_entry_is_changeable(vector, new)) 416 /* may not change if vectors are different */ 417 return rsvd; 418 rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new); 419 } while (rsvd != new); 420 421 rsvd &= ~APIC_EILVT_MASKED; 422 if (rsvd && rsvd != vector) 423 pr_info("LVT offset %d assigned for vector 0x%02x\n", 424 offset, rsvd); 425 426 return new; 427 } 428 429 /* 430 * If mask=1, the LVT entry does not generate interrupts while mask=0 431 * enables the vector. See also the BKDGs. Must be called with 432 * preemption disabled. 433 */ 434 435 int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask) 436 { 437 unsigned long reg = APIC_EILVTn(offset); 438 unsigned int new, old, reserved; 439 440 new = (mask << 16) | (msg_type << 8) | vector; 441 old = apic_read(reg); 442 reserved = reserve_eilvt_offset(offset, new); 443 444 if (reserved != new) { 445 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for " 446 "vector 0x%x, but the register is already in use for " 447 "vector 0x%x on another cpu\n", 448 smp_processor_id(), reg, offset, new, reserved); 449 return -EINVAL; 450 } 451 452 if (!eilvt_entry_is_changeable(old, new)) { 453 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for " 454 "vector 0x%x, but the register is already in use for " 455 "vector 0x%x on this cpu\n", 456 smp_processor_id(), reg, offset, new, old); 457 return -EBUSY; 458 } 459 460 apic_write(reg, new); 461 462 return 0; 463 } 464 EXPORT_SYMBOL_GPL(setup_APIC_eilvt); 465 466 /* 467 * Program the next event, relative to now 468 */ 469 static int lapic_next_event(unsigned long delta, 470 struct clock_event_device *evt) 471 { 472 apic_write(APIC_TMICT, delta); 473 return 0; 474 } 475 476 static int lapic_next_deadline(unsigned long delta, 477 struct clock_event_device *evt) 478 { 479 u64 tsc; 480 481 /* This MSR is special and need a special fence: */ 482 weak_wrmsr_fence(); 483 484 tsc = rdtsc(); 485 wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR)); 486 return 0; 487 } 488 489 static int lapic_timer_shutdown(struct clock_event_device *evt) 490 { 491 unsigned int v; 492 493 /* Lapic used as dummy for broadcast ? */ 494 if (evt->features & CLOCK_EVT_FEAT_DUMMY) 495 return 0; 496 497 v = apic_read(APIC_LVTT); 498 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); 499 apic_write(APIC_LVTT, v); 500 apic_write(APIC_TMICT, 0); 501 return 0; 502 } 503 504 static inline int 505 lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot) 506 { 507 /* Lapic used as dummy for broadcast ? */ 508 if (evt->features & CLOCK_EVT_FEAT_DUMMY) 509 return 0; 510 511 __setup_APIC_LVTT(lapic_timer_period, oneshot, 1); 512 return 0; 513 } 514 515 static int lapic_timer_set_periodic(struct clock_event_device *evt) 516 { 517 return lapic_timer_set_periodic_oneshot(evt, false); 518 } 519 520 static int lapic_timer_set_oneshot(struct clock_event_device *evt) 521 { 522 return lapic_timer_set_periodic_oneshot(evt, true); 523 } 524 525 /* 526 * Local APIC timer broadcast function 527 */ 528 static void lapic_timer_broadcast(const struct cpumask *mask) 529 { 530 #ifdef CONFIG_SMP 531 apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR); 532 #endif 533 } 534 535 536 /* 537 * The local apic timer can be used for any function which is CPU local. 538 */ 539 static struct clock_event_device lapic_clockevent = { 540 .name = "lapic", 541 .features = CLOCK_EVT_FEAT_PERIODIC | 542 CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP 543 | CLOCK_EVT_FEAT_DUMMY, 544 .shift = 32, 545 .set_state_shutdown = lapic_timer_shutdown, 546 .set_state_periodic = lapic_timer_set_periodic, 547 .set_state_oneshot = lapic_timer_set_oneshot, 548 .set_state_oneshot_stopped = lapic_timer_shutdown, 549 .set_next_event = lapic_next_event, 550 .broadcast = lapic_timer_broadcast, 551 .rating = 100, 552 .irq = -1, 553 }; 554 static DEFINE_PER_CPU(struct clock_event_device, lapic_events); 555 556 static const struct x86_cpu_id deadline_match[] __initconst = { 557 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */ 558 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x4, 0x4), 0x0f), /* EX */ 559 560 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_X, 0x0b000020), 561 562 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x2, 0x2), 0x00000011), 563 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x3, 0x3), 0x0700000e), 564 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x4, 0x4), 0x0f00000c), 565 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x5, 0x5), 0x0e000003), 566 567 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x3, 0x3), 0x01000136), 568 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x4, 0x4), 0x02000014), 569 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x5, 0xf), 0), 570 571 X86_MATCH_INTEL_FAM6_MODEL( HASWELL, 0x22), 572 X86_MATCH_INTEL_FAM6_MODEL( HASWELL_L, 0x20), 573 X86_MATCH_INTEL_FAM6_MODEL( HASWELL_G, 0x17), 574 575 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL, 0x25), 576 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_G, 0x17), 577 578 X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE_L, 0xb2), 579 X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE, 0xb2), 580 581 X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE_L, 0x52), 582 X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE, 0x52), 583 584 {}, 585 }; 586 587 static __init bool apic_validate_deadline_timer(void) 588 { 589 const struct x86_cpu_id *m; 590 u32 rev; 591 592 if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 593 return false; 594 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 595 return true; 596 597 m = x86_match_cpu(deadline_match); 598 if (!m) 599 return true; 600 601 rev = (u32)m->driver_data; 602 603 if (boot_cpu_data.microcode >= rev) 604 return true; 605 606 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); 607 pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; " 608 "please update microcode to version: 0x%x (or later)\n", rev); 609 return false; 610 } 611 612 /* 613 * Setup the local APIC timer for this CPU. Copy the initialized values 614 * of the boot CPU and register the clock event in the framework. 615 */ 616 static void setup_APIC_timer(void) 617 { 618 struct clock_event_device *levt = this_cpu_ptr(&lapic_events); 619 620 if (this_cpu_has(X86_FEATURE_ARAT)) { 621 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP; 622 /* Make LAPIC timer preferrable over percpu HPET */ 623 lapic_clockevent.rating = 150; 624 } 625 626 memcpy(levt, &lapic_clockevent, sizeof(*levt)); 627 levt->cpumask = cpumask_of(smp_processor_id()); 628 629 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) { 630 levt->name = "lapic-deadline"; 631 levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC | 632 CLOCK_EVT_FEAT_DUMMY); 633 levt->set_next_event = lapic_next_deadline; 634 clockevents_config_and_register(levt, 635 tsc_khz * (1000 / TSC_DIVISOR), 636 0xF, ~0UL); 637 } else 638 clockevents_register_device(levt); 639 } 640 641 /* 642 * Install the updated TSC frequency from recalibration at the TSC 643 * deadline clockevent devices. 644 */ 645 static void __lapic_update_tsc_freq(void *info) 646 { 647 struct clock_event_device *levt = this_cpu_ptr(&lapic_events); 648 649 if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 650 return; 651 652 clockevents_update_freq(levt, tsc_khz * (1000 / TSC_DIVISOR)); 653 } 654 655 void lapic_update_tsc_freq(void) 656 { 657 /* 658 * The clockevent device's ->mult and ->shift can both be 659 * changed. In order to avoid races, schedule the frequency 660 * update code on each CPU. 661 */ 662 on_each_cpu(__lapic_update_tsc_freq, NULL, 0); 663 } 664 665 /* 666 * In this functions we calibrate APIC bus clocks to the external timer. 667 * 668 * We want to do the calibration only once since we want to have local timer 669 * irqs syncron. CPUs connected by the same APIC bus have the very same bus 670 * frequency. 671 * 672 * This was previously done by reading the PIT/HPET and waiting for a wrap 673 * around to find out, that a tick has elapsed. I have a box, where the PIT 674 * readout is broken, so it never gets out of the wait loop again. This was 675 * also reported by others. 676 * 677 * Monitoring the jiffies value is inaccurate and the clockevents 678 * infrastructure allows us to do a simple substitution of the interrupt 679 * handler. 680 * 681 * The calibration routine also uses the pm_timer when possible, as the PIT 682 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes 683 * back to normal later in the boot process). 684 */ 685 686 #define LAPIC_CAL_LOOPS (HZ/10) 687 688 static __initdata int lapic_cal_loops = -1; 689 static __initdata long lapic_cal_t1, lapic_cal_t2; 690 static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2; 691 static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2; 692 static __initdata unsigned long lapic_cal_j1, lapic_cal_j2; 693 694 /* 695 * Temporary interrupt handler and polled calibration function. 696 */ 697 static void __init lapic_cal_handler(struct clock_event_device *dev) 698 { 699 unsigned long long tsc = 0; 700 long tapic = apic_read(APIC_TMCCT); 701 unsigned long pm = acpi_pm_read_early(); 702 703 if (boot_cpu_has(X86_FEATURE_TSC)) 704 tsc = rdtsc(); 705 706 switch (lapic_cal_loops++) { 707 case 0: 708 lapic_cal_t1 = tapic; 709 lapic_cal_tsc1 = tsc; 710 lapic_cal_pm1 = pm; 711 lapic_cal_j1 = jiffies; 712 break; 713 714 case LAPIC_CAL_LOOPS: 715 lapic_cal_t2 = tapic; 716 lapic_cal_tsc2 = tsc; 717 if (pm < lapic_cal_pm1) 718 pm += ACPI_PM_OVRRUN; 719 lapic_cal_pm2 = pm; 720 lapic_cal_j2 = jiffies; 721 break; 722 } 723 } 724 725 static int __init 726 calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc) 727 { 728 const long pm_100ms = PMTMR_TICKS_PER_SEC / 10; 729 const long pm_thresh = pm_100ms / 100; 730 unsigned long mult; 731 u64 res; 732 733 #ifndef CONFIG_X86_PM_TIMER 734 return -1; 735 #endif 736 737 apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm); 738 739 /* Check, if the PM timer is available */ 740 if (!deltapm) 741 return -1; 742 743 mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22); 744 745 if (deltapm > (pm_100ms - pm_thresh) && 746 deltapm < (pm_100ms + pm_thresh)) { 747 apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n"); 748 return 0; 749 } 750 751 res = (((u64)deltapm) * mult) >> 22; 752 do_div(res, 1000000); 753 pr_warn("APIC calibration not consistent " 754 "with PM-Timer: %ldms instead of 100ms\n", (long)res); 755 756 /* Correct the lapic counter value */ 757 res = (((u64)(*delta)) * pm_100ms); 758 do_div(res, deltapm); 759 pr_info("APIC delta adjusted to PM-Timer: " 760 "%lu (%ld)\n", (unsigned long)res, *delta); 761 *delta = (long)res; 762 763 /* Correct the tsc counter value */ 764 if (boot_cpu_has(X86_FEATURE_TSC)) { 765 res = (((u64)(*deltatsc)) * pm_100ms); 766 do_div(res, deltapm); 767 apic_printk(APIC_VERBOSE, "TSC delta adjusted to " 768 "PM-Timer: %lu (%ld)\n", 769 (unsigned long)res, *deltatsc); 770 *deltatsc = (long)res; 771 } 772 773 return 0; 774 } 775 776 static int __init lapic_init_clockevent(void) 777 { 778 if (!lapic_timer_period) 779 return -1; 780 781 /* Calculate the scaled math multiplication factor */ 782 lapic_clockevent.mult = div_sc(lapic_timer_period/APIC_DIVISOR, 783 TICK_NSEC, lapic_clockevent.shift); 784 lapic_clockevent.max_delta_ns = 785 clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent); 786 lapic_clockevent.max_delta_ticks = 0x7FFFFFFF; 787 lapic_clockevent.min_delta_ns = 788 clockevent_delta2ns(0xF, &lapic_clockevent); 789 lapic_clockevent.min_delta_ticks = 0xF; 790 791 return 0; 792 } 793 794 bool __init apic_needs_pit(void) 795 { 796 /* 797 * If the frequencies are not known, PIT is required for both TSC 798 * and apic timer calibration. 799 */ 800 if (!tsc_khz || !cpu_khz) 801 return true; 802 803 /* Is there an APIC at all or is it disabled? */ 804 if (!boot_cpu_has(X86_FEATURE_APIC) || disable_apic) 805 return true; 806 807 /* 808 * If interrupt delivery mode is legacy PIC or virtual wire without 809 * configuration, the local APIC timer wont be set up. Make sure 810 * that the PIT is initialized. 811 */ 812 if (apic_intr_mode == APIC_PIC || 813 apic_intr_mode == APIC_VIRTUAL_WIRE_NO_CONFIG) 814 return true; 815 816 /* Virt guests may lack ARAT, but still have DEADLINE */ 817 if (!boot_cpu_has(X86_FEATURE_ARAT)) 818 return true; 819 820 /* Deadline timer is based on TSC so no further PIT action required */ 821 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 822 return false; 823 824 /* APIC timer disabled? */ 825 if (disable_apic_timer) 826 return true; 827 /* 828 * The APIC timer frequency is known already, no PIT calibration 829 * required. If unknown, let the PIT be initialized. 830 */ 831 return lapic_timer_period == 0; 832 } 833 834 static int __init calibrate_APIC_clock(void) 835 { 836 struct clock_event_device *levt = this_cpu_ptr(&lapic_events); 837 u64 tsc_perj = 0, tsc_start = 0; 838 unsigned long jif_start; 839 unsigned long deltaj; 840 long delta, deltatsc; 841 int pm_referenced = 0; 842 843 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 844 return 0; 845 846 /* 847 * Check if lapic timer has already been calibrated by platform 848 * specific routine, such as tsc calibration code. If so just fill 849 * in the clockevent structure and return. 850 */ 851 if (!lapic_init_clockevent()) { 852 apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n", 853 lapic_timer_period); 854 /* 855 * Direct calibration methods must have an always running 856 * local APIC timer, no need for broadcast timer. 857 */ 858 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY; 859 return 0; 860 } 861 862 apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n" 863 "calibrating APIC timer ...\n"); 864 865 /* 866 * There are platforms w/o global clockevent devices. Instead of 867 * making the calibration conditional on that, use a polling based 868 * approach everywhere. 869 */ 870 local_irq_disable(); 871 872 /* 873 * Setup the APIC counter to maximum. There is no way the lapic 874 * can underflow in the 100ms detection time frame 875 */ 876 __setup_APIC_LVTT(0xffffffff, 0, 0); 877 878 /* 879 * Methods to terminate the calibration loop: 880 * 1) Global clockevent if available (jiffies) 881 * 2) TSC if available and frequency is known 882 */ 883 jif_start = READ_ONCE(jiffies); 884 885 if (tsc_khz) { 886 tsc_start = rdtsc(); 887 tsc_perj = div_u64((u64)tsc_khz * 1000, HZ); 888 } 889 890 /* 891 * Enable interrupts so the tick can fire, if a global 892 * clockevent device is available 893 */ 894 local_irq_enable(); 895 896 while (lapic_cal_loops <= LAPIC_CAL_LOOPS) { 897 /* Wait for a tick to elapse */ 898 while (1) { 899 if (tsc_khz) { 900 u64 tsc_now = rdtsc(); 901 if ((tsc_now - tsc_start) >= tsc_perj) { 902 tsc_start += tsc_perj; 903 break; 904 } 905 } else { 906 unsigned long jif_now = READ_ONCE(jiffies); 907 908 if (time_after(jif_now, jif_start)) { 909 jif_start = jif_now; 910 break; 911 } 912 } 913 cpu_relax(); 914 } 915 916 /* Invoke the calibration routine */ 917 local_irq_disable(); 918 lapic_cal_handler(NULL); 919 local_irq_enable(); 920 } 921 922 local_irq_disable(); 923 924 /* Build delta t1-t2 as apic timer counts down */ 925 delta = lapic_cal_t1 - lapic_cal_t2; 926 apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta); 927 928 deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1); 929 930 /* we trust the PM based calibration if possible */ 931 pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1, 932 &delta, &deltatsc); 933 934 lapic_timer_period = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS; 935 lapic_init_clockevent(); 936 937 apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta); 938 apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult); 939 apic_printk(APIC_VERBOSE, "..... calibration result: %u\n", 940 lapic_timer_period); 941 942 if (boot_cpu_has(X86_FEATURE_TSC)) { 943 apic_printk(APIC_VERBOSE, "..... CPU clock speed is " 944 "%ld.%04ld MHz.\n", 945 (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ), 946 (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ)); 947 } 948 949 apic_printk(APIC_VERBOSE, "..... host bus clock speed is " 950 "%u.%04u MHz.\n", 951 lapic_timer_period / (1000000 / HZ), 952 lapic_timer_period % (1000000 / HZ)); 953 954 /* 955 * Do a sanity check on the APIC calibration result 956 */ 957 if (lapic_timer_period < (1000000 / HZ)) { 958 local_irq_enable(); 959 pr_warn("APIC frequency too slow, disabling apic timer\n"); 960 return -1; 961 } 962 963 levt->features &= ~CLOCK_EVT_FEAT_DUMMY; 964 965 /* 966 * PM timer calibration failed or not turned on so lets try APIC 967 * timer based calibration, if a global clockevent device is 968 * available. 969 */ 970 if (!pm_referenced && global_clock_event) { 971 apic_printk(APIC_VERBOSE, "... verify APIC timer\n"); 972 973 /* 974 * Setup the apic timer manually 975 */ 976 levt->event_handler = lapic_cal_handler; 977 lapic_timer_set_periodic(levt); 978 lapic_cal_loops = -1; 979 980 /* Let the interrupts run */ 981 local_irq_enable(); 982 983 while (lapic_cal_loops <= LAPIC_CAL_LOOPS) 984 cpu_relax(); 985 986 /* Stop the lapic timer */ 987 local_irq_disable(); 988 lapic_timer_shutdown(levt); 989 990 /* Jiffies delta */ 991 deltaj = lapic_cal_j2 - lapic_cal_j1; 992 apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj); 993 994 /* Check, if the jiffies result is consistent */ 995 if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2) 996 apic_printk(APIC_VERBOSE, "... jiffies result ok\n"); 997 else 998 levt->features |= CLOCK_EVT_FEAT_DUMMY; 999 } 1000 local_irq_enable(); 1001 1002 if (levt->features & CLOCK_EVT_FEAT_DUMMY) { 1003 pr_warn("APIC timer disabled due to verification failure\n"); 1004 return -1; 1005 } 1006 1007 return 0; 1008 } 1009 1010 /* 1011 * Setup the boot APIC 1012 * 1013 * Calibrate and verify the result. 1014 */ 1015 void __init setup_boot_APIC_clock(void) 1016 { 1017 /* 1018 * The local apic timer can be disabled via the kernel 1019 * commandline or from the CPU detection code. Register the lapic 1020 * timer as a dummy clock event source on SMP systems, so the 1021 * broadcast mechanism is used. On UP systems simply ignore it. 1022 */ 1023 if (disable_apic_timer) { 1024 pr_info("Disabling APIC timer\n"); 1025 /* No broadcast on UP ! */ 1026 if (num_possible_cpus() > 1) { 1027 lapic_clockevent.mult = 1; 1028 setup_APIC_timer(); 1029 } 1030 return; 1031 } 1032 1033 if (calibrate_APIC_clock()) { 1034 /* No broadcast on UP ! */ 1035 if (num_possible_cpus() > 1) 1036 setup_APIC_timer(); 1037 return; 1038 } 1039 1040 /* 1041 * If nmi_watchdog is set to IO_APIC, we need the 1042 * PIT/HPET going. Otherwise register lapic as a dummy 1043 * device. 1044 */ 1045 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY; 1046 1047 /* Setup the lapic or request the broadcast */ 1048 setup_APIC_timer(); 1049 amd_e400_c1e_apic_setup(); 1050 } 1051 1052 void setup_secondary_APIC_clock(void) 1053 { 1054 setup_APIC_timer(); 1055 amd_e400_c1e_apic_setup(); 1056 } 1057 1058 /* 1059 * The guts of the apic timer interrupt 1060 */ 1061 static void local_apic_timer_interrupt(void) 1062 { 1063 struct clock_event_device *evt = this_cpu_ptr(&lapic_events); 1064 1065 /* 1066 * Normally we should not be here till LAPIC has been initialized but 1067 * in some cases like kdump, its possible that there is a pending LAPIC 1068 * timer interrupt from previous kernel's context and is delivered in 1069 * new kernel the moment interrupts are enabled. 1070 * 1071 * Interrupts are enabled early and LAPIC is setup much later, hence 1072 * its possible that when we get here evt->event_handler is NULL. 1073 * Check for event_handler being NULL and discard the interrupt as 1074 * spurious. 1075 */ 1076 if (!evt->event_handler) { 1077 pr_warn("Spurious LAPIC timer interrupt on cpu %d\n", 1078 smp_processor_id()); 1079 /* Switch it off */ 1080 lapic_timer_shutdown(evt); 1081 return; 1082 } 1083 1084 /* 1085 * the NMI deadlock-detector uses this. 1086 */ 1087 inc_irq_stat(apic_timer_irqs); 1088 1089 evt->event_handler(evt); 1090 } 1091 1092 /* 1093 * Local APIC timer interrupt. This is the most natural way for doing 1094 * local interrupts, but local timer interrupts can be emulated by 1095 * broadcast interrupts too. [in case the hw doesn't support APIC timers] 1096 * 1097 * [ if a single-CPU system runs an SMP kernel then we call the local 1098 * interrupt as well. Thus we cannot inline the local irq ... ] 1099 */ 1100 DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt) 1101 { 1102 struct pt_regs *old_regs = set_irq_regs(regs); 1103 1104 ack_APIC_irq(); 1105 trace_local_timer_entry(LOCAL_TIMER_VECTOR); 1106 local_apic_timer_interrupt(); 1107 trace_local_timer_exit(LOCAL_TIMER_VECTOR); 1108 1109 set_irq_regs(old_regs); 1110 } 1111 1112 int setup_profiling_timer(unsigned int multiplier) 1113 { 1114 return -EINVAL; 1115 } 1116 1117 /* 1118 * Local APIC start and shutdown 1119 */ 1120 1121 /** 1122 * clear_local_APIC - shutdown the local APIC 1123 * 1124 * This is called, when a CPU is disabled and before rebooting, so the state of 1125 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS 1126 * leftovers during boot. 1127 */ 1128 void clear_local_APIC(void) 1129 { 1130 int maxlvt; 1131 u32 v; 1132 1133 /* APIC hasn't been mapped yet */ 1134 if (!x2apic_mode && !apic_phys) 1135 return; 1136 1137 maxlvt = lapic_get_maxlvt(); 1138 /* 1139 * Masking an LVT entry can trigger a local APIC error 1140 * if the vector is zero. Mask LVTERR first to prevent this. 1141 */ 1142 if (maxlvt >= 3) { 1143 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */ 1144 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED); 1145 } 1146 /* 1147 * Careful: we have to set masks only first to deassert 1148 * any level-triggered sources. 1149 */ 1150 v = apic_read(APIC_LVTT); 1151 apic_write(APIC_LVTT, v | APIC_LVT_MASKED); 1152 v = apic_read(APIC_LVT0); 1153 apic_write(APIC_LVT0, v | APIC_LVT_MASKED); 1154 v = apic_read(APIC_LVT1); 1155 apic_write(APIC_LVT1, v | APIC_LVT_MASKED); 1156 if (maxlvt >= 4) { 1157 v = apic_read(APIC_LVTPC); 1158 apic_write(APIC_LVTPC, v | APIC_LVT_MASKED); 1159 } 1160 1161 /* lets not touch this if we didn't frob it */ 1162 #ifdef CONFIG_X86_THERMAL_VECTOR 1163 if (maxlvt >= 5) { 1164 v = apic_read(APIC_LVTTHMR); 1165 apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED); 1166 } 1167 #endif 1168 #ifdef CONFIG_X86_MCE_INTEL 1169 if (maxlvt >= 6) { 1170 v = apic_read(APIC_LVTCMCI); 1171 if (!(v & APIC_LVT_MASKED)) 1172 apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED); 1173 } 1174 #endif 1175 1176 /* 1177 * Clean APIC state for other OSs: 1178 */ 1179 apic_write(APIC_LVTT, APIC_LVT_MASKED); 1180 apic_write(APIC_LVT0, APIC_LVT_MASKED); 1181 apic_write(APIC_LVT1, APIC_LVT_MASKED); 1182 if (maxlvt >= 3) 1183 apic_write(APIC_LVTERR, APIC_LVT_MASKED); 1184 if (maxlvt >= 4) 1185 apic_write(APIC_LVTPC, APIC_LVT_MASKED); 1186 1187 /* Integrated APIC (!82489DX) ? */ 1188 if (lapic_is_integrated()) { 1189 if (maxlvt > 3) 1190 /* Clear ESR due to Pentium errata 3AP and 11AP */ 1191 apic_write(APIC_ESR, 0); 1192 apic_read(APIC_ESR); 1193 } 1194 } 1195 1196 /** 1197 * apic_soft_disable - Clears and software disables the local APIC on hotplug 1198 * 1199 * Contrary to disable_local_APIC() this does not touch the enable bit in 1200 * MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC 1201 * bus would require a hardware reset as the APIC would lose track of bus 1202 * arbitration. On systems with FSB delivery APICBASE could be disabled, 1203 * but it has to be guaranteed that no interrupt is sent to the APIC while 1204 * in that state and it's not clear from the SDM whether it still responds 1205 * to INIT/SIPI messages. Stay on the safe side and use software disable. 1206 */ 1207 void apic_soft_disable(void) 1208 { 1209 u32 value; 1210 1211 clear_local_APIC(); 1212 1213 /* Soft disable APIC (implies clearing of registers for 82489DX!). */ 1214 value = apic_read(APIC_SPIV); 1215 value &= ~APIC_SPIV_APIC_ENABLED; 1216 apic_write(APIC_SPIV, value); 1217 } 1218 1219 /** 1220 * disable_local_APIC - clear and disable the local APIC 1221 */ 1222 void disable_local_APIC(void) 1223 { 1224 /* APIC hasn't been mapped yet */ 1225 if (!x2apic_mode && !apic_phys) 1226 return; 1227 1228 apic_soft_disable(); 1229 1230 #ifdef CONFIG_X86_32 1231 /* 1232 * When LAPIC was disabled by the BIOS and enabled by the kernel, 1233 * restore the disabled state. 1234 */ 1235 if (enabled_via_apicbase) { 1236 unsigned int l, h; 1237 1238 rdmsr(MSR_IA32_APICBASE, l, h); 1239 l &= ~MSR_IA32_APICBASE_ENABLE; 1240 wrmsr(MSR_IA32_APICBASE, l, h); 1241 } 1242 #endif 1243 } 1244 1245 /* 1246 * If Linux enabled the LAPIC against the BIOS default disable it down before 1247 * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and 1248 * not power-off. Additionally clear all LVT entries before disable_local_APIC 1249 * for the case where Linux didn't enable the LAPIC. 1250 */ 1251 void lapic_shutdown(void) 1252 { 1253 unsigned long flags; 1254 1255 if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config()) 1256 return; 1257 1258 local_irq_save(flags); 1259 1260 #ifdef CONFIG_X86_32 1261 if (!enabled_via_apicbase) 1262 clear_local_APIC(); 1263 else 1264 #endif 1265 disable_local_APIC(); 1266 1267 1268 local_irq_restore(flags); 1269 } 1270 1271 /** 1272 * sync_Arb_IDs - synchronize APIC bus arbitration IDs 1273 */ 1274 void __init sync_Arb_IDs(void) 1275 { 1276 /* 1277 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not 1278 * needed on AMD. 1279 */ 1280 if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 1281 return; 1282 1283 /* 1284 * Wait for idle. 1285 */ 1286 apic_wait_icr_idle(); 1287 1288 apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n"); 1289 apic_write(APIC_ICR, APIC_DEST_ALLINC | 1290 APIC_INT_LEVELTRIG | APIC_DM_INIT); 1291 } 1292 1293 enum apic_intr_mode_id apic_intr_mode __ro_after_init; 1294 1295 static int __init __apic_intr_mode_select(void) 1296 { 1297 /* Check kernel option */ 1298 if (disable_apic) { 1299 pr_info("APIC disabled via kernel command line\n"); 1300 return APIC_PIC; 1301 } 1302 1303 /* Check BIOS */ 1304 #ifdef CONFIG_X86_64 1305 /* On 64-bit, the APIC must be integrated, Check local APIC only */ 1306 if (!boot_cpu_has(X86_FEATURE_APIC)) { 1307 disable_apic = 1; 1308 pr_info("APIC disabled by BIOS\n"); 1309 return APIC_PIC; 1310 } 1311 #else 1312 /* On 32-bit, the APIC may be integrated APIC or 82489DX */ 1313 1314 /* Neither 82489DX nor integrated APIC ? */ 1315 if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) { 1316 disable_apic = 1; 1317 return APIC_PIC; 1318 } 1319 1320 /* If the BIOS pretends there is an integrated APIC ? */ 1321 if (!boot_cpu_has(X86_FEATURE_APIC) && 1322 APIC_INTEGRATED(boot_cpu_apic_version)) { 1323 disable_apic = 1; 1324 pr_err(FW_BUG "Local APIC %d not detected, force emulation\n", 1325 boot_cpu_physical_apicid); 1326 return APIC_PIC; 1327 } 1328 #endif 1329 1330 /* Check MP table or ACPI MADT configuration */ 1331 if (!smp_found_config) { 1332 disable_ioapic_support(); 1333 if (!acpi_lapic) { 1334 pr_info("APIC: ACPI MADT or MP tables are not detected\n"); 1335 return APIC_VIRTUAL_WIRE_NO_CONFIG; 1336 } 1337 return APIC_VIRTUAL_WIRE; 1338 } 1339 1340 #ifdef CONFIG_SMP 1341 /* If SMP should be disabled, then really disable it! */ 1342 if (!setup_max_cpus) { 1343 pr_info("APIC: SMP mode deactivated\n"); 1344 return APIC_SYMMETRIC_IO_NO_ROUTING; 1345 } 1346 1347 if (read_apic_id() != boot_cpu_physical_apicid) { 1348 panic("Boot APIC ID in local APIC unexpected (%d vs %d)", 1349 read_apic_id(), boot_cpu_physical_apicid); 1350 /* Or can we switch back to PIC here? */ 1351 } 1352 #endif 1353 1354 return APIC_SYMMETRIC_IO; 1355 } 1356 1357 /* Select the interrupt delivery mode for the BSP */ 1358 void __init apic_intr_mode_select(void) 1359 { 1360 apic_intr_mode = __apic_intr_mode_select(); 1361 } 1362 1363 /* 1364 * An initial setup of the virtual wire mode. 1365 */ 1366 void __init init_bsp_APIC(void) 1367 { 1368 unsigned int value; 1369 1370 /* 1371 * Don't do the setup now if we have a SMP BIOS as the 1372 * through-I/O-APIC virtual wire mode might be active. 1373 */ 1374 if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC)) 1375 return; 1376 1377 /* 1378 * Do not trust the local APIC being empty at bootup. 1379 */ 1380 clear_local_APIC(); 1381 1382 /* 1383 * Enable APIC. 1384 */ 1385 value = apic_read(APIC_SPIV); 1386 value &= ~APIC_VECTOR_MASK; 1387 value |= APIC_SPIV_APIC_ENABLED; 1388 1389 #ifdef CONFIG_X86_32 1390 /* This bit is reserved on P4/Xeon and should be cleared */ 1391 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && 1392 (boot_cpu_data.x86 == 15)) 1393 value &= ~APIC_SPIV_FOCUS_DISABLED; 1394 else 1395 #endif 1396 value |= APIC_SPIV_FOCUS_DISABLED; 1397 value |= SPURIOUS_APIC_VECTOR; 1398 apic_write(APIC_SPIV, value); 1399 1400 /* 1401 * Set up the virtual wire mode. 1402 */ 1403 apic_write(APIC_LVT0, APIC_DM_EXTINT); 1404 value = APIC_DM_NMI; 1405 if (!lapic_is_integrated()) /* 82489DX */ 1406 value |= APIC_LVT_LEVEL_TRIGGER; 1407 if (apic_extnmi == APIC_EXTNMI_NONE) 1408 value |= APIC_LVT_MASKED; 1409 apic_write(APIC_LVT1, value); 1410 } 1411 1412 static void __init apic_bsp_setup(bool upmode); 1413 1414 /* Init the interrupt delivery mode for the BSP */ 1415 void __init apic_intr_mode_init(void) 1416 { 1417 bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT); 1418 1419 switch (apic_intr_mode) { 1420 case APIC_PIC: 1421 pr_info("APIC: Keep in PIC mode(8259)\n"); 1422 return; 1423 case APIC_VIRTUAL_WIRE: 1424 pr_info("APIC: Switch to virtual wire mode setup\n"); 1425 default_setup_apic_routing(); 1426 break; 1427 case APIC_VIRTUAL_WIRE_NO_CONFIG: 1428 pr_info("APIC: Switch to virtual wire mode setup with no configuration\n"); 1429 upmode = true; 1430 default_setup_apic_routing(); 1431 break; 1432 case APIC_SYMMETRIC_IO: 1433 pr_info("APIC: Switch to symmetric I/O mode setup\n"); 1434 default_setup_apic_routing(); 1435 break; 1436 case APIC_SYMMETRIC_IO_NO_ROUTING: 1437 pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n"); 1438 break; 1439 } 1440 1441 if (x86_platform.apic_post_init) 1442 x86_platform.apic_post_init(); 1443 1444 apic_bsp_setup(upmode); 1445 } 1446 1447 static void lapic_setup_esr(void) 1448 { 1449 unsigned int oldvalue, value, maxlvt; 1450 1451 if (!lapic_is_integrated()) { 1452 pr_info("No ESR for 82489DX.\n"); 1453 return; 1454 } 1455 1456 if (apic->disable_esr) { 1457 /* 1458 * Something untraceable is creating bad interrupts on 1459 * secondary quads ... for the moment, just leave the 1460 * ESR disabled - we can't do anything useful with the 1461 * errors anyway - mbligh 1462 */ 1463 pr_info("Leaving ESR disabled.\n"); 1464 return; 1465 } 1466 1467 maxlvt = lapic_get_maxlvt(); 1468 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ 1469 apic_write(APIC_ESR, 0); 1470 oldvalue = apic_read(APIC_ESR); 1471 1472 /* enables sending errors */ 1473 value = ERROR_APIC_VECTOR; 1474 apic_write(APIC_LVTERR, value); 1475 1476 /* 1477 * spec says clear errors after enabling vector. 1478 */ 1479 if (maxlvt > 3) 1480 apic_write(APIC_ESR, 0); 1481 value = apic_read(APIC_ESR); 1482 if (value != oldvalue) 1483 apic_printk(APIC_VERBOSE, "ESR value before enabling " 1484 "vector: 0x%08x after: 0x%08x\n", 1485 oldvalue, value); 1486 } 1487 1488 #define APIC_IR_REGS APIC_ISR_NR 1489 #define APIC_IR_BITS (APIC_IR_REGS * 32) 1490 #define APIC_IR_MAPSIZE (APIC_IR_BITS / BITS_PER_LONG) 1491 1492 union apic_ir { 1493 unsigned long map[APIC_IR_MAPSIZE]; 1494 u32 regs[APIC_IR_REGS]; 1495 }; 1496 1497 static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr) 1498 { 1499 int i, bit; 1500 1501 /* Read the IRRs */ 1502 for (i = 0; i < APIC_IR_REGS; i++) 1503 irr->regs[i] = apic_read(APIC_IRR + i * 0x10); 1504 1505 /* Read the ISRs */ 1506 for (i = 0; i < APIC_IR_REGS; i++) 1507 isr->regs[i] = apic_read(APIC_ISR + i * 0x10); 1508 1509 /* 1510 * If the ISR map is not empty. ACK the APIC and run another round 1511 * to verify whether a pending IRR has been unblocked and turned 1512 * into a ISR. 1513 */ 1514 if (!bitmap_empty(isr->map, APIC_IR_BITS)) { 1515 /* 1516 * There can be multiple ISR bits set when a high priority 1517 * interrupt preempted a lower priority one. Issue an ACK 1518 * per set bit. 1519 */ 1520 for_each_set_bit(bit, isr->map, APIC_IR_BITS) 1521 ack_APIC_irq(); 1522 return true; 1523 } 1524 1525 return !bitmap_empty(irr->map, APIC_IR_BITS); 1526 } 1527 1528 /* 1529 * After a crash, we no longer service the interrupts and a pending 1530 * interrupt from previous kernel might still have ISR bit set. 1531 * 1532 * Most probably by now the CPU has serviced that pending interrupt and it 1533 * might not have done the ack_APIC_irq() because it thought, interrupt 1534 * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear 1535 * the ISR bit and cpu thinks it has already serivced the interrupt. Hence 1536 * a vector might get locked. It was noticed for timer irq (vector 1537 * 0x31). Issue an extra EOI to clear ISR. 1538 * 1539 * If there are pending IRR bits they turn into ISR bits after a higher 1540 * priority ISR bit has been acked. 1541 */ 1542 static void apic_pending_intr_clear(void) 1543 { 1544 union apic_ir irr, isr; 1545 unsigned int i; 1546 1547 /* 512 loops are way oversized and give the APIC a chance to obey. */ 1548 for (i = 0; i < 512; i++) { 1549 if (!apic_check_and_ack(&irr, &isr)) 1550 return; 1551 } 1552 /* Dump the IRR/ISR content if that failed */ 1553 pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr.map, isr.map); 1554 } 1555 1556 /** 1557 * setup_local_APIC - setup the local APIC 1558 * 1559 * Used to setup local APIC while initializing BSP or bringing up APs. 1560 * Always called with preemption disabled. 1561 */ 1562 static void setup_local_APIC(void) 1563 { 1564 int cpu = smp_processor_id(); 1565 unsigned int value; 1566 1567 if (disable_apic) { 1568 disable_ioapic_support(); 1569 return; 1570 } 1571 1572 /* 1573 * If this comes from kexec/kcrash the APIC might be enabled in 1574 * SPIV. Soft disable it before doing further initialization. 1575 */ 1576 value = apic_read(APIC_SPIV); 1577 value &= ~APIC_SPIV_APIC_ENABLED; 1578 apic_write(APIC_SPIV, value); 1579 1580 #ifdef CONFIG_X86_32 1581 /* Pound the ESR really hard over the head with a big hammer - mbligh */ 1582 if (lapic_is_integrated() && apic->disable_esr) { 1583 apic_write(APIC_ESR, 0); 1584 apic_write(APIC_ESR, 0); 1585 apic_write(APIC_ESR, 0); 1586 apic_write(APIC_ESR, 0); 1587 } 1588 #endif 1589 /* 1590 * Double-check whether this APIC is really registered. 1591 * This is meaningless in clustered apic mode, so we skip it. 1592 */ 1593 BUG_ON(!apic->apic_id_registered()); 1594 1595 /* 1596 * Intel recommends to set DFR, LDR and TPR before enabling 1597 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel 1598 * document number 292116). So here it goes... 1599 */ 1600 apic->init_apic_ldr(); 1601 1602 #ifdef CONFIG_X86_32 1603 if (apic->dest_mode_logical) { 1604 int logical_apicid, ldr_apicid; 1605 1606 /* 1607 * APIC LDR is initialized. If logical_apicid mapping was 1608 * initialized during get_smp_config(), make sure it matches 1609 * the actual value. 1610 */ 1611 logical_apicid = early_per_cpu(x86_cpu_to_logical_apicid, cpu); 1612 ldr_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR)); 1613 if (logical_apicid != BAD_APICID) 1614 WARN_ON(logical_apicid != ldr_apicid); 1615 /* Always use the value from LDR. */ 1616 early_per_cpu(x86_cpu_to_logical_apicid, cpu) = ldr_apicid; 1617 } 1618 #endif 1619 1620 /* 1621 * Set Task Priority to 'accept all except vectors 0-31'. An APIC 1622 * vector in the 16-31 range could be delivered if TPR == 0, but we 1623 * would think it's an exception and terrible things will happen. We 1624 * never change this later on. 1625 */ 1626 value = apic_read(APIC_TASKPRI); 1627 value &= ~APIC_TPRI_MASK; 1628 value |= 0x10; 1629 apic_write(APIC_TASKPRI, value); 1630 1631 /* Clear eventually stale ISR/IRR bits */ 1632 apic_pending_intr_clear(); 1633 1634 /* 1635 * Now that we are all set up, enable the APIC 1636 */ 1637 value = apic_read(APIC_SPIV); 1638 value &= ~APIC_VECTOR_MASK; 1639 /* 1640 * Enable APIC 1641 */ 1642 value |= APIC_SPIV_APIC_ENABLED; 1643 1644 #ifdef CONFIG_X86_32 1645 /* 1646 * Some unknown Intel IO/APIC (or APIC) errata is biting us with 1647 * certain networking cards. If high frequency interrupts are 1648 * happening on a particular IOAPIC pin, plus the IOAPIC routing 1649 * entry is masked/unmasked at a high rate as well then sooner or 1650 * later IOAPIC line gets 'stuck', no more interrupts are received 1651 * from the device. If focus CPU is disabled then the hang goes 1652 * away, oh well :-( 1653 * 1654 * [ This bug can be reproduced easily with a level-triggered 1655 * PCI Ne2000 networking cards and PII/PIII processors, dual 1656 * BX chipset. ] 1657 */ 1658 /* 1659 * Actually disabling the focus CPU check just makes the hang less 1660 * frequent as it makes the interrupt distributon model be more 1661 * like LRU than MRU (the short-term load is more even across CPUs). 1662 */ 1663 1664 /* 1665 * - enable focus processor (bit==0) 1666 * - 64bit mode always use processor focus 1667 * so no need to set it 1668 */ 1669 value &= ~APIC_SPIV_FOCUS_DISABLED; 1670 #endif 1671 1672 /* 1673 * Set spurious IRQ vector 1674 */ 1675 value |= SPURIOUS_APIC_VECTOR; 1676 apic_write(APIC_SPIV, value); 1677 1678 perf_events_lapic_init(); 1679 1680 /* 1681 * Set up LVT0, LVT1: 1682 * 1683 * set up through-local-APIC on the boot CPU's LINT0. This is not 1684 * strictly necessary in pure symmetric-IO mode, but sometimes 1685 * we delegate interrupts to the 8259A. 1686 */ 1687 /* 1688 * TODO: set up through-local-APIC from through-I/O-APIC? --macro 1689 */ 1690 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED; 1691 if (!cpu && (pic_mode || !value || skip_ioapic_setup)) { 1692 value = APIC_DM_EXTINT; 1693 apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu); 1694 } else { 1695 value = APIC_DM_EXTINT | APIC_LVT_MASKED; 1696 apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu); 1697 } 1698 apic_write(APIC_LVT0, value); 1699 1700 /* 1701 * Only the BSP sees the LINT1 NMI signal by default. This can be 1702 * modified by apic_extnmi= boot option. 1703 */ 1704 if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) || 1705 apic_extnmi == APIC_EXTNMI_ALL) 1706 value = APIC_DM_NMI; 1707 else 1708 value = APIC_DM_NMI | APIC_LVT_MASKED; 1709 1710 /* Is 82489DX ? */ 1711 if (!lapic_is_integrated()) 1712 value |= APIC_LVT_LEVEL_TRIGGER; 1713 apic_write(APIC_LVT1, value); 1714 1715 #ifdef CONFIG_X86_MCE_INTEL 1716 /* Recheck CMCI information after local APIC is up on CPU #0 */ 1717 if (!cpu) 1718 cmci_recheck(); 1719 #endif 1720 } 1721 1722 static void end_local_APIC_setup(void) 1723 { 1724 lapic_setup_esr(); 1725 1726 #ifdef CONFIG_X86_32 1727 { 1728 unsigned int value; 1729 /* Disable the local apic timer */ 1730 value = apic_read(APIC_LVTT); 1731 value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); 1732 apic_write(APIC_LVTT, value); 1733 } 1734 #endif 1735 1736 apic_pm_activate(); 1737 } 1738 1739 /* 1740 * APIC setup function for application processors. Called from smpboot.c 1741 */ 1742 void apic_ap_setup(void) 1743 { 1744 setup_local_APIC(); 1745 end_local_APIC_setup(); 1746 } 1747 1748 #ifdef CONFIG_X86_X2APIC 1749 int x2apic_mode; 1750 EXPORT_SYMBOL_GPL(x2apic_mode); 1751 1752 enum { 1753 X2APIC_OFF, 1754 X2APIC_ON, 1755 X2APIC_DISABLED, 1756 }; 1757 static int x2apic_state; 1758 1759 static void __x2apic_disable(void) 1760 { 1761 u64 msr; 1762 1763 if (!boot_cpu_has(X86_FEATURE_APIC)) 1764 return; 1765 1766 rdmsrl(MSR_IA32_APICBASE, msr); 1767 if (!(msr & X2APIC_ENABLE)) 1768 return; 1769 /* Disable xapic and x2apic first and then reenable xapic mode */ 1770 wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE)); 1771 wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE); 1772 printk_once(KERN_INFO "x2apic disabled\n"); 1773 } 1774 1775 static void __x2apic_enable(void) 1776 { 1777 u64 msr; 1778 1779 rdmsrl(MSR_IA32_APICBASE, msr); 1780 if (msr & X2APIC_ENABLE) 1781 return; 1782 wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE); 1783 printk_once(KERN_INFO "x2apic enabled\n"); 1784 } 1785 1786 static int __init setup_nox2apic(char *str) 1787 { 1788 if (x2apic_enabled()) { 1789 int apicid = native_apic_msr_read(APIC_ID); 1790 1791 if (apicid >= 255) { 1792 pr_warn("Apicid: %08x, cannot enforce nox2apic\n", 1793 apicid); 1794 return 0; 1795 } 1796 pr_warn("x2apic already enabled.\n"); 1797 __x2apic_disable(); 1798 } 1799 setup_clear_cpu_cap(X86_FEATURE_X2APIC); 1800 x2apic_state = X2APIC_DISABLED; 1801 x2apic_mode = 0; 1802 return 0; 1803 } 1804 early_param("nox2apic", setup_nox2apic); 1805 1806 /* Called from cpu_init() to enable x2apic on (secondary) cpus */ 1807 void x2apic_setup(void) 1808 { 1809 /* 1810 * If x2apic is not in ON state, disable it if already enabled 1811 * from BIOS. 1812 */ 1813 if (x2apic_state != X2APIC_ON) { 1814 __x2apic_disable(); 1815 return; 1816 } 1817 __x2apic_enable(); 1818 } 1819 1820 static __init void x2apic_disable(void) 1821 { 1822 u32 x2apic_id, state = x2apic_state; 1823 1824 x2apic_mode = 0; 1825 x2apic_state = X2APIC_DISABLED; 1826 1827 if (state != X2APIC_ON) 1828 return; 1829 1830 x2apic_id = read_apic_id(); 1831 if (x2apic_id >= 255) 1832 panic("Cannot disable x2apic, id: %08x\n", x2apic_id); 1833 1834 __x2apic_disable(); 1835 register_lapic_address(mp_lapic_addr); 1836 } 1837 1838 static __init void x2apic_enable(void) 1839 { 1840 if (x2apic_state != X2APIC_OFF) 1841 return; 1842 1843 x2apic_mode = 1; 1844 x2apic_state = X2APIC_ON; 1845 __x2apic_enable(); 1846 } 1847 1848 static __init void try_to_enable_x2apic(int remap_mode) 1849 { 1850 if (x2apic_state == X2APIC_DISABLED) 1851 return; 1852 1853 if (remap_mode != IRQ_REMAP_X2APIC_MODE) { 1854 u32 apic_limit = 255; 1855 1856 /* 1857 * Using X2APIC without IR is not architecturally supported 1858 * on bare metal but may be supported in guests. 1859 */ 1860 if (!x86_init.hyper.x2apic_available()) { 1861 pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n"); 1862 x2apic_disable(); 1863 return; 1864 } 1865 1866 /* 1867 * If the hypervisor supports extended destination ID in 1868 * MSI, that increases the maximum APIC ID that can be 1869 * used for non-remapped IRQ domains. 1870 */ 1871 if (x86_init.hyper.msi_ext_dest_id()) { 1872 virt_ext_dest_id = 1; 1873 apic_limit = 32767; 1874 } 1875 1876 /* 1877 * Without IR, all CPUs can be addressed by IOAPIC/MSI only 1878 * in physical mode, and CPUs with an APIC ID that cannnot 1879 * be addressed must not be brought online. 1880 */ 1881 x2apic_set_max_apicid(apic_limit); 1882 x2apic_phys = 1; 1883 } 1884 x2apic_enable(); 1885 } 1886 1887 void __init check_x2apic(void) 1888 { 1889 if (x2apic_enabled()) { 1890 pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n"); 1891 x2apic_mode = 1; 1892 x2apic_state = X2APIC_ON; 1893 } else if (!boot_cpu_has(X86_FEATURE_X2APIC)) { 1894 x2apic_state = X2APIC_DISABLED; 1895 } 1896 } 1897 #else /* CONFIG_X86_X2APIC */ 1898 static int __init validate_x2apic(void) 1899 { 1900 if (!apic_is_x2apic_enabled()) 1901 return 0; 1902 /* 1903 * Checkme: Can we simply turn off x2apic here instead of panic? 1904 */ 1905 panic("BIOS has enabled x2apic but kernel doesn't support x2apic, please disable x2apic in BIOS.\n"); 1906 } 1907 early_initcall(validate_x2apic); 1908 1909 static inline void try_to_enable_x2apic(int remap_mode) { } 1910 static inline void __x2apic_enable(void) { } 1911 #endif /* !CONFIG_X86_X2APIC */ 1912 1913 void __init enable_IR_x2apic(void) 1914 { 1915 unsigned long flags; 1916 int ret, ir_stat; 1917 1918 if (skip_ioapic_setup) { 1919 pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n"); 1920 return; 1921 } 1922 1923 ir_stat = irq_remapping_prepare(); 1924 if (ir_stat < 0 && !x2apic_supported()) 1925 return; 1926 1927 ret = save_ioapic_entries(); 1928 if (ret) { 1929 pr_info("Saving IO-APIC state failed: %d\n", ret); 1930 return; 1931 } 1932 1933 local_irq_save(flags); 1934 legacy_pic->mask_all(); 1935 mask_ioapic_entries(); 1936 1937 /* If irq_remapping_prepare() succeeded, try to enable it */ 1938 if (ir_stat >= 0) 1939 ir_stat = irq_remapping_enable(); 1940 /* ir_stat contains the remap mode or an error code */ 1941 try_to_enable_x2apic(ir_stat); 1942 1943 if (ir_stat < 0) 1944 restore_ioapic_entries(); 1945 legacy_pic->restore_mask(); 1946 local_irq_restore(flags); 1947 } 1948 1949 #ifdef CONFIG_X86_64 1950 /* 1951 * Detect and enable local APICs on non-SMP boards. 1952 * Original code written by Keir Fraser. 1953 * On AMD64 we trust the BIOS - if it says no APIC it is likely 1954 * not correctly set up (usually the APIC timer won't work etc.) 1955 */ 1956 static int __init detect_init_APIC(void) 1957 { 1958 if (!boot_cpu_has(X86_FEATURE_APIC)) { 1959 pr_info("No local APIC present\n"); 1960 return -1; 1961 } 1962 1963 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; 1964 return 0; 1965 } 1966 #else 1967 1968 static int __init apic_verify(void) 1969 { 1970 u32 features, h, l; 1971 1972 /* 1973 * The APIC feature bit should now be enabled 1974 * in `cpuid' 1975 */ 1976 features = cpuid_edx(1); 1977 if (!(features & (1 << X86_FEATURE_APIC))) { 1978 pr_warn("Could not enable APIC!\n"); 1979 return -1; 1980 } 1981 set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC); 1982 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; 1983 1984 /* The BIOS may have set up the APIC at some other address */ 1985 if (boot_cpu_data.x86 >= 6) { 1986 rdmsr(MSR_IA32_APICBASE, l, h); 1987 if (l & MSR_IA32_APICBASE_ENABLE) 1988 mp_lapic_addr = l & MSR_IA32_APICBASE_BASE; 1989 } 1990 1991 pr_info("Found and enabled local APIC!\n"); 1992 return 0; 1993 } 1994 1995 int __init apic_force_enable(unsigned long addr) 1996 { 1997 u32 h, l; 1998 1999 if (disable_apic) 2000 return -1; 2001 2002 /* 2003 * Some BIOSes disable the local APIC in the APIC_BASE 2004 * MSR. This can only be done in software for Intel P6 or later 2005 * and AMD K7 (Model > 1) or later. 2006 */ 2007 if (boot_cpu_data.x86 >= 6) { 2008 rdmsr(MSR_IA32_APICBASE, l, h); 2009 if (!(l & MSR_IA32_APICBASE_ENABLE)) { 2010 pr_info("Local APIC disabled by BIOS -- reenabling.\n"); 2011 l &= ~MSR_IA32_APICBASE_BASE; 2012 l |= MSR_IA32_APICBASE_ENABLE | addr; 2013 wrmsr(MSR_IA32_APICBASE, l, h); 2014 enabled_via_apicbase = 1; 2015 } 2016 } 2017 return apic_verify(); 2018 } 2019 2020 /* 2021 * Detect and initialize APIC 2022 */ 2023 static int __init detect_init_APIC(void) 2024 { 2025 /* Disabled by kernel option? */ 2026 if (disable_apic) 2027 return -1; 2028 2029 switch (boot_cpu_data.x86_vendor) { 2030 case X86_VENDOR_AMD: 2031 if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) || 2032 (boot_cpu_data.x86 >= 15)) 2033 break; 2034 goto no_apic; 2035 case X86_VENDOR_HYGON: 2036 break; 2037 case X86_VENDOR_INTEL: 2038 if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 || 2039 (boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC))) 2040 break; 2041 goto no_apic; 2042 default: 2043 goto no_apic; 2044 } 2045 2046 if (!boot_cpu_has(X86_FEATURE_APIC)) { 2047 /* 2048 * Over-ride BIOS and try to enable the local APIC only if 2049 * "lapic" specified. 2050 */ 2051 if (!force_enable_local_apic) { 2052 pr_info("Local APIC disabled by BIOS -- " 2053 "you can enable it with \"lapic\"\n"); 2054 return -1; 2055 } 2056 if (apic_force_enable(APIC_DEFAULT_PHYS_BASE)) 2057 return -1; 2058 } else { 2059 if (apic_verify()) 2060 return -1; 2061 } 2062 2063 apic_pm_activate(); 2064 2065 return 0; 2066 2067 no_apic: 2068 pr_info("No local APIC present or hardware disabled\n"); 2069 return -1; 2070 } 2071 #endif 2072 2073 /** 2074 * init_apic_mappings - initialize APIC mappings 2075 */ 2076 void __init init_apic_mappings(void) 2077 { 2078 unsigned int new_apicid; 2079 2080 if (apic_validate_deadline_timer()) 2081 pr_info("TSC deadline timer available\n"); 2082 2083 if (x2apic_mode) { 2084 boot_cpu_physical_apicid = read_apic_id(); 2085 return; 2086 } 2087 2088 /* If no local APIC can be found return early */ 2089 if (!smp_found_config && detect_init_APIC()) { 2090 /* lets NOP'ify apic operations */ 2091 pr_info("APIC: disable apic facility\n"); 2092 apic_disable(); 2093 } else { 2094 apic_phys = mp_lapic_addr; 2095 2096 /* 2097 * If the system has ACPI MADT tables or MP info, the LAPIC 2098 * address is already registered. 2099 */ 2100 if (!acpi_lapic && !smp_found_config) 2101 register_lapic_address(apic_phys); 2102 } 2103 2104 /* 2105 * Fetch the APIC ID of the BSP in case we have a 2106 * default configuration (or the MP table is broken). 2107 */ 2108 new_apicid = read_apic_id(); 2109 if (boot_cpu_physical_apicid != new_apicid) { 2110 boot_cpu_physical_apicid = new_apicid; 2111 /* 2112 * yeah -- we lie about apic_version 2113 * in case if apic was disabled via boot option 2114 * but it's not a problem for SMP compiled kernel 2115 * since apic_intr_mode_select is prepared for such 2116 * a case and disable smp mode 2117 */ 2118 boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR)); 2119 } 2120 } 2121 2122 void __init register_lapic_address(unsigned long address) 2123 { 2124 mp_lapic_addr = address; 2125 2126 if (!x2apic_mode) { 2127 set_fixmap_nocache(FIX_APIC_BASE, address); 2128 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", 2129 APIC_BASE, address); 2130 } 2131 if (boot_cpu_physical_apicid == -1U) { 2132 boot_cpu_physical_apicid = read_apic_id(); 2133 boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR)); 2134 } 2135 } 2136 2137 /* 2138 * Local APIC interrupts 2139 */ 2140 2141 /* 2142 * Common handling code for spurious_interrupt and spurious_vector entry 2143 * points below. No point in allowing the compiler to inline it twice. 2144 */ 2145 static noinline void handle_spurious_interrupt(u8 vector) 2146 { 2147 u32 v; 2148 2149 trace_spurious_apic_entry(vector); 2150 2151 inc_irq_stat(irq_spurious_count); 2152 2153 /* 2154 * If this is a spurious interrupt then do not acknowledge 2155 */ 2156 if (vector == SPURIOUS_APIC_VECTOR) { 2157 /* See SDM vol 3 */ 2158 pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n", 2159 smp_processor_id()); 2160 goto out; 2161 } 2162 2163 /* 2164 * If it is a vectored one, verify it's set in the ISR. If set, 2165 * acknowledge it. 2166 */ 2167 v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1)); 2168 if (v & (1 << (vector & 0x1f))) { 2169 pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n", 2170 vector, smp_processor_id()); 2171 ack_APIC_irq(); 2172 } else { 2173 pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n", 2174 vector, smp_processor_id()); 2175 } 2176 out: 2177 trace_spurious_apic_exit(vector); 2178 } 2179 2180 /** 2181 * spurious_interrupt - Catch all for interrupts raised on unused vectors 2182 * @regs: Pointer to pt_regs on stack 2183 * @vector: The vector number 2184 * 2185 * This is invoked from ASM entry code to catch all interrupts which 2186 * trigger on an entry which is routed to the common_spurious idtentry 2187 * point. 2188 */ 2189 DEFINE_IDTENTRY_IRQ(spurious_interrupt) 2190 { 2191 handle_spurious_interrupt(vector); 2192 } 2193 2194 DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt) 2195 { 2196 handle_spurious_interrupt(SPURIOUS_APIC_VECTOR); 2197 } 2198 2199 /* 2200 * This interrupt should never happen with our APIC/SMP architecture 2201 */ 2202 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt) 2203 { 2204 static const char * const error_interrupt_reason[] = { 2205 "Send CS error", /* APIC Error Bit 0 */ 2206 "Receive CS error", /* APIC Error Bit 1 */ 2207 "Send accept error", /* APIC Error Bit 2 */ 2208 "Receive accept error", /* APIC Error Bit 3 */ 2209 "Redirectable IPI", /* APIC Error Bit 4 */ 2210 "Send illegal vector", /* APIC Error Bit 5 */ 2211 "Received illegal vector", /* APIC Error Bit 6 */ 2212 "Illegal register address", /* APIC Error Bit 7 */ 2213 }; 2214 u32 v, i = 0; 2215 2216 trace_error_apic_entry(ERROR_APIC_VECTOR); 2217 2218 /* First tickle the hardware, only then report what went on. -- REW */ 2219 if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */ 2220 apic_write(APIC_ESR, 0); 2221 v = apic_read(APIC_ESR); 2222 ack_APIC_irq(); 2223 atomic_inc(&irq_err_count); 2224 2225 apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x", 2226 smp_processor_id(), v); 2227 2228 v &= 0xff; 2229 while (v) { 2230 if (v & 0x1) 2231 apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]); 2232 i++; 2233 v >>= 1; 2234 } 2235 2236 apic_printk(APIC_DEBUG, KERN_CONT "\n"); 2237 2238 trace_error_apic_exit(ERROR_APIC_VECTOR); 2239 } 2240 2241 /** 2242 * connect_bsp_APIC - attach the APIC to the interrupt system 2243 */ 2244 static void __init connect_bsp_APIC(void) 2245 { 2246 #ifdef CONFIG_X86_32 2247 if (pic_mode) { 2248 /* 2249 * Do not trust the local APIC being empty at bootup. 2250 */ 2251 clear_local_APIC(); 2252 /* 2253 * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's 2254 * local APIC to INT and NMI lines. 2255 */ 2256 apic_printk(APIC_VERBOSE, "leaving PIC mode, " 2257 "enabling APIC mode.\n"); 2258 imcr_pic_to_apic(); 2259 } 2260 #endif 2261 } 2262 2263 /** 2264 * disconnect_bsp_APIC - detach the APIC from the interrupt system 2265 * @virt_wire_setup: indicates, whether virtual wire mode is selected 2266 * 2267 * Virtual wire mode is necessary to deliver legacy interrupts even when the 2268 * APIC is disabled. 2269 */ 2270 void disconnect_bsp_APIC(int virt_wire_setup) 2271 { 2272 unsigned int value; 2273 2274 #ifdef CONFIG_X86_32 2275 if (pic_mode) { 2276 /* 2277 * Put the board back into PIC mode (has an effect only on 2278 * certain older boards). Note that APIC interrupts, including 2279 * IPIs, won't work beyond this point! The only exception are 2280 * INIT IPIs. 2281 */ 2282 apic_printk(APIC_VERBOSE, "disabling APIC mode, " 2283 "entering PIC mode.\n"); 2284 imcr_apic_to_pic(); 2285 return; 2286 } 2287 #endif 2288 2289 /* Go back to Virtual Wire compatibility mode */ 2290 2291 /* For the spurious interrupt use vector F, and enable it */ 2292 value = apic_read(APIC_SPIV); 2293 value &= ~APIC_VECTOR_MASK; 2294 value |= APIC_SPIV_APIC_ENABLED; 2295 value |= 0xf; 2296 apic_write(APIC_SPIV, value); 2297 2298 if (!virt_wire_setup) { 2299 /* 2300 * For LVT0 make it edge triggered, active high, 2301 * external and enabled 2302 */ 2303 value = apic_read(APIC_LVT0); 2304 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | 2305 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | 2306 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED); 2307 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; 2308 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT); 2309 apic_write(APIC_LVT0, value); 2310 } else { 2311 /* Disable LVT0 */ 2312 apic_write(APIC_LVT0, APIC_LVT_MASKED); 2313 } 2314 2315 /* 2316 * For LVT1 make it edge triggered, active high, 2317 * nmi and enabled 2318 */ 2319 value = apic_read(APIC_LVT1); 2320 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | 2321 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | 2322 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED); 2323 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; 2324 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI); 2325 apic_write(APIC_LVT1, value); 2326 } 2327 2328 /* 2329 * The number of allocated logical CPU IDs. Since logical CPU IDs are allocated 2330 * contiguously, it equals to current allocated max logical CPU ID plus 1. 2331 * All allocated CPU IDs should be in the [0, nr_logical_cpuids) range, 2332 * so the maximum of nr_logical_cpuids is nr_cpu_ids. 2333 * 2334 * NOTE: Reserve 0 for BSP. 2335 */ 2336 static int nr_logical_cpuids = 1; 2337 2338 /* 2339 * Used to store mapping between logical CPU IDs and APIC IDs. 2340 */ 2341 static int cpuid_to_apicid[] = { 2342 [0 ... NR_CPUS - 1] = -1, 2343 }; 2344 2345 #ifdef CONFIG_SMP 2346 /** 2347 * apic_id_is_primary_thread - Check whether APIC ID belongs to a primary thread 2348 * @apicid: APIC ID to check 2349 */ 2350 bool apic_id_is_primary_thread(unsigned int apicid) 2351 { 2352 u32 mask; 2353 2354 if (smp_num_siblings == 1) 2355 return true; 2356 /* Isolate the SMT bit(s) in the APICID and check for 0 */ 2357 mask = (1U << (fls(smp_num_siblings) - 1)) - 1; 2358 return !(apicid & mask); 2359 } 2360 #endif 2361 2362 /* 2363 * Should use this API to allocate logical CPU IDs to keep nr_logical_cpuids 2364 * and cpuid_to_apicid[] synchronized. 2365 */ 2366 static int allocate_logical_cpuid(int apicid) 2367 { 2368 int i; 2369 2370 /* 2371 * cpuid <-> apicid mapping is persistent, so when a cpu is up, 2372 * check if the kernel has allocated a cpuid for it. 2373 */ 2374 for (i = 0; i < nr_logical_cpuids; i++) { 2375 if (cpuid_to_apicid[i] == apicid) 2376 return i; 2377 } 2378 2379 /* Allocate a new cpuid. */ 2380 if (nr_logical_cpuids >= nr_cpu_ids) { 2381 WARN_ONCE(1, "APIC: NR_CPUS/possible_cpus limit of %u reached. " 2382 "Processor %d/0x%x and the rest are ignored.\n", 2383 nr_cpu_ids, nr_logical_cpuids, apicid); 2384 return -EINVAL; 2385 } 2386 2387 cpuid_to_apicid[nr_logical_cpuids] = apicid; 2388 return nr_logical_cpuids++; 2389 } 2390 2391 int generic_processor_info(int apicid, int version) 2392 { 2393 int cpu, max = nr_cpu_ids; 2394 bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid, 2395 phys_cpu_present_map); 2396 2397 /* 2398 * boot_cpu_physical_apicid is designed to have the apicid 2399 * returned by read_apic_id(), i.e, the apicid of the 2400 * currently booting-up processor. However, on some platforms, 2401 * it is temporarily modified by the apicid reported as BSP 2402 * through MP table. Concretely: 2403 * 2404 * - arch/x86/kernel/mpparse.c: MP_processor_info() 2405 * - arch/x86/mm/amdtopology.c: amd_numa_init() 2406 * 2407 * This function is executed with the modified 2408 * boot_cpu_physical_apicid. So, disabled_cpu_apicid kernel 2409 * parameter doesn't work to disable APs on kdump 2nd kernel. 2410 * 2411 * Since fixing handling of boot_cpu_physical_apicid requires 2412 * another discussion and tests on each platform, we leave it 2413 * for now and here we use read_apic_id() directly in this 2414 * function, generic_processor_info(). 2415 */ 2416 if (disabled_cpu_apicid != BAD_APICID && 2417 disabled_cpu_apicid != read_apic_id() && 2418 disabled_cpu_apicid == apicid) { 2419 int thiscpu = num_processors + disabled_cpus; 2420 2421 pr_warn("APIC: Disabling requested cpu." 2422 " Processor %d/0x%x ignored.\n", thiscpu, apicid); 2423 2424 disabled_cpus++; 2425 return -ENODEV; 2426 } 2427 2428 /* 2429 * If boot cpu has not been detected yet, then only allow upto 2430 * nr_cpu_ids - 1 processors and keep one slot free for boot cpu 2431 */ 2432 if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 && 2433 apicid != boot_cpu_physical_apicid) { 2434 int thiscpu = max + disabled_cpus - 1; 2435 2436 pr_warn("APIC: NR_CPUS/possible_cpus limit of %i almost" 2437 " reached. Keeping one slot for boot cpu." 2438 " Processor %d/0x%x ignored.\n", max, thiscpu, apicid); 2439 2440 disabled_cpus++; 2441 return -ENODEV; 2442 } 2443 2444 if (num_processors >= nr_cpu_ids) { 2445 int thiscpu = max + disabled_cpus; 2446 2447 pr_warn("APIC: NR_CPUS/possible_cpus limit of %i reached. " 2448 "Processor %d/0x%x ignored.\n", max, thiscpu, apicid); 2449 2450 disabled_cpus++; 2451 return -EINVAL; 2452 } 2453 2454 if (apicid == boot_cpu_physical_apicid) { 2455 /* 2456 * x86_bios_cpu_apicid is required to have processors listed 2457 * in same order as logical cpu numbers. Hence the first 2458 * entry is BSP, and so on. 2459 * boot_cpu_init() already hold bit 0 in cpu_present_mask 2460 * for BSP. 2461 */ 2462 cpu = 0; 2463 2464 /* Logical cpuid 0 is reserved for BSP. */ 2465 cpuid_to_apicid[0] = apicid; 2466 } else { 2467 cpu = allocate_logical_cpuid(apicid); 2468 if (cpu < 0) { 2469 disabled_cpus++; 2470 return -EINVAL; 2471 } 2472 } 2473 2474 /* 2475 * Validate version 2476 */ 2477 if (version == 0x0) { 2478 pr_warn("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n", 2479 cpu, apicid); 2480 version = 0x10; 2481 } 2482 2483 if (version != boot_cpu_apic_version) { 2484 pr_warn("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n", 2485 boot_cpu_apic_version, cpu, version); 2486 } 2487 2488 if (apicid > max_physical_apicid) 2489 max_physical_apicid = apicid; 2490 2491 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64) 2492 early_per_cpu(x86_cpu_to_apicid, cpu) = apicid; 2493 early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid; 2494 #endif 2495 #ifdef CONFIG_X86_32 2496 early_per_cpu(x86_cpu_to_logical_apicid, cpu) = 2497 apic->x86_32_early_logical_apicid(cpu); 2498 #endif 2499 set_cpu_possible(cpu, true); 2500 physid_set(apicid, phys_cpu_present_map); 2501 set_cpu_present(cpu, true); 2502 num_processors++; 2503 2504 return cpu; 2505 } 2506 2507 int hard_smp_processor_id(void) 2508 { 2509 return read_apic_id(); 2510 } 2511 2512 void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg, 2513 bool dmar) 2514 { 2515 memset(msg, 0, sizeof(*msg)); 2516 2517 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW; 2518 msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical; 2519 msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF; 2520 2521 msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED; 2522 msg->arch_data.vector = cfg->vector; 2523 2524 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH; 2525 /* 2526 * Only the IOMMU itself can use the trick of putting destination 2527 * APIC ID into the high bits of the address. Anything else would 2528 * just be writing to memory if it tried that, and needs IR to 2529 * address APICs which can't be addressed in the normal 32-bit 2530 * address range at 0xFFExxxxx. That is typically just 8 bits, but 2531 * some hypervisors allow the extended destination ID field in bits 2532 * 5-11 to be used, giving support for 15 bits of APIC IDs in total. 2533 */ 2534 if (dmar) 2535 msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8; 2536 else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000) 2537 msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8; 2538 else 2539 WARN_ON_ONCE(cfg->dest_apicid > 0xFF); 2540 } 2541 2542 u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid) 2543 { 2544 u32 dest = msg->arch_addr_lo.destid_0_7; 2545 2546 if (extid) 2547 dest |= msg->arch_addr_hi.destid_8_31 << 8; 2548 return dest; 2549 } 2550 EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid); 2551 2552 /* 2553 * Override the generic EOI implementation with an optimized version. 2554 * Only called during early boot when only one CPU is active and with 2555 * interrupts disabled, so we know this does not race with actual APIC driver 2556 * use. 2557 */ 2558 void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)) 2559 { 2560 struct apic **drv; 2561 2562 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { 2563 /* Should happen once for each apic */ 2564 WARN_ON((*drv)->eoi_write == eoi_write); 2565 (*drv)->native_eoi_write = (*drv)->eoi_write; 2566 (*drv)->eoi_write = eoi_write; 2567 } 2568 } 2569 2570 static void __init apic_bsp_up_setup(void) 2571 { 2572 #ifdef CONFIG_X86_64 2573 apic_write(APIC_ID, apic->set_apic_id(boot_cpu_physical_apicid)); 2574 #else 2575 /* 2576 * Hack: In case of kdump, after a crash, kernel might be booting 2577 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid 2578 * might be zero if read from MP tables. Get it from LAPIC. 2579 */ 2580 # ifdef CONFIG_CRASH_DUMP 2581 boot_cpu_physical_apicid = read_apic_id(); 2582 # endif 2583 #endif 2584 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map); 2585 } 2586 2587 /** 2588 * apic_bsp_setup - Setup function for local apic and io-apic 2589 * @upmode: Force UP mode (for APIC_init_uniprocessor) 2590 */ 2591 static void __init apic_bsp_setup(bool upmode) 2592 { 2593 connect_bsp_APIC(); 2594 if (upmode) 2595 apic_bsp_up_setup(); 2596 setup_local_APIC(); 2597 2598 enable_IO_APIC(); 2599 end_local_APIC_setup(); 2600 irq_remap_enable_fault_handling(); 2601 setup_IO_APIC(); 2602 } 2603 2604 #ifdef CONFIG_UP_LATE_INIT 2605 void __init up_late_init(void) 2606 { 2607 if (apic_intr_mode == APIC_PIC) 2608 return; 2609 2610 /* Setup local timer */ 2611 x86_init.timers.setup_percpu_clockev(); 2612 } 2613 #endif 2614 2615 /* 2616 * Power management 2617 */ 2618 #ifdef CONFIG_PM 2619 2620 static struct { 2621 /* 2622 * 'active' is true if the local APIC was enabled by us and 2623 * not the BIOS; this signifies that we are also responsible 2624 * for disabling it before entering apm/acpi suspend 2625 */ 2626 int active; 2627 /* r/w apic fields */ 2628 unsigned int apic_id; 2629 unsigned int apic_taskpri; 2630 unsigned int apic_ldr; 2631 unsigned int apic_dfr; 2632 unsigned int apic_spiv; 2633 unsigned int apic_lvtt; 2634 unsigned int apic_lvtpc; 2635 unsigned int apic_lvt0; 2636 unsigned int apic_lvt1; 2637 unsigned int apic_lvterr; 2638 unsigned int apic_tmict; 2639 unsigned int apic_tdcr; 2640 unsigned int apic_thmr; 2641 unsigned int apic_cmci; 2642 } apic_pm_state; 2643 2644 static int lapic_suspend(void) 2645 { 2646 unsigned long flags; 2647 int maxlvt; 2648 2649 if (!apic_pm_state.active) 2650 return 0; 2651 2652 maxlvt = lapic_get_maxlvt(); 2653 2654 apic_pm_state.apic_id = apic_read(APIC_ID); 2655 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI); 2656 apic_pm_state.apic_ldr = apic_read(APIC_LDR); 2657 apic_pm_state.apic_dfr = apic_read(APIC_DFR); 2658 apic_pm_state.apic_spiv = apic_read(APIC_SPIV); 2659 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT); 2660 if (maxlvt >= 4) 2661 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC); 2662 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0); 2663 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1); 2664 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR); 2665 apic_pm_state.apic_tmict = apic_read(APIC_TMICT); 2666 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR); 2667 #ifdef CONFIG_X86_THERMAL_VECTOR 2668 if (maxlvt >= 5) 2669 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR); 2670 #endif 2671 #ifdef CONFIG_X86_MCE_INTEL 2672 if (maxlvt >= 6) 2673 apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI); 2674 #endif 2675 2676 local_irq_save(flags); 2677 2678 /* 2679 * Mask IOAPIC before disabling the local APIC to prevent stale IRR 2680 * entries on some implementations. 2681 */ 2682 mask_ioapic_entries(); 2683 2684 disable_local_APIC(); 2685 2686 irq_remapping_disable(); 2687 2688 local_irq_restore(flags); 2689 return 0; 2690 } 2691 2692 static void lapic_resume(void) 2693 { 2694 unsigned int l, h; 2695 unsigned long flags; 2696 int maxlvt; 2697 2698 if (!apic_pm_state.active) 2699 return; 2700 2701 local_irq_save(flags); 2702 2703 /* 2704 * IO-APIC and PIC have their own resume routines. 2705 * We just mask them here to make sure the interrupt 2706 * subsystem is completely quiet while we enable x2apic 2707 * and interrupt-remapping. 2708 */ 2709 mask_ioapic_entries(); 2710 legacy_pic->mask_all(); 2711 2712 if (x2apic_mode) { 2713 __x2apic_enable(); 2714 } else { 2715 /* 2716 * Make sure the APICBASE points to the right address 2717 * 2718 * FIXME! This will be wrong if we ever support suspend on 2719 * SMP! We'll need to do this as part of the CPU restore! 2720 */ 2721 if (boot_cpu_data.x86 >= 6) { 2722 rdmsr(MSR_IA32_APICBASE, l, h); 2723 l &= ~MSR_IA32_APICBASE_BASE; 2724 l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; 2725 wrmsr(MSR_IA32_APICBASE, l, h); 2726 } 2727 } 2728 2729 maxlvt = lapic_get_maxlvt(); 2730 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED); 2731 apic_write(APIC_ID, apic_pm_state.apic_id); 2732 apic_write(APIC_DFR, apic_pm_state.apic_dfr); 2733 apic_write(APIC_LDR, apic_pm_state.apic_ldr); 2734 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri); 2735 apic_write(APIC_SPIV, apic_pm_state.apic_spiv); 2736 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0); 2737 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1); 2738 #ifdef CONFIG_X86_THERMAL_VECTOR 2739 if (maxlvt >= 5) 2740 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr); 2741 #endif 2742 #ifdef CONFIG_X86_MCE_INTEL 2743 if (maxlvt >= 6) 2744 apic_write(APIC_LVTCMCI, apic_pm_state.apic_cmci); 2745 #endif 2746 if (maxlvt >= 4) 2747 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc); 2748 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt); 2749 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr); 2750 apic_write(APIC_TMICT, apic_pm_state.apic_tmict); 2751 apic_write(APIC_ESR, 0); 2752 apic_read(APIC_ESR); 2753 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr); 2754 apic_write(APIC_ESR, 0); 2755 apic_read(APIC_ESR); 2756 2757 irq_remapping_reenable(x2apic_mode); 2758 2759 local_irq_restore(flags); 2760 } 2761 2762 /* 2763 * This device has no shutdown method - fully functioning local APICs 2764 * are needed on every CPU up until machine_halt/restart/poweroff. 2765 */ 2766 2767 static struct syscore_ops lapic_syscore_ops = { 2768 .resume = lapic_resume, 2769 .suspend = lapic_suspend, 2770 }; 2771 2772 static void apic_pm_activate(void) 2773 { 2774 apic_pm_state.active = 1; 2775 } 2776 2777 static int __init init_lapic_sysfs(void) 2778 { 2779 /* XXX: remove suspend/resume procs if !apic_pm_state.active? */ 2780 if (boot_cpu_has(X86_FEATURE_APIC)) 2781 register_syscore_ops(&lapic_syscore_ops); 2782 2783 return 0; 2784 } 2785 2786 /* local apic needs to resume before other devices access its registers. */ 2787 core_initcall(init_lapic_sysfs); 2788 2789 #else /* CONFIG_PM */ 2790 2791 static void apic_pm_activate(void) { } 2792 2793 #endif /* CONFIG_PM */ 2794 2795 #ifdef CONFIG_X86_64 2796 2797 static int multi_checked; 2798 static int multi; 2799 2800 static int set_multi(const struct dmi_system_id *d) 2801 { 2802 if (multi) 2803 return 0; 2804 pr_info("APIC: %s detected, Multi Chassis\n", d->ident); 2805 multi = 1; 2806 return 0; 2807 } 2808 2809 static const struct dmi_system_id multi_dmi_table[] = { 2810 { 2811 .callback = set_multi, 2812 .ident = "IBM System Summit2", 2813 .matches = { 2814 DMI_MATCH(DMI_SYS_VENDOR, "IBM"), 2815 DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"), 2816 }, 2817 }, 2818 {} 2819 }; 2820 2821 static void dmi_check_multi(void) 2822 { 2823 if (multi_checked) 2824 return; 2825 2826 dmi_check_system(multi_dmi_table); 2827 multi_checked = 1; 2828 } 2829 2830 /* 2831 * apic_is_clustered_box() -- Check if we can expect good TSC 2832 * 2833 * Thus far, the major user of this is IBM's Summit2 series: 2834 * Clustered boxes may have unsynced TSC problems if they are 2835 * multi-chassis. 2836 * Use DMI to check them 2837 */ 2838 int apic_is_clustered_box(void) 2839 { 2840 dmi_check_multi(); 2841 return multi; 2842 } 2843 #endif 2844 2845 /* 2846 * APIC command line parameters 2847 */ 2848 static int __init setup_disableapic(char *arg) 2849 { 2850 disable_apic = 1; 2851 setup_clear_cpu_cap(X86_FEATURE_APIC); 2852 return 0; 2853 } 2854 early_param("disableapic", setup_disableapic); 2855 2856 /* same as disableapic, for compatibility */ 2857 static int __init setup_nolapic(char *arg) 2858 { 2859 return setup_disableapic(arg); 2860 } 2861 early_param("nolapic", setup_nolapic); 2862 2863 static int __init parse_lapic_timer_c2_ok(char *arg) 2864 { 2865 local_apic_timer_c2_ok = 1; 2866 return 0; 2867 } 2868 early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok); 2869 2870 static int __init parse_disable_apic_timer(char *arg) 2871 { 2872 disable_apic_timer = 1; 2873 return 0; 2874 } 2875 early_param("noapictimer", parse_disable_apic_timer); 2876 2877 static int __init parse_nolapic_timer(char *arg) 2878 { 2879 disable_apic_timer = 1; 2880 return 0; 2881 } 2882 early_param("nolapic_timer", parse_nolapic_timer); 2883 2884 static int __init apic_set_verbosity(char *arg) 2885 { 2886 if (!arg) { 2887 #ifdef CONFIG_X86_64 2888 skip_ioapic_setup = 0; 2889 return 0; 2890 #endif 2891 return -EINVAL; 2892 } 2893 2894 if (strcmp("debug", arg) == 0) 2895 apic_verbosity = APIC_DEBUG; 2896 else if (strcmp("verbose", arg) == 0) 2897 apic_verbosity = APIC_VERBOSE; 2898 #ifdef CONFIG_X86_64 2899 else { 2900 pr_warn("APIC Verbosity level %s not recognised" 2901 " use apic=verbose or apic=debug\n", arg); 2902 return -EINVAL; 2903 } 2904 #endif 2905 2906 return 0; 2907 } 2908 early_param("apic", apic_set_verbosity); 2909 2910 static int __init lapic_insert_resource(void) 2911 { 2912 if (!apic_phys) 2913 return -1; 2914 2915 /* Put local APIC into the resource map. */ 2916 lapic_resource.start = apic_phys; 2917 lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1; 2918 insert_resource(&iomem_resource, &lapic_resource); 2919 2920 return 0; 2921 } 2922 2923 /* 2924 * need call insert after e820__reserve_resources() 2925 * that is using request_resource 2926 */ 2927 late_initcall(lapic_insert_resource); 2928 2929 static int __init apic_set_disabled_cpu_apicid(char *arg) 2930 { 2931 if (!arg || !get_option(&arg, &disabled_cpu_apicid)) 2932 return -EINVAL; 2933 2934 return 0; 2935 } 2936 early_param("disable_cpu_apicid", apic_set_disabled_cpu_apicid); 2937 2938 static int __init apic_set_extnmi(char *arg) 2939 { 2940 if (!arg) 2941 return -EINVAL; 2942 2943 if (!strncmp("all", arg, 3)) 2944 apic_extnmi = APIC_EXTNMI_ALL; 2945 else if (!strncmp("none", arg, 4)) 2946 apic_extnmi = APIC_EXTNMI_NONE; 2947 else if (!strncmp("bsp", arg, 3)) 2948 apic_extnmi = APIC_EXTNMI_BSP; 2949 else { 2950 pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg); 2951 return -EINVAL; 2952 } 2953 2954 return 0; 2955 } 2956 early_param("apic_extnmi", apic_set_extnmi); 2957