1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Local APIC handling, local APIC timers 4 * 5 * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com> 6 * 7 * Fixes 8 * Maciej W. Rozycki : Bits for genuine 82489DX APICs; 9 * thanks to Eric Gilmore 10 * and Rolf G. Tews 11 * for testing these extensively. 12 * Maciej W. Rozycki : Various updates and fixes. 13 * Mikael Pettersson : Power Management for UP-APIC. 14 * Pavel Machek and 15 * Mikael Pettersson : PM converted to driver model. 16 */ 17 18 #include <linux/perf_event.h> 19 #include <linux/kernel_stat.h> 20 #include <linux/mc146818rtc.h> 21 #include <linux/acpi_pmtmr.h> 22 #include <linux/clockchips.h> 23 #include <linux/interrupt.h> 24 #include <linux/memblock.h> 25 #include <linux/ftrace.h> 26 #include <linux/ioport.h> 27 #include <linux/export.h> 28 #include <linux/syscore_ops.h> 29 #include <linux/delay.h> 30 #include <linux/timex.h> 31 #include <linux/i8253.h> 32 #include <linux/dmar.h> 33 #include <linux/init.h> 34 #include <linux/cpu.h> 35 #include <linux/dmi.h> 36 #include <linux/smp.h> 37 #include <linux/mm.h> 38 39 #include <asm/trace/irq_vectors.h> 40 #include <asm/irq_remapping.h> 41 #include <asm/perf_event.h> 42 #include <asm/x86_init.h> 43 #include <linux/atomic.h> 44 #include <asm/barrier.h> 45 #include <asm/mpspec.h> 46 #include <asm/i8259.h> 47 #include <asm/proto.h> 48 #include <asm/traps.h> 49 #include <asm/apic.h> 50 #include <asm/acpi.h> 51 #include <asm/io_apic.h> 52 #include <asm/desc.h> 53 #include <asm/hpet.h> 54 #include <asm/mtrr.h> 55 #include <asm/time.h> 56 #include <asm/smp.h> 57 #include <asm/mce.h> 58 #include <asm/tsc.h> 59 #include <asm/hypervisor.h> 60 #include <asm/cpu_device_id.h> 61 #include <asm/intel-family.h> 62 #include <asm/irq_regs.h> 63 64 unsigned int num_processors; 65 66 unsigned disabled_cpus; 67 68 /* Processor that is doing the boot up */ 69 unsigned int boot_cpu_physical_apicid __ro_after_init = -1U; 70 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid); 71 72 u8 boot_cpu_apic_version __ro_after_init; 73 74 /* 75 * The highest APIC ID seen during enumeration. 76 */ 77 static unsigned int max_physical_apicid; 78 79 /* 80 * Bitmask of physically existing CPUs: 81 */ 82 physid_mask_t phys_cpu_present_map; 83 84 /* 85 * Processor to be disabled specified by kernel parameter 86 * disable_cpu_apicid=<int>, mostly used for the kdump 2nd kernel to 87 * avoid undefined behaviour caused by sending INIT from AP to BSP. 88 */ 89 static unsigned int disabled_cpu_apicid __ro_after_init = BAD_APICID; 90 91 /* 92 * This variable controls which CPUs receive external NMIs. By default, 93 * external NMIs are delivered only to the BSP. 94 */ 95 static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP; 96 97 /* 98 * Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID 99 */ 100 static bool virt_ext_dest_id __ro_after_init; 101 102 /* 103 * Map cpu index to physical APIC ID 104 */ 105 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID); 106 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID); 107 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid, U32_MAX); 108 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid); 109 EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid); 110 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid); 111 112 #ifdef CONFIG_X86_32 113 114 /* 115 * On x86_32, the mapping between cpu and logical apicid may vary 116 * depending on apic in use. The following early percpu variable is 117 * used for the mapping. This is where the behaviors of x86_64 and 32 118 * actually diverge. Let's keep it ugly for now. 119 */ 120 DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID); 121 122 /* Local APIC was disabled by the BIOS and enabled by the kernel */ 123 static int enabled_via_apicbase __ro_after_init; 124 125 /* 126 * Handle interrupt mode configuration register (IMCR). 127 * This register controls whether the interrupt signals 128 * that reach the BSP come from the master PIC or from the 129 * local APIC. Before entering Symmetric I/O Mode, either 130 * the BIOS or the operating system must switch out of 131 * PIC Mode by changing the IMCR. 132 */ 133 static inline void imcr_pic_to_apic(void) 134 { 135 /* select IMCR register */ 136 outb(0x70, 0x22); 137 /* NMI and 8259 INTR go through APIC */ 138 outb(0x01, 0x23); 139 } 140 141 static inline void imcr_apic_to_pic(void) 142 { 143 /* select IMCR register */ 144 outb(0x70, 0x22); 145 /* NMI and 8259 INTR go directly to BSP */ 146 outb(0x00, 0x23); 147 } 148 #endif 149 150 /* 151 * Knob to control our willingness to enable the local APIC. 152 * 153 * +1=force-enable 154 */ 155 static int force_enable_local_apic __initdata; 156 157 /* 158 * APIC command line parameters 159 */ 160 static int __init parse_lapic(char *arg) 161 { 162 if (IS_ENABLED(CONFIG_X86_32) && !arg) 163 force_enable_local_apic = 1; 164 else if (arg && !strncmp(arg, "notscdeadline", 13)) 165 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); 166 return 0; 167 } 168 early_param("lapic", parse_lapic); 169 170 #ifdef CONFIG_X86_64 171 static int apic_calibrate_pmtmr __initdata; 172 static __init int setup_apicpmtimer(char *s) 173 { 174 apic_calibrate_pmtmr = 1; 175 notsc_setup(NULL); 176 return 0; 177 } 178 __setup("apicpmtimer", setup_apicpmtimer); 179 #endif 180 181 unsigned long mp_lapic_addr __ro_after_init; 182 int disable_apic __ro_after_init; 183 /* Disable local APIC timer from the kernel commandline or via dmi quirk */ 184 static int disable_apic_timer __initdata; 185 /* Local APIC timer works in C2 */ 186 int local_apic_timer_c2_ok __ro_after_init; 187 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok); 188 189 /* 190 * Debug level, exported for io_apic.c 191 */ 192 int apic_verbosity __ro_after_init; 193 194 int pic_mode __ro_after_init; 195 196 /* Have we found an MP table */ 197 int smp_found_config __ro_after_init; 198 199 static struct resource lapic_resource = { 200 .name = "Local APIC", 201 .flags = IORESOURCE_MEM | IORESOURCE_BUSY, 202 }; 203 204 unsigned int lapic_timer_period = 0; 205 206 static void apic_pm_activate(void); 207 208 static unsigned long apic_phys __ro_after_init; 209 210 /* 211 * Get the LAPIC version 212 */ 213 static inline int lapic_get_version(void) 214 { 215 return GET_APIC_VERSION(apic_read(APIC_LVR)); 216 } 217 218 /* 219 * Check, if the APIC is integrated or a separate chip 220 */ 221 static inline int lapic_is_integrated(void) 222 { 223 return APIC_INTEGRATED(lapic_get_version()); 224 } 225 226 /* 227 * Check, whether this is a modern or a first generation APIC 228 */ 229 static int modern_apic(void) 230 { 231 /* AMD systems use old APIC versions, so check the CPU */ 232 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && 233 boot_cpu_data.x86 >= 0xf) 234 return 1; 235 236 /* Hygon systems use modern APIC */ 237 if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) 238 return 1; 239 240 return lapic_get_version() >= 0x14; 241 } 242 243 /* 244 * right after this call apic become NOOP driven 245 * so apic->write/read doesn't do anything 246 */ 247 static void __init apic_disable(void) 248 { 249 pr_info("APIC: switched to apic NOOP\n"); 250 apic = &apic_noop; 251 } 252 253 void native_apic_wait_icr_idle(void) 254 { 255 while (apic_read(APIC_ICR) & APIC_ICR_BUSY) 256 cpu_relax(); 257 } 258 259 u32 native_safe_apic_wait_icr_idle(void) 260 { 261 u32 send_status; 262 int timeout; 263 264 timeout = 0; 265 do { 266 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY; 267 if (!send_status) 268 break; 269 inc_irq_stat(icr_read_retry_count); 270 udelay(100); 271 } while (timeout++ < 1000); 272 273 return send_status; 274 } 275 276 void native_apic_icr_write(u32 low, u32 id) 277 { 278 unsigned long flags; 279 280 local_irq_save(flags); 281 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id)); 282 apic_write(APIC_ICR, low); 283 local_irq_restore(flags); 284 } 285 286 u64 native_apic_icr_read(void) 287 { 288 u32 icr1, icr2; 289 290 icr2 = apic_read(APIC_ICR2); 291 icr1 = apic_read(APIC_ICR); 292 293 return icr1 | ((u64)icr2 << 32); 294 } 295 296 #ifdef CONFIG_X86_32 297 /** 298 * get_physical_broadcast - Get number of physical broadcast IDs 299 */ 300 int get_physical_broadcast(void) 301 { 302 return modern_apic() ? 0xff : 0xf; 303 } 304 #endif 305 306 /** 307 * lapic_get_maxlvt - get the maximum number of local vector table entries 308 */ 309 int lapic_get_maxlvt(void) 310 { 311 /* 312 * - we always have APIC integrated on 64bit mode 313 * - 82489DXs do not report # of LVT entries 314 */ 315 return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2; 316 } 317 318 /* 319 * Local APIC timer 320 */ 321 322 /* Clock divisor */ 323 #define APIC_DIVISOR 16 324 #define TSC_DIVISOR 8 325 326 /* 327 * This function sets up the local APIC timer, with a timeout of 328 * 'clocks' APIC bus clock. During calibration we actually call 329 * this function twice on the boot CPU, once with a bogus timeout 330 * value, second time for real. The other (noncalibrating) CPUs 331 * call this function only once, with the real, calibrated value. 332 * 333 * We do reads before writes even if unnecessary, to get around the 334 * P5 APIC double write bug. 335 */ 336 static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen) 337 { 338 unsigned int lvtt_value, tmp_value; 339 340 lvtt_value = LOCAL_TIMER_VECTOR; 341 if (!oneshot) 342 lvtt_value |= APIC_LVT_TIMER_PERIODIC; 343 else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 344 lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE; 345 346 if (!lapic_is_integrated()) 347 lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV); 348 349 if (!irqen) 350 lvtt_value |= APIC_LVT_MASKED; 351 352 apic_write(APIC_LVTT, lvtt_value); 353 354 if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) { 355 /* 356 * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode, 357 * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized. 358 * According to Intel, MFENCE can do the serialization here. 359 */ 360 asm volatile("mfence" : : : "memory"); 361 return; 362 } 363 364 /* 365 * Divide PICLK by 16 366 */ 367 tmp_value = apic_read(APIC_TDCR); 368 apic_write(APIC_TDCR, 369 (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) | 370 APIC_TDR_DIV_16); 371 372 if (!oneshot) 373 apic_write(APIC_TMICT, clocks / APIC_DIVISOR); 374 } 375 376 /* 377 * Setup extended LVT, AMD specific 378 * 379 * Software should use the LVT offsets the BIOS provides. The offsets 380 * are determined by the subsystems using it like those for MCE 381 * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts 382 * are supported. Beginning with family 10h at least 4 offsets are 383 * available. 384 * 385 * Since the offsets must be consistent for all cores, we keep track 386 * of the LVT offsets in software and reserve the offset for the same 387 * vector also to be used on other cores. An offset is freed by 388 * setting the entry to APIC_EILVT_MASKED. 389 * 390 * If the BIOS is right, there should be no conflicts. Otherwise a 391 * "[Firmware Bug]: ..." error message is generated. However, if 392 * software does not properly determines the offsets, it is not 393 * necessarily a BIOS bug. 394 */ 395 396 static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX]; 397 398 static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new) 399 { 400 return (old & APIC_EILVT_MASKED) 401 || (new == APIC_EILVT_MASKED) 402 || ((new & ~APIC_EILVT_MASKED) == old); 403 } 404 405 static unsigned int reserve_eilvt_offset(int offset, unsigned int new) 406 { 407 unsigned int rsvd, vector; 408 409 if (offset >= APIC_EILVT_NR_MAX) 410 return ~0; 411 412 rsvd = atomic_read(&eilvt_offsets[offset]); 413 do { 414 vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */ 415 if (vector && !eilvt_entry_is_changeable(vector, new)) 416 /* may not change if vectors are different */ 417 return rsvd; 418 rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new); 419 } while (rsvd != new); 420 421 rsvd &= ~APIC_EILVT_MASKED; 422 if (rsvd && rsvd != vector) 423 pr_info("LVT offset %d assigned for vector 0x%02x\n", 424 offset, rsvd); 425 426 return new; 427 } 428 429 /* 430 * If mask=1, the LVT entry does not generate interrupts while mask=0 431 * enables the vector. See also the BKDGs. Must be called with 432 * preemption disabled. 433 */ 434 435 int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask) 436 { 437 unsigned long reg = APIC_EILVTn(offset); 438 unsigned int new, old, reserved; 439 440 new = (mask << 16) | (msg_type << 8) | vector; 441 old = apic_read(reg); 442 reserved = reserve_eilvt_offset(offset, new); 443 444 if (reserved != new) { 445 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for " 446 "vector 0x%x, but the register is already in use for " 447 "vector 0x%x on another cpu\n", 448 smp_processor_id(), reg, offset, new, reserved); 449 return -EINVAL; 450 } 451 452 if (!eilvt_entry_is_changeable(old, new)) { 453 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for " 454 "vector 0x%x, but the register is already in use for " 455 "vector 0x%x on this cpu\n", 456 smp_processor_id(), reg, offset, new, old); 457 return -EBUSY; 458 } 459 460 apic_write(reg, new); 461 462 return 0; 463 } 464 EXPORT_SYMBOL_GPL(setup_APIC_eilvt); 465 466 /* 467 * Program the next event, relative to now 468 */ 469 static int lapic_next_event(unsigned long delta, 470 struct clock_event_device *evt) 471 { 472 apic_write(APIC_TMICT, delta); 473 return 0; 474 } 475 476 static int lapic_next_deadline(unsigned long delta, 477 struct clock_event_device *evt) 478 { 479 u64 tsc; 480 481 /* This MSR is special and need a special fence: */ 482 weak_wrmsr_fence(); 483 484 tsc = rdtsc(); 485 wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR)); 486 return 0; 487 } 488 489 static int lapic_timer_shutdown(struct clock_event_device *evt) 490 { 491 unsigned int v; 492 493 /* Lapic used as dummy for broadcast ? */ 494 if (evt->features & CLOCK_EVT_FEAT_DUMMY) 495 return 0; 496 497 v = apic_read(APIC_LVTT); 498 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); 499 apic_write(APIC_LVTT, v); 500 apic_write(APIC_TMICT, 0); 501 return 0; 502 } 503 504 static inline int 505 lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot) 506 { 507 /* Lapic used as dummy for broadcast ? */ 508 if (evt->features & CLOCK_EVT_FEAT_DUMMY) 509 return 0; 510 511 __setup_APIC_LVTT(lapic_timer_period, oneshot, 1); 512 return 0; 513 } 514 515 static int lapic_timer_set_periodic(struct clock_event_device *evt) 516 { 517 return lapic_timer_set_periodic_oneshot(evt, false); 518 } 519 520 static int lapic_timer_set_oneshot(struct clock_event_device *evt) 521 { 522 return lapic_timer_set_periodic_oneshot(evt, true); 523 } 524 525 /* 526 * Local APIC timer broadcast function 527 */ 528 static void lapic_timer_broadcast(const struct cpumask *mask) 529 { 530 #ifdef CONFIG_SMP 531 apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR); 532 #endif 533 } 534 535 536 /* 537 * The local apic timer can be used for any function which is CPU local. 538 */ 539 static struct clock_event_device lapic_clockevent = { 540 .name = "lapic", 541 .features = CLOCK_EVT_FEAT_PERIODIC | 542 CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP 543 | CLOCK_EVT_FEAT_DUMMY, 544 .shift = 32, 545 .set_state_shutdown = lapic_timer_shutdown, 546 .set_state_periodic = lapic_timer_set_periodic, 547 .set_state_oneshot = lapic_timer_set_oneshot, 548 .set_state_oneshot_stopped = lapic_timer_shutdown, 549 .set_next_event = lapic_next_event, 550 .broadcast = lapic_timer_broadcast, 551 .rating = 100, 552 .irq = -1, 553 }; 554 static DEFINE_PER_CPU(struct clock_event_device, lapic_events); 555 556 static const struct x86_cpu_id deadline_match[] __initconst = { 557 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */ 558 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x4, 0x4), 0x0f), /* EX */ 559 560 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_X, 0x0b000020), 561 562 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x2, 0x2), 0x00000011), 563 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x3, 0x3), 0x0700000e), 564 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x4, 0x4), 0x0f00000c), 565 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x5, 0x5), 0x0e000003), 566 567 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x3, 0x3), 0x01000136), 568 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x4, 0x4), 0x02000014), 569 X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x5, 0xf), 0), 570 571 X86_MATCH_INTEL_FAM6_MODEL( HASWELL, 0x22), 572 X86_MATCH_INTEL_FAM6_MODEL( HASWELL_L, 0x20), 573 X86_MATCH_INTEL_FAM6_MODEL( HASWELL_G, 0x17), 574 575 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL, 0x25), 576 X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_G, 0x17), 577 578 X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE_L, 0xb2), 579 X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE, 0xb2), 580 581 X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE_L, 0x52), 582 X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE, 0x52), 583 584 {}, 585 }; 586 587 static __init bool apic_validate_deadline_timer(void) 588 { 589 const struct x86_cpu_id *m; 590 u32 rev; 591 592 if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 593 return false; 594 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 595 return true; 596 597 m = x86_match_cpu(deadline_match); 598 if (!m) 599 return true; 600 601 rev = (u32)m->driver_data; 602 603 if (boot_cpu_data.microcode >= rev) 604 return true; 605 606 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); 607 pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; " 608 "please update microcode to version: 0x%x (or later)\n", rev); 609 return false; 610 } 611 612 /* 613 * Setup the local APIC timer for this CPU. Copy the initialized values 614 * of the boot CPU and register the clock event in the framework. 615 */ 616 static void setup_APIC_timer(void) 617 { 618 struct clock_event_device *levt = this_cpu_ptr(&lapic_events); 619 620 if (this_cpu_has(X86_FEATURE_ARAT)) { 621 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP; 622 /* Make LAPIC timer preferrable over percpu HPET */ 623 lapic_clockevent.rating = 150; 624 } 625 626 memcpy(levt, &lapic_clockevent, sizeof(*levt)); 627 levt->cpumask = cpumask_of(smp_processor_id()); 628 629 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) { 630 levt->name = "lapic-deadline"; 631 levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC | 632 CLOCK_EVT_FEAT_DUMMY); 633 levt->set_next_event = lapic_next_deadline; 634 clockevents_config_and_register(levt, 635 tsc_khz * (1000 / TSC_DIVISOR), 636 0xF, ~0UL); 637 } else 638 clockevents_register_device(levt); 639 } 640 641 /* 642 * Install the updated TSC frequency from recalibration at the TSC 643 * deadline clockevent devices. 644 */ 645 static void __lapic_update_tsc_freq(void *info) 646 { 647 struct clock_event_device *levt = this_cpu_ptr(&lapic_events); 648 649 if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 650 return; 651 652 clockevents_update_freq(levt, tsc_khz * (1000 / TSC_DIVISOR)); 653 } 654 655 void lapic_update_tsc_freq(void) 656 { 657 /* 658 * The clockevent device's ->mult and ->shift can both be 659 * changed. In order to avoid races, schedule the frequency 660 * update code on each CPU. 661 */ 662 on_each_cpu(__lapic_update_tsc_freq, NULL, 0); 663 } 664 665 /* 666 * In this functions we calibrate APIC bus clocks to the external timer. 667 * 668 * We want to do the calibration only once since we want to have local timer 669 * irqs syncron. CPUs connected by the same APIC bus have the very same bus 670 * frequency. 671 * 672 * This was previously done by reading the PIT/HPET and waiting for a wrap 673 * around to find out, that a tick has elapsed. I have a box, where the PIT 674 * readout is broken, so it never gets out of the wait loop again. This was 675 * also reported by others. 676 * 677 * Monitoring the jiffies value is inaccurate and the clockevents 678 * infrastructure allows us to do a simple substitution of the interrupt 679 * handler. 680 * 681 * The calibration routine also uses the pm_timer when possible, as the PIT 682 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes 683 * back to normal later in the boot process). 684 */ 685 686 #define LAPIC_CAL_LOOPS (HZ/10) 687 688 static __initdata int lapic_cal_loops = -1; 689 static __initdata long lapic_cal_t1, lapic_cal_t2; 690 static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2; 691 static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2; 692 static __initdata unsigned long lapic_cal_j1, lapic_cal_j2; 693 694 /* 695 * Temporary interrupt handler and polled calibration function. 696 */ 697 static void __init lapic_cal_handler(struct clock_event_device *dev) 698 { 699 unsigned long long tsc = 0; 700 long tapic = apic_read(APIC_TMCCT); 701 unsigned long pm = acpi_pm_read_early(); 702 703 if (boot_cpu_has(X86_FEATURE_TSC)) 704 tsc = rdtsc(); 705 706 switch (lapic_cal_loops++) { 707 case 0: 708 lapic_cal_t1 = tapic; 709 lapic_cal_tsc1 = tsc; 710 lapic_cal_pm1 = pm; 711 lapic_cal_j1 = jiffies; 712 break; 713 714 case LAPIC_CAL_LOOPS: 715 lapic_cal_t2 = tapic; 716 lapic_cal_tsc2 = tsc; 717 if (pm < lapic_cal_pm1) 718 pm += ACPI_PM_OVRRUN; 719 lapic_cal_pm2 = pm; 720 lapic_cal_j2 = jiffies; 721 break; 722 } 723 } 724 725 static int __init 726 calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc) 727 { 728 const long pm_100ms = PMTMR_TICKS_PER_SEC / 10; 729 const long pm_thresh = pm_100ms / 100; 730 unsigned long mult; 731 u64 res; 732 733 #ifndef CONFIG_X86_PM_TIMER 734 return -1; 735 #endif 736 737 apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm); 738 739 /* Check, if the PM timer is available */ 740 if (!deltapm) 741 return -1; 742 743 mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22); 744 745 if (deltapm > (pm_100ms - pm_thresh) && 746 deltapm < (pm_100ms + pm_thresh)) { 747 apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n"); 748 return 0; 749 } 750 751 res = (((u64)deltapm) * mult) >> 22; 752 do_div(res, 1000000); 753 pr_warn("APIC calibration not consistent " 754 "with PM-Timer: %ldms instead of 100ms\n", (long)res); 755 756 /* Correct the lapic counter value */ 757 res = (((u64)(*delta)) * pm_100ms); 758 do_div(res, deltapm); 759 pr_info("APIC delta adjusted to PM-Timer: " 760 "%lu (%ld)\n", (unsigned long)res, *delta); 761 *delta = (long)res; 762 763 /* Correct the tsc counter value */ 764 if (boot_cpu_has(X86_FEATURE_TSC)) { 765 res = (((u64)(*deltatsc)) * pm_100ms); 766 do_div(res, deltapm); 767 apic_printk(APIC_VERBOSE, "TSC delta adjusted to " 768 "PM-Timer: %lu (%ld)\n", 769 (unsigned long)res, *deltatsc); 770 *deltatsc = (long)res; 771 } 772 773 return 0; 774 } 775 776 static int __init lapic_init_clockevent(void) 777 { 778 if (!lapic_timer_period) 779 return -1; 780 781 /* Calculate the scaled math multiplication factor */ 782 lapic_clockevent.mult = div_sc(lapic_timer_period/APIC_DIVISOR, 783 TICK_NSEC, lapic_clockevent.shift); 784 lapic_clockevent.max_delta_ns = 785 clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent); 786 lapic_clockevent.max_delta_ticks = 0x7FFFFFFF; 787 lapic_clockevent.min_delta_ns = 788 clockevent_delta2ns(0xF, &lapic_clockevent); 789 lapic_clockevent.min_delta_ticks = 0xF; 790 791 return 0; 792 } 793 794 bool __init apic_needs_pit(void) 795 { 796 /* 797 * If the frequencies are not known, PIT is required for both TSC 798 * and apic timer calibration. 799 */ 800 if (!tsc_khz || !cpu_khz) 801 return true; 802 803 /* Is there an APIC at all or is it disabled? */ 804 if (!boot_cpu_has(X86_FEATURE_APIC) || disable_apic) 805 return true; 806 807 /* 808 * If interrupt delivery mode is legacy PIC or virtual wire without 809 * configuration, the local APIC timer wont be set up. Make sure 810 * that the PIT is initialized. 811 */ 812 if (apic_intr_mode == APIC_PIC || 813 apic_intr_mode == APIC_VIRTUAL_WIRE_NO_CONFIG) 814 return true; 815 816 /* Virt guests may lack ARAT, but still have DEADLINE */ 817 if (!boot_cpu_has(X86_FEATURE_ARAT)) 818 return true; 819 820 /* Deadline timer is based on TSC so no further PIT action required */ 821 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 822 return false; 823 824 /* APIC timer disabled? */ 825 if (disable_apic_timer) 826 return true; 827 /* 828 * The APIC timer frequency is known already, no PIT calibration 829 * required. If unknown, let the PIT be initialized. 830 */ 831 return lapic_timer_period == 0; 832 } 833 834 static int __init calibrate_APIC_clock(void) 835 { 836 struct clock_event_device *levt = this_cpu_ptr(&lapic_events); 837 u64 tsc_perj = 0, tsc_start = 0; 838 unsigned long jif_start; 839 unsigned long deltaj; 840 long delta, deltatsc; 841 int pm_referenced = 0; 842 843 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 844 return 0; 845 846 /* 847 * Check if lapic timer has already been calibrated by platform 848 * specific routine, such as tsc calibration code. If so just fill 849 * in the clockevent structure and return. 850 */ 851 if (!lapic_init_clockevent()) { 852 apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n", 853 lapic_timer_period); 854 /* 855 * Direct calibration methods must have an always running 856 * local APIC timer, no need for broadcast timer. 857 */ 858 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY; 859 return 0; 860 } 861 862 apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n" 863 "calibrating APIC timer ...\n"); 864 865 /* 866 * There are platforms w/o global clockevent devices. Instead of 867 * making the calibration conditional on that, use a polling based 868 * approach everywhere. 869 */ 870 local_irq_disable(); 871 872 /* 873 * Setup the APIC counter to maximum. There is no way the lapic 874 * can underflow in the 100ms detection time frame 875 */ 876 __setup_APIC_LVTT(0xffffffff, 0, 0); 877 878 /* 879 * Methods to terminate the calibration loop: 880 * 1) Global clockevent if available (jiffies) 881 * 2) TSC if available and frequency is known 882 */ 883 jif_start = READ_ONCE(jiffies); 884 885 if (tsc_khz) { 886 tsc_start = rdtsc(); 887 tsc_perj = div_u64((u64)tsc_khz * 1000, HZ); 888 } 889 890 /* 891 * Enable interrupts so the tick can fire, if a global 892 * clockevent device is available 893 */ 894 local_irq_enable(); 895 896 while (lapic_cal_loops <= LAPIC_CAL_LOOPS) { 897 /* Wait for a tick to elapse */ 898 while (1) { 899 if (tsc_khz) { 900 u64 tsc_now = rdtsc(); 901 if ((tsc_now - tsc_start) >= tsc_perj) { 902 tsc_start += tsc_perj; 903 break; 904 } 905 } else { 906 unsigned long jif_now = READ_ONCE(jiffies); 907 908 if (time_after(jif_now, jif_start)) { 909 jif_start = jif_now; 910 break; 911 } 912 } 913 cpu_relax(); 914 } 915 916 /* Invoke the calibration routine */ 917 local_irq_disable(); 918 lapic_cal_handler(NULL); 919 local_irq_enable(); 920 } 921 922 local_irq_disable(); 923 924 /* Build delta t1-t2 as apic timer counts down */ 925 delta = lapic_cal_t1 - lapic_cal_t2; 926 apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta); 927 928 deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1); 929 930 /* we trust the PM based calibration if possible */ 931 pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1, 932 &delta, &deltatsc); 933 934 lapic_timer_period = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS; 935 lapic_init_clockevent(); 936 937 apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta); 938 apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult); 939 apic_printk(APIC_VERBOSE, "..... calibration result: %u\n", 940 lapic_timer_period); 941 942 if (boot_cpu_has(X86_FEATURE_TSC)) { 943 apic_printk(APIC_VERBOSE, "..... CPU clock speed is " 944 "%ld.%04ld MHz.\n", 945 (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ), 946 (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ)); 947 } 948 949 apic_printk(APIC_VERBOSE, "..... host bus clock speed is " 950 "%u.%04u MHz.\n", 951 lapic_timer_period / (1000000 / HZ), 952 lapic_timer_period % (1000000 / HZ)); 953 954 /* 955 * Do a sanity check on the APIC calibration result 956 */ 957 if (lapic_timer_period < (1000000 / HZ)) { 958 local_irq_enable(); 959 pr_warn("APIC frequency too slow, disabling apic timer\n"); 960 return -1; 961 } 962 963 levt->features &= ~CLOCK_EVT_FEAT_DUMMY; 964 965 /* 966 * PM timer calibration failed or not turned on so lets try APIC 967 * timer based calibration, if a global clockevent device is 968 * available. 969 */ 970 if (!pm_referenced && global_clock_event) { 971 apic_printk(APIC_VERBOSE, "... verify APIC timer\n"); 972 973 /* 974 * Setup the apic timer manually 975 */ 976 levt->event_handler = lapic_cal_handler; 977 lapic_timer_set_periodic(levt); 978 lapic_cal_loops = -1; 979 980 /* Let the interrupts run */ 981 local_irq_enable(); 982 983 while (lapic_cal_loops <= LAPIC_CAL_LOOPS) 984 cpu_relax(); 985 986 /* Stop the lapic timer */ 987 local_irq_disable(); 988 lapic_timer_shutdown(levt); 989 990 /* Jiffies delta */ 991 deltaj = lapic_cal_j2 - lapic_cal_j1; 992 apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj); 993 994 /* Check, if the jiffies result is consistent */ 995 if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2) 996 apic_printk(APIC_VERBOSE, "... jiffies result ok\n"); 997 else 998 levt->features |= CLOCK_EVT_FEAT_DUMMY; 999 } 1000 local_irq_enable(); 1001 1002 if (levt->features & CLOCK_EVT_FEAT_DUMMY) { 1003 pr_warn("APIC timer disabled due to verification failure\n"); 1004 return -1; 1005 } 1006 1007 return 0; 1008 } 1009 1010 /* 1011 * Setup the boot APIC 1012 * 1013 * Calibrate and verify the result. 1014 */ 1015 void __init setup_boot_APIC_clock(void) 1016 { 1017 /* 1018 * The local apic timer can be disabled via the kernel 1019 * commandline or from the CPU detection code. Register the lapic 1020 * timer as a dummy clock event source on SMP systems, so the 1021 * broadcast mechanism is used. On UP systems simply ignore it. 1022 */ 1023 if (disable_apic_timer) { 1024 pr_info("Disabling APIC timer\n"); 1025 /* No broadcast on UP ! */ 1026 if (num_possible_cpus() > 1) { 1027 lapic_clockevent.mult = 1; 1028 setup_APIC_timer(); 1029 } 1030 return; 1031 } 1032 1033 if (calibrate_APIC_clock()) { 1034 /* No broadcast on UP ! */ 1035 if (num_possible_cpus() > 1) 1036 setup_APIC_timer(); 1037 return; 1038 } 1039 1040 /* 1041 * If nmi_watchdog is set to IO_APIC, we need the 1042 * PIT/HPET going. Otherwise register lapic as a dummy 1043 * device. 1044 */ 1045 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY; 1046 1047 /* Setup the lapic or request the broadcast */ 1048 setup_APIC_timer(); 1049 amd_e400_c1e_apic_setup(); 1050 } 1051 1052 void setup_secondary_APIC_clock(void) 1053 { 1054 setup_APIC_timer(); 1055 amd_e400_c1e_apic_setup(); 1056 } 1057 1058 /* 1059 * The guts of the apic timer interrupt 1060 */ 1061 static void local_apic_timer_interrupt(void) 1062 { 1063 struct clock_event_device *evt = this_cpu_ptr(&lapic_events); 1064 1065 /* 1066 * Normally we should not be here till LAPIC has been initialized but 1067 * in some cases like kdump, its possible that there is a pending LAPIC 1068 * timer interrupt from previous kernel's context and is delivered in 1069 * new kernel the moment interrupts are enabled. 1070 * 1071 * Interrupts are enabled early and LAPIC is setup much later, hence 1072 * its possible that when we get here evt->event_handler is NULL. 1073 * Check for event_handler being NULL and discard the interrupt as 1074 * spurious. 1075 */ 1076 if (!evt->event_handler) { 1077 pr_warn("Spurious LAPIC timer interrupt on cpu %d\n", 1078 smp_processor_id()); 1079 /* Switch it off */ 1080 lapic_timer_shutdown(evt); 1081 return; 1082 } 1083 1084 /* 1085 * the NMI deadlock-detector uses this. 1086 */ 1087 inc_irq_stat(apic_timer_irqs); 1088 1089 evt->event_handler(evt); 1090 } 1091 1092 /* 1093 * Local APIC timer interrupt. This is the most natural way for doing 1094 * local interrupts, but local timer interrupts can be emulated by 1095 * broadcast interrupts too. [in case the hw doesn't support APIC timers] 1096 * 1097 * [ if a single-CPU system runs an SMP kernel then we call the local 1098 * interrupt as well. Thus we cannot inline the local irq ... ] 1099 */ 1100 DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt) 1101 { 1102 struct pt_regs *old_regs = set_irq_regs(regs); 1103 1104 ack_APIC_irq(); 1105 trace_local_timer_entry(LOCAL_TIMER_VECTOR); 1106 local_apic_timer_interrupt(); 1107 trace_local_timer_exit(LOCAL_TIMER_VECTOR); 1108 1109 set_irq_regs(old_regs); 1110 } 1111 1112 int setup_profiling_timer(unsigned int multiplier) 1113 { 1114 return -EINVAL; 1115 } 1116 1117 /* 1118 * Local APIC start and shutdown 1119 */ 1120 1121 /** 1122 * clear_local_APIC - shutdown the local APIC 1123 * 1124 * This is called, when a CPU is disabled and before rebooting, so the state of 1125 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS 1126 * leftovers during boot. 1127 */ 1128 void clear_local_APIC(void) 1129 { 1130 int maxlvt; 1131 u32 v; 1132 1133 /* APIC hasn't been mapped yet */ 1134 if (!x2apic_mode && !apic_phys) 1135 return; 1136 1137 maxlvt = lapic_get_maxlvt(); 1138 /* 1139 * Masking an LVT entry can trigger a local APIC error 1140 * if the vector is zero. Mask LVTERR first to prevent this. 1141 */ 1142 if (maxlvt >= 3) { 1143 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */ 1144 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED); 1145 } 1146 /* 1147 * Careful: we have to set masks only first to deassert 1148 * any level-triggered sources. 1149 */ 1150 v = apic_read(APIC_LVTT); 1151 apic_write(APIC_LVTT, v | APIC_LVT_MASKED); 1152 v = apic_read(APIC_LVT0); 1153 apic_write(APIC_LVT0, v | APIC_LVT_MASKED); 1154 v = apic_read(APIC_LVT1); 1155 apic_write(APIC_LVT1, v | APIC_LVT_MASKED); 1156 if (maxlvt >= 4) { 1157 v = apic_read(APIC_LVTPC); 1158 apic_write(APIC_LVTPC, v | APIC_LVT_MASKED); 1159 } 1160 1161 /* lets not touch this if we didn't frob it */ 1162 #ifdef CONFIG_X86_THERMAL_VECTOR 1163 if (maxlvt >= 5) { 1164 v = apic_read(APIC_LVTTHMR); 1165 apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED); 1166 } 1167 #endif 1168 #ifdef CONFIG_X86_MCE_INTEL 1169 if (maxlvt >= 6) { 1170 v = apic_read(APIC_LVTCMCI); 1171 if (!(v & APIC_LVT_MASKED)) 1172 apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED); 1173 } 1174 #endif 1175 1176 /* 1177 * Clean APIC state for other OSs: 1178 */ 1179 apic_write(APIC_LVTT, APIC_LVT_MASKED); 1180 apic_write(APIC_LVT0, APIC_LVT_MASKED); 1181 apic_write(APIC_LVT1, APIC_LVT_MASKED); 1182 if (maxlvt >= 3) 1183 apic_write(APIC_LVTERR, APIC_LVT_MASKED); 1184 if (maxlvt >= 4) 1185 apic_write(APIC_LVTPC, APIC_LVT_MASKED); 1186 1187 /* Integrated APIC (!82489DX) ? */ 1188 if (lapic_is_integrated()) { 1189 if (maxlvt > 3) 1190 /* Clear ESR due to Pentium errata 3AP and 11AP */ 1191 apic_write(APIC_ESR, 0); 1192 apic_read(APIC_ESR); 1193 } 1194 } 1195 1196 /** 1197 * apic_soft_disable - Clears and software disables the local APIC on hotplug 1198 * 1199 * Contrary to disable_local_APIC() this does not touch the enable bit in 1200 * MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC 1201 * bus would require a hardware reset as the APIC would lose track of bus 1202 * arbitration. On systems with FSB delivery APICBASE could be disabled, 1203 * but it has to be guaranteed that no interrupt is sent to the APIC while 1204 * in that state and it's not clear from the SDM whether it still responds 1205 * to INIT/SIPI messages. Stay on the safe side and use software disable. 1206 */ 1207 void apic_soft_disable(void) 1208 { 1209 u32 value; 1210 1211 clear_local_APIC(); 1212 1213 /* Soft disable APIC (implies clearing of registers for 82489DX!). */ 1214 value = apic_read(APIC_SPIV); 1215 value &= ~APIC_SPIV_APIC_ENABLED; 1216 apic_write(APIC_SPIV, value); 1217 } 1218 1219 /** 1220 * disable_local_APIC - clear and disable the local APIC 1221 */ 1222 void disable_local_APIC(void) 1223 { 1224 /* APIC hasn't been mapped yet */ 1225 if (!x2apic_mode && !apic_phys) 1226 return; 1227 1228 apic_soft_disable(); 1229 1230 #ifdef CONFIG_X86_32 1231 /* 1232 * When LAPIC was disabled by the BIOS and enabled by the kernel, 1233 * restore the disabled state. 1234 */ 1235 if (enabled_via_apicbase) { 1236 unsigned int l, h; 1237 1238 rdmsr(MSR_IA32_APICBASE, l, h); 1239 l &= ~MSR_IA32_APICBASE_ENABLE; 1240 wrmsr(MSR_IA32_APICBASE, l, h); 1241 } 1242 #endif 1243 } 1244 1245 /* 1246 * If Linux enabled the LAPIC against the BIOS default disable it down before 1247 * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and 1248 * not power-off. Additionally clear all LVT entries before disable_local_APIC 1249 * for the case where Linux didn't enable the LAPIC. 1250 */ 1251 void lapic_shutdown(void) 1252 { 1253 unsigned long flags; 1254 1255 if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config()) 1256 return; 1257 1258 local_irq_save(flags); 1259 1260 #ifdef CONFIG_X86_32 1261 if (!enabled_via_apicbase) 1262 clear_local_APIC(); 1263 else 1264 #endif 1265 disable_local_APIC(); 1266 1267 1268 local_irq_restore(flags); 1269 } 1270 1271 /** 1272 * sync_Arb_IDs - synchronize APIC bus arbitration IDs 1273 */ 1274 void __init sync_Arb_IDs(void) 1275 { 1276 /* 1277 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not 1278 * needed on AMD. 1279 */ 1280 if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 1281 return; 1282 1283 /* 1284 * Wait for idle. 1285 */ 1286 apic_wait_icr_idle(); 1287 1288 apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n"); 1289 apic_write(APIC_ICR, APIC_DEST_ALLINC | 1290 APIC_INT_LEVELTRIG | APIC_DM_INIT); 1291 } 1292 1293 enum apic_intr_mode_id apic_intr_mode __ro_after_init; 1294 1295 static int __init __apic_intr_mode_select(void) 1296 { 1297 /* Check kernel option */ 1298 if (disable_apic) { 1299 pr_info("APIC disabled via kernel command line\n"); 1300 return APIC_PIC; 1301 } 1302 1303 /* Check BIOS */ 1304 #ifdef CONFIG_X86_64 1305 /* On 64-bit, the APIC must be integrated, Check local APIC only */ 1306 if (!boot_cpu_has(X86_FEATURE_APIC)) { 1307 disable_apic = 1; 1308 pr_info("APIC disabled by BIOS\n"); 1309 return APIC_PIC; 1310 } 1311 #else 1312 /* On 32-bit, the APIC may be integrated APIC or 82489DX */ 1313 1314 /* Neither 82489DX nor integrated APIC ? */ 1315 if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) { 1316 disable_apic = 1; 1317 return APIC_PIC; 1318 } 1319 1320 /* If the BIOS pretends there is an integrated APIC ? */ 1321 if (!boot_cpu_has(X86_FEATURE_APIC) && 1322 APIC_INTEGRATED(boot_cpu_apic_version)) { 1323 disable_apic = 1; 1324 pr_err(FW_BUG "Local APIC %d not detected, force emulation\n", 1325 boot_cpu_physical_apicid); 1326 return APIC_PIC; 1327 } 1328 #endif 1329 1330 /* Check MP table or ACPI MADT configuration */ 1331 if (!smp_found_config) { 1332 disable_ioapic_support(); 1333 if (!acpi_lapic) { 1334 pr_info("APIC: ACPI MADT or MP tables are not detected\n"); 1335 return APIC_VIRTUAL_WIRE_NO_CONFIG; 1336 } 1337 return APIC_VIRTUAL_WIRE; 1338 } 1339 1340 #ifdef CONFIG_SMP 1341 /* If SMP should be disabled, then really disable it! */ 1342 if (!setup_max_cpus) { 1343 pr_info("APIC: SMP mode deactivated\n"); 1344 return APIC_SYMMETRIC_IO_NO_ROUTING; 1345 } 1346 1347 if (read_apic_id() != boot_cpu_physical_apicid) { 1348 panic("Boot APIC ID in local APIC unexpected (%d vs %d)", 1349 read_apic_id(), boot_cpu_physical_apicid); 1350 /* Or can we switch back to PIC here? */ 1351 } 1352 #endif 1353 1354 return APIC_SYMMETRIC_IO; 1355 } 1356 1357 /* Select the interrupt delivery mode for the BSP */ 1358 void __init apic_intr_mode_select(void) 1359 { 1360 apic_intr_mode = __apic_intr_mode_select(); 1361 } 1362 1363 /* 1364 * An initial setup of the virtual wire mode. 1365 */ 1366 void __init init_bsp_APIC(void) 1367 { 1368 unsigned int value; 1369 1370 /* 1371 * Don't do the setup now if we have a SMP BIOS as the 1372 * through-I/O-APIC virtual wire mode might be active. 1373 */ 1374 if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC)) 1375 return; 1376 1377 /* 1378 * Do not trust the local APIC being empty at bootup. 1379 */ 1380 clear_local_APIC(); 1381 1382 /* 1383 * Enable APIC. 1384 */ 1385 value = apic_read(APIC_SPIV); 1386 value &= ~APIC_VECTOR_MASK; 1387 value |= APIC_SPIV_APIC_ENABLED; 1388 1389 #ifdef CONFIG_X86_32 1390 /* This bit is reserved on P4/Xeon and should be cleared */ 1391 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && 1392 (boot_cpu_data.x86 == 15)) 1393 value &= ~APIC_SPIV_FOCUS_DISABLED; 1394 else 1395 #endif 1396 value |= APIC_SPIV_FOCUS_DISABLED; 1397 value |= SPURIOUS_APIC_VECTOR; 1398 apic_write(APIC_SPIV, value); 1399 1400 /* 1401 * Set up the virtual wire mode. 1402 */ 1403 apic_write(APIC_LVT0, APIC_DM_EXTINT); 1404 value = APIC_DM_NMI; 1405 if (!lapic_is_integrated()) /* 82489DX */ 1406 value |= APIC_LVT_LEVEL_TRIGGER; 1407 if (apic_extnmi == APIC_EXTNMI_NONE) 1408 value |= APIC_LVT_MASKED; 1409 apic_write(APIC_LVT1, value); 1410 } 1411 1412 static void __init apic_bsp_setup(bool upmode); 1413 1414 /* Init the interrupt delivery mode for the BSP */ 1415 void __init apic_intr_mode_init(void) 1416 { 1417 bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT); 1418 1419 switch (apic_intr_mode) { 1420 case APIC_PIC: 1421 pr_info("APIC: Keep in PIC mode(8259)\n"); 1422 return; 1423 case APIC_VIRTUAL_WIRE: 1424 pr_info("APIC: Switch to virtual wire mode setup\n"); 1425 default_setup_apic_routing(); 1426 break; 1427 case APIC_VIRTUAL_WIRE_NO_CONFIG: 1428 pr_info("APIC: Switch to virtual wire mode setup with no configuration\n"); 1429 upmode = true; 1430 default_setup_apic_routing(); 1431 break; 1432 case APIC_SYMMETRIC_IO: 1433 pr_info("APIC: Switch to symmetric I/O mode setup\n"); 1434 default_setup_apic_routing(); 1435 break; 1436 case APIC_SYMMETRIC_IO_NO_ROUTING: 1437 pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n"); 1438 break; 1439 } 1440 1441 if (x86_platform.apic_post_init) 1442 x86_platform.apic_post_init(); 1443 1444 apic_bsp_setup(upmode); 1445 } 1446 1447 static void lapic_setup_esr(void) 1448 { 1449 unsigned int oldvalue, value, maxlvt; 1450 1451 if (!lapic_is_integrated()) { 1452 pr_info("No ESR for 82489DX.\n"); 1453 return; 1454 } 1455 1456 if (apic->disable_esr) { 1457 /* 1458 * Something untraceable is creating bad interrupts on 1459 * secondary quads ... for the moment, just leave the 1460 * ESR disabled - we can't do anything useful with the 1461 * errors anyway - mbligh 1462 */ 1463 pr_info("Leaving ESR disabled.\n"); 1464 return; 1465 } 1466 1467 maxlvt = lapic_get_maxlvt(); 1468 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ 1469 apic_write(APIC_ESR, 0); 1470 oldvalue = apic_read(APIC_ESR); 1471 1472 /* enables sending errors */ 1473 value = ERROR_APIC_VECTOR; 1474 apic_write(APIC_LVTERR, value); 1475 1476 /* 1477 * spec says clear errors after enabling vector. 1478 */ 1479 if (maxlvt > 3) 1480 apic_write(APIC_ESR, 0); 1481 value = apic_read(APIC_ESR); 1482 if (value != oldvalue) 1483 apic_printk(APIC_VERBOSE, "ESR value before enabling " 1484 "vector: 0x%08x after: 0x%08x\n", 1485 oldvalue, value); 1486 } 1487 1488 #define APIC_IR_REGS APIC_ISR_NR 1489 #define APIC_IR_BITS (APIC_IR_REGS * 32) 1490 #define APIC_IR_MAPSIZE (APIC_IR_BITS / BITS_PER_LONG) 1491 1492 union apic_ir { 1493 unsigned long map[APIC_IR_MAPSIZE]; 1494 u32 regs[APIC_IR_REGS]; 1495 }; 1496 1497 static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr) 1498 { 1499 int i, bit; 1500 1501 /* Read the IRRs */ 1502 for (i = 0; i < APIC_IR_REGS; i++) 1503 irr->regs[i] = apic_read(APIC_IRR + i * 0x10); 1504 1505 /* Read the ISRs */ 1506 for (i = 0; i < APIC_IR_REGS; i++) 1507 isr->regs[i] = apic_read(APIC_ISR + i * 0x10); 1508 1509 /* 1510 * If the ISR map is not empty. ACK the APIC and run another round 1511 * to verify whether a pending IRR has been unblocked and turned 1512 * into a ISR. 1513 */ 1514 if (!bitmap_empty(isr->map, APIC_IR_BITS)) { 1515 /* 1516 * There can be multiple ISR bits set when a high priority 1517 * interrupt preempted a lower priority one. Issue an ACK 1518 * per set bit. 1519 */ 1520 for_each_set_bit(bit, isr->map, APIC_IR_BITS) 1521 ack_APIC_irq(); 1522 return true; 1523 } 1524 1525 return !bitmap_empty(irr->map, APIC_IR_BITS); 1526 } 1527 1528 /* 1529 * After a crash, we no longer service the interrupts and a pending 1530 * interrupt from previous kernel might still have ISR bit set. 1531 * 1532 * Most probably by now the CPU has serviced that pending interrupt and it 1533 * might not have done the ack_APIC_irq() because it thought, interrupt 1534 * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear 1535 * the ISR bit and cpu thinks it has already serivced the interrupt. Hence 1536 * a vector might get locked. It was noticed for timer irq (vector 1537 * 0x31). Issue an extra EOI to clear ISR. 1538 * 1539 * If there are pending IRR bits they turn into ISR bits after a higher 1540 * priority ISR bit has been acked. 1541 */ 1542 static void apic_pending_intr_clear(void) 1543 { 1544 union apic_ir irr, isr; 1545 unsigned int i; 1546 1547 /* 512 loops are way oversized and give the APIC a chance to obey. */ 1548 for (i = 0; i < 512; i++) { 1549 if (!apic_check_and_ack(&irr, &isr)) 1550 return; 1551 } 1552 /* Dump the IRR/ISR content if that failed */ 1553 pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr.map, isr.map); 1554 } 1555 1556 /** 1557 * setup_local_APIC - setup the local APIC 1558 * 1559 * Used to setup local APIC while initializing BSP or bringing up APs. 1560 * Always called with preemption disabled. 1561 */ 1562 static void setup_local_APIC(void) 1563 { 1564 int cpu = smp_processor_id(); 1565 unsigned int value; 1566 1567 if (disable_apic) { 1568 disable_ioapic_support(); 1569 return; 1570 } 1571 1572 /* 1573 * If this comes from kexec/kcrash the APIC might be enabled in 1574 * SPIV. Soft disable it before doing further initialization. 1575 */ 1576 value = apic_read(APIC_SPIV); 1577 value &= ~APIC_SPIV_APIC_ENABLED; 1578 apic_write(APIC_SPIV, value); 1579 1580 #ifdef CONFIG_X86_32 1581 /* Pound the ESR really hard over the head with a big hammer - mbligh */ 1582 if (lapic_is_integrated() && apic->disable_esr) { 1583 apic_write(APIC_ESR, 0); 1584 apic_write(APIC_ESR, 0); 1585 apic_write(APIC_ESR, 0); 1586 apic_write(APIC_ESR, 0); 1587 } 1588 #endif 1589 /* 1590 * Double-check whether this APIC is really registered. 1591 * This is meaningless in clustered apic mode, so we skip it. 1592 */ 1593 BUG_ON(!apic->apic_id_registered()); 1594 1595 /* 1596 * Intel recommends to set DFR, LDR and TPR before enabling 1597 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel 1598 * document number 292116). So here it goes... 1599 */ 1600 apic->init_apic_ldr(); 1601 1602 #ifdef CONFIG_X86_32 1603 if (apic->dest_mode_logical) { 1604 int logical_apicid, ldr_apicid; 1605 1606 /* 1607 * APIC LDR is initialized. If logical_apicid mapping was 1608 * initialized during get_smp_config(), make sure it matches 1609 * the actual value. 1610 */ 1611 logical_apicid = early_per_cpu(x86_cpu_to_logical_apicid, cpu); 1612 ldr_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR)); 1613 if (logical_apicid != BAD_APICID) 1614 WARN_ON(logical_apicid != ldr_apicid); 1615 /* Always use the value from LDR. */ 1616 early_per_cpu(x86_cpu_to_logical_apicid, cpu) = ldr_apicid; 1617 } 1618 #endif 1619 1620 /* 1621 * Set Task Priority to 'accept all except vectors 0-31'. An APIC 1622 * vector in the 16-31 range could be delivered if TPR == 0, but we 1623 * would think it's an exception and terrible things will happen. We 1624 * never change this later on. 1625 */ 1626 value = apic_read(APIC_TASKPRI); 1627 value &= ~APIC_TPRI_MASK; 1628 value |= 0x10; 1629 apic_write(APIC_TASKPRI, value); 1630 1631 /* Clear eventually stale ISR/IRR bits */ 1632 apic_pending_intr_clear(); 1633 1634 /* 1635 * Now that we are all set up, enable the APIC 1636 */ 1637 value = apic_read(APIC_SPIV); 1638 value &= ~APIC_VECTOR_MASK; 1639 /* 1640 * Enable APIC 1641 */ 1642 value |= APIC_SPIV_APIC_ENABLED; 1643 1644 #ifdef CONFIG_X86_32 1645 /* 1646 * Some unknown Intel IO/APIC (or APIC) errata is biting us with 1647 * certain networking cards. If high frequency interrupts are 1648 * happening on a particular IOAPIC pin, plus the IOAPIC routing 1649 * entry is masked/unmasked at a high rate as well then sooner or 1650 * later IOAPIC line gets 'stuck', no more interrupts are received 1651 * from the device. If focus CPU is disabled then the hang goes 1652 * away, oh well :-( 1653 * 1654 * [ This bug can be reproduced easily with a level-triggered 1655 * PCI Ne2000 networking cards and PII/PIII processors, dual 1656 * BX chipset. ] 1657 */ 1658 /* 1659 * Actually disabling the focus CPU check just makes the hang less 1660 * frequent as it makes the interrupt distributon model be more 1661 * like LRU than MRU (the short-term load is more even across CPUs). 1662 */ 1663 1664 /* 1665 * - enable focus processor (bit==0) 1666 * - 64bit mode always use processor focus 1667 * so no need to set it 1668 */ 1669 value &= ~APIC_SPIV_FOCUS_DISABLED; 1670 #endif 1671 1672 /* 1673 * Set spurious IRQ vector 1674 */ 1675 value |= SPURIOUS_APIC_VECTOR; 1676 apic_write(APIC_SPIV, value); 1677 1678 perf_events_lapic_init(); 1679 1680 /* 1681 * Set up LVT0, LVT1: 1682 * 1683 * set up through-local-APIC on the boot CPU's LINT0. This is not 1684 * strictly necessary in pure symmetric-IO mode, but sometimes 1685 * we delegate interrupts to the 8259A. 1686 */ 1687 /* 1688 * TODO: set up through-local-APIC from through-I/O-APIC? --macro 1689 */ 1690 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED; 1691 if (!cpu && (pic_mode || !value || skip_ioapic_setup)) { 1692 value = APIC_DM_EXTINT; 1693 apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu); 1694 } else { 1695 value = APIC_DM_EXTINT | APIC_LVT_MASKED; 1696 apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu); 1697 } 1698 apic_write(APIC_LVT0, value); 1699 1700 /* 1701 * Only the BSP sees the LINT1 NMI signal by default. This can be 1702 * modified by apic_extnmi= boot option. 1703 */ 1704 if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) || 1705 apic_extnmi == APIC_EXTNMI_ALL) 1706 value = APIC_DM_NMI; 1707 else 1708 value = APIC_DM_NMI | APIC_LVT_MASKED; 1709 1710 /* Is 82489DX ? */ 1711 if (!lapic_is_integrated()) 1712 value |= APIC_LVT_LEVEL_TRIGGER; 1713 apic_write(APIC_LVT1, value); 1714 1715 #ifdef CONFIG_X86_MCE_INTEL 1716 /* Recheck CMCI information after local APIC is up on CPU #0 */ 1717 if (!cpu) 1718 cmci_recheck(); 1719 #endif 1720 } 1721 1722 static void end_local_APIC_setup(void) 1723 { 1724 lapic_setup_esr(); 1725 1726 #ifdef CONFIG_X86_32 1727 { 1728 unsigned int value; 1729 /* Disable the local apic timer */ 1730 value = apic_read(APIC_LVTT); 1731 value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); 1732 apic_write(APIC_LVTT, value); 1733 } 1734 #endif 1735 1736 apic_pm_activate(); 1737 } 1738 1739 /* 1740 * APIC setup function for application processors. Called from smpboot.c 1741 */ 1742 void apic_ap_setup(void) 1743 { 1744 setup_local_APIC(); 1745 end_local_APIC_setup(); 1746 } 1747 1748 #ifdef CONFIG_X86_X2APIC 1749 int x2apic_mode; 1750 1751 enum { 1752 X2APIC_OFF, 1753 X2APIC_ON, 1754 X2APIC_DISABLED, 1755 }; 1756 static int x2apic_state; 1757 1758 static void __x2apic_disable(void) 1759 { 1760 u64 msr; 1761 1762 if (!boot_cpu_has(X86_FEATURE_APIC)) 1763 return; 1764 1765 rdmsrl(MSR_IA32_APICBASE, msr); 1766 if (!(msr & X2APIC_ENABLE)) 1767 return; 1768 /* Disable xapic and x2apic first and then reenable xapic mode */ 1769 wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE)); 1770 wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE); 1771 printk_once(KERN_INFO "x2apic disabled\n"); 1772 } 1773 1774 static void __x2apic_enable(void) 1775 { 1776 u64 msr; 1777 1778 rdmsrl(MSR_IA32_APICBASE, msr); 1779 if (msr & X2APIC_ENABLE) 1780 return; 1781 wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE); 1782 printk_once(KERN_INFO "x2apic enabled\n"); 1783 } 1784 1785 static int __init setup_nox2apic(char *str) 1786 { 1787 if (x2apic_enabled()) { 1788 int apicid = native_apic_msr_read(APIC_ID); 1789 1790 if (apicid >= 255) { 1791 pr_warn("Apicid: %08x, cannot enforce nox2apic\n", 1792 apicid); 1793 return 0; 1794 } 1795 pr_warn("x2apic already enabled.\n"); 1796 __x2apic_disable(); 1797 } 1798 setup_clear_cpu_cap(X86_FEATURE_X2APIC); 1799 x2apic_state = X2APIC_DISABLED; 1800 x2apic_mode = 0; 1801 return 0; 1802 } 1803 early_param("nox2apic", setup_nox2apic); 1804 1805 /* Called from cpu_init() to enable x2apic on (secondary) cpus */ 1806 void x2apic_setup(void) 1807 { 1808 /* 1809 * If x2apic is not in ON state, disable it if already enabled 1810 * from BIOS. 1811 */ 1812 if (x2apic_state != X2APIC_ON) { 1813 __x2apic_disable(); 1814 return; 1815 } 1816 __x2apic_enable(); 1817 } 1818 1819 static __init void x2apic_disable(void) 1820 { 1821 u32 x2apic_id, state = x2apic_state; 1822 1823 x2apic_mode = 0; 1824 x2apic_state = X2APIC_DISABLED; 1825 1826 if (state != X2APIC_ON) 1827 return; 1828 1829 x2apic_id = read_apic_id(); 1830 if (x2apic_id >= 255) 1831 panic("Cannot disable x2apic, id: %08x\n", x2apic_id); 1832 1833 __x2apic_disable(); 1834 register_lapic_address(mp_lapic_addr); 1835 } 1836 1837 static __init void x2apic_enable(void) 1838 { 1839 if (x2apic_state != X2APIC_OFF) 1840 return; 1841 1842 x2apic_mode = 1; 1843 x2apic_state = X2APIC_ON; 1844 __x2apic_enable(); 1845 } 1846 1847 static __init void try_to_enable_x2apic(int remap_mode) 1848 { 1849 if (x2apic_state == X2APIC_DISABLED) 1850 return; 1851 1852 if (remap_mode != IRQ_REMAP_X2APIC_MODE) { 1853 u32 apic_limit = 255; 1854 1855 /* 1856 * Using X2APIC without IR is not architecturally supported 1857 * on bare metal but may be supported in guests. 1858 */ 1859 if (!x86_init.hyper.x2apic_available()) { 1860 pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n"); 1861 x2apic_disable(); 1862 return; 1863 } 1864 1865 /* 1866 * If the hypervisor supports extended destination ID in 1867 * MSI, that increases the maximum APIC ID that can be 1868 * used for non-remapped IRQ domains. 1869 */ 1870 if (x86_init.hyper.msi_ext_dest_id()) { 1871 virt_ext_dest_id = 1; 1872 apic_limit = 32767; 1873 } 1874 1875 /* 1876 * Without IR, all CPUs can be addressed by IOAPIC/MSI only 1877 * in physical mode, and CPUs with an APIC ID that cannnot 1878 * be addressed must not be brought online. 1879 */ 1880 x2apic_set_max_apicid(apic_limit); 1881 x2apic_phys = 1; 1882 } 1883 x2apic_enable(); 1884 } 1885 1886 void __init check_x2apic(void) 1887 { 1888 if (x2apic_enabled()) { 1889 pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n"); 1890 x2apic_mode = 1; 1891 x2apic_state = X2APIC_ON; 1892 } else if (!boot_cpu_has(X86_FEATURE_X2APIC)) { 1893 x2apic_state = X2APIC_DISABLED; 1894 } 1895 } 1896 #else /* CONFIG_X86_X2APIC */ 1897 static int __init validate_x2apic(void) 1898 { 1899 if (!apic_is_x2apic_enabled()) 1900 return 0; 1901 /* 1902 * Checkme: Can we simply turn off x2apic here instead of panic? 1903 */ 1904 panic("BIOS has enabled x2apic but kernel doesn't support x2apic, please disable x2apic in BIOS.\n"); 1905 } 1906 early_initcall(validate_x2apic); 1907 1908 static inline void try_to_enable_x2apic(int remap_mode) { } 1909 static inline void __x2apic_enable(void) { } 1910 #endif /* !CONFIG_X86_X2APIC */ 1911 1912 void __init enable_IR_x2apic(void) 1913 { 1914 unsigned long flags; 1915 int ret, ir_stat; 1916 1917 if (skip_ioapic_setup) { 1918 pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n"); 1919 return; 1920 } 1921 1922 ir_stat = irq_remapping_prepare(); 1923 if (ir_stat < 0 && !x2apic_supported()) 1924 return; 1925 1926 ret = save_ioapic_entries(); 1927 if (ret) { 1928 pr_info("Saving IO-APIC state failed: %d\n", ret); 1929 return; 1930 } 1931 1932 local_irq_save(flags); 1933 legacy_pic->mask_all(); 1934 mask_ioapic_entries(); 1935 1936 /* If irq_remapping_prepare() succeeded, try to enable it */ 1937 if (ir_stat >= 0) 1938 ir_stat = irq_remapping_enable(); 1939 /* ir_stat contains the remap mode or an error code */ 1940 try_to_enable_x2apic(ir_stat); 1941 1942 if (ir_stat < 0) 1943 restore_ioapic_entries(); 1944 legacy_pic->restore_mask(); 1945 local_irq_restore(flags); 1946 } 1947 1948 #ifdef CONFIG_X86_64 1949 /* 1950 * Detect and enable local APICs on non-SMP boards. 1951 * Original code written by Keir Fraser. 1952 * On AMD64 we trust the BIOS - if it says no APIC it is likely 1953 * not correctly set up (usually the APIC timer won't work etc.) 1954 */ 1955 static int __init detect_init_APIC(void) 1956 { 1957 if (!boot_cpu_has(X86_FEATURE_APIC)) { 1958 pr_info("No local APIC present\n"); 1959 return -1; 1960 } 1961 1962 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; 1963 return 0; 1964 } 1965 #else 1966 1967 static int __init apic_verify(void) 1968 { 1969 u32 features, h, l; 1970 1971 /* 1972 * The APIC feature bit should now be enabled 1973 * in `cpuid' 1974 */ 1975 features = cpuid_edx(1); 1976 if (!(features & (1 << X86_FEATURE_APIC))) { 1977 pr_warn("Could not enable APIC!\n"); 1978 return -1; 1979 } 1980 set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC); 1981 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; 1982 1983 /* The BIOS may have set up the APIC at some other address */ 1984 if (boot_cpu_data.x86 >= 6) { 1985 rdmsr(MSR_IA32_APICBASE, l, h); 1986 if (l & MSR_IA32_APICBASE_ENABLE) 1987 mp_lapic_addr = l & MSR_IA32_APICBASE_BASE; 1988 } 1989 1990 pr_info("Found and enabled local APIC!\n"); 1991 return 0; 1992 } 1993 1994 int __init apic_force_enable(unsigned long addr) 1995 { 1996 u32 h, l; 1997 1998 if (disable_apic) 1999 return -1; 2000 2001 /* 2002 * Some BIOSes disable the local APIC in the APIC_BASE 2003 * MSR. This can only be done in software for Intel P6 or later 2004 * and AMD K7 (Model > 1) or later. 2005 */ 2006 if (boot_cpu_data.x86 >= 6) { 2007 rdmsr(MSR_IA32_APICBASE, l, h); 2008 if (!(l & MSR_IA32_APICBASE_ENABLE)) { 2009 pr_info("Local APIC disabled by BIOS -- reenabling.\n"); 2010 l &= ~MSR_IA32_APICBASE_BASE; 2011 l |= MSR_IA32_APICBASE_ENABLE | addr; 2012 wrmsr(MSR_IA32_APICBASE, l, h); 2013 enabled_via_apicbase = 1; 2014 } 2015 } 2016 return apic_verify(); 2017 } 2018 2019 /* 2020 * Detect and initialize APIC 2021 */ 2022 static int __init detect_init_APIC(void) 2023 { 2024 /* Disabled by kernel option? */ 2025 if (disable_apic) 2026 return -1; 2027 2028 switch (boot_cpu_data.x86_vendor) { 2029 case X86_VENDOR_AMD: 2030 if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) || 2031 (boot_cpu_data.x86 >= 15)) 2032 break; 2033 goto no_apic; 2034 case X86_VENDOR_HYGON: 2035 break; 2036 case X86_VENDOR_INTEL: 2037 if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 || 2038 (boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC))) 2039 break; 2040 goto no_apic; 2041 default: 2042 goto no_apic; 2043 } 2044 2045 if (!boot_cpu_has(X86_FEATURE_APIC)) { 2046 /* 2047 * Over-ride BIOS and try to enable the local APIC only if 2048 * "lapic" specified. 2049 */ 2050 if (!force_enable_local_apic) { 2051 pr_info("Local APIC disabled by BIOS -- " 2052 "you can enable it with \"lapic\"\n"); 2053 return -1; 2054 } 2055 if (apic_force_enable(APIC_DEFAULT_PHYS_BASE)) 2056 return -1; 2057 } else { 2058 if (apic_verify()) 2059 return -1; 2060 } 2061 2062 apic_pm_activate(); 2063 2064 return 0; 2065 2066 no_apic: 2067 pr_info("No local APIC present or hardware disabled\n"); 2068 return -1; 2069 } 2070 #endif 2071 2072 /** 2073 * init_apic_mappings - initialize APIC mappings 2074 */ 2075 void __init init_apic_mappings(void) 2076 { 2077 unsigned int new_apicid; 2078 2079 if (apic_validate_deadline_timer()) 2080 pr_info("TSC deadline timer available\n"); 2081 2082 if (x2apic_mode) { 2083 boot_cpu_physical_apicid = read_apic_id(); 2084 return; 2085 } 2086 2087 /* If no local APIC can be found return early */ 2088 if (!smp_found_config && detect_init_APIC()) { 2089 /* lets NOP'ify apic operations */ 2090 pr_info("APIC: disable apic facility\n"); 2091 apic_disable(); 2092 } else { 2093 apic_phys = mp_lapic_addr; 2094 2095 /* 2096 * If the system has ACPI MADT tables or MP info, the LAPIC 2097 * address is already registered. 2098 */ 2099 if (!acpi_lapic && !smp_found_config) 2100 register_lapic_address(apic_phys); 2101 } 2102 2103 /* 2104 * Fetch the APIC ID of the BSP in case we have a 2105 * default configuration (or the MP table is broken). 2106 */ 2107 new_apicid = read_apic_id(); 2108 if (boot_cpu_physical_apicid != new_apicid) { 2109 boot_cpu_physical_apicid = new_apicid; 2110 /* 2111 * yeah -- we lie about apic_version 2112 * in case if apic was disabled via boot option 2113 * but it's not a problem for SMP compiled kernel 2114 * since apic_intr_mode_select is prepared for such 2115 * a case and disable smp mode 2116 */ 2117 boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR)); 2118 } 2119 } 2120 2121 void __init register_lapic_address(unsigned long address) 2122 { 2123 mp_lapic_addr = address; 2124 2125 if (!x2apic_mode) { 2126 set_fixmap_nocache(FIX_APIC_BASE, address); 2127 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", 2128 APIC_BASE, address); 2129 } 2130 if (boot_cpu_physical_apicid == -1U) { 2131 boot_cpu_physical_apicid = read_apic_id(); 2132 boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR)); 2133 } 2134 } 2135 2136 /* 2137 * Local APIC interrupts 2138 */ 2139 2140 /** 2141 * spurious_interrupt - Catch all for interrupts raised on unused vectors 2142 * @regs: Pointer to pt_regs on stack 2143 * @vector: The vector number 2144 * 2145 * This is invoked from ASM entry code to catch all interrupts which 2146 * trigger on an entry which is routed to the common_spurious idtentry 2147 * point. 2148 * 2149 * Also called from sysvec_spurious_apic_interrupt(). 2150 */ 2151 DEFINE_IDTENTRY_IRQ(spurious_interrupt) 2152 { 2153 u32 v; 2154 2155 trace_spurious_apic_entry(vector); 2156 2157 inc_irq_stat(irq_spurious_count); 2158 2159 /* 2160 * If this is a spurious interrupt then do not acknowledge 2161 */ 2162 if (vector == SPURIOUS_APIC_VECTOR) { 2163 /* See SDM vol 3 */ 2164 pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n", 2165 smp_processor_id()); 2166 goto out; 2167 } 2168 2169 /* 2170 * If it is a vectored one, verify it's set in the ISR. If set, 2171 * acknowledge it. 2172 */ 2173 v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1)); 2174 if (v & (1 << (vector & 0x1f))) { 2175 pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n", 2176 vector, smp_processor_id()); 2177 ack_APIC_irq(); 2178 } else { 2179 pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n", 2180 vector, smp_processor_id()); 2181 } 2182 out: 2183 trace_spurious_apic_exit(vector); 2184 } 2185 2186 DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt) 2187 { 2188 __spurious_interrupt(regs, SPURIOUS_APIC_VECTOR); 2189 } 2190 2191 /* 2192 * This interrupt should never happen with our APIC/SMP architecture 2193 */ 2194 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt) 2195 { 2196 static const char * const error_interrupt_reason[] = { 2197 "Send CS error", /* APIC Error Bit 0 */ 2198 "Receive CS error", /* APIC Error Bit 1 */ 2199 "Send accept error", /* APIC Error Bit 2 */ 2200 "Receive accept error", /* APIC Error Bit 3 */ 2201 "Redirectable IPI", /* APIC Error Bit 4 */ 2202 "Send illegal vector", /* APIC Error Bit 5 */ 2203 "Received illegal vector", /* APIC Error Bit 6 */ 2204 "Illegal register address", /* APIC Error Bit 7 */ 2205 }; 2206 u32 v, i = 0; 2207 2208 trace_error_apic_entry(ERROR_APIC_VECTOR); 2209 2210 /* First tickle the hardware, only then report what went on. -- REW */ 2211 if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */ 2212 apic_write(APIC_ESR, 0); 2213 v = apic_read(APIC_ESR); 2214 ack_APIC_irq(); 2215 atomic_inc(&irq_err_count); 2216 2217 apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x", 2218 smp_processor_id(), v); 2219 2220 v &= 0xff; 2221 while (v) { 2222 if (v & 0x1) 2223 apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]); 2224 i++; 2225 v >>= 1; 2226 } 2227 2228 apic_printk(APIC_DEBUG, KERN_CONT "\n"); 2229 2230 trace_error_apic_exit(ERROR_APIC_VECTOR); 2231 } 2232 2233 /** 2234 * connect_bsp_APIC - attach the APIC to the interrupt system 2235 */ 2236 static void __init connect_bsp_APIC(void) 2237 { 2238 #ifdef CONFIG_X86_32 2239 if (pic_mode) { 2240 /* 2241 * Do not trust the local APIC being empty at bootup. 2242 */ 2243 clear_local_APIC(); 2244 /* 2245 * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's 2246 * local APIC to INT and NMI lines. 2247 */ 2248 apic_printk(APIC_VERBOSE, "leaving PIC mode, " 2249 "enabling APIC mode.\n"); 2250 imcr_pic_to_apic(); 2251 } 2252 #endif 2253 } 2254 2255 /** 2256 * disconnect_bsp_APIC - detach the APIC from the interrupt system 2257 * @virt_wire_setup: indicates, whether virtual wire mode is selected 2258 * 2259 * Virtual wire mode is necessary to deliver legacy interrupts even when the 2260 * APIC is disabled. 2261 */ 2262 void disconnect_bsp_APIC(int virt_wire_setup) 2263 { 2264 unsigned int value; 2265 2266 #ifdef CONFIG_X86_32 2267 if (pic_mode) { 2268 /* 2269 * Put the board back into PIC mode (has an effect only on 2270 * certain older boards). Note that APIC interrupts, including 2271 * IPIs, won't work beyond this point! The only exception are 2272 * INIT IPIs. 2273 */ 2274 apic_printk(APIC_VERBOSE, "disabling APIC mode, " 2275 "entering PIC mode.\n"); 2276 imcr_apic_to_pic(); 2277 return; 2278 } 2279 #endif 2280 2281 /* Go back to Virtual Wire compatibility mode */ 2282 2283 /* For the spurious interrupt use vector F, and enable it */ 2284 value = apic_read(APIC_SPIV); 2285 value &= ~APIC_VECTOR_MASK; 2286 value |= APIC_SPIV_APIC_ENABLED; 2287 value |= 0xf; 2288 apic_write(APIC_SPIV, value); 2289 2290 if (!virt_wire_setup) { 2291 /* 2292 * For LVT0 make it edge triggered, active high, 2293 * external and enabled 2294 */ 2295 value = apic_read(APIC_LVT0); 2296 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | 2297 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | 2298 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED); 2299 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; 2300 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT); 2301 apic_write(APIC_LVT0, value); 2302 } else { 2303 /* Disable LVT0 */ 2304 apic_write(APIC_LVT0, APIC_LVT_MASKED); 2305 } 2306 2307 /* 2308 * For LVT1 make it edge triggered, active high, 2309 * nmi and enabled 2310 */ 2311 value = apic_read(APIC_LVT1); 2312 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | 2313 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | 2314 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED); 2315 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; 2316 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI); 2317 apic_write(APIC_LVT1, value); 2318 } 2319 2320 /* 2321 * The number of allocated logical CPU IDs. Since logical CPU IDs are allocated 2322 * contiguously, it equals to current allocated max logical CPU ID plus 1. 2323 * All allocated CPU IDs should be in the [0, nr_logical_cpuids) range, 2324 * so the maximum of nr_logical_cpuids is nr_cpu_ids. 2325 * 2326 * NOTE: Reserve 0 for BSP. 2327 */ 2328 static int nr_logical_cpuids = 1; 2329 2330 /* 2331 * Used to store mapping between logical CPU IDs and APIC IDs. 2332 */ 2333 static int cpuid_to_apicid[] = { 2334 [0 ... NR_CPUS - 1] = -1, 2335 }; 2336 2337 #ifdef CONFIG_SMP 2338 /** 2339 * apic_id_is_primary_thread - Check whether APIC ID belongs to a primary thread 2340 * @apicid: APIC ID to check 2341 */ 2342 bool apic_id_is_primary_thread(unsigned int apicid) 2343 { 2344 u32 mask; 2345 2346 if (smp_num_siblings == 1) 2347 return true; 2348 /* Isolate the SMT bit(s) in the APICID and check for 0 */ 2349 mask = (1U << (fls(smp_num_siblings) - 1)) - 1; 2350 return !(apicid & mask); 2351 } 2352 #endif 2353 2354 /* 2355 * Should use this API to allocate logical CPU IDs to keep nr_logical_cpuids 2356 * and cpuid_to_apicid[] synchronized. 2357 */ 2358 static int allocate_logical_cpuid(int apicid) 2359 { 2360 int i; 2361 2362 /* 2363 * cpuid <-> apicid mapping is persistent, so when a cpu is up, 2364 * check if the kernel has allocated a cpuid for it. 2365 */ 2366 for (i = 0; i < nr_logical_cpuids; i++) { 2367 if (cpuid_to_apicid[i] == apicid) 2368 return i; 2369 } 2370 2371 /* Allocate a new cpuid. */ 2372 if (nr_logical_cpuids >= nr_cpu_ids) { 2373 WARN_ONCE(1, "APIC: NR_CPUS/possible_cpus limit of %u reached. " 2374 "Processor %d/0x%x and the rest are ignored.\n", 2375 nr_cpu_ids, nr_logical_cpuids, apicid); 2376 return -EINVAL; 2377 } 2378 2379 cpuid_to_apicid[nr_logical_cpuids] = apicid; 2380 return nr_logical_cpuids++; 2381 } 2382 2383 int generic_processor_info(int apicid, int version) 2384 { 2385 int cpu, max = nr_cpu_ids; 2386 bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid, 2387 phys_cpu_present_map); 2388 2389 /* 2390 * boot_cpu_physical_apicid is designed to have the apicid 2391 * returned by read_apic_id(), i.e, the apicid of the 2392 * currently booting-up processor. However, on some platforms, 2393 * it is temporarily modified by the apicid reported as BSP 2394 * through MP table. Concretely: 2395 * 2396 * - arch/x86/kernel/mpparse.c: MP_processor_info() 2397 * - arch/x86/mm/amdtopology.c: amd_numa_init() 2398 * 2399 * This function is executed with the modified 2400 * boot_cpu_physical_apicid. So, disabled_cpu_apicid kernel 2401 * parameter doesn't work to disable APs on kdump 2nd kernel. 2402 * 2403 * Since fixing handling of boot_cpu_physical_apicid requires 2404 * another discussion and tests on each platform, we leave it 2405 * for now and here we use read_apic_id() directly in this 2406 * function, generic_processor_info(). 2407 */ 2408 if (disabled_cpu_apicid != BAD_APICID && 2409 disabled_cpu_apicid != read_apic_id() && 2410 disabled_cpu_apicid == apicid) { 2411 int thiscpu = num_processors + disabled_cpus; 2412 2413 pr_warn("APIC: Disabling requested cpu." 2414 " Processor %d/0x%x ignored.\n", thiscpu, apicid); 2415 2416 disabled_cpus++; 2417 return -ENODEV; 2418 } 2419 2420 /* 2421 * If boot cpu has not been detected yet, then only allow upto 2422 * nr_cpu_ids - 1 processors and keep one slot free for boot cpu 2423 */ 2424 if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 && 2425 apicid != boot_cpu_physical_apicid) { 2426 int thiscpu = max + disabled_cpus - 1; 2427 2428 pr_warn("APIC: NR_CPUS/possible_cpus limit of %i almost" 2429 " reached. Keeping one slot for boot cpu." 2430 " Processor %d/0x%x ignored.\n", max, thiscpu, apicid); 2431 2432 disabled_cpus++; 2433 return -ENODEV; 2434 } 2435 2436 if (num_processors >= nr_cpu_ids) { 2437 int thiscpu = max + disabled_cpus; 2438 2439 pr_warn("APIC: NR_CPUS/possible_cpus limit of %i reached. " 2440 "Processor %d/0x%x ignored.\n", max, thiscpu, apicid); 2441 2442 disabled_cpus++; 2443 return -EINVAL; 2444 } 2445 2446 if (apicid == boot_cpu_physical_apicid) { 2447 /* 2448 * x86_bios_cpu_apicid is required to have processors listed 2449 * in same order as logical cpu numbers. Hence the first 2450 * entry is BSP, and so on. 2451 * boot_cpu_init() already hold bit 0 in cpu_present_mask 2452 * for BSP. 2453 */ 2454 cpu = 0; 2455 2456 /* Logical cpuid 0 is reserved for BSP. */ 2457 cpuid_to_apicid[0] = apicid; 2458 } else { 2459 cpu = allocate_logical_cpuid(apicid); 2460 if (cpu < 0) { 2461 disabled_cpus++; 2462 return -EINVAL; 2463 } 2464 } 2465 2466 /* 2467 * Validate version 2468 */ 2469 if (version == 0x0) { 2470 pr_warn("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n", 2471 cpu, apicid); 2472 version = 0x10; 2473 } 2474 2475 if (version != boot_cpu_apic_version) { 2476 pr_warn("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n", 2477 boot_cpu_apic_version, cpu, version); 2478 } 2479 2480 if (apicid > max_physical_apicid) 2481 max_physical_apicid = apicid; 2482 2483 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64) 2484 early_per_cpu(x86_cpu_to_apicid, cpu) = apicid; 2485 early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid; 2486 #endif 2487 #ifdef CONFIG_X86_32 2488 early_per_cpu(x86_cpu_to_logical_apicid, cpu) = 2489 apic->x86_32_early_logical_apicid(cpu); 2490 #endif 2491 set_cpu_possible(cpu, true); 2492 physid_set(apicid, phys_cpu_present_map); 2493 set_cpu_present(cpu, true); 2494 num_processors++; 2495 2496 return cpu; 2497 } 2498 2499 int hard_smp_processor_id(void) 2500 { 2501 return read_apic_id(); 2502 } 2503 2504 void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg, 2505 bool dmar) 2506 { 2507 memset(msg, 0, sizeof(*msg)); 2508 2509 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW; 2510 msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical; 2511 msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF; 2512 2513 msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED; 2514 msg->arch_data.vector = cfg->vector; 2515 2516 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH; 2517 /* 2518 * Only the IOMMU itself can use the trick of putting destination 2519 * APIC ID into the high bits of the address. Anything else would 2520 * just be writing to memory if it tried that, and needs IR to 2521 * address APICs which can't be addressed in the normal 32-bit 2522 * address range at 0xFFExxxxx. That is typically just 8 bits, but 2523 * some hypervisors allow the extended destination ID field in bits 2524 * 5-11 to be used, giving support for 15 bits of APIC IDs in total. 2525 */ 2526 if (dmar) 2527 msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8; 2528 else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000) 2529 msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8; 2530 else 2531 WARN_ON_ONCE(cfg->dest_apicid > 0xFF); 2532 } 2533 2534 u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid) 2535 { 2536 u32 dest = msg->arch_addr_lo.destid_0_7; 2537 2538 if (extid) 2539 dest |= msg->arch_addr_hi.destid_8_31 << 8; 2540 return dest; 2541 } 2542 EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid); 2543 2544 /* 2545 * Override the generic EOI implementation with an optimized version. 2546 * Only called during early boot when only one CPU is active and with 2547 * interrupts disabled, so we know this does not race with actual APIC driver 2548 * use. 2549 */ 2550 void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)) 2551 { 2552 struct apic **drv; 2553 2554 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { 2555 /* Should happen once for each apic */ 2556 WARN_ON((*drv)->eoi_write == eoi_write); 2557 (*drv)->native_eoi_write = (*drv)->eoi_write; 2558 (*drv)->eoi_write = eoi_write; 2559 } 2560 } 2561 2562 static void __init apic_bsp_up_setup(void) 2563 { 2564 #ifdef CONFIG_X86_64 2565 apic_write(APIC_ID, apic->set_apic_id(boot_cpu_physical_apicid)); 2566 #else 2567 /* 2568 * Hack: In case of kdump, after a crash, kernel might be booting 2569 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid 2570 * might be zero if read from MP tables. Get it from LAPIC. 2571 */ 2572 # ifdef CONFIG_CRASH_DUMP 2573 boot_cpu_physical_apicid = read_apic_id(); 2574 # endif 2575 #endif 2576 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map); 2577 } 2578 2579 /** 2580 * apic_bsp_setup - Setup function for local apic and io-apic 2581 * @upmode: Force UP mode (for APIC_init_uniprocessor) 2582 */ 2583 static void __init apic_bsp_setup(bool upmode) 2584 { 2585 connect_bsp_APIC(); 2586 if (upmode) 2587 apic_bsp_up_setup(); 2588 setup_local_APIC(); 2589 2590 enable_IO_APIC(); 2591 end_local_APIC_setup(); 2592 irq_remap_enable_fault_handling(); 2593 setup_IO_APIC(); 2594 } 2595 2596 #ifdef CONFIG_UP_LATE_INIT 2597 void __init up_late_init(void) 2598 { 2599 if (apic_intr_mode == APIC_PIC) 2600 return; 2601 2602 /* Setup local timer */ 2603 x86_init.timers.setup_percpu_clockev(); 2604 } 2605 #endif 2606 2607 /* 2608 * Power management 2609 */ 2610 #ifdef CONFIG_PM 2611 2612 static struct { 2613 /* 2614 * 'active' is true if the local APIC was enabled by us and 2615 * not the BIOS; this signifies that we are also responsible 2616 * for disabling it before entering apm/acpi suspend 2617 */ 2618 int active; 2619 /* r/w apic fields */ 2620 unsigned int apic_id; 2621 unsigned int apic_taskpri; 2622 unsigned int apic_ldr; 2623 unsigned int apic_dfr; 2624 unsigned int apic_spiv; 2625 unsigned int apic_lvtt; 2626 unsigned int apic_lvtpc; 2627 unsigned int apic_lvt0; 2628 unsigned int apic_lvt1; 2629 unsigned int apic_lvterr; 2630 unsigned int apic_tmict; 2631 unsigned int apic_tdcr; 2632 unsigned int apic_thmr; 2633 unsigned int apic_cmci; 2634 } apic_pm_state; 2635 2636 static int lapic_suspend(void) 2637 { 2638 unsigned long flags; 2639 int maxlvt; 2640 2641 if (!apic_pm_state.active) 2642 return 0; 2643 2644 maxlvt = lapic_get_maxlvt(); 2645 2646 apic_pm_state.apic_id = apic_read(APIC_ID); 2647 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI); 2648 apic_pm_state.apic_ldr = apic_read(APIC_LDR); 2649 apic_pm_state.apic_dfr = apic_read(APIC_DFR); 2650 apic_pm_state.apic_spiv = apic_read(APIC_SPIV); 2651 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT); 2652 if (maxlvt >= 4) 2653 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC); 2654 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0); 2655 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1); 2656 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR); 2657 apic_pm_state.apic_tmict = apic_read(APIC_TMICT); 2658 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR); 2659 #ifdef CONFIG_X86_THERMAL_VECTOR 2660 if (maxlvt >= 5) 2661 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR); 2662 #endif 2663 #ifdef CONFIG_X86_MCE_INTEL 2664 if (maxlvt >= 6) 2665 apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI); 2666 #endif 2667 2668 local_irq_save(flags); 2669 2670 /* 2671 * Mask IOAPIC before disabling the local APIC to prevent stale IRR 2672 * entries on some implementations. 2673 */ 2674 mask_ioapic_entries(); 2675 2676 disable_local_APIC(); 2677 2678 irq_remapping_disable(); 2679 2680 local_irq_restore(flags); 2681 return 0; 2682 } 2683 2684 static void lapic_resume(void) 2685 { 2686 unsigned int l, h; 2687 unsigned long flags; 2688 int maxlvt; 2689 2690 if (!apic_pm_state.active) 2691 return; 2692 2693 local_irq_save(flags); 2694 2695 /* 2696 * IO-APIC and PIC have their own resume routines. 2697 * We just mask them here to make sure the interrupt 2698 * subsystem is completely quiet while we enable x2apic 2699 * and interrupt-remapping. 2700 */ 2701 mask_ioapic_entries(); 2702 legacy_pic->mask_all(); 2703 2704 if (x2apic_mode) { 2705 __x2apic_enable(); 2706 } else { 2707 /* 2708 * Make sure the APICBASE points to the right address 2709 * 2710 * FIXME! This will be wrong if we ever support suspend on 2711 * SMP! We'll need to do this as part of the CPU restore! 2712 */ 2713 if (boot_cpu_data.x86 >= 6) { 2714 rdmsr(MSR_IA32_APICBASE, l, h); 2715 l &= ~MSR_IA32_APICBASE_BASE; 2716 l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; 2717 wrmsr(MSR_IA32_APICBASE, l, h); 2718 } 2719 } 2720 2721 maxlvt = lapic_get_maxlvt(); 2722 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED); 2723 apic_write(APIC_ID, apic_pm_state.apic_id); 2724 apic_write(APIC_DFR, apic_pm_state.apic_dfr); 2725 apic_write(APIC_LDR, apic_pm_state.apic_ldr); 2726 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri); 2727 apic_write(APIC_SPIV, apic_pm_state.apic_spiv); 2728 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0); 2729 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1); 2730 #ifdef CONFIG_X86_THERMAL_VECTOR 2731 if (maxlvt >= 5) 2732 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr); 2733 #endif 2734 #ifdef CONFIG_X86_MCE_INTEL 2735 if (maxlvt >= 6) 2736 apic_write(APIC_LVTCMCI, apic_pm_state.apic_cmci); 2737 #endif 2738 if (maxlvt >= 4) 2739 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc); 2740 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt); 2741 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr); 2742 apic_write(APIC_TMICT, apic_pm_state.apic_tmict); 2743 apic_write(APIC_ESR, 0); 2744 apic_read(APIC_ESR); 2745 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr); 2746 apic_write(APIC_ESR, 0); 2747 apic_read(APIC_ESR); 2748 2749 irq_remapping_reenable(x2apic_mode); 2750 2751 local_irq_restore(flags); 2752 } 2753 2754 /* 2755 * This device has no shutdown method - fully functioning local APICs 2756 * are needed on every CPU up until machine_halt/restart/poweroff. 2757 */ 2758 2759 static struct syscore_ops lapic_syscore_ops = { 2760 .resume = lapic_resume, 2761 .suspend = lapic_suspend, 2762 }; 2763 2764 static void apic_pm_activate(void) 2765 { 2766 apic_pm_state.active = 1; 2767 } 2768 2769 static int __init init_lapic_sysfs(void) 2770 { 2771 /* XXX: remove suspend/resume procs if !apic_pm_state.active? */ 2772 if (boot_cpu_has(X86_FEATURE_APIC)) 2773 register_syscore_ops(&lapic_syscore_ops); 2774 2775 return 0; 2776 } 2777 2778 /* local apic needs to resume before other devices access its registers. */ 2779 core_initcall(init_lapic_sysfs); 2780 2781 #else /* CONFIG_PM */ 2782 2783 static void apic_pm_activate(void) { } 2784 2785 #endif /* CONFIG_PM */ 2786 2787 #ifdef CONFIG_X86_64 2788 2789 static int multi_checked; 2790 static int multi; 2791 2792 static int set_multi(const struct dmi_system_id *d) 2793 { 2794 if (multi) 2795 return 0; 2796 pr_info("APIC: %s detected, Multi Chassis\n", d->ident); 2797 multi = 1; 2798 return 0; 2799 } 2800 2801 static const struct dmi_system_id multi_dmi_table[] = { 2802 { 2803 .callback = set_multi, 2804 .ident = "IBM System Summit2", 2805 .matches = { 2806 DMI_MATCH(DMI_SYS_VENDOR, "IBM"), 2807 DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"), 2808 }, 2809 }, 2810 {} 2811 }; 2812 2813 static void dmi_check_multi(void) 2814 { 2815 if (multi_checked) 2816 return; 2817 2818 dmi_check_system(multi_dmi_table); 2819 multi_checked = 1; 2820 } 2821 2822 /* 2823 * apic_is_clustered_box() -- Check if we can expect good TSC 2824 * 2825 * Thus far, the major user of this is IBM's Summit2 series: 2826 * Clustered boxes may have unsynced TSC problems if they are 2827 * multi-chassis. 2828 * Use DMI to check them 2829 */ 2830 int apic_is_clustered_box(void) 2831 { 2832 dmi_check_multi(); 2833 return multi; 2834 } 2835 #endif 2836 2837 /* 2838 * APIC command line parameters 2839 */ 2840 static int __init setup_disableapic(char *arg) 2841 { 2842 disable_apic = 1; 2843 setup_clear_cpu_cap(X86_FEATURE_APIC); 2844 return 0; 2845 } 2846 early_param("disableapic", setup_disableapic); 2847 2848 /* same as disableapic, for compatibility */ 2849 static int __init setup_nolapic(char *arg) 2850 { 2851 return setup_disableapic(arg); 2852 } 2853 early_param("nolapic", setup_nolapic); 2854 2855 static int __init parse_lapic_timer_c2_ok(char *arg) 2856 { 2857 local_apic_timer_c2_ok = 1; 2858 return 0; 2859 } 2860 early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok); 2861 2862 static int __init parse_disable_apic_timer(char *arg) 2863 { 2864 disable_apic_timer = 1; 2865 return 0; 2866 } 2867 early_param("noapictimer", parse_disable_apic_timer); 2868 2869 static int __init parse_nolapic_timer(char *arg) 2870 { 2871 disable_apic_timer = 1; 2872 return 0; 2873 } 2874 early_param("nolapic_timer", parse_nolapic_timer); 2875 2876 static int __init apic_set_verbosity(char *arg) 2877 { 2878 if (!arg) { 2879 #ifdef CONFIG_X86_64 2880 skip_ioapic_setup = 0; 2881 return 0; 2882 #endif 2883 return -EINVAL; 2884 } 2885 2886 if (strcmp("debug", arg) == 0) 2887 apic_verbosity = APIC_DEBUG; 2888 else if (strcmp("verbose", arg) == 0) 2889 apic_verbosity = APIC_VERBOSE; 2890 #ifdef CONFIG_X86_64 2891 else { 2892 pr_warn("APIC Verbosity level %s not recognised" 2893 " use apic=verbose or apic=debug\n", arg); 2894 return -EINVAL; 2895 } 2896 #endif 2897 2898 return 0; 2899 } 2900 early_param("apic", apic_set_verbosity); 2901 2902 static int __init lapic_insert_resource(void) 2903 { 2904 if (!apic_phys) 2905 return -1; 2906 2907 /* Put local APIC into the resource map. */ 2908 lapic_resource.start = apic_phys; 2909 lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1; 2910 insert_resource(&iomem_resource, &lapic_resource); 2911 2912 return 0; 2913 } 2914 2915 /* 2916 * need call insert after e820__reserve_resources() 2917 * that is using request_resource 2918 */ 2919 late_initcall(lapic_insert_resource); 2920 2921 static int __init apic_set_disabled_cpu_apicid(char *arg) 2922 { 2923 if (!arg || !get_option(&arg, &disabled_cpu_apicid)) 2924 return -EINVAL; 2925 2926 return 0; 2927 } 2928 early_param("disable_cpu_apicid", apic_set_disabled_cpu_apicid); 2929 2930 static int __init apic_set_extnmi(char *arg) 2931 { 2932 if (!arg) 2933 return -EINVAL; 2934 2935 if (!strncmp("all", arg, 3)) 2936 apic_extnmi = APIC_EXTNMI_ALL; 2937 else if (!strncmp("none", arg, 4)) 2938 apic_extnmi = APIC_EXTNMI_NONE; 2939 else if (!strncmp("bsp", arg, 3)) 2940 apic_extnmi = APIC_EXTNMI_BSP; 2941 else { 2942 pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg); 2943 return -EINVAL; 2944 } 2945 2946 return 0; 2947 } 2948 early_param("apic_extnmi", apic_set_extnmi); 2949