1 /* 2 * Local APIC handling, local APIC timers 3 * 4 * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com> 5 * 6 * Fixes 7 * Maciej W. Rozycki : Bits for genuine 82489DX APICs; 8 * thanks to Eric Gilmore 9 * and Rolf G. Tews 10 * for testing these extensively. 11 * Maciej W. Rozycki : Various updates and fixes. 12 * Mikael Pettersson : Power Management for UP-APIC. 13 * Pavel Machek and 14 * Mikael Pettersson : PM converted to driver model. 15 */ 16 17 #include <linux/perf_event.h> 18 #include <linux/kernel_stat.h> 19 #include <linux/mc146818rtc.h> 20 #include <linux/acpi_pmtmr.h> 21 #include <linux/clockchips.h> 22 #include <linux/interrupt.h> 23 #include <linux/bootmem.h> 24 #include <linux/ftrace.h> 25 #include <linux/ioport.h> 26 #include <linux/module.h> 27 #include <linux/syscore_ops.h> 28 #include <linux/delay.h> 29 #include <linux/timex.h> 30 #include <linux/i8253.h> 31 #include <linux/dmar.h> 32 #include <linux/init.h> 33 #include <linux/cpu.h> 34 #include <linux/dmi.h> 35 #include <linux/smp.h> 36 #include <linux/mm.h> 37 38 #include <asm/trace/irq_vectors.h> 39 #include <asm/irq_remapping.h> 40 #include <asm/perf_event.h> 41 #include <asm/x86_init.h> 42 #include <asm/pgalloc.h> 43 #include <linux/atomic.h> 44 #include <asm/mpspec.h> 45 #include <asm/i8259.h> 46 #include <asm/proto.h> 47 #include <asm/apic.h> 48 #include <asm/io_apic.h> 49 #include <asm/desc.h> 50 #include <asm/hpet.h> 51 #include <asm/idle.h> 52 #include <asm/mtrr.h> 53 #include <asm/time.h> 54 #include <asm/smp.h> 55 #include <asm/mce.h> 56 #include <asm/tsc.h> 57 #include <asm/hypervisor.h> 58 59 unsigned int num_processors; 60 61 unsigned disabled_cpus; 62 63 /* Processor that is doing the boot up */ 64 unsigned int boot_cpu_physical_apicid = -1U; 65 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid); 66 67 /* 68 * The highest APIC ID seen during enumeration. 69 */ 70 unsigned int max_physical_apicid; 71 72 /* 73 * Bitmask of physically existing CPUs: 74 */ 75 physid_mask_t phys_cpu_present_map; 76 77 /* 78 * Map cpu index to physical APIC ID 79 */ 80 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID); 81 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID); 82 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid); 83 EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid); 84 85 #ifdef CONFIG_X86_32 86 87 /* 88 * On x86_32, the mapping between cpu and logical apicid may vary 89 * depending on apic in use. The following early percpu variable is 90 * used for the mapping. This is where the behaviors of x86_64 and 32 91 * actually diverge. Let's keep it ugly for now. 92 */ 93 DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID); 94 95 /* Local APIC was disabled by the BIOS and enabled by the kernel */ 96 static int enabled_via_apicbase; 97 98 /* 99 * Handle interrupt mode configuration register (IMCR). 100 * This register controls whether the interrupt signals 101 * that reach the BSP come from the master PIC or from the 102 * local APIC. Before entering Symmetric I/O Mode, either 103 * the BIOS or the operating system must switch out of 104 * PIC Mode by changing the IMCR. 105 */ 106 static inline void imcr_pic_to_apic(void) 107 { 108 /* select IMCR register */ 109 outb(0x70, 0x22); 110 /* NMI and 8259 INTR go through APIC */ 111 outb(0x01, 0x23); 112 } 113 114 static inline void imcr_apic_to_pic(void) 115 { 116 /* select IMCR register */ 117 outb(0x70, 0x22); 118 /* NMI and 8259 INTR go directly to BSP */ 119 outb(0x00, 0x23); 120 } 121 #endif 122 123 /* 124 * Knob to control our willingness to enable the local APIC. 125 * 126 * +1=force-enable 127 */ 128 static int force_enable_local_apic __initdata; 129 /* 130 * APIC command line parameters 131 */ 132 static int __init parse_lapic(char *arg) 133 { 134 if (config_enabled(CONFIG_X86_32) && !arg) 135 force_enable_local_apic = 1; 136 else if (arg && !strncmp(arg, "notscdeadline", 13)) 137 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); 138 return 0; 139 } 140 early_param("lapic", parse_lapic); 141 142 #ifdef CONFIG_X86_64 143 static int apic_calibrate_pmtmr __initdata; 144 static __init int setup_apicpmtimer(char *s) 145 { 146 apic_calibrate_pmtmr = 1; 147 notsc_setup(NULL); 148 return 0; 149 } 150 __setup("apicpmtimer", setup_apicpmtimer); 151 #endif 152 153 int x2apic_mode; 154 #ifdef CONFIG_X86_X2APIC 155 /* x2apic enabled before OS handover */ 156 int x2apic_preenabled; 157 static int x2apic_disabled; 158 static int nox2apic; 159 static __init int setup_nox2apic(char *str) 160 { 161 if (x2apic_enabled()) { 162 int apicid = native_apic_msr_read(APIC_ID); 163 164 if (apicid >= 255) { 165 pr_warning("Apicid: %08x, cannot enforce nox2apic\n", 166 apicid); 167 return 0; 168 } 169 170 pr_warning("x2apic already enabled. will disable it\n"); 171 } else 172 setup_clear_cpu_cap(X86_FEATURE_X2APIC); 173 174 nox2apic = 1; 175 176 return 0; 177 } 178 early_param("nox2apic", setup_nox2apic); 179 #endif 180 181 unsigned long mp_lapic_addr; 182 int disable_apic; 183 /* Disable local APIC timer from the kernel commandline or via dmi quirk */ 184 static int disable_apic_timer __initdata; 185 /* Local APIC timer works in C2 */ 186 int local_apic_timer_c2_ok; 187 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok); 188 189 int first_system_vector = 0xfe; 190 191 /* 192 * Debug level, exported for io_apic.c 193 */ 194 unsigned int apic_verbosity; 195 196 int pic_mode; 197 198 /* Have we found an MP table */ 199 int smp_found_config; 200 201 static struct resource lapic_resource = { 202 .name = "Local APIC", 203 .flags = IORESOURCE_MEM | IORESOURCE_BUSY, 204 }; 205 206 unsigned int lapic_timer_frequency = 0; 207 208 static void apic_pm_activate(void); 209 210 static unsigned long apic_phys; 211 212 /* 213 * Get the LAPIC version 214 */ 215 static inline int lapic_get_version(void) 216 { 217 return GET_APIC_VERSION(apic_read(APIC_LVR)); 218 } 219 220 /* 221 * Check, if the APIC is integrated or a separate chip 222 */ 223 static inline int lapic_is_integrated(void) 224 { 225 #ifdef CONFIG_X86_64 226 return 1; 227 #else 228 return APIC_INTEGRATED(lapic_get_version()); 229 #endif 230 } 231 232 /* 233 * Check, whether this is a modern or a first generation APIC 234 */ 235 static int modern_apic(void) 236 { 237 /* AMD systems use old APIC versions, so check the CPU */ 238 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && 239 boot_cpu_data.x86 >= 0xf) 240 return 1; 241 return lapic_get_version() >= 0x14; 242 } 243 244 /* 245 * right after this call apic become NOOP driven 246 * so apic->write/read doesn't do anything 247 */ 248 static void __init apic_disable(void) 249 { 250 pr_info("APIC: switched to apic NOOP\n"); 251 apic = &apic_noop; 252 } 253 254 void native_apic_wait_icr_idle(void) 255 { 256 while (apic_read(APIC_ICR) & APIC_ICR_BUSY) 257 cpu_relax(); 258 } 259 260 u32 native_safe_apic_wait_icr_idle(void) 261 { 262 u32 send_status; 263 int timeout; 264 265 timeout = 0; 266 do { 267 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY; 268 if (!send_status) 269 break; 270 inc_irq_stat(icr_read_retry_count); 271 udelay(100); 272 } while (timeout++ < 1000); 273 274 return send_status; 275 } 276 277 void native_apic_icr_write(u32 low, u32 id) 278 { 279 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id)); 280 apic_write(APIC_ICR, low); 281 } 282 283 u64 native_apic_icr_read(void) 284 { 285 u32 icr1, icr2; 286 287 icr2 = apic_read(APIC_ICR2); 288 icr1 = apic_read(APIC_ICR); 289 290 return icr1 | ((u64)icr2 << 32); 291 } 292 293 #ifdef CONFIG_X86_32 294 /** 295 * get_physical_broadcast - Get number of physical broadcast IDs 296 */ 297 int get_physical_broadcast(void) 298 { 299 return modern_apic() ? 0xff : 0xf; 300 } 301 #endif 302 303 /** 304 * lapic_get_maxlvt - get the maximum number of local vector table entries 305 */ 306 int lapic_get_maxlvt(void) 307 { 308 unsigned int v; 309 310 v = apic_read(APIC_LVR); 311 /* 312 * - we always have APIC integrated on 64bit mode 313 * - 82489DXs do not report # of LVT entries 314 */ 315 return APIC_INTEGRATED(GET_APIC_VERSION(v)) ? GET_APIC_MAXLVT(v) : 2; 316 } 317 318 /* 319 * Local APIC timer 320 */ 321 322 /* Clock divisor */ 323 #define APIC_DIVISOR 16 324 #define TSC_DIVISOR 32 325 326 /* 327 * This function sets up the local APIC timer, with a timeout of 328 * 'clocks' APIC bus clock. During calibration we actually call 329 * this function twice on the boot CPU, once with a bogus timeout 330 * value, second time for real. The other (noncalibrating) CPUs 331 * call this function only once, with the real, calibrated value. 332 * 333 * We do reads before writes even if unnecessary, to get around the 334 * P5 APIC double write bug. 335 */ 336 static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen) 337 { 338 unsigned int lvtt_value, tmp_value; 339 340 lvtt_value = LOCAL_TIMER_VECTOR; 341 if (!oneshot) 342 lvtt_value |= APIC_LVT_TIMER_PERIODIC; 343 else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) 344 lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE; 345 346 if (!lapic_is_integrated()) 347 lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV); 348 349 if (!irqen) 350 lvtt_value |= APIC_LVT_MASKED; 351 352 apic_write(APIC_LVTT, lvtt_value); 353 354 if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) { 355 printk_once(KERN_DEBUG "TSC deadline timer enabled\n"); 356 return; 357 } 358 359 /* 360 * Divide PICLK by 16 361 */ 362 tmp_value = apic_read(APIC_TDCR); 363 apic_write(APIC_TDCR, 364 (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) | 365 APIC_TDR_DIV_16); 366 367 if (!oneshot) 368 apic_write(APIC_TMICT, clocks / APIC_DIVISOR); 369 } 370 371 /* 372 * Setup extended LVT, AMD specific 373 * 374 * Software should use the LVT offsets the BIOS provides. The offsets 375 * are determined by the subsystems using it like those for MCE 376 * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts 377 * are supported. Beginning with family 10h at least 4 offsets are 378 * available. 379 * 380 * Since the offsets must be consistent for all cores, we keep track 381 * of the LVT offsets in software and reserve the offset for the same 382 * vector also to be used on other cores. An offset is freed by 383 * setting the entry to APIC_EILVT_MASKED. 384 * 385 * If the BIOS is right, there should be no conflicts. Otherwise a 386 * "[Firmware Bug]: ..." error message is generated. However, if 387 * software does not properly determines the offsets, it is not 388 * necessarily a BIOS bug. 389 */ 390 391 static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX]; 392 393 static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new) 394 { 395 return (old & APIC_EILVT_MASKED) 396 || (new == APIC_EILVT_MASKED) 397 || ((new & ~APIC_EILVT_MASKED) == old); 398 } 399 400 static unsigned int reserve_eilvt_offset(int offset, unsigned int new) 401 { 402 unsigned int rsvd, vector; 403 404 if (offset >= APIC_EILVT_NR_MAX) 405 return ~0; 406 407 rsvd = atomic_read(&eilvt_offsets[offset]); 408 do { 409 vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */ 410 if (vector && !eilvt_entry_is_changeable(vector, new)) 411 /* may not change if vectors are different */ 412 return rsvd; 413 rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new); 414 } while (rsvd != new); 415 416 rsvd &= ~APIC_EILVT_MASKED; 417 if (rsvd && rsvd != vector) 418 pr_info("LVT offset %d assigned for vector 0x%02x\n", 419 offset, rsvd); 420 421 return new; 422 } 423 424 /* 425 * If mask=1, the LVT entry does not generate interrupts while mask=0 426 * enables the vector. See also the BKDGs. Must be called with 427 * preemption disabled. 428 */ 429 430 int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask) 431 { 432 unsigned long reg = APIC_EILVTn(offset); 433 unsigned int new, old, reserved; 434 435 new = (mask << 16) | (msg_type << 8) | vector; 436 old = apic_read(reg); 437 reserved = reserve_eilvt_offset(offset, new); 438 439 if (reserved != new) { 440 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for " 441 "vector 0x%x, but the register is already in use for " 442 "vector 0x%x on another cpu\n", 443 smp_processor_id(), reg, offset, new, reserved); 444 return -EINVAL; 445 } 446 447 if (!eilvt_entry_is_changeable(old, new)) { 448 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for " 449 "vector 0x%x, but the register is already in use for " 450 "vector 0x%x on this cpu\n", 451 smp_processor_id(), reg, offset, new, old); 452 return -EBUSY; 453 } 454 455 apic_write(reg, new); 456 457 return 0; 458 } 459 EXPORT_SYMBOL_GPL(setup_APIC_eilvt); 460 461 /* 462 * Program the next event, relative to now 463 */ 464 static int lapic_next_event(unsigned long delta, 465 struct clock_event_device *evt) 466 { 467 apic_write(APIC_TMICT, delta); 468 return 0; 469 } 470 471 static int lapic_next_deadline(unsigned long delta, 472 struct clock_event_device *evt) 473 { 474 u64 tsc; 475 476 rdtscll(tsc); 477 wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR)); 478 return 0; 479 } 480 481 /* 482 * Setup the lapic timer in periodic or oneshot mode 483 */ 484 static void lapic_timer_setup(enum clock_event_mode mode, 485 struct clock_event_device *evt) 486 { 487 unsigned long flags; 488 unsigned int v; 489 490 /* Lapic used as dummy for broadcast ? */ 491 if (evt->features & CLOCK_EVT_FEAT_DUMMY) 492 return; 493 494 local_irq_save(flags); 495 496 switch (mode) { 497 case CLOCK_EVT_MODE_PERIODIC: 498 case CLOCK_EVT_MODE_ONESHOT: 499 __setup_APIC_LVTT(lapic_timer_frequency, 500 mode != CLOCK_EVT_MODE_PERIODIC, 1); 501 break; 502 case CLOCK_EVT_MODE_UNUSED: 503 case CLOCK_EVT_MODE_SHUTDOWN: 504 v = apic_read(APIC_LVTT); 505 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); 506 apic_write(APIC_LVTT, v); 507 apic_write(APIC_TMICT, 0); 508 break; 509 case CLOCK_EVT_MODE_RESUME: 510 /* Nothing to do here */ 511 break; 512 } 513 514 local_irq_restore(flags); 515 } 516 517 /* 518 * Local APIC timer broadcast function 519 */ 520 static void lapic_timer_broadcast(const struct cpumask *mask) 521 { 522 #ifdef CONFIG_SMP 523 apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR); 524 #endif 525 } 526 527 528 /* 529 * The local apic timer can be used for any function which is CPU local. 530 */ 531 static struct clock_event_device lapic_clockevent = { 532 .name = "lapic", 533 .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT 534 | CLOCK_EVT_FEAT_C3STOP | CLOCK_EVT_FEAT_DUMMY, 535 .shift = 32, 536 .set_mode = lapic_timer_setup, 537 .set_next_event = lapic_next_event, 538 .broadcast = lapic_timer_broadcast, 539 .rating = 100, 540 .irq = -1, 541 }; 542 static DEFINE_PER_CPU(struct clock_event_device, lapic_events); 543 544 /* 545 * Setup the local APIC timer for this CPU. Copy the initialized values 546 * of the boot CPU and register the clock event in the framework. 547 */ 548 static void setup_APIC_timer(void) 549 { 550 struct clock_event_device *levt = &__get_cpu_var(lapic_events); 551 552 if (this_cpu_has(X86_FEATURE_ARAT)) { 553 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP; 554 /* Make LAPIC timer preferrable over percpu HPET */ 555 lapic_clockevent.rating = 150; 556 } 557 558 memcpy(levt, &lapic_clockevent, sizeof(*levt)); 559 levt->cpumask = cpumask_of(smp_processor_id()); 560 561 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) { 562 levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC | 563 CLOCK_EVT_FEAT_DUMMY); 564 levt->set_next_event = lapic_next_deadline; 565 clockevents_config_and_register(levt, 566 (tsc_khz / TSC_DIVISOR) * 1000, 567 0xF, ~0UL); 568 } else 569 clockevents_register_device(levt); 570 } 571 572 /* 573 * In this functions we calibrate APIC bus clocks to the external timer. 574 * 575 * We want to do the calibration only once since we want to have local timer 576 * irqs syncron. CPUs connected by the same APIC bus have the very same bus 577 * frequency. 578 * 579 * This was previously done by reading the PIT/HPET and waiting for a wrap 580 * around to find out, that a tick has elapsed. I have a box, where the PIT 581 * readout is broken, so it never gets out of the wait loop again. This was 582 * also reported by others. 583 * 584 * Monitoring the jiffies value is inaccurate and the clockevents 585 * infrastructure allows us to do a simple substitution of the interrupt 586 * handler. 587 * 588 * The calibration routine also uses the pm_timer when possible, as the PIT 589 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes 590 * back to normal later in the boot process). 591 */ 592 593 #define LAPIC_CAL_LOOPS (HZ/10) 594 595 static __initdata int lapic_cal_loops = -1; 596 static __initdata long lapic_cal_t1, lapic_cal_t2; 597 static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2; 598 static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2; 599 static __initdata unsigned long lapic_cal_j1, lapic_cal_j2; 600 601 /* 602 * Temporary interrupt handler. 603 */ 604 static void __init lapic_cal_handler(struct clock_event_device *dev) 605 { 606 unsigned long long tsc = 0; 607 long tapic = apic_read(APIC_TMCCT); 608 unsigned long pm = acpi_pm_read_early(); 609 610 if (cpu_has_tsc) 611 rdtscll(tsc); 612 613 switch (lapic_cal_loops++) { 614 case 0: 615 lapic_cal_t1 = tapic; 616 lapic_cal_tsc1 = tsc; 617 lapic_cal_pm1 = pm; 618 lapic_cal_j1 = jiffies; 619 break; 620 621 case LAPIC_CAL_LOOPS: 622 lapic_cal_t2 = tapic; 623 lapic_cal_tsc2 = tsc; 624 if (pm < lapic_cal_pm1) 625 pm += ACPI_PM_OVRRUN; 626 lapic_cal_pm2 = pm; 627 lapic_cal_j2 = jiffies; 628 break; 629 } 630 } 631 632 static int __init 633 calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc) 634 { 635 const long pm_100ms = PMTMR_TICKS_PER_SEC / 10; 636 const long pm_thresh = pm_100ms / 100; 637 unsigned long mult; 638 u64 res; 639 640 #ifndef CONFIG_X86_PM_TIMER 641 return -1; 642 #endif 643 644 apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm); 645 646 /* Check, if the PM timer is available */ 647 if (!deltapm) 648 return -1; 649 650 mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22); 651 652 if (deltapm > (pm_100ms - pm_thresh) && 653 deltapm < (pm_100ms + pm_thresh)) { 654 apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n"); 655 return 0; 656 } 657 658 res = (((u64)deltapm) * mult) >> 22; 659 do_div(res, 1000000); 660 pr_warning("APIC calibration not consistent " 661 "with PM-Timer: %ldms instead of 100ms\n",(long)res); 662 663 /* Correct the lapic counter value */ 664 res = (((u64)(*delta)) * pm_100ms); 665 do_div(res, deltapm); 666 pr_info("APIC delta adjusted to PM-Timer: " 667 "%lu (%ld)\n", (unsigned long)res, *delta); 668 *delta = (long)res; 669 670 /* Correct the tsc counter value */ 671 if (cpu_has_tsc) { 672 res = (((u64)(*deltatsc)) * pm_100ms); 673 do_div(res, deltapm); 674 apic_printk(APIC_VERBOSE, "TSC delta adjusted to " 675 "PM-Timer: %lu (%ld)\n", 676 (unsigned long)res, *deltatsc); 677 *deltatsc = (long)res; 678 } 679 680 return 0; 681 } 682 683 static int __init calibrate_APIC_clock(void) 684 { 685 struct clock_event_device *levt = &__get_cpu_var(lapic_events); 686 void (*real_handler)(struct clock_event_device *dev); 687 unsigned long deltaj; 688 long delta, deltatsc; 689 int pm_referenced = 0; 690 691 /** 692 * check if lapic timer has already been calibrated by platform 693 * specific routine, such as tsc calibration code. if so, we just fill 694 * in the clockevent structure and return. 695 */ 696 697 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) { 698 return 0; 699 } else if (lapic_timer_frequency) { 700 apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n", 701 lapic_timer_frequency); 702 lapic_clockevent.mult = div_sc(lapic_timer_frequency/APIC_DIVISOR, 703 TICK_NSEC, lapic_clockevent.shift); 704 lapic_clockevent.max_delta_ns = 705 clockevent_delta2ns(0x7FFFFF, &lapic_clockevent); 706 lapic_clockevent.min_delta_ns = 707 clockevent_delta2ns(0xF, &lapic_clockevent); 708 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY; 709 return 0; 710 } 711 712 apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n" 713 "calibrating APIC timer ...\n"); 714 715 local_irq_disable(); 716 717 /* Replace the global interrupt handler */ 718 real_handler = global_clock_event->event_handler; 719 global_clock_event->event_handler = lapic_cal_handler; 720 721 /* 722 * Setup the APIC counter to maximum. There is no way the lapic 723 * can underflow in the 100ms detection time frame 724 */ 725 __setup_APIC_LVTT(0xffffffff, 0, 0); 726 727 /* Let the interrupts run */ 728 local_irq_enable(); 729 730 while (lapic_cal_loops <= LAPIC_CAL_LOOPS) 731 cpu_relax(); 732 733 local_irq_disable(); 734 735 /* Restore the real event handler */ 736 global_clock_event->event_handler = real_handler; 737 738 /* Build delta t1-t2 as apic timer counts down */ 739 delta = lapic_cal_t1 - lapic_cal_t2; 740 apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta); 741 742 deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1); 743 744 /* we trust the PM based calibration if possible */ 745 pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1, 746 &delta, &deltatsc); 747 748 /* Calculate the scaled math multiplication factor */ 749 lapic_clockevent.mult = div_sc(delta, TICK_NSEC * LAPIC_CAL_LOOPS, 750 lapic_clockevent.shift); 751 lapic_clockevent.max_delta_ns = 752 clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent); 753 lapic_clockevent.min_delta_ns = 754 clockevent_delta2ns(0xF, &lapic_clockevent); 755 756 lapic_timer_frequency = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS; 757 758 apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta); 759 apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult); 760 apic_printk(APIC_VERBOSE, "..... calibration result: %u\n", 761 lapic_timer_frequency); 762 763 if (cpu_has_tsc) { 764 apic_printk(APIC_VERBOSE, "..... CPU clock speed is " 765 "%ld.%04ld MHz.\n", 766 (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ), 767 (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ)); 768 } 769 770 apic_printk(APIC_VERBOSE, "..... host bus clock speed is " 771 "%u.%04u MHz.\n", 772 lapic_timer_frequency / (1000000 / HZ), 773 lapic_timer_frequency % (1000000 / HZ)); 774 775 /* 776 * Do a sanity check on the APIC calibration result 777 */ 778 if (lapic_timer_frequency < (1000000 / HZ)) { 779 local_irq_enable(); 780 pr_warning("APIC frequency too slow, disabling apic timer\n"); 781 return -1; 782 } 783 784 levt->features &= ~CLOCK_EVT_FEAT_DUMMY; 785 786 /* 787 * PM timer calibration failed or not turned on 788 * so lets try APIC timer based calibration 789 */ 790 if (!pm_referenced) { 791 apic_printk(APIC_VERBOSE, "... verify APIC timer\n"); 792 793 /* 794 * Setup the apic timer manually 795 */ 796 levt->event_handler = lapic_cal_handler; 797 lapic_timer_setup(CLOCK_EVT_MODE_PERIODIC, levt); 798 lapic_cal_loops = -1; 799 800 /* Let the interrupts run */ 801 local_irq_enable(); 802 803 while (lapic_cal_loops <= LAPIC_CAL_LOOPS) 804 cpu_relax(); 805 806 /* Stop the lapic timer */ 807 lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, levt); 808 809 /* Jiffies delta */ 810 deltaj = lapic_cal_j2 - lapic_cal_j1; 811 apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj); 812 813 /* Check, if the jiffies result is consistent */ 814 if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2) 815 apic_printk(APIC_VERBOSE, "... jiffies result ok\n"); 816 else 817 levt->features |= CLOCK_EVT_FEAT_DUMMY; 818 } else 819 local_irq_enable(); 820 821 if (levt->features & CLOCK_EVT_FEAT_DUMMY) { 822 pr_warning("APIC timer disabled due to verification failure\n"); 823 return -1; 824 } 825 826 return 0; 827 } 828 829 /* 830 * Setup the boot APIC 831 * 832 * Calibrate and verify the result. 833 */ 834 void __init setup_boot_APIC_clock(void) 835 { 836 /* 837 * The local apic timer can be disabled via the kernel 838 * commandline or from the CPU detection code. Register the lapic 839 * timer as a dummy clock event source on SMP systems, so the 840 * broadcast mechanism is used. On UP systems simply ignore it. 841 */ 842 if (disable_apic_timer) { 843 pr_info("Disabling APIC timer\n"); 844 /* No broadcast on UP ! */ 845 if (num_possible_cpus() > 1) { 846 lapic_clockevent.mult = 1; 847 setup_APIC_timer(); 848 } 849 return; 850 } 851 852 if (calibrate_APIC_clock()) { 853 /* No broadcast on UP ! */ 854 if (num_possible_cpus() > 1) 855 setup_APIC_timer(); 856 return; 857 } 858 859 /* 860 * If nmi_watchdog is set to IO_APIC, we need the 861 * PIT/HPET going. Otherwise register lapic as a dummy 862 * device. 863 */ 864 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY; 865 866 /* Setup the lapic or request the broadcast */ 867 setup_APIC_timer(); 868 } 869 870 void setup_secondary_APIC_clock(void) 871 { 872 setup_APIC_timer(); 873 } 874 875 /* 876 * The guts of the apic timer interrupt 877 */ 878 static void local_apic_timer_interrupt(void) 879 { 880 int cpu = smp_processor_id(); 881 struct clock_event_device *evt = &per_cpu(lapic_events, cpu); 882 883 /* 884 * Normally we should not be here till LAPIC has been initialized but 885 * in some cases like kdump, its possible that there is a pending LAPIC 886 * timer interrupt from previous kernel's context and is delivered in 887 * new kernel the moment interrupts are enabled. 888 * 889 * Interrupts are enabled early and LAPIC is setup much later, hence 890 * its possible that when we get here evt->event_handler is NULL. 891 * Check for event_handler being NULL and discard the interrupt as 892 * spurious. 893 */ 894 if (!evt->event_handler) { 895 pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu); 896 /* Switch it off */ 897 lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, evt); 898 return; 899 } 900 901 /* 902 * the NMI deadlock-detector uses this. 903 */ 904 inc_irq_stat(apic_timer_irqs); 905 906 evt->event_handler(evt); 907 } 908 909 /* 910 * Local APIC timer interrupt. This is the most natural way for doing 911 * local interrupts, but local timer interrupts can be emulated by 912 * broadcast interrupts too. [in case the hw doesn't support APIC timers] 913 * 914 * [ if a single-CPU system runs an SMP kernel then we call the local 915 * interrupt as well. Thus we cannot inline the local irq ... ] 916 */ 917 __visible void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs) 918 { 919 struct pt_regs *old_regs = set_irq_regs(regs); 920 921 /* 922 * NOTE! We'd better ACK the irq immediately, 923 * because timer handling can be slow. 924 * 925 * update_process_times() expects us to have done irq_enter(). 926 * Besides, if we don't timer interrupts ignore the global 927 * interrupt lock, which is the WrongThing (tm) to do. 928 */ 929 entering_ack_irq(); 930 local_apic_timer_interrupt(); 931 exiting_irq(); 932 933 set_irq_regs(old_regs); 934 } 935 936 __visible void __irq_entry smp_trace_apic_timer_interrupt(struct pt_regs *regs) 937 { 938 struct pt_regs *old_regs = set_irq_regs(regs); 939 940 /* 941 * NOTE! We'd better ACK the irq immediately, 942 * because timer handling can be slow. 943 * 944 * update_process_times() expects us to have done irq_enter(). 945 * Besides, if we don't timer interrupts ignore the global 946 * interrupt lock, which is the WrongThing (tm) to do. 947 */ 948 entering_ack_irq(); 949 trace_local_timer_entry(LOCAL_TIMER_VECTOR); 950 local_apic_timer_interrupt(); 951 trace_local_timer_exit(LOCAL_TIMER_VECTOR); 952 exiting_irq(); 953 954 set_irq_regs(old_regs); 955 } 956 957 int setup_profiling_timer(unsigned int multiplier) 958 { 959 return -EINVAL; 960 } 961 962 /* 963 * Local APIC start and shutdown 964 */ 965 966 /** 967 * clear_local_APIC - shutdown the local APIC 968 * 969 * This is called, when a CPU is disabled and before rebooting, so the state of 970 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS 971 * leftovers during boot. 972 */ 973 void clear_local_APIC(void) 974 { 975 int maxlvt; 976 u32 v; 977 978 /* APIC hasn't been mapped yet */ 979 if (!x2apic_mode && !apic_phys) 980 return; 981 982 maxlvt = lapic_get_maxlvt(); 983 /* 984 * Masking an LVT entry can trigger a local APIC error 985 * if the vector is zero. Mask LVTERR first to prevent this. 986 */ 987 if (maxlvt >= 3) { 988 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */ 989 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED); 990 } 991 /* 992 * Careful: we have to set masks only first to deassert 993 * any level-triggered sources. 994 */ 995 v = apic_read(APIC_LVTT); 996 apic_write(APIC_LVTT, v | APIC_LVT_MASKED); 997 v = apic_read(APIC_LVT0); 998 apic_write(APIC_LVT0, v | APIC_LVT_MASKED); 999 v = apic_read(APIC_LVT1); 1000 apic_write(APIC_LVT1, v | APIC_LVT_MASKED); 1001 if (maxlvt >= 4) { 1002 v = apic_read(APIC_LVTPC); 1003 apic_write(APIC_LVTPC, v | APIC_LVT_MASKED); 1004 } 1005 1006 /* lets not touch this if we didn't frob it */ 1007 #ifdef CONFIG_X86_THERMAL_VECTOR 1008 if (maxlvt >= 5) { 1009 v = apic_read(APIC_LVTTHMR); 1010 apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED); 1011 } 1012 #endif 1013 #ifdef CONFIG_X86_MCE_INTEL 1014 if (maxlvt >= 6) { 1015 v = apic_read(APIC_LVTCMCI); 1016 if (!(v & APIC_LVT_MASKED)) 1017 apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED); 1018 } 1019 #endif 1020 1021 /* 1022 * Clean APIC state for other OSs: 1023 */ 1024 apic_write(APIC_LVTT, APIC_LVT_MASKED); 1025 apic_write(APIC_LVT0, APIC_LVT_MASKED); 1026 apic_write(APIC_LVT1, APIC_LVT_MASKED); 1027 if (maxlvt >= 3) 1028 apic_write(APIC_LVTERR, APIC_LVT_MASKED); 1029 if (maxlvt >= 4) 1030 apic_write(APIC_LVTPC, APIC_LVT_MASKED); 1031 1032 /* Integrated APIC (!82489DX) ? */ 1033 if (lapic_is_integrated()) { 1034 if (maxlvt > 3) 1035 /* Clear ESR due to Pentium errata 3AP and 11AP */ 1036 apic_write(APIC_ESR, 0); 1037 apic_read(APIC_ESR); 1038 } 1039 } 1040 1041 /** 1042 * disable_local_APIC - clear and disable the local APIC 1043 */ 1044 void disable_local_APIC(void) 1045 { 1046 unsigned int value; 1047 1048 /* APIC hasn't been mapped yet */ 1049 if (!x2apic_mode && !apic_phys) 1050 return; 1051 1052 clear_local_APIC(); 1053 1054 /* 1055 * Disable APIC (implies clearing of registers 1056 * for 82489DX!). 1057 */ 1058 value = apic_read(APIC_SPIV); 1059 value &= ~APIC_SPIV_APIC_ENABLED; 1060 apic_write(APIC_SPIV, value); 1061 1062 #ifdef CONFIG_X86_32 1063 /* 1064 * When LAPIC was disabled by the BIOS and enabled by the kernel, 1065 * restore the disabled state. 1066 */ 1067 if (enabled_via_apicbase) { 1068 unsigned int l, h; 1069 1070 rdmsr(MSR_IA32_APICBASE, l, h); 1071 l &= ~MSR_IA32_APICBASE_ENABLE; 1072 wrmsr(MSR_IA32_APICBASE, l, h); 1073 } 1074 #endif 1075 } 1076 1077 /* 1078 * If Linux enabled the LAPIC against the BIOS default disable it down before 1079 * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and 1080 * not power-off. Additionally clear all LVT entries before disable_local_APIC 1081 * for the case where Linux didn't enable the LAPIC. 1082 */ 1083 void lapic_shutdown(void) 1084 { 1085 unsigned long flags; 1086 1087 if (!cpu_has_apic && !apic_from_smp_config()) 1088 return; 1089 1090 local_irq_save(flags); 1091 1092 #ifdef CONFIG_X86_32 1093 if (!enabled_via_apicbase) 1094 clear_local_APIC(); 1095 else 1096 #endif 1097 disable_local_APIC(); 1098 1099 1100 local_irq_restore(flags); 1101 } 1102 1103 /* 1104 * This is to verify that we're looking at a real local APIC. 1105 * Check these against your board if the CPUs aren't getting 1106 * started for no apparent reason. 1107 */ 1108 int __init verify_local_APIC(void) 1109 { 1110 unsigned int reg0, reg1; 1111 1112 /* 1113 * The version register is read-only in a real APIC. 1114 */ 1115 reg0 = apic_read(APIC_LVR); 1116 apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0); 1117 apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK); 1118 reg1 = apic_read(APIC_LVR); 1119 apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1); 1120 1121 /* 1122 * The two version reads above should print the same 1123 * numbers. If the second one is different, then we 1124 * poke at a non-APIC. 1125 */ 1126 if (reg1 != reg0) 1127 return 0; 1128 1129 /* 1130 * Check if the version looks reasonably. 1131 */ 1132 reg1 = GET_APIC_VERSION(reg0); 1133 if (reg1 == 0x00 || reg1 == 0xff) 1134 return 0; 1135 reg1 = lapic_get_maxlvt(); 1136 if (reg1 < 0x02 || reg1 == 0xff) 1137 return 0; 1138 1139 /* 1140 * The ID register is read/write in a real APIC. 1141 */ 1142 reg0 = apic_read(APIC_ID); 1143 apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0); 1144 apic_write(APIC_ID, reg0 ^ apic->apic_id_mask); 1145 reg1 = apic_read(APIC_ID); 1146 apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg1); 1147 apic_write(APIC_ID, reg0); 1148 if (reg1 != (reg0 ^ apic->apic_id_mask)) 1149 return 0; 1150 1151 /* 1152 * The next two are just to see if we have sane values. 1153 * They're only really relevant if we're in Virtual Wire 1154 * compatibility mode, but most boxes are anymore. 1155 */ 1156 reg0 = apic_read(APIC_LVT0); 1157 apic_printk(APIC_DEBUG, "Getting LVT0: %x\n", reg0); 1158 reg1 = apic_read(APIC_LVT1); 1159 apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1); 1160 1161 return 1; 1162 } 1163 1164 /** 1165 * sync_Arb_IDs - synchronize APIC bus arbitration IDs 1166 */ 1167 void __init sync_Arb_IDs(void) 1168 { 1169 /* 1170 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not 1171 * needed on AMD. 1172 */ 1173 if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 1174 return; 1175 1176 /* 1177 * Wait for idle. 1178 */ 1179 apic_wait_icr_idle(); 1180 1181 apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n"); 1182 apic_write(APIC_ICR, APIC_DEST_ALLINC | 1183 APIC_INT_LEVELTRIG | APIC_DM_INIT); 1184 } 1185 1186 /* 1187 * An initial setup of the virtual wire mode. 1188 */ 1189 void __init init_bsp_APIC(void) 1190 { 1191 unsigned int value; 1192 1193 /* 1194 * Don't do the setup now if we have a SMP BIOS as the 1195 * through-I/O-APIC virtual wire mode might be active. 1196 */ 1197 if (smp_found_config || !cpu_has_apic) 1198 return; 1199 1200 /* 1201 * Do not trust the local APIC being empty at bootup. 1202 */ 1203 clear_local_APIC(); 1204 1205 /* 1206 * Enable APIC. 1207 */ 1208 value = apic_read(APIC_SPIV); 1209 value &= ~APIC_VECTOR_MASK; 1210 value |= APIC_SPIV_APIC_ENABLED; 1211 1212 #ifdef CONFIG_X86_32 1213 /* This bit is reserved on P4/Xeon and should be cleared */ 1214 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && 1215 (boot_cpu_data.x86 == 15)) 1216 value &= ~APIC_SPIV_FOCUS_DISABLED; 1217 else 1218 #endif 1219 value |= APIC_SPIV_FOCUS_DISABLED; 1220 value |= SPURIOUS_APIC_VECTOR; 1221 apic_write(APIC_SPIV, value); 1222 1223 /* 1224 * Set up the virtual wire mode. 1225 */ 1226 apic_write(APIC_LVT0, APIC_DM_EXTINT); 1227 value = APIC_DM_NMI; 1228 if (!lapic_is_integrated()) /* 82489DX */ 1229 value |= APIC_LVT_LEVEL_TRIGGER; 1230 apic_write(APIC_LVT1, value); 1231 } 1232 1233 static void lapic_setup_esr(void) 1234 { 1235 unsigned int oldvalue, value, maxlvt; 1236 1237 if (!lapic_is_integrated()) { 1238 pr_info("No ESR for 82489DX.\n"); 1239 return; 1240 } 1241 1242 if (apic->disable_esr) { 1243 /* 1244 * Something untraceable is creating bad interrupts on 1245 * secondary quads ... for the moment, just leave the 1246 * ESR disabled - we can't do anything useful with the 1247 * errors anyway - mbligh 1248 */ 1249 pr_info("Leaving ESR disabled.\n"); 1250 return; 1251 } 1252 1253 maxlvt = lapic_get_maxlvt(); 1254 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ 1255 apic_write(APIC_ESR, 0); 1256 oldvalue = apic_read(APIC_ESR); 1257 1258 /* enables sending errors */ 1259 value = ERROR_APIC_VECTOR; 1260 apic_write(APIC_LVTERR, value); 1261 1262 /* 1263 * spec says clear errors after enabling vector. 1264 */ 1265 if (maxlvt > 3) 1266 apic_write(APIC_ESR, 0); 1267 value = apic_read(APIC_ESR); 1268 if (value != oldvalue) 1269 apic_printk(APIC_VERBOSE, "ESR value before enabling " 1270 "vector: 0x%08x after: 0x%08x\n", 1271 oldvalue, value); 1272 } 1273 1274 /** 1275 * setup_local_APIC - setup the local APIC 1276 * 1277 * Used to setup local APIC while initializing BSP or bringin up APs. 1278 * Always called with preemption disabled. 1279 */ 1280 void setup_local_APIC(void) 1281 { 1282 int cpu = smp_processor_id(); 1283 unsigned int value, queued; 1284 int i, j, acked = 0; 1285 unsigned long long tsc = 0, ntsc; 1286 long long max_loops = cpu_khz; 1287 1288 if (cpu_has_tsc) 1289 rdtscll(tsc); 1290 1291 if (disable_apic) { 1292 disable_ioapic_support(); 1293 return; 1294 } 1295 1296 #ifdef CONFIG_X86_32 1297 /* Pound the ESR really hard over the head with a big hammer - mbligh */ 1298 if (lapic_is_integrated() && apic->disable_esr) { 1299 apic_write(APIC_ESR, 0); 1300 apic_write(APIC_ESR, 0); 1301 apic_write(APIC_ESR, 0); 1302 apic_write(APIC_ESR, 0); 1303 } 1304 #endif 1305 perf_events_lapic_init(); 1306 1307 /* 1308 * Double-check whether this APIC is really registered. 1309 * This is meaningless in clustered apic mode, so we skip it. 1310 */ 1311 BUG_ON(!apic->apic_id_registered()); 1312 1313 /* 1314 * Intel recommends to set DFR, LDR and TPR before enabling 1315 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel 1316 * document number 292116). So here it goes... 1317 */ 1318 apic->init_apic_ldr(); 1319 1320 #ifdef CONFIG_X86_32 1321 /* 1322 * APIC LDR is initialized. If logical_apicid mapping was 1323 * initialized during get_smp_config(), make sure it matches the 1324 * actual value. 1325 */ 1326 i = early_per_cpu(x86_cpu_to_logical_apicid, cpu); 1327 WARN_ON(i != BAD_APICID && i != logical_smp_processor_id()); 1328 /* always use the value from LDR */ 1329 early_per_cpu(x86_cpu_to_logical_apicid, cpu) = 1330 logical_smp_processor_id(); 1331 1332 /* 1333 * Some NUMA implementations (NUMAQ) don't initialize apicid to 1334 * node mapping during NUMA init. Now that logical apicid is 1335 * guaranteed to be known, give it another chance. This is already 1336 * a bit too late - percpu allocation has already happened without 1337 * proper NUMA affinity. 1338 */ 1339 if (apic->x86_32_numa_cpu_node) 1340 set_apicid_to_node(early_per_cpu(x86_cpu_to_apicid, cpu), 1341 apic->x86_32_numa_cpu_node(cpu)); 1342 #endif 1343 1344 /* 1345 * Set Task Priority to 'accept all'. We never change this 1346 * later on. 1347 */ 1348 value = apic_read(APIC_TASKPRI); 1349 value &= ~APIC_TPRI_MASK; 1350 apic_write(APIC_TASKPRI, value); 1351 1352 /* 1353 * After a crash, we no longer service the interrupts and a pending 1354 * interrupt from previous kernel might still have ISR bit set. 1355 * 1356 * Most probably by now CPU has serviced that pending interrupt and 1357 * it might not have done the ack_APIC_irq() because it thought, 1358 * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it 1359 * does not clear the ISR bit and cpu thinks it has already serivced 1360 * the interrupt. Hence a vector might get locked. It was noticed 1361 * for timer irq (vector 0x31). Issue an extra EOI to clear ISR. 1362 */ 1363 do { 1364 queued = 0; 1365 for (i = APIC_ISR_NR - 1; i >= 0; i--) 1366 queued |= apic_read(APIC_IRR + i*0x10); 1367 1368 for (i = APIC_ISR_NR - 1; i >= 0; i--) { 1369 value = apic_read(APIC_ISR + i*0x10); 1370 for (j = 31; j >= 0; j--) { 1371 if (value & (1<<j)) { 1372 ack_APIC_irq(); 1373 acked++; 1374 } 1375 } 1376 } 1377 if (acked > 256) { 1378 printk(KERN_ERR "LAPIC pending interrupts after %d EOI\n", 1379 acked); 1380 break; 1381 } 1382 if (queued) { 1383 if (cpu_has_tsc) { 1384 rdtscll(ntsc); 1385 max_loops = (cpu_khz << 10) - (ntsc - tsc); 1386 } else 1387 max_loops--; 1388 } 1389 } while (queued && max_loops > 0); 1390 WARN_ON(max_loops <= 0); 1391 1392 /* 1393 * Now that we are all set up, enable the APIC 1394 */ 1395 value = apic_read(APIC_SPIV); 1396 value &= ~APIC_VECTOR_MASK; 1397 /* 1398 * Enable APIC 1399 */ 1400 value |= APIC_SPIV_APIC_ENABLED; 1401 1402 #ifdef CONFIG_X86_32 1403 /* 1404 * Some unknown Intel IO/APIC (or APIC) errata is biting us with 1405 * certain networking cards. If high frequency interrupts are 1406 * happening on a particular IOAPIC pin, plus the IOAPIC routing 1407 * entry is masked/unmasked at a high rate as well then sooner or 1408 * later IOAPIC line gets 'stuck', no more interrupts are received 1409 * from the device. If focus CPU is disabled then the hang goes 1410 * away, oh well :-( 1411 * 1412 * [ This bug can be reproduced easily with a level-triggered 1413 * PCI Ne2000 networking cards and PII/PIII processors, dual 1414 * BX chipset. ] 1415 */ 1416 /* 1417 * Actually disabling the focus CPU check just makes the hang less 1418 * frequent as it makes the interrupt distributon model be more 1419 * like LRU than MRU (the short-term load is more even across CPUs). 1420 * See also the comment in end_level_ioapic_irq(). --macro 1421 */ 1422 1423 /* 1424 * - enable focus processor (bit==0) 1425 * - 64bit mode always use processor focus 1426 * so no need to set it 1427 */ 1428 value &= ~APIC_SPIV_FOCUS_DISABLED; 1429 #endif 1430 1431 /* 1432 * Set spurious IRQ vector 1433 */ 1434 value |= SPURIOUS_APIC_VECTOR; 1435 apic_write(APIC_SPIV, value); 1436 1437 /* 1438 * Set up LVT0, LVT1: 1439 * 1440 * set up through-local-APIC on the BP's LINT0. This is not 1441 * strictly necessary in pure symmetric-IO mode, but sometimes 1442 * we delegate interrupts to the 8259A. 1443 */ 1444 /* 1445 * TODO: set up through-local-APIC from through-I/O-APIC? --macro 1446 */ 1447 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED; 1448 if (!cpu && (pic_mode || !value)) { 1449 value = APIC_DM_EXTINT; 1450 apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu); 1451 } else { 1452 value = APIC_DM_EXTINT | APIC_LVT_MASKED; 1453 apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu); 1454 } 1455 apic_write(APIC_LVT0, value); 1456 1457 /* 1458 * only the BP should see the LINT1 NMI signal, obviously. 1459 */ 1460 if (!cpu) 1461 value = APIC_DM_NMI; 1462 else 1463 value = APIC_DM_NMI | APIC_LVT_MASKED; 1464 if (!lapic_is_integrated()) /* 82489DX */ 1465 value |= APIC_LVT_LEVEL_TRIGGER; 1466 apic_write(APIC_LVT1, value); 1467 1468 #ifdef CONFIG_X86_MCE_INTEL 1469 /* Recheck CMCI information after local APIC is up on CPU #0 */ 1470 if (!cpu) 1471 cmci_recheck(); 1472 #endif 1473 } 1474 1475 void end_local_APIC_setup(void) 1476 { 1477 lapic_setup_esr(); 1478 1479 #ifdef CONFIG_X86_32 1480 { 1481 unsigned int value; 1482 /* Disable the local apic timer */ 1483 value = apic_read(APIC_LVTT); 1484 value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); 1485 apic_write(APIC_LVTT, value); 1486 } 1487 #endif 1488 1489 apic_pm_activate(); 1490 } 1491 1492 void __init bsp_end_local_APIC_setup(void) 1493 { 1494 end_local_APIC_setup(); 1495 1496 /* 1497 * Now that local APIC setup is completed for BP, configure the fault 1498 * handling for interrupt remapping. 1499 */ 1500 irq_remap_enable_fault_handling(); 1501 1502 } 1503 1504 #ifdef CONFIG_X86_X2APIC 1505 /* 1506 * Need to disable xapic and x2apic at the same time and then enable xapic mode 1507 */ 1508 static inline void __disable_x2apic(u64 msr) 1509 { 1510 wrmsrl(MSR_IA32_APICBASE, 1511 msr & ~(X2APIC_ENABLE | XAPIC_ENABLE)); 1512 wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE); 1513 } 1514 1515 static __init void disable_x2apic(void) 1516 { 1517 u64 msr; 1518 1519 if (!cpu_has_x2apic) 1520 return; 1521 1522 rdmsrl(MSR_IA32_APICBASE, msr); 1523 if (msr & X2APIC_ENABLE) { 1524 u32 x2apic_id = read_apic_id(); 1525 1526 if (x2apic_id >= 255) 1527 panic("Cannot disable x2apic, id: %08x\n", x2apic_id); 1528 1529 pr_info("Disabling x2apic\n"); 1530 __disable_x2apic(msr); 1531 1532 if (nox2apic) { 1533 clear_cpu_cap(&cpu_data(0), X86_FEATURE_X2APIC); 1534 setup_clear_cpu_cap(X86_FEATURE_X2APIC); 1535 } 1536 1537 x2apic_disabled = 1; 1538 x2apic_mode = 0; 1539 1540 register_lapic_address(mp_lapic_addr); 1541 } 1542 } 1543 1544 void check_x2apic(void) 1545 { 1546 if (x2apic_enabled()) { 1547 pr_info("x2apic enabled by BIOS, switching to x2apic ops\n"); 1548 x2apic_preenabled = x2apic_mode = 1; 1549 } 1550 } 1551 1552 void enable_x2apic(void) 1553 { 1554 u64 msr; 1555 1556 rdmsrl(MSR_IA32_APICBASE, msr); 1557 if (x2apic_disabled) { 1558 __disable_x2apic(msr); 1559 return; 1560 } 1561 1562 if (!x2apic_mode) 1563 return; 1564 1565 if (!(msr & X2APIC_ENABLE)) { 1566 printk_once(KERN_INFO "Enabling x2apic\n"); 1567 wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE); 1568 } 1569 } 1570 #endif /* CONFIG_X86_X2APIC */ 1571 1572 int __init enable_IR(void) 1573 { 1574 #ifdef CONFIG_IRQ_REMAP 1575 if (!irq_remapping_supported()) { 1576 pr_debug("intr-remapping not supported\n"); 1577 return -1; 1578 } 1579 1580 if (!x2apic_preenabled && skip_ioapic_setup) { 1581 pr_info("Skipped enabling intr-remap because of skipping " 1582 "io-apic setup\n"); 1583 return -1; 1584 } 1585 1586 return irq_remapping_enable(); 1587 #endif 1588 return -1; 1589 } 1590 1591 void __init enable_IR_x2apic(void) 1592 { 1593 unsigned long flags; 1594 int ret, x2apic_enabled = 0; 1595 int hardware_init_ret; 1596 1597 /* Make sure irq_remap_ops are initialized */ 1598 setup_irq_remapping_ops(); 1599 1600 hardware_init_ret = irq_remapping_prepare(); 1601 if (hardware_init_ret && !x2apic_supported()) 1602 return; 1603 1604 ret = save_ioapic_entries(); 1605 if (ret) { 1606 pr_info("Saving IO-APIC state failed: %d\n", ret); 1607 return; 1608 } 1609 1610 local_irq_save(flags); 1611 legacy_pic->mask_all(); 1612 mask_ioapic_entries(); 1613 1614 if (x2apic_preenabled && nox2apic) 1615 disable_x2apic(); 1616 1617 if (hardware_init_ret) 1618 ret = -1; 1619 else 1620 ret = enable_IR(); 1621 1622 if (!x2apic_supported()) 1623 goto skip_x2apic; 1624 1625 if (ret < 0) { 1626 /* IR is required if there is APIC ID > 255 even when running 1627 * under KVM 1628 */ 1629 if (max_physical_apicid > 255 || 1630 !hypervisor_x2apic_available()) { 1631 if (x2apic_preenabled) 1632 disable_x2apic(); 1633 goto skip_x2apic; 1634 } 1635 /* 1636 * without IR all CPUs can be addressed by IOAPIC/MSI 1637 * only in physical mode 1638 */ 1639 x2apic_force_phys(); 1640 } 1641 1642 if (ret == IRQ_REMAP_XAPIC_MODE) { 1643 pr_info("x2apic not enabled, IRQ remapping is in xapic mode\n"); 1644 goto skip_x2apic; 1645 } 1646 1647 x2apic_enabled = 1; 1648 1649 if (x2apic_supported() && !x2apic_mode) { 1650 x2apic_mode = 1; 1651 enable_x2apic(); 1652 pr_info("Enabled x2apic\n"); 1653 } 1654 1655 skip_x2apic: 1656 if (ret < 0) /* IR enabling failed */ 1657 restore_ioapic_entries(); 1658 legacy_pic->restore_mask(); 1659 local_irq_restore(flags); 1660 } 1661 1662 #ifdef CONFIG_X86_64 1663 /* 1664 * Detect and enable local APICs on non-SMP boards. 1665 * Original code written by Keir Fraser. 1666 * On AMD64 we trust the BIOS - if it says no APIC it is likely 1667 * not correctly set up (usually the APIC timer won't work etc.) 1668 */ 1669 static int __init detect_init_APIC(void) 1670 { 1671 if (!cpu_has_apic) { 1672 pr_info("No local APIC present\n"); 1673 return -1; 1674 } 1675 1676 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; 1677 return 0; 1678 } 1679 #else 1680 1681 static int __init apic_verify(void) 1682 { 1683 u32 features, h, l; 1684 1685 /* 1686 * The APIC feature bit should now be enabled 1687 * in `cpuid' 1688 */ 1689 features = cpuid_edx(1); 1690 if (!(features & (1 << X86_FEATURE_APIC))) { 1691 pr_warning("Could not enable APIC!\n"); 1692 return -1; 1693 } 1694 set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC); 1695 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; 1696 1697 /* The BIOS may have set up the APIC at some other address */ 1698 if (boot_cpu_data.x86 >= 6) { 1699 rdmsr(MSR_IA32_APICBASE, l, h); 1700 if (l & MSR_IA32_APICBASE_ENABLE) 1701 mp_lapic_addr = l & MSR_IA32_APICBASE_BASE; 1702 } 1703 1704 pr_info("Found and enabled local APIC!\n"); 1705 return 0; 1706 } 1707 1708 int __init apic_force_enable(unsigned long addr) 1709 { 1710 u32 h, l; 1711 1712 if (disable_apic) 1713 return -1; 1714 1715 /* 1716 * Some BIOSes disable the local APIC in the APIC_BASE 1717 * MSR. This can only be done in software for Intel P6 or later 1718 * and AMD K7 (Model > 1) or later. 1719 */ 1720 if (boot_cpu_data.x86 >= 6) { 1721 rdmsr(MSR_IA32_APICBASE, l, h); 1722 if (!(l & MSR_IA32_APICBASE_ENABLE)) { 1723 pr_info("Local APIC disabled by BIOS -- reenabling.\n"); 1724 l &= ~MSR_IA32_APICBASE_BASE; 1725 l |= MSR_IA32_APICBASE_ENABLE | addr; 1726 wrmsr(MSR_IA32_APICBASE, l, h); 1727 enabled_via_apicbase = 1; 1728 } 1729 } 1730 return apic_verify(); 1731 } 1732 1733 /* 1734 * Detect and initialize APIC 1735 */ 1736 static int __init detect_init_APIC(void) 1737 { 1738 /* Disabled by kernel option? */ 1739 if (disable_apic) 1740 return -1; 1741 1742 switch (boot_cpu_data.x86_vendor) { 1743 case X86_VENDOR_AMD: 1744 if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) || 1745 (boot_cpu_data.x86 >= 15)) 1746 break; 1747 goto no_apic; 1748 case X86_VENDOR_INTEL: 1749 if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 || 1750 (boot_cpu_data.x86 == 5 && cpu_has_apic)) 1751 break; 1752 goto no_apic; 1753 default: 1754 goto no_apic; 1755 } 1756 1757 if (!cpu_has_apic) { 1758 /* 1759 * Over-ride BIOS and try to enable the local APIC only if 1760 * "lapic" specified. 1761 */ 1762 if (!force_enable_local_apic) { 1763 pr_info("Local APIC disabled by BIOS -- " 1764 "you can enable it with \"lapic\"\n"); 1765 return -1; 1766 } 1767 if (apic_force_enable(APIC_DEFAULT_PHYS_BASE)) 1768 return -1; 1769 } else { 1770 if (apic_verify()) 1771 return -1; 1772 } 1773 1774 apic_pm_activate(); 1775 1776 return 0; 1777 1778 no_apic: 1779 pr_info("No local APIC present or hardware disabled\n"); 1780 return -1; 1781 } 1782 #endif 1783 1784 /** 1785 * init_apic_mappings - initialize APIC mappings 1786 */ 1787 void __init init_apic_mappings(void) 1788 { 1789 unsigned int new_apicid; 1790 1791 if (x2apic_mode) { 1792 boot_cpu_physical_apicid = read_apic_id(); 1793 return; 1794 } 1795 1796 /* If no local APIC can be found return early */ 1797 if (!smp_found_config && detect_init_APIC()) { 1798 /* lets NOP'ify apic operations */ 1799 pr_info("APIC: disable apic facility\n"); 1800 apic_disable(); 1801 } else { 1802 apic_phys = mp_lapic_addr; 1803 1804 /* 1805 * acpi lapic path already maps that address in 1806 * acpi_register_lapic_address() 1807 */ 1808 if (!acpi_lapic && !smp_found_config) 1809 register_lapic_address(apic_phys); 1810 } 1811 1812 /* 1813 * Fetch the APIC ID of the BSP in case we have a 1814 * default configuration (or the MP table is broken). 1815 */ 1816 new_apicid = read_apic_id(); 1817 if (boot_cpu_physical_apicid != new_apicid) { 1818 boot_cpu_physical_apicid = new_apicid; 1819 /* 1820 * yeah -- we lie about apic_version 1821 * in case if apic was disabled via boot option 1822 * but it's not a problem for SMP compiled kernel 1823 * since smp_sanity_check is prepared for such a case 1824 * and disable smp mode 1825 */ 1826 apic_version[new_apicid] = 1827 GET_APIC_VERSION(apic_read(APIC_LVR)); 1828 } 1829 } 1830 1831 void __init register_lapic_address(unsigned long address) 1832 { 1833 mp_lapic_addr = address; 1834 1835 if (!x2apic_mode) { 1836 set_fixmap_nocache(FIX_APIC_BASE, address); 1837 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", 1838 APIC_BASE, mp_lapic_addr); 1839 } 1840 if (boot_cpu_physical_apicid == -1U) { 1841 boot_cpu_physical_apicid = read_apic_id(); 1842 apic_version[boot_cpu_physical_apicid] = 1843 GET_APIC_VERSION(apic_read(APIC_LVR)); 1844 } 1845 } 1846 1847 /* 1848 * This initializes the IO-APIC and APIC hardware if this is 1849 * a UP kernel. 1850 */ 1851 int apic_version[MAX_LOCAL_APIC]; 1852 1853 int __init APIC_init_uniprocessor(void) 1854 { 1855 if (disable_apic) { 1856 pr_info("Apic disabled\n"); 1857 return -1; 1858 } 1859 #ifdef CONFIG_X86_64 1860 if (!cpu_has_apic) { 1861 disable_apic = 1; 1862 pr_info("Apic disabled by BIOS\n"); 1863 return -1; 1864 } 1865 #else 1866 if (!smp_found_config && !cpu_has_apic) 1867 return -1; 1868 1869 /* 1870 * Complain if the BIOS pretends there is one. 1871 */ 1872 if (!cpu_has_apic && 1873 APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) { 1874 pr_err("BIOS bug, local APIC 0x%x not detected!...\n", 1875 boot_cpu_physical_apicid); 1876 return -1; 1877 } 1878 #endif 1879 1880 default_setup_apic_routing(); 1881 1882 verify_local_APIC(); 1883 connect_bsp_APIC(); 1884 1885 #ifdef CONFIG_X86_64 1886 apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid)); 1887 #else 1888 /* 1889 * Hack: In case of kdump, after a crash, kernel might be booting 1890 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid 1891 * might be zero if read from MP tables. Get it from LAPIC. 1892 */ 1893 # ifdef CONFIG_CRASH_DUMP 1894 boot_cpu_physical_apicid = read_apic_id(); 1895 # endif 1896 #endif 1897 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map); 1898 setup_local_APIC(); 1899 1900 #ifdef CONFIG_X86_IO_APIC 1901 /* 1902 * Now enable IO-APICs, actually call clear_IO_APIC 1903 * We need clear_IO_APIC before enabling error vector 1904 */ 1905 if (!skip_ioapic_setup && nr_ioapics) 1906 enable_IO_APIC(); 1907 #endif 1908 1909 bsp_end_local_APIC_setup(); 1910 1911 #ifdef CONFIG_X86_IO_APIC 1912 if (smp_found_config && !skip_ioapic_setup && nr_ioapics) 1913 setup_IO_APIC(); 1914 else { 1915 nr_ioapics = 0; 1916 } 1917 #endif 1918 1919 x86_init.timers.setup_percpu_clockev(); 1920 return 0; 1921 } 1922 1923 /* 1924 * Local APIC interrupts 1925 */ 1926 1927 /* 1928 * This interrupt should _never_ happen with our APIC/SMP architecture 1929 */ 1930 static inline void __smp_spurious_interrupt(void) 1931 { 1932 u32 v; 1933 1934 /* 1935 * Check if this really is a spurious interrupt and ACK it 1936 * if it is a vectored one. Just in case... 1937 * Spurious interrupts should not be ACKed. 1938 */ 1939 v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1)); 1940 if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f))) 1941 ack_APIC_irq(); 1942 1943 inc_irq_stat(irq_spurious_count); 1944 1945 /* see sw-dev-man vol 3, chapter 7.4.13.5 */ 1946 pr_info("spurious APIC interrupt on CPU#%d, " 1947 "should never happen.\n", smp_processor_id()); 1948 } 1949 1950 __visible void smp_spurious_interrupt(struct pt_regs *regs) 1951 { 1952 entering_irq(); 1953 __smp_spurious_interrupt(); 1954 exiting_irq(); 1955 } 1956 1957 __visible void smp_trace_spurious_interrupt(struct pt_regs *regs) 1958 { 1959 entering_irq(); 1960 trace_spurious_apic_entry(SPURIOUS_APIC_VECTOR); 1961 __smp_spurious_interrupt(); 1962 trace_spurious_apic_exit(SPURIOUS_APIC_VECTOR); 1963 exiting_irq(); 1964 } 1965 1966 /* 1967 * This interrupt should never happen with our APIC/SMP architecture 1968 */ 1969 static inline void __smp_error_interrupt(struct pt_regs *regs) 1970 { 1971 u32 v0, v1; 1972 u32 i = 0; 1973 static const char * const error_interrupt_reason[] = { 1974 "Send CS error", /* APIC Error Bit 0 */ 1975 "Receive CS error", /* APIC Error Bit 1 */ 1976 "Send accept error", /* APIC Error Bit 2 */ 1977 "Receive accept error", /* APIC Error Bit 3 */ 1978 "Redirectable IPI", /* APIC Error Bit 4 */ 1979 "Send illegal vector", /* APIC Error Bit 5 */ 1980 "Received illegal vector", /* APIC Error Bit 6 */ 1981 "Illegal register address", /* APIC Error Bit 7 */ 1982 }; 1983 1984 /* First tickle the hardware, only then report what went on. -- REW */ 1985 v0 = apic_read(APIC_ESR); 1986 apic_write(APIC_ESR, 0); 1987 v1 = apic_read(APIC_ESR); 1988 ack_APIC_irq(); 1989 atomic_inc(&irq_err_count); 1990 1991 apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x(%02x)", 1992 smp_processor_id(), v0 , v1); 1993 1994 v1 = v1 & 0xff; 1995 while (v1) { 1996 if (v1 & 0x1) 1997 apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]); 1998 i++; 1999 v1 >>= 1; 2000 } 2001 2002 apic_printk(APIC_DEBUG, KERN_CONT "\n"); 2003 2004 } 2005 2006 __visible void smp_error_interrupt(struct pt_regs *regs) 2007 { 2008 entering_irq(); 2009 __smp_error_interrupt(regs); 2010 exiting_irq(); 2011 } 2012 2013 __visible void smp_trace_error_interrupt(struct pt_regs *regs) 2014 { 2015 entering_irq(); 2016 trace_error_apic_entry(ERROR_APIC_VECTOR); 2017 __smp_error_interrupt(regs); 2018 trace_error_apic_exit(ERROR_APIC_VECTOR); 2019 exiting_irq(); 2020 } 2021 2022 /** 2023 * connect_bsp_APIC - attach the APIC to the interrupt system 2024 */ 2025 void __init connect_bsp_APIC(void) 2026 { 2027 #ifdef CONFIG_X86_32 2028 if (pic_mode) { 2029 /* 2030 * Do not trust the local APIC being empty at bootup. 2031 */ 2032 clear_local_APIC(); 2033 /* 2034 * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's 2035 * local APIC to INT and NMI lines. 2036 */ 2037 apic_printk(APIC_VERBOSE, "leaving PIC mode, " 2038 "enabling APIC mode.\n"); 2039 imcr_pic_to_apic(); 2040 } 2041 #endif 2042 if (apic->enable_apic_mode) 2043 apic->enable_apic_mode(); 2044 } 2045 2046 /** 2047 * disconnect_bsp_APIC - detach the APIC from the interrupt system 2048 * @virt_wire_setup: indicates, whether virtual wire mode is selected 2049 * 2050 * Virtual wire mode is necessary to deliver legacy interrupts even when the 2051 * APIC is disabled. 2052 */ 2053 void disconnect_bsp_APIC(int virt_wire_setup) 2054 { 2055 unsigned int value; 2056 2057 #ifdef CONFIG_X86_32 2058 if (pic_mode) { 2059 /* 2060 * Put the board back into PIC mode (has an effect only on 2061 * certain older boards). Note that APIC interrupts, including 2062 * IPIs, won't work beyond this point! The only exception are 2063 * INIT IPIs. 2064 */ 2065 apic_printk(APIC_VERBOSE, "disabling APIC mode, " 2066 "entering PIC mode.\n"); 2067 imcr_apic_to_pic(); 2068 return; 2069 } 2070 #endif 2071 2072 /* Go back to Virtual Wire compatibility mode */ 2073 2074 /* For the spurious interrupt use vector F, and enable it */ 2075 value = apic_read(APIC_SPIV); 2076 value &= ~APIC_VECTOR_MASK; 2077 value |= APIC_SPIV_APIC_ENABLED; 2078 value |= 0xf; 2079 apic_write(APIC_SPIV, value); 2080 2081 if (!virt_wire_setup) { 2082 /* 2083 * For LVT0 make it edge triggered, active high, 2084 * external and enabled 2085 */ 2086 value = apic_read(APIC_LVT0); 2087 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | 2088 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | 2089 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED); 2090 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; 2091 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT); 2092 apic_write(APIC_LVT0, value); 2093 } else { 2094 /* Disable LVT0 */ 2095 apic_write(APIC_LVT0, APIC_LVT_MASKED); 2096 } 2097 2098 /* 2099 * For LVT1 make it edge triggered, active high, 2100 * nmi and enabled 2101 */ 2102 value = apic_read(APIC_LVT1); 2103 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | 2104 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | 2105 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED); 2106 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; 2107 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI); 2108 apic_write(APIC_LVT1, value); 2109 } 2110 2111 int generic_processor_info(int apicid, int version) 2112 { 2113 int cpu, max = nr_cpu_ids; 2114 bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid, 2115 phys_cpu_present_map); 2116 2117 /* 2118 * If boot cpu has not been detected yet, then only allow upto 2119 * nr_cpu_ids - 1 processors and keep one slot free for boot cpu 2120 */ 2121 if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 && 2122 apicid != boot_cpu_physical_apicid) { 2123 int thiscpu = max + disabled_cpus - 1; 2124 2125 pr_warning( 2126 "ACPI: NR_CPUS/possible_cpus limit of %i almost" 2127 " reached. Keeping one slot for boot cpu." 2128 " Processor %d/0x%x ignored.\n", max, thiscpu, apicid); 2129 2130 disabled_cpus++; 2131 return -ENODEV; 2132 } 2133 2134 if (num_processors >= nr_cpu_ids) { 2135 int thiscpu = max + disabled_cpus; 2136 2137 pr_warning( 2138 "ACPI: NR_CPUS/possible_cpus limit of %i reached." 2139 " Processor %d/0x%x ignored.\n", max, thiscpu, apicid); 2140 2141 disabled_cpus++; 2142 return -EINVAL; 2143 } 2144 2145 num_processors++; 2146 if (apicid == boot_cpu_physical_apicid) { 2147 /* 2148 * x86_bios_cpu_apicid is required to have processors listed 2149 * in same order as logical cpu numbers. Hence the first 2150 * entry is BSP, and so on. 2151 * boot_cpu_init() already hold bit 0 in cpu_present_mask 2152 * for BSP. 2153 */ 2154 cpu = 0; 2155 } else 2156 cpu = cpumask_next_zero(-1, cpu_present_mask); 2157 2158 /* 2159 * Validate version 2160 */ 2161 if (version == 0x0) { 2162 pr_warning("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n", 2163 cpu, apicid); 2164 version = 0x10; 2165 } 2166 apic_version[apicid] = version; 2167 2168 if (version != apic_version[boot_cpu_physical_apicid]) { 2169 pr_warning("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n", 2170 apic_version[boot_cpu_physical_apicid], cpu, version); 2171 } 2172 2173 physid_set(apicid, phys_cpu_present_map); 2174 if (apicid > max_physical_apicid) 2175 max_physical_apicid = apicid; 2176 2177 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64) 2178 early_per_cpu(x86_cpu_to_apicid, cpu) = apicid; 2179 early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid; 2180 #endif 2181 #ifdef CONFIG_X86_32 2182 early_per_cpu(x86_cpu_to_logical_apicid, cpu) = 2183 apic->x86_32_early_logical_apicid(cpu); 2184 #endif 2185 set_cpu_possible(cpu, true); 2186 set_cpu_present(cpu, true); 2187 2188 return cpu; 2189 } 2190 2191 int hard_smp_processor_id(void) 2192 { 2193 return read_apic_id(); 2194 } 2195 2196 void default_init_apic_ldr(void) 2197 { 2198 unsigned long val; 2199 2200 apic_write(APIC_DFR, APIC_DFR_VALUE); 2201 val = apic_read(APIC_LDR) & ~APIC_LDR_MASK; 2202 val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id()); 2203 apic_write(APIC_LDR, val); 2204 } 2205 2206 int default_cpu_mask_to_apicid_and(const struct cpumask *cpumask, 2207 const struct cpumask *andmask, 2208 unsigned int *apicid) 2209 { 2210 unsigned int cpu; 2211 2212 for_each_cpu_and(cpu, cpumask, andmask) { 2213 if (cpumask_test_cpu(cpu, cpu_online_mask)) 2214 break; 2215 } 2216 2217 if (likely(cpu < nr_cpu_ids)) { 2218 *apicid = per_cpu(x86_cpu_to_apicid, cpu); 2219 return 0; 2220 } 2221 2222 return -EINVAL; 2223 } 2224 2225 /* 2226 * Override the generic EOI implementation with an optimized version. 2227 * Only called during early boot when only one CPU is active and with 2228 * interrupts disabled, so we know this does not race with actual APIC driver 2229 * use. 2230 */ 2231 void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)) 2232 { 2233 struct apic **drv; 2234 2235 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { 2236 /* Should happen once for each apic */ 2237 WARN_ON((*drv)->eoi_write == eoi_write); 2238 (*drv)->eoi_write = eoi_write; 2239 } 2240 } 2241 2242 /* 2243 * Power management 2244 */ 2245 #ifdef CONFIG_PM 2246 2247 static struct { 2248 /* 2249 * 'active' is true if the local APIC was enabled by us and 2250 * not the BIOS; this signifies that we are also responsible 2251 * for disabling it before entering apm/acpi suspend 2252 */ 2253 int active; 2254 /* r/w apic fields */ 2255 unsigned int apic_id; 2256 unsigned int apic_taskpri; 2257 unsigned int apic_ldr; 2258 unsigned int apic_dfr; 2259 unsigned int apic_spiv; 2260 unsigned int apic_lvtt; 2261 unsigned int apic_lvtpc; 2262 unsigned int apic_lvt0; 2263 unsigned int apic_lvt1; 2264 unsigned int apic_lvterr; 2265 unsigned int apic_tmict; 2266 unsigned int apic_tdcr; 2267 unsigned int apic_thmr; 2268 } apic_pm_state; 2269 2270 static int lapic_suspend(void) 2271 { 2272 unsigned long flags; 2273 int maxlvt; 2274 2275 if (!apic_pm_state.active) 2276 return 0; 2277 2278 maxlvt = lapic_get_maxlvt(); 2279 2280 apic_pm_state.apic_id = apic_read(APIC_ID); 2281 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI); 2282 apic_pm_state.apic_ldr = apic_read(APIC_LDR); 2283 apic_pm_state.apic_dfr = apic_read(APIC_DFR); 2284 apic_pm_state.apic_spiv = apic_read(APIC_SPIV); 2285 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT); 2286 if (maxlvt >= 4) 2287 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC); 2288 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0); 2289 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1); 2290 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR); 2291 apic_pm_state.apic_tmict = apic_read(APIC_TMICT); 2292 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR); 2293 #ifdef CONFIG_X86_THERMAL_VECTOR 2294 if (maxlvt >= 5) 2295 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR); 2296 #endif 2297 2298 local_irq_save(flags); 2299 disable_local_APIC(); 2300 2301 irq_remapping_disable(); 2302 2303 local_irq_restore(flags); 2304 return 0; 2305 } 2306 2307 static void lapic_resume(void) 2308 { 2309 unsigned int l, h; 2310 unsigned long flags; 2311 int maxlvt; 2312 2313 if (!apic_pm_state.active) 2314 return; 2315 2316 local_irq_save(flags); 2317 2318 /* 2319 * IO-APIC and PIC have their own resume routines. 2320 * We just mask them here to make sure the interrupt 2321 * subsystem is completely quiet while we enable x2apic 2322 * and interrupt-remapping. 2323 */ 2324 mask_ioapic_entries(); 2325 legacy_pic->mask_all(); 2326 2327 if (x2apic_mode) 2328 enable_x2apic(); 2329 else { 2330 /* 2331 * Make sure the APICBASE points to the right address 2332 * 2333 * FIXME! This will be wrong if we ever support suspend on 2334 * SMP! We'll need to do this as part of the CPU restore! 2335 */ 2336 if (boot_cpu_data.x86 >= 6) { 2337 rdmsr(MSR_IA32_APICBASE, l, h); 2338 l &= ~MSR_IA32_APICBASE_BASE; 2339 l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; 2340 wrmsr(MSR_IA32_APICBASE, l, h); 2341 } 2342 } 2343 2344 maxlvt = lapic_get_maxlvt(); 2345 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED); 2346 apic_write(APIC_ID, apic_pm_state.apic_id); 2347 apic_write(APIC_DFR, apic_pm_state.apic_dfr); 2348 apic_write(APIC_LDR, apic_pm_state.apic_ldr); 2349 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri); 2350 apic_write(APIC_SPIV, apic_pm_state.apic_spiv); 2351 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0); 2352 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1); 2353 #if defined(CONFIG_X86_MCE_INTEL) 2354 if (maxlvt >= 5) 2355 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr); 2356 #endif 2357 if (maxlvt >= 4) 2358 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc); 2359 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt); 2360 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr); 2361 apic_write(APIC_TMICT, apic_pm_state.apic_tmict); 2362 apic_write(APIC_ESR, 0); 2363 apic_read(APIC_ESR); 2364 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr); 2365 apic_write(APIC_ESR, 0); 2366 apic_read(APIC_ESR); 2367 2368 irq_remapping_reenable(x2apic_mode); 2369 2370 local_irq_restore(flags); 2371 } 2372 2373 /* 2374 * This device has no shutdown method - fully functioning local APICs 2375 * are needed on every CPU up until machine_halt/restart/poweroff. 2376 */ 2377 2378 static struct syscore_ops lapic_syscore_ops = { 2379 .resume = lapic_resume, 2380 .suspend = lapic_suspend, 2381 }; 2382 2383 static void apic_pm_activate(void) 2384 { 2385 apic_pm_state.active = 1; 2386 } 2387 2388 static int __init init_lapic_sysfs(void) 2389 { 2390 /* XXX: remove suspend/resume procs if !apic_pm_state.active? */ 2391 if (cpu_has_apic) 2392 register_syscore_ops(&lapic_syscore_ops); 2393 2394 return 0; 2395 } 2396 2397 /* local apic needs to resume before other devices access its registers. */ 2398 core_initcall(init_lapic_sysfs); 2399 2400 #else /* CONFIG_PM */ 2401 2402 static void apic_pm_activate(void) { } 2403 2404 #endif /* CONFIG_PM */ 2405 2406 #ifdef CONFIG_X86_64 2407 2408 static int apic_cluster_num(void) 2409 { 2410 int i, clusters, zeros; 2411 unsigned id; 2412 u16 *bios_cpu_apicid; 2413 DECLARE_BITMAP(clustermap, NUM_APIC_CLUSTERS); 2414 2415 bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid); 2416 bitmap_zero(clustermap, NUM_APIC_CLUSTERS); 2417 2418 for (i = 0; i < nr_cpu_ids; i++) { 2419 /* are we being called early in kernel startup? */ 2420 if (bios_cpu_apicid) { 2421 id = bios_cpu_apicid[i]; 2422 } else if (i < nr_cpu_ids) { 2423 if (cpu_present(i)) 2424 id = per_cpu(x86_bios_cpu_apicid, i); 2425 else 2426 continue; 2427 } else 2428 break; 2429 2430 if (id != BAD_APICID) 2431 __set_bit(APIC_CLUSTERID(id), clustermap); 2432 } 2433 2434 /* Problem: Partially populated chassis may not have CPUs in some of 2435 * the APIC clusters they have been allocated. Only present CPUs have 2436 * x86_bios_cpu_apicid entries, thus causing zeroes in the bitmap. 2437 * Since clusters are allocated sequentially, count zeros only if 2438 * they are bounded by ones. 2439 */ 2440 clusters = 0; 2441 zeros = 0; 2442 for (i = 0; i < NUM_APIC_CLUSTERS; i++) { 2443 if (test_bit(i, clustermap)) { 2444 clusters += 1 + zeros; 2445 zeros = 0; 2446 } else 2447 ++zeros; 2448 } 2449 2450 return clusters; 2451 } 2452 2453 static int multi_checked; 2454 static int multi; 2455 2456 static int set_multi(const struct dmi_system_id *d) 2457 { 2458 if (multi) 2459 return 0; 2460 pr_info("APIC: %s detected, Multi Chassis\n", d->ident); 2461 multi = 1; 2462 return 0; 2463 } 2464 2465 static const struct dmi_system_id multi_dmi_table[] = { 2466 { 2467 .callback = set_multi, 2468 .ident = "IBM System Summit2", 2469 .matches = { 2470 DMI_MATCH(DMI_SYS_VENDOR, "IBM"), 2471 DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"), 2472 }, 2473 }, 2474 {} 2475 }; 2476 2477 static void dmi_check_multi(void) 2478 { 2479 if (multi_checked) 2480 return; 2481 2482 dmi_check_system(multi_dmi_table); 2483 multi_checked = 1; 2484 } 2485 2486 /* 2487 * apic_is_clustered_box() -- Check if we can expect good TSC 2488 * 2489 * Thus far, the major user of this is IBM's Summit2 series: 2490 * Clustered boxes may have unsynced TSC problems if they are 2491 * multi-chassis. 2492 * Use DMI to check them 2493 */ 2494 int apic_is_clustered_box(void) 2495 { 2496 dmi_check_multi(); 2497 if (multi) 2498 return 1; 2499 2500 if (!is_vsmp_box()) 2501 return 0; 2502 2503 /* 2504 * ScaleMP vSMPowered boxes have one cluster per board and TSCs are 2505 * not guaranteed to be synced between boards 2506 */ 2507 if (apic_cluster_num() > 1) 2508 return 1; 2509 2510 return 0; 2511 } 2512 #endif 2513 2514 /* 2515 * APIC command line parameters 2516 */ 2517 static int __init setup_disableapic(char *arg) 2518 { 2519 disable_apic = 1; 2520 setup_clear_cpu_cap(X86_FEATURE_APIC); 2521 return 0; 2522 } 2523 early_param("disableapic", setup_disableapic); 2524 2525 /* same as disableapic, for compatibility */ 2526 static int __init setup_nolapic(char *arg) 2527 { 2528 return setup_disableapic(arg); 2529 } 2530 early_param("nolapic", setup_nolapic); 2531 2532 static int __init parse_lapic_timer_c2_ok(char *arg) 2533 { 2534 local_apic_timer_c2_ok = 1; 2535 return 0; 2536 } 2537 early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok); 2538 2539 static int __init parse_disable_apic_timer(char *arg) 2540 { 2541 disable_apic_timer = 1; 2542 return 0; 2543 } 2544 early_param("noapictimer", parse_disable_apic_timer); 2545 2546 static int __init parse_nolapic_timer(char *arg) 2547 { 2548 disable_apic_timer = 1; 2549 return 0; 2550 } 2551 early_param("nolapic_timer", parse_nolapic_timer); 2552 2553 static int __init apic_set_verbosity(char *arg) 2554 { 2555 if (!arg) { 2556 #ifdef CONFIG_X86_64 2557 skip_ioapic_setup = 0; 2558 return 0; 2559 #endif 2560 return -EINVAL; 2561 } 2562 2563 if (strcmp("debug", arg) == 0) 2564 apic_verbosity = APIC_DEBUG; 2565 else if (strcmp("verbose", arg) == 0) 2566 apic_verbosity = APIC_VERBOSE; 2567 else { 2568 pr_warning("APIC Verbosity level %s not recognised" 2569 " use apic=verbose or apic=debug\n", arg); 2570 return -EINVAL; 2571 } 2572 2573 return 0; 2574 } 2575 early_param("apic", apic_set_verbosity); 2576 2577 static int __init lapic_insert_resource(void) 2578 { 2579 if (!apic_phys) 2580 return -1; 2581 2582 /* Put local APIC into the resource map. */ 2583 lapic_resource.start = apic_phys; 2584 lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1; 2585 insert_resource(&iomem_resource, &lapic_resource); 2586 2587 return 0; 2588 } 2589 2590 /* 2591 * need call insert after e820_reserve_resources() 2592 * that is using request_resource 2593 */ 2594 late_initcall(lapic_insert_resource); 2595