1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2f62bae50SIngo Molnar /*
3f62bae50SIngo Molnar * Local APIC handling, local APIC timers
4f62bae50SIngo Molnar *
5f62bae50SIngo Molnar * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6f62bae50SIngo Molnar *
7f62bae50SIngo Molnar * Fixes
8f62bae50SIngo Molnar * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
9f62bae50SIngo Molnar * thanks to Eric Gilmore
10f62bae50SIngo Molnar * and Rolf G. Tews
11f62bae50SIngo Molnar * for testing these extensively.
12f62bae50SIngo Molnar * Maciej W. Rozycki : Various updates and fixes.
13f62bae50SIngo Molnar * Mikael Pettersson : Power Management for UP-APIC.
14f62bae50SIngo Molnar * Pavel Machek and
15f62bae50SIngo Molnar * Mikael Pettersson : PM converted to driver model.
16f62bae50SIngo Molnar */
17f62bae50SIngo Molnar
18cdd6c482SIngo Molnar #include <linux/perf_event.h>
19f62bae50SIngo Molnar #include <linux/kernel_stat.h>
20f62bae50SIngo Molnar #include <linux/mc146818rtc.h>
21f62bae50SIngo Molnar #include <linux/acpi_pmtmr.h>
22f62bae50SIngo Molnar #include <linux/clockchips.h>
23f62bae50SIngo Molnar #include <linux/interrupt.h>
2457c8a661SMike Rapoport #include <linux/memblock.h>
25f62bae50SIngo Molnar #include <linux/ftrace.h>
26f62bae50SIngo Molnar #include <linux/ioport.h>
27186f4360SPaul Gortmaker #include <linux/export.h>
28f3c6ea1bSRafael J. Wysocki #include <linux/syscore_ops.h>
29f62bae50SIngo Molnar #include <linux/delay.h>
30f62bae50SIngo Molnar #include <linux/timex.h>
31334955efSRalf Baechle #include <linux/i8253.h>
32f62bae50SIngo Molnar #include <linux/dmar.h>
33f62bae50SIngo Molnar #include <linux/init.h>
34f62bae50SIngo Molnar #include <linux/cpu.h>
35f62bae50SIngo Molnar #include <linux/dmi.h>
36f62bae50SIngo Molnar #include <linux/smp.h>
37f62bae50SIngo Molnar #include <linux/mm.h>
38f62bae50SIngo Molnar
39e0621d5dSThomas Gleixner #include <xen/xen.h>
40e0621d5dSThomas Gleixner
4183ab8514SSteven Rostedt (Red Hat) #include <asm/trace/irq_vectors.h>
428a8f422dSSuresh Siddha #include <asm/irq_remapping.h>
43fb6a0408SMaciej W. Rozycki #include <asm/pc-conf-reg.h>
44cdd6c482SIngo Molnar #include <asm/perf_event.h>
45736decacSThomas Gleixner #include <asm/x86_init.h>
4660063497SArun Sharma #include <linux/atomic.h>
4725a068b8SDave Hansen #include <asm/barrier.h>
48f62bae50SIngo Molnar #include <asm/mpspec.h>
49f62bae50SIngo Molnar #include <asm/i8259.h>
50f62bae50SIngo Molnar #include <asm/proto.h>
51ad3bc25aSBorislav Petkov #include <asm/traps.h>
52f62bae50SIngo Molnar #include <asm/apic.h>
5313c01139SIngo Molnar #include <asm/acpi.h>
547167d08eSHenrik Kretzschmar #include <asm/io_apic.h>
55f62bae50SIngo Molnar #include <asm/desc.h>
56f62bae50SIngo Molnar #include <asm/hpet.h>
57f62bae50SIngo Molnar #include <asm/mtrr.h>
5816f871bcSRalf Baechle #include <asm/time.h>
59f62bae50SIngo Molnar #include <asm/smp.h>
60638bee71SH. Peter Anvin #include <asm/mce.h>
618c3ba8d0SKerstin Jonsson #include <asm/tsc.h>
622904ed8dSSheng Yang #include <asm/hypervisor.h>
63bd9240a1SPeter Zijlstra #include <asm/cpu_device_id.h>
64bd9240a1SPeter Zijlstra #include <asm/intel-family.h>
65447ae316SNicolai Stange #include <asm/irq_regs.h>
66b8d1d163SDaniel Sneddon #include <asm/cpu.h>
67f62bae50SIngo Molnar
6879c9a17cSThomas Gleixner #include "local.h"
6979c9a17cSThomas Gleixner
70f62bae50SIngo Molnar unsigned int num_processors;
71f62bae50SIngo Molnar
72148f9bb8SPaul Gortmaker unsigned disabled_cpus;
73f62bae50SIngo Molnar
74f62bae50SIngo Molnar /* Processor that is doing the boot up */
756444b40eSSean Christopherson unsigned int boot_cpu_physical_apicid __ro_after_init = -1U;
76cc08e04cSDavid Rientjes EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
77f62bae50SIngo Molnar
786444b40eSSean Christopherson u8 boot_cpu_apic_version __ro_after_init;
79cff9ab2bSDenys Vlasenko
80f62bae50SIngo Molnar /*
81f62bae50SIngo Molnar * Bitmask of physically existing CPUs:
82f62bae50SIngo Molnar */
83f62bae50SIngo Molnar physid_mask_t phys_cpu_present_map;
84f62bae50SIngo Molnar
85f62bae50SIngo Molnar /*
86151e0c7dSHATAYAMA Daisuke * Processor to be disabled specified by kernel parameter
87151e0c7dSHATAYAMA Daisuke * disable_cpu_apicid=<int>, mostly used for the kdump 2nd kernel to
88151e0c7dSHATAYAMA Daisuke * avoid undefined behaviour caused by sending INIT from AP to BSP.
89151e0c7dSHATAYAMA Daisuke */
906444b40eSSean Christopherson static unsigned int disabled_cpu_apicid __ro_after_init = BAD_APICID;
91151e0c7dSHATAYAMA Daisuke
92151e0c7dSHATAYAMA Daisuke /*
93b7c4948eSHidehiro Kawai * This variable controls which CPUs receive external NMIs. By default,
94b7c4948eSHidehiro Kawai * external NMIs are delivered only to the BSP.
95b7c4948eSHidehiro Kawai */
966444b40eSSean Christopherson static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP;
97b7c4948eSHidehiro Kawai
98b7c4948eSHidehiro Kawai /*
99ab0f59c6SDavid Woodhouse * Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID
100ab0f59c6SDavid Woodhouse */
101ab0f59c6SDavid Woodhouse static bool virt_ext_dest_id __ro_after_init;
102ab0f59c6SDavid Woodhouse
103bea629d5SThomas Gleixner /* For parallel bootup. */
104bea629d5SThomas Gleixner unsigned long apic_mmio_base __ro_after_init;
105bea629d5SThomas Gleixner
apic_accessible(void)10678c32000SThomas Gleixner static inline bool apic_accessible(void)
10778c32000SThomas Gleixner {
10878c32000SThomas Gleixner return x2apic_mode || apic_mmio_base;
10978c32000SThomas Gleixner }
11078c32000SThomas Gleixner
111ab0f59c6SDavid Woodhouse /*
112f62bae50SIngo Molnar * Map cpu index to physical APIC ID
113f62bae50SIngo Molnar */
1140816b0f0SVlad Zolotarov DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);
1153e9e57faSVitaly Kuznetsov DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid, U32_MAX);
116f62bae50SIngo Molnar EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
1173e9e57faSVitaly Kuznetsov EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid);
118f62bae50SIngo Molnar
119f62bae50SIngo Molnar #ifdef CONFIG_X86_32
120f62bae50SIngo Molnar /* Local APIC was disabled by the BIOS and enabled by the kernel */
1216444b40eSSean Christopherson static int enabled_via_apicbase __ro_after_init;
122f62bae50SIngo Molnar
123c0eaa453SCyrill Gorcunov /*
124c0eaa453SCyrill Gorcunov * Handle interrupt mode configuration register (IMCR).
125c0eaa453SCyrill Gorcunov * This register controls whether the interrupt signals
126c0eaa453SCyrill Gorcunov * that reach the BSP come from the master PIC or from the
127c0eaa453SCyrill Gorcunov * local APIC. Before entering Symmetric I/O Mode, either
128c0eaa453SCyrill Gorcunov * the BIOS or the operating system must switch out of
129c0eaa453SCyrill Gorcunov * PIC Mode by changing the IMCR.
130c0eaa453SCyrill Gorcunov */
imcr_pic_to_apic(void)1315cda395fSAlexander van Heukelum static inline void imcr_pic_to_apic(void)
132c0eaa453SCyrill Gorcunov {
133c0eaa453SCyrill Gorcunov /* NMI and 8259 INTR go through APIC */
134fb6a0408SMaciej W. Rozycki pc_conf_set(PC_CONF_MPS_IMCR, 0x01);
135c0eaa453SCyrill Gorcunov }
136c0eaa453SCyrill Gorcunov
imcr_apic_to_pic(void)1375cda395fSAlexander van Heukelum static inline void imcr_apic_to_pic(void)
138c0eaa453SCyrill Gorcunov {
139c0eaa453SCyrill Gorcunov /* NMI and 8259 INTR go directly to BSP */
140fb6a0408SMaciej W. Rozycki pc_conf_set(PC_CONF_MPS_IMCR, 0x00);
141c0eaa453SCyrill Gorcunov }
142f62bae50SIngo Molnar #endif
143f62bae50SIngo Molnar
144279f1461SSuresh Siddha /*
145279f1461SSuresh Siddha * Knob to control our willingness to enable the local APIC.
146279f1461SSuresh Siddha *
147279f1461SSuresh Siddha * +1=force-enable
148279f1461SSuresh Siddha */
149279f1461SSuresh Siddha static int force_enable_local_apic __initdata;
150dc9788f4SDavid Rientjes
151279f1461SSuresh Siddha /*
152279f1461SSuresh Siddha * APIC command line parameters
153279f1461SSuresh Siddha */
parse_lapic(char * arg)154279f1461SSuresh Siddha static int __init parse_lapic(char *arg)
155279f1461SSuresh Siddha {
15697f2645fSMasahiro Yamada if (IS_ENABLED(CONFIG_X86_32) && !arg)
157279f1461SSuresh Siddha force_enable_local_apic = 1;
15827cf9298SMathias Krause else if (arg && !strncmp(arg, "notscdeadline", 13))
159279f1461SSuresh Siddha setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
160279f1461SSuresh Siddha return 0;
161279f1461SSuresh Siddha }
162279f1461SSuresh Siddha early_param("lapic", parse_lapic);
163279f1461SSuresh Siddha
164f62bae50SIngo Molnar #ifdef CONFIG_X86_64
165f62bae50SIngo Molnar static int apic_calibrate_pmtmr __initdata;
setup_apicpmtimer(char * s)166f62bae50SIngo Molnar static __init int setup_apicpmtimer(char *s)
167f62bae50SIngo Molnar {
168f62bae50SIngo Molnar apic_calibrate_pmtmr = 1;
169f62bae50SIngo Molnar notsc_setup(NULL);
17012441ccdSRandy Dunlap return 1;
171f62bae50SIngo Molnar }
172f62bae50SIngo Molnar __setup("apicpmtimer", setup_apicpmtimer);
173f62bae50SIngo Molnar #endif
174f62bae50SIngo Molnar
17581287ad6SThomas Gleixner static unsigned long mp_lapic_addr __ro_after_init;
17649062454SThomas Gleixner bool apic_is_disabled __ro_after_init;
177f62bae50SIngo Molnar /* Disable local APIC timer from the kernel commandline or via dmi quirk */
17825874a29SHenrik Kretzschmar static int disable_apic_timer __initdata;
179f62bae50SIngo Molnar /* Local APIC timer works in C2 */
1806444b40eSSean Christopherson int local_apic_timer_c2_ok __ro_after_init;
181f62bae50SIngo Molnar EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
182f62bae50SIngo Molnar
183f62bae50SIngo Molnar /*
184f62bae50SIngo Molnar * Debug level, exported for io_apic.c
185f62bae50SIngo Molnar */
1866444b40eSSean Christopherson int apic_verbosity __ro_after_init;
187f62bae50SIngo Molnar
1886444b40eSSean Christopherson int pic_mode __ro_after_init;
189f62bae50SIngo Molnar
190f62bae50SIngo Molnar /* Have we found an MP table */
1916444b40eSSean Christopherson int smp_found_config __ro_after_init;
192f62bae50SIngo Molnar
193f62bae50SIngo Molnar static struct resource lapic_resource = {
194f62bae50SIngo Molnar .name = "Local APIC",
195f62bae50SIngo Molnar .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
196f62bae50SIngo Molnar };
197f62bae50SIngo Molnar
19852ae346bSDaniel Drake unsigned int lapic_timer_period = 0;
199f62bae50SIngo Molnar
200f62bae50SIngo Molnar static void apic_pm_activate(void);
201f62bae50SIngo Molnar
202f62bae50SIngo Molnar /*
203f62bae50SIngo Molnar * Get the LAPIC version
204f62bae50SIngo Molnar */
lapic_get_version(void)205f62bae50SIngo Molnar static inline int lapic_get_version(void)
206f62bae50SIngo Molnar {
207f62bae50SIngo Molnar return GET_APIC_VERSION(apic_read(APIC_LVR));
208f62bae50SIngo Molnar }
209f62bae50SIngo Molnar
210f62bae50SIngo Molnar /*
211f62bae50SIngo Molnar * Check, if the APIC is integrated or a separate chip
212f62bae50SIngo Molnar */
lapic_is_integrated(void)213f62bae50SIngo Molnar static inline int lapic_is_integrated(void)
214f62bae50SIngo Molnar {
215f62bae50SIngo Molnar return APIC_INTEGRATED(lapic_get_version());
216f62bae50SIngo Molnar }
217f62bae50SIngo Molnar
218f62bae50SIngo Molnar /*
219f62bae50SIngo Molnar * Check, whether this is a modern or a first generation APIC
220f62bae50SIngo Molnar */
modern_apic(void)221f62bae50SIngo Molnar static int modern_apic(void)
222f62bae50SIngo Molnar {
223f62bae50SIngo Molnar /* AMD systems use old APIC versions, so check the CPU */
224f62bae50SIngo Molnar if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
225f62bae50SIngo Molnar boot_cpu_data.x86 >= 0xf)
226f62bae50SIngo Molnar return 1;
227da33dfefSPu Wen
228da33dfefSPu Wen /* Hygon systems use modern APIC */
229da33dfefSPu Wen if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
230da33dfefSPu Wen return 1;
231da33dfefSPu Wen
232f62bae50SIngo Molnar return lapic_get_version() >= 0x14;
233f62bae50SIngo Molnar }
234f62bae50SIngo Molnar
23508306ce6SCyrill Gorcunov /*
236a933c618SCyrill Gorcunov * right after this call apic become NOOP driven
237a933c618SCyrill Gorcunov * so apic->write/read doesn't do anything
23808306ce6SCyrill Gorcunov */
apic_disable(void)23925874a29SHenrik Kretzschmar static void __init apic_disable(void)
24008306ce6SCyrill Gorcunov {
2413af1e415SThomas Gleixner apic_install_driver(&apic_noop);
24208306ce6SCyrill Gorcunov }
24308306ce6SCyrill Gorcunov
native_apic_icr_write(u32 low,u32 id)244f62bae50SIngo Molnar void native_apic_icr_write(u32 low, u32 id)
245f62bae50SIngo Molnar {
246ea7bdc65SJan Kiszka unsigned long flags;
247ea7bdc65SJan Kiszka
248ea7bdc65SJan Kiszka local_irq_save(flags);
249bf348f66SSuravee Suthikulpanit apic_write(APIC_ICR2, SET_XAPIC_DEST_FIELD(id));
250f62bae50SIngo Molnar apic_write(APIC_ICR, low);
251ea7bdc65SJan Kiszka local_irq_restore(flags);
252f62bae50SIngo Molnar }
253f62bae50SIngo Molnar
native_apic_icr_read(void)254f62bae50SIngo Molnar u64 native_apic_icr_read(void)
255f62bae50SIngo Molnar {
256f62bae50SIngo Molnar u32 icr1, icr2;
257f62bae50SIngo Molnar
258f62bae50SIngo Molnar icr2 = apic_read(APIC_ICR2);
259f62bae50SIngo Molnar icr1 = apic_read(APIC_ICR);
260f62bae50SIngo Molnar
261f62bae50SIngo Molnar return icr1 | ((u64)icr2 << 32);
262f62bae50SIngo Molnar }
263f62bae50SIngo Molnar
264f62bae50SIngo Molnar #ifdef CONFIG_X86_32
265f62bae50SIngo Molnar /**
266f62bae50SIngo Molnar * get_physical_broadcast - Get number of physical broadcast IDs
267f62bae50SIngo Molnar */
get_physical_broadcast(void)268f62bae50SIngo Molnar int get_physical_broadcast(void)
269f62bae50SIngo Molnar {
270f62bae50SIngo Molnar return modern_apic() ? 0xff : 0xf;
271f62bae50SIngo Molnar }
272f62bae50SIngo Molnar #endif
273f62bae50SIngo Molnar
274f62bae50SIngo Molnar /**
275f62bae50SIngo Molnar * lapic_get_maxlvt - get the maximum number of local vector table entries
276f62bae50SIngo Molnar */
lapic_get_maxlvt(void)277f62bae50SIngo Molnar int lapic_get_maxlvt(void)
278f62bae50SIngo Molnar {
279f62bae50SIngo Molnar /*
280f62bae50SIngo Molnar * - we always have APIC integrated on 64bit mode
281f62bae50SIngo Molnar * - 82489DXs do not report # of LVT entries
282f62bae50SIngo Molnar */
283ae41a2a4SDou Liyang return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2;
284f62bae50SIngo Molnar }
285f62bae50SIngo Molnar
286f62bae50SIngo Molnar /*
287f62bae50SIngo Molnar * Local APIC timer
288f62bae50SIngo Molnar */
289f62bae50SIngo Molnar
290f62bae50SIngo Molnar /* Clock divisor */
291f62bae50SIngo Molnar #define APIC_DIVISOR 16
2921a9e4c56SNicolai Stange #define TSC_DIVISOR 8
293f62bae50SIngo Molnar
294daf3af47SThomas Gleixner /* i82489DX specific */
295daf3af47SThomas Gleixner #define I82489DX_BASE_DIVIDER (((0x2) << 18))
296daf3af47SThomas Gleixner
297f62bae50SIngo Molnar /*
298f62bae50SIngo Molnar * This function sets up the local APIC timer, with a timeout of
299f62bae50SIngo Molnar * 'clocks' APIC bus clock. During calibration we actually call
300f62bae50SIngo Molnar * this function twice on the boot CPU, once with a bogus timeout
301f62bae50SIngo Molnar * value, second time for real. The other (noncalibrating) CPUs
302f62bae50SIngo Molnar * call this function only once, with the real, calibrated value.
303f62bae50SIngo Molnar *
304f62bae50SIngo Molnar * We do reads before writes even if unnecessary, to get around the
305f62bae50SIngo Molnar * P5 APIC double write bug.
306f62bae50SIngo Molnar */
__setup_APIC_LVTT(unsigned int clocks,int oneshot,int irqen)307f62bae50SIngo Molnar static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
308f62bae50SIngo Molnar {
309f62bae50SIngo Molnar unsigned int lvtt_value, tmp_value;
310f62bae50SIngo Molnar
311f62bae50SIngo Molnar lvtt_value = LOCAL_TIMER_VECTOR;
312f62bae50SIngo Molnar if (!oneshot)
313f62bae50SIngo Molnar lvtt_value |= APIC_LVT_TIMER_PERIODIC;
314279f1461SSuresh Siddha else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
315279f1461SSuresh Siddha lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE;
316279f1461SSuresh Siddha
317daf3af47SThomas Gleixner /*
318daf3af47SThomas Gleixner * The i82489DX APIC uses bit 18 and 19 for the base divider. This
319daf3af47SThomas Gleixner * overlaps with bit 18 on integrated APICs, but is not documented
320daf3af47SThomas Gleixner * in the SDM. No problem though. i82489DX equipped systems do not
321daf3af47SThomas Gleixner * have TSC deadline timer.
322daf3af47SThomas Gleixner */
323f62bae50SIngo Molnar if (!lapic_is_integrated())
324daf3af47SThomas Gleixner lvtt_value |= I82489DX_BASE_DIVIDER;
325f62bae50SIngo Molnar
326f62bae50SIngo Molnar if (!irqen)
327f62bae50SIngo Molnar lvtt_value |= APIC_LVT_MASKED;
328f62bae50SIngo Molnar
329f62bae50SIngo Molnar apic_write(APIC_LVTT, lvtt_value);
330f62bae50SIngo Molnar
331279f1461SSuresh Siddha if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
3325d7c631dSShaohua Li /*
3335d7c631dSShaohua Li * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
3345d7c631dSShaohua Li * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
3355d7c631dSShaohua Li * According to Intel, MFENCE can do the serialization here.
3365d7c631dSShaohua Li */
3375d7c631dSShaohua Li asm volatile("mfence" : : : "memory");
338279f1461SSuresh Siddha return;
339279f1461SSuresh Siddha }
340279f1461SSuresh Siddha
341f62bae50SIngo Molnar /*
342f62bae50SIngo Molnar * Divide PICLK by 16
343f62bae50SIngo Molnar */
344f62bae50SIngo Molnar tmp_value = apic_read(APIC_TDCR);
345f62bae50SIngo Molnar apic_write(APIC_TDCR,
346f62bae50SIngo Molnar (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
347f62bae50SIngo Molnar APIC_TDR_DIV_16);
348f62bae50SIngo Molnar
349f62bae50SIngo Molnar if (!oneshot)
350f62bae50SIngo Molnar apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
351f62bae50SIngo Molnar }
352f62bae50SIngo Molnar
353f62bae50SIngo Molnar /*
354a68c439bSRobert Richter * Setup extended LVT, AMD specific
355f62bae50SIngo Molnar *
356a68c439bSRobert Richter * Software should use the LVT offsets the BIOS provides. The offsets
357a68c439bSRobert Richter * are determined by the subsystems using it like those for MCE
358a68c439bSRobert Richter * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
359a68c439bSRobert Richter * are supported. Beginning with family 10h at least 4 offsets are
360a68c439bSRobert Richter * available.
361f62bae50SIngo Molnar *
362a68c439bSRobert Richter * Since the offsets must be consistent for all cores, we keep track
363a68c439bSRobert Richter * of the LVT offsets in software and reserve the offset for the same
364a68c439bSRobert Richter * vector also to be used on other cores. An offset is freed by
365a68c439bSRobert Richter * setting the entry to APIC_EILVT_MASKED.
366a68c439bSRobert Richter *
367a68c439bSRobert Richter * If the BIOS is right, there should be no conflicts. Otherwise a
368a68c439bSRobert Richter * "[Firmware Bug]: ..." error message is generated. However, if
369a68c439bSRobert Richter * software does not properly determines the offsets, it is not
370a68c439bSRobert Richter * necessarily a BIOS bug.
371f62bae50SIngo Molnar */
372f62bae50SIngo Molnar
373a68c439bSRobert Richter static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
374f62bae50SIngo Molnar
eilvt_entry_is_changeable(unsigned int old,unsigned int new)375a68c439bSRobert Richter static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
376a68c439bSRobert Richter {
377a68c439bSRobert Richter return (old & APIC_EILVT_MASKED)
378a68c439bSRobert Richter || (new == APIC_EILVT_MASKED)
379a68c439bSRobert Richter || ((new & ~APIC_EILVT_MASKED) == old);
380a68c439bSRobert Richter }
381a68c439bSRobert Richter
reserve_eilvt_offset(int offset,unsigned int new)382a68c439bSRobert Richter static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
383a68c439bSRobert Richter {
3848abc3122SRobert Richter unsigned int rsvd, vector;
385a68c439bSRobert Richter
386a68c439bSRobert Richter if (offset >= APIC_EILVT_NR_MAX)
387a68c439bSRobert Richter return ~0;
388a68c439bSRobert Richter
3898abc3122SRobert Richter rsvd = atomic_read(&eilvt_offsets[offset]);
390a68c439bSRobert Richter do {
3918abc3122SRobert Richter vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */
3928abc3122SRobert Richter if (vector && !eilvt_entry_is_changeable(vector, new))
393a68c439bSRobert Richter /* may not change if vectors are different */
394a68c439bSRobert Richter return rsvd;
395f96fb2dfSUros Bizjak } while (!atomic_try_cmpxchg(&eilvt_offsets[offset], &rsvd, new));
396a68c439bSRobert Richter
397f96fb2dfSUros Bizjak rsvd = new & ~APIC_EILVT_MASKED;
3988abc3122SRobert Richter if (rsvd && rsvd != vector)
3998abc3122SRobert Richter pr_info("LVT offset %d assigned for vector 0x%02x\n",
4008abc3122SRobert Richter offset, rsvd);
4018abc3122SRobert Richter
402a68c439bSRobert Richter return new;
403a68c439bSRobert Richter }
404a68c439bSRobert Richter
405a68c439bSRobert Richter /*
406a68c439bSRobert Richter * If mask=1, the LVT entry does not generate interrupts while mask=0
407cbf74ceaSRobert Richter * enables the vector. See also the BKDGs. Must be called with
408cbf74ceaSRobert Richter * preemption disabled.
409a68c439bSRobert Richter */
410a68c439bSRobert Richter
setup_APIC_eilvt(u8 offset,u8 vector,u8 msg_type,u8 mask)41127afdf20SRobert Richter int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
412a68c439bSRobert Richter {
413a68c439bSRobert Richter unsigned long reg = APIC_EILVTn(offset);
414a68c439bSRobert Richter unsigned int new, old, reserved;
415a68c439bSRobert Richter
416a68c439bSRobert Richter new = (mask << 16) | (msg_type << 8) | vector;
417a68c439bSRobert Richter old = apic_read(reg);
418a68c439bSRobert Richter reserved = reserve_eilvt_offset(offset, new);
419a68c439bSRobert Richter
420a68c439bSRobert Richter if (reserved != new) {
421eb48c9cbSRobert Richter pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
422eb48c9cbSRobert Richter "vector 0x%x, but the register is already in use for "
423eb48c9cbSRobert Richter "vector 0x%x on another cpu\n",
424eb48c9cbSRobert Richter smp_processor_id(), reg, offset, new, reserved);
425a68c439bSRobert Richter return -EINVAL;
426a68c439bSRobert Richter }
427a68c439bSRobert Richter
428a68c439bSRobert Richter if (!eilvt_entry_is_changeable(old, new)) {
429eb48c9cbSRobert Richter pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
430eb48c9cbSRobert Richter "vector 0x%x, but the register is already in use for "
431eb48c9cbSRobert Richter "vector 0x%x on this cpu\n",
432eb48c9cbSRobert Richter smp_processor_id(), reg, offset, new, old);
433a68c439bSRobert Richter return -EBUSY;
434a68c439bSRobert Richter }
435a68c439bSRobert Richter
436a68c439bSRobert Richter apic_write(reg, new);
437a68c439bSRobert Richter
438a68c439bSRobert Richter return 0;
439f62bae50SIngo Molnar }
44027afdf20SRobert Richter EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
441f62bae50SIngo Molnar
442f62bae50SIngo Molnar /*
443f62bae50SIngo Molnar * Program the next event, relative to now
444f62bae50SIngo Molnar */
lapic_next_event(unsigned long delta,struct clock_event_device * evt)445f62bae50SIngo Molnar static int lapic_next_event(unsigned long delta,
446f62bae50SIngo Molnar struct clock_event_device *evt)
447f62bae50SIngo Molnar {
448f62bae50SIngo Molnar apic_write(APIC_TMICT, delta);
449f62bae50SIngo Molnar return 0;
450f62bae50SIngo Molnar }
451f62bae50SIngo Molnar
lapic_next_deadline(unsigned long delta,struct clock_event_device * evt)452279f1461SSuresh Siddha static int lapic_next_deadline(unsigned long delta,
453279f1461SSuresh Siddha struct clock_event_device *evt)
454279f1461SSuresh Siddha {
455279f1461SSuresh Siddha u64 tsc;
456279f1461SSuresh Siddha
45725a068b8SDave Hansen /* This MSR is special and need a special fence: */
45825a068b8SDave Hansen weak_wrmsr_fence();
45925a068b8SDave Hansen
4604ea1636bSAndy Lutomirski tsc = rdtsc();
461279f1461SSuresh Siddha wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
462279f1461SSuresh Siddha return 0;
463279f1461SSuresh Siddha }
464279f1461SSuresh Siddha
lapic_timer_shutdown(struct clock_event_device * evt)465b23d8e52SViresh Kumar static int lapic_timer_shutdown(struct clock_event_device *evt)
466f62bae50SIngo Molnar {
467f62bae50SIngo Molnar unsigned int v;
468f62bae50SIngo Molnar
469f62bae50SIngo Molnar /* Lapic used as dummy for broadcast ? */
470f62bae50SIngo Molnar if (evt->features & CLOCK_EVT_FEAT_DUMMY)
471b23d8e52SViresh Kumar return 0;
472f62bae50SIngo Molnar
473f62bae50SIngo Molnar v = apic_read(APIC_LVTT);
474f62bae50SIngo Molnar v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
475f62bae50SIngo Molnar apic_write(APIC_LVTT, v);
4766f9b4100SAndreas Herrmann apic_write(APIC_TMICT, 0);
477b23d8e52SViresh Kumar return 0;
478f62bae50SIngo Molnar }
479f62bae50SIngo Molnar
480b23d8e52SViresh Kumar static inline int
lapic_timer_set_periodic_oneshot(struct clock_event_device * evt,bool oneshot)481b23d8e52SViresh Kumar lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot)
482b23d8e52SViresh Kumar {
483b23d8e52SViresh Kumar /* Lapic used as dummy for broadcast ? */
484b23d8e52SViresh Kumar if (evt->features & CLOCK_EVT_FEAT_DUMMY)
485b23d8e52SViresh Kumar return 0;
486b23d8e52SViresh Kumar
48752ae346bSDaniel Drake __setup_APIC_LVTT(lapic_timer_period, oneshot, 1);
488b23d8e52SViresh Kumar return 0;
489b23d8e52SViresh Kumar }
490b23d8e52SViresh Kumar
lapic_timer_set_periodic(struct clock_event_device * evt)491b23d8e52SViresh Kumar static int lapic_timer_set_periodic(struct clock_event_device *evt)
492b23d8e52SViresh Kumar {
493b23d8e52SViresh Kumar return lapic_timer_set_periodic_oneshot(evt, false);
494b23d8e52SViresh Kumar }
495b23d8e52SViresh Kumar
lapic_timer_set_oneshot(struct clock_event_device * evt)496b23d8e52SViresh Kumar static int lapic_timer_set_oneshot(struct clock_event_device *evt)
497b23d8e52SViresh Kumar {
498b23d8e52SViresh Kumar return lapic_timer_set_periodic_oneshot(evt, true);
499f62bae50SIngo Molnar }
500f62bae50SIngo Molnar
501f62bae50SIngo Molnar /*
502f62bae50SIngo Molnar * Local APIC timer broadcast function
503f62bae50SIngo Molnar */
lapic_timer_broadcast(const struct cpumask * mask)504f62bae50SIngo Molnar static void lapic_timer_broadcast(const struct cpumask *mask)
505f62bae50SIngo Molnar {
506f62bae50SIngo Molnar #ifdef CONFIG_SMP
50728b82352SDave Hansen __apic_send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
508f62bae50SIngo Molnar #endif
509f62bae50SIngo Molnar }
510f62bae50SIngo Molnar
51125874a29SHenrik Kretzschmar
51225874a29SHenrik Kretzschmar /*
51325874a29SHenrik Kretzschmar * The local apic timer can be used for any function which is CPU local.
51425874a29SHenrik Kretzschmar */
51525874a29SHenrik Kretzschmar static struct clock_event_device lapic_clockevent = {
51625874a29SHenrik Kretzschmar .name = "lapic",
517b23d8e52SViresh Kumar .features = CLOCK_EVT_FEAT_PERIODIC |
518b23d8e52SViresh Kumar CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
519b23d8e52SViresh Kumar | CLOCK_EVT_FEAT_DUMMY,
52025874a29SHenrik Kretzschmar .shift = 32,
521b23d8e52SViresh Kumar .set_state_shutdown = lapic_timer_shutdown,
522b23d8e52SViresh Kumar .set_state_periodic = lapic_timer_set_periodic,
523b23d8e52SViresh Kumar .set_state_oneshot = lapic_timer_set_oneshot,
524914122c3SFrederic Weisbecker .set_state_oneshot_stopped = lapic_timer_shutdown,
52525874a29SHenrik Kretzschmar .set_next_event = lapic_next_event,
52625874a29SHenrik Kretzschmar .broadcast = lapic_timer_broadcast,
52725874a29SHenrik Kretzschmar .rating = 100,
52825874a29SHenrik Kretzschmar .irq = -1,
52925874a29SHenrik Kretzschmar };
53025874a29SHenrik Kretzschmar static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
53125874a29SHenrik Kretzschmar
532c84cb373SThomas Gleixner static const struct x86_cpu_id deadline_match[] __initconst = {
53366abf238SBorislav Petkov X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */
53466abf238SBorislav Petkov X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x4, 0x4), 0x0f), /* EX */
53566abf238SBorislav Petkov
536adefe55eSThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_X, 0x0b000020),
53766abf238SBorislav Petkov
53866abf238SBorislav Petkov X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x2, 0x2), 0x00000011),
53966abf238SBorislav Petkov X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x3, 0x3), 0x0700000e),
54066abf238SBorislav Petkov X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x4, 0x4), 0x0f00000c),
54166abf238SBorislav Petkov X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x5, 0x5), 0x0e000003),
54266abf238SBorislav Petkov
54366abf238SBorislav Petkov X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x3, 0x3), 0x01000136),
54466abf238SBorislav Petkov X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x4, 0x4), 0x02000014),
54566abf238SBorislav Petkov X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x5, 0xf), 0),
546bd9240a1SPeter Zijlstra
547adefe55eSThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL( HASWELL, 0x22),
548adefe55eSThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL( HASWELL_L, 0x20),
549adefe55eSThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL( HASWELL_G, 0x17),
550bd9240a1SPeter Zijlstra
551adefe55eSThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL( BROADWELL, 0x25),
552adefe55eSThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_G, 0x17),
553bd9240a1SPeter Zijlstra
554adefe55eSThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE_L, 0xb2),
555adefe55eSThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE, 0xb2),
556bd9240a1SPeter Zijlstra
557adefe55eSThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE_L, 0x52),
558adefe55eSThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE, 0x52),
559bd9240a1SPeter Zijlstra
560bd9240a1SPeter Zijlstra {},
561bd9240a1SPeter Zijlstra };
562bd9240a1SPeter Zijlstra
apic_validate_deadline_timer(void)563c84cb373SThomas Gleixner static __init bool apic_validate_deadline_timer(void)
564bd9240a1SPeter Zijlstra {
565594a30fbSHans de Goede const struct x86_cpu_id *m;
566bd9240a1SPeter Zijlstra u32 rev;
567bd9240a1SPeter Zijlstra
568c84cb373SThomas Gleixner if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
569c84cb373SThomas Gleixner return false;
570c84cb373SThomas Gleixner if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
571c84cb373SThomas Gleixner return true;
572594a30fbSHans de Goede
573594a30fbSHans de Goede m = x86_match_cpu(deadline_match);
574bd9240a1SPeter Zijlstra if (!m)
575c84cb373SThomas Gleixner return true;
576bd9240a1SPeter Zijlstra
577bd9240a1SPeter Zijlstra rev = (u32)m->driver_data;
578bd9240a1SPeter Zijlstra
579bd9240a1SPeter Zijlstra if (boot_cpu_data.microcode >= rev)
580c84cb373SThomas Gleixner return true;
581bd9240a1SPeter Zijlstra
582bd9240a1SPeter Zijlstra setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
583bd9240a1SPeter Zijlstra pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; "
584bd9240a1SPeter Zijlstra "please update microcode to version: 0x%x (or later)\n", rev);
585c84cb373SThomas Gleixner return false;
586bd9240a1SPeter Zijlstra }
587bd9240a1SPeter Zijlstra
588f62bae50SIngo Molnar /*
589421f91d2SUwe Kleine-König * Setup the local APIC timer for this CPU. Copy the initialized values
590f62bae50SIngo Molnar * of the boot CPU and register the clock event in the framework.
591f62bae50SIngo Molnar */
setup_APIC_timer(void)592148f9bb8SPaul Gortmaker static void setup_APIC_timer(void)
593f62bae50SIngo Molnar {
59489cbc767SChristoph Lameter struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
595f62bae50SIngo Molnar
596349c004eSChristoph Lameter if (this_cpu_has(X86_FEATURE_ARAT)) {
597db954b58SVenkatesh Pallipadi lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
598d9f6e12fSIngo Molnar /* Make LAPIC timer preferable over percpu HPET */
599db954b58SVenkatesh Pallipadi lapic_clockevent.rating = 150;
600db954b58SVenkatesh Pallipadi }
601db954b58SVenkatesh Pallipadi
602f62bae50SIngo Molnar memcpy(levt, &lapic_clockevent, sizeof(*levt));
603f62bae50SIngo Molnar levt->cpumask = cpumask_of(smp_processor_id());
604f62bae50SIngo Molnar
605279f1461SSuresh Siddha if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
606c6e9f42bSPeter Zijlstra levt->name = "lapic-deadline";
607279f1461SSuresh Siddha levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
608279f1461SSuresh Siddha CLOCK_EVT_FEAT_DUMMY);
609279f1461SSuresh Siddha levt->set_next_event = lapic_next_deadline;
610279f1461SSuresh Siddha clockevents_config_and_register(levt,
6111a9e4c56SNicolai Stange tsc_khz * (1000 / TSC_DIVISOR),
612279f1461SSuresh Siddha 0xF, ~0UL);
613279f1461SSuresh Siddha } else
614f62bae50SIngo Molnar clockevents_register_device(levt);
615f62bae50SIngo Molnar }
616f62bae50SIngo Molnar
617f62bae50SIngo Molnar /*
6186731b0d6SNicolai Stange * Install the updated TSC frequency from recalibration at the TSC
6196731b0d6SNicolai Stange * deadline clockevent devices.
6206731b0d6SNicolai Stange */
__lapic_update_tsc_freq(void * info)6216731b0d6SNicolai Stange static void __lapic_update_tsc_freq(void *info)
6226731b0d6SNicolai Stange {
6236731b0d6SNicolai Stange struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
6246731b0d6SNicolai Stange
6256731b0d6SNicolai Stange if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
6266731b0d6SNicolai Stange return;
6276731b0d6SNicolai Stange
6286731b0d6SNicolai Stange clockevents_update_freq(levt, tsc_khz * (1000 / TSC_DIVISOR));
6296731b0d6SNicolai Stange }
6306731b0d6SNicolai Stange
lapic_update_tsc_freq(void)6316731b0d6SNicolai Stange void lapic_update_tsc_freq(void)
6326731b0d6SNicolai Stange {
6336731b0d6SNicolai Stange /*
6346731b0d6SNicolai Stange * The clockevent device's ->mult and ->shift can both be
6356731b0d6SNicolai Stange * changed. In order to avoid races, schedule the frequency
6366731b0d6SNicolai Stange * update code on each CPU.
6376731b0d6SNicolai Stange */
6386731b0d6SNicolai Stange on_each_cpu(__lapic_update_tsc_freq, NULL, 0);
6396731b0d6SNicolai Stange }
6406731b0d6SNicolai Stange
6416731b0d6SNicolai Stange /*
642f62bae50SIngo Molnar * In this functions we calibrate APIC bus clocks to the external timer.
643f62bae50SIngo Molnar *
644f62bae50SIngo Molnar * We want to do the calibration only once since we want to have local timer
645d9f6e12fSIngo Molnar * irqs synchronous. CPUs connected by the same APIC bus have the very same bus
646f62bae50SIngo Molnar * frequency.
647f62bae50SIngo Molnar *
648f62bae50SIngo Molnar * This was previously done by reading the PIT/HPET and waiting for a wrap
649f62bae50SIngo Molnar * around to find out, that a tick has elapsed. I have a box, where the PIT
650f62bae50SIngo Molnar * readout is broken, so it never gets out of the wait loop again. This was
651f62bae50SIngo Molnar * also reported by others.
652f62bae50SIngo Molnar *
653f62bae50SIngo Molnar * Monitoring the jiffies value is inaccurate and the clockevents
654f62bae50SIngo Molnar * infrastructure allows us to do a simple substitution of the interrupt
655f62bae50SIngo Molnar * handler.
656f62bae50SIngo Molnar *
657f62bae50SIngo Molnar * The calibration routine also uses the pm_timer when possible, as the PIT
658f62bae50SIngo Molnar * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
659f62bae50SIngo Molnar * back to normal later in the boot process).
660f62bae50SIngo Molnar */
661f62bae50SIngo Molnar
662f62bae50SIngo Molnar #define LAPIC_CAL_LOOPS (HZ/10)
663f62bae50SIngo Molnar
664f62bae50SIngo Molnar static __initdata int lapic_cal_loops = -1;
665f62bae50SIngo Molnar static __initdata long lapic_cal_t1, lapic_cal_t2;
666f62bae50SIngo Molnar static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
667f62bae50SIngo Molnar static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
668f62bae50SIngo Molnar static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
669f62bae50SIngo Molnar
670f62bae50SIngo Molnar /*
671f897e60aSThomas Gleixner * Temporary interrupt handler and polled calibration function.
672f62bae50SIngo Molnar */
lapic_cal_handler(struct clock_event_device * dev)673f62bae50SIngo Molnar static void __init lapic_cal_handler(struct clock_event_device *dev)
674f62bae50SIngo Molnar {
675f62bae50SIngo Molnar unsigned long long tsc = 0;
676f62bae50SIngo Molnar long tapic = apic_read(APIC_TMCCT);
677f62bae50SIngo Molnar unsigned long pm = acpi_pm_read_early();
678f62bae50SIngo Molnar
67959e21e3dSBorislav Petkov if (boot_cpu_has(X86_FEATURE_TSC))
6804ea1636bSAndy Lutomirski tsc = rdtsc();
681f62bae50SIngo Molnar
682f62bae50SIngo Molnar switch (lapic_cal_loops++) {
683f62bae50SIngo Molnar case 0:
684f62bae50SIngo Molnar lapic_cal_t1 = tapic;
685f62bae50SIngo Molnar lapic_cal_tsc1 = tsc;
686f62bae50SIngo Molnar lapic_cal_pm1 = pm;
687f62bae50SIngo Molnar lapic_cal_j1 = jiffies;
688f62bae50SIngo Molnar break;
689f62bae50SIngo Molnar
690f62bae50SIngo Molnar case LAPIC_CAL_LOOPS:
691f62bae50SIngo Molnar lapic_cal_t2 = tapic;
692f62bae50SIngo Molnar lapic_cal_tsc2 = tsc;
693f62bae50SIngo Molnar if (pm < lapic_cal_pm1)
694f62bae50SIngo Molnar pm += ACPI_PM_OVRRUN;
695f62bae50SIngo Molnar lapic_cal_pm2 = pm;
696f62bae50SIngo Molnar lapic_cal_j2 = jiffies;
697f62bae50SIngo Molnar break;
698f62bae50SIngo Molnar }
699f62bae50SIngo Molnar }
700f62bae50SIngo Molnar
701f62bae50SIngo Molnar static int __init
calibrate_by_pmtimer(long deltapm,long * delta,long * deltatsc)702f62bae50SIngo Molnar calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
703f62bae50SIngo Molnar {
704f62bae50SIngo Molnar const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
705f62bae50SIngo Molnar const long pm_thresh = pm_100ms / 100;
706f62bae50SIngo Molnar unsigned long mult;
707f62bae50SIngo Molnar u64 res;
708f62bae50SIngo Molnar
709f62bae50SIngo Molnar #ifndef CONFIG_X86_PM_TIMER
710f62bae50SIngo Molnar return -1;
711f62bae50SIngo Molnar #endif
712f62bae50SIngo Molnar
713f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
714f62bae50SIngo Molnar
715f62bae50SIngo Molnar /* Check, if the PM timer is available */
716f62bae50SIngo Molnar if (!deltapm)
717f62bae50SIngo Molnar return -1;
718f62bae50SIngo Molnar
719f62bae50SIngo Molnar mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
720f62bae50SIngo Molnar
721f62bae50SIngo Molnar if (deltapm > (pm_100ms - pm_thresh) &&
722f62bae50SIngo Molnar deltapm < (pm_100ms + pm_thresh)) {
723f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
724f62bae50SIngo Molnar return 0;
725f62bae50SIngo Molnar }
726f62bae50SIngo Molnar
727f62bae50SIngo Molnar res = (((u64)deltapm) * mult) >> 22;
728f62bae50SIngo Molnar do_div(res, 1000000);
7298d3bcc44SKefeng Wang pr_warn("APIC calibration not consistent "
730f62bae50SIngo Molnar "with PM-Timer: %ldms instead of 100ms\n", (long)res);
731f62bae50SIngo Molnar
732f62bae50SIngo Molnar /* Correct the lapic counter value */
733f62bae50SIngo Molnar res = (((u64)(*delta)) * pm_100ms);
734f62bae50SIngo Molnar do_div(res, deltapm);
735f62bae50SIngo Molnar pr_info("APIC delta adjusted to PM-Timer: "
736f62bae50SIngo Molnar "%lu (%ld)\n", (unsigned long)res, *delta);
737f62bae50SIngo Molnar *delta = (long)res;
738f62bae50SIngo Molnar
739f62bae50SIngo Molnar /* Correct the tsc counter value */
74059e21e3dSBorislav Petkov if (boot_cpu_has(X86_FEATURE_TSC)) {
741f62bae50SIngo Molnar res = (((u64)(*deltatsc)) * pm_100ms);
742f62bae50SIngo Molnar do_div(res, deltapm);
743f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
744f62bae50SIngo Molnar "PM-Timer: %lu (%ld)\n",
745f62bae50SIngo Molnar (unsigned long)res, *deltatsc);
746f62bae50SIngo Molnar *deltatsc = (long)res;
747f62bae50SIngo Molnar }
748f62bae50SIngo Molnar
749f62bae50SIngo Molnar return 0;
750f62bae50SIngo Molnar }
751f62bae50SIngo Molnar
lapic_init_clockevent(void)7526eb4f082SJacob Pan static int __init lapic_init_clockevent(void)
7536eb4f082SJacob Pan {
75452ae346bSDaniel Drake if (!lapic_timer_period)
7556eb4f082SJacob Pan return -1;
7566eb4f082SJacob Pan
7576eb4f082SJacob Pan /* Calculate the scaled math multiplication factor */
75852ae346bSDaniel Drake lapic_clockevent.mult = div_sc(lapic_timer_period/APIC_DIVISOR,
7596eb4f082SJacob Pan TICK_NSEC, lapic_clockevent.shift);
7606eb4f082SJacob Pan lapic_clockevent.max_delta_ns =
7616eb4f082SJacob Pan clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
7626eb4f082SJacob Pan lapic_clockevent.max_delta_ticks = 0x7FFFFFFF;
7636eb4f082SJacob Pan lapic_clockevent.min_delta_ns =
7646eb4f082SJacob Pan clockevent_delta2ns(0xF, &lapic_clockevent);
7656eb4f082SJacob Pan lapic_clockevent.min_delta_ticks = 0xF;
7666eb4f082SJacob Pan
7676eb4f082SJacob Pan return 0;
7686eb4f082SJacob Pan }
7696eb4f082SJacob Pan
apic_needs_pit(void)770c8c40767SThomas Gleixner bool __init apic_needs_pit(void)
771c8c40767SThomas Gleixner {
772c8c40767SThomas Gleixner /*
773c8c40767SThomas Gleixner * If the frequencies are not known, PIT is required for both TSC
774c8c40767SThomas Gleixner * and apic timer calibration.
775c8c40767SThomas Gleixner */
776c8c40767SThomas Gleixner if (!tsc_khz || !cpu_khz)
777c8c40767SThomas Gleixner return true;
778c8c40767SThomas Gleixner
77997992387SThomas Gleixner /* Is there an APIC at all or is it disabled? */
78049062454SThomas Gleixner if (!boot_cpu_has(X86_FEATURE_APIC) || apic_is_disabled)
78197992387SThomas Gleixner return true;
78297992387SThomas Gleixner
78397992387SThomas Gleixner /*
78497992387SThomas Gleixner * If interrupt delivery mode is legacy PIC or virtual wire without
78597992387SThomas Gleixner * configuration, the local APIC timer wont be set up. Make sure
78697992387SThomas Gleixner * that the PIT is initialized.
78797992387SThomas Gleixner */
78897992387SThomas Gleixner if (apic_intr_mode == APIC_PIC ||
78997992387SThomas Gleixner apic_intr_mode == APIC_VIRTUAL_WIRE_NO_CONFIG)
790c8c40767SThomas Gleixner return true;
791c8c40767SThomas Gleixner
792afa8b475SJan Stancek /* Virt guests may lack ARAT, but still have DEADLINE */
793afa8b475SJan Stancek if (!boot_cpu_has(X86_FEATURE_ARAT))
794afa8b475SJan Stancek return true;
795afa8b475SJan Stancek
796c8c40767SThomas Gleixner /* Deadline timer is based on TSC so no further PIT action required */
797c8c40767SThomas Gleixner if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
798c8c40767SThomas Gleixner return false;
799c8c40767SThomas Gleixner
800c8c40767SThomas Gleixner /* APIC timer disabled? */
801c8c40767SThomas Gleixner if (disable_apic_timer)
802c8c40767SThomas Gleixner return true;
803c8c40767SThomas Gleixner /*
804c8c40767SThomas Gleixner * The APIC timer frequency is known already, no PIT calibration
805c8c40767SThomas Gleixner * required. If unknown, let the PIT be initialized.
806c8c40767SThomas Gleixner */
807c8c40767SThomas Gleixner return lapic_timer_period == 0;
808c8c40767SThomas Gleixner }
809c8c40767SThomas Gleixner
calibrate_APIC_clock(void)810f62bae50SIngo Molnar static int __init calibrate_APIC_clock(void)
811f62bae50SIngo Molnar {
81289cbc767SChristoph Lameter struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
813f897e60aSThomas Gleixner u64 tsc_perj = 0, tsc_start = 0;
814f897e60aSThomas Gleixner unsigned long jif_start;
815f62bae50SIngo Molnar unsigned long deltaj;
816f62bae50SIngo Molnar long delta, deltatsc;
817f62bae50SIngo Molnar int pm_referenced = 0;
818f62bae50SIngo Molnar
8196eb4f082SJacob Pan if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
8206eb4f082SJacob Pan return 0;
8216eb4f082SJacob Pan
8226eb4f082SJacob Pan /*
8236eb4f082SJacob Pan * Check if lapic timer has already been calibrated by platform
8246eb4f082SJacob Pan * specific routine, such as tsc calibration code. If so just fill
8251ade93efSJacob Pan * in the clockevent structure and return.
8261ade93efSJacob Pan */
8276eb4f082SJacob Pan if (!lapic_init_clockevent()) {
8281ade93efSJacob Pan apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n",
82952ae346bSDaniel Drake lapic_timer_period);
8306eb4f082SJacob Pan /*
8316eb4f082SJacob Pan * Direct calibration methods must have an always running
8326eb4f082SJacob Pan * local APIC timer, no need for broadcast timer.
8336eb4f082SJacob Pan */
8341ade93efSJacob Pan lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
8351ade93efSJacob Pan return 0;
8361ade93efSJacob Pan }
8371ade93efSJacob Pan
838279f1461SSuresh Siddha apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
839279f1461SSuresh Siddha "calibrating APIC timer ...\n");
840279f1461SSuresh Siddha
841f897e60aSThomas Gleixner /*
842f897e60aSThomas Gleixner * There are platforms w/o global clockevent devices. Instead of
843f897e60aSThomas Gleixner * making the calibration conditional on that, use a polling based
844f897e60aSThomas Gleixner * approach everywhere.
845f897e60aSThomas Gleixner */
846f62bae50SIngo Molnar local_irq_disable();
847f62bae50SIngo Molnar
848f62bae50SIngo Molnar /*
849f62bae50SIngo Molnar * Setup the APIC counter to maximum. There is no way the lapic
850f62bae50SIngo Molnar * can underflow in the 100ms detection time frame
851f62bae50SIngo Molnar */
852f62bae50SIngo Molnar __setup_APIC_LVTT(0xffffffff, 0, 0);
853f62bae50SIngo Molnar
854f897e60aSThomas Gleixner /*
855f897e60aSThomas Gleixner * Methods to terminate the calibration loop:
856f897e60aSThomas Gleixner * 1) Global clockevent if available (jiffies)
857f897e60aSThomas Gleixner * 2) TSC if available and frequency is known
858f897e60aSThomas Gleixner */
859f897e60aSThomas Gleixner jif_start = READ_ONCE(jiffies);
860f897e60aSThomas Gleixner
861f897e60aSThomas Gleixner if (tsc_khz) {
862f897e60aSThomas Gleixner tsc_start = rdtsc();
863f897e60aSThomas Gleixner tsc_perj = div_u64((u64)tsc_khz * 1000, HZ);
864f897e60aSThomas Gleixner }
865f897e60aSThomas Gleixner
866f897e60aSThomas Gleixner /*
867f897e60aSThomas Gleixner * Enable interrupts so the tick can fire, if a global
868f897e60aSThomas Gleixner * clockevent device is available
869f897e60aSThomas Gleixner */
870f62bae50SIngo Molnar local_irq_enable();
871f62bae50SIngo Molnar
872f897e60aSThomas Gleixner while (lapic_cal_loops <= LAPIC_CAL_LOOPS) {
873f897e60aSThomas Gleixner /* Wait for a tick to elapse */
874f897e60aSThomas Gleixner while (1) {
875f897e60aSThomas Gleixner if (tsc_khz) {
876f897e60aSThomas Gleixner u64 tsc_now = rdtsc();
877f897e60aSThomas Gleixner if ((tsc_now - tsc_start) >= tsc_perj) {
878f897e60aSThomas Gleixner tsc_start += tsc_perj;
879f897e60aSThomas Gleixner break;
880f897e60aSThomas Gleixner }
881f897e60aSThomas Gleixner } else {
882f897e60aSThomas Gleixner unsigned long jif_now = READ_ONCE(jiffies);
883f897e60aSThomas Gleixner
884f897e60aSThomas Gleixner if (time_after(jif_now, jif_start)) {
885f897e60aSThomas Gleixner jif_start = jif_now;
886f897e60aSThomas Gleixner break;
887f897e60aSThomas Gleixner }
888f897e60aSThomas Gleixner }
889f62bae50SIngo Molnar cpu_relax();
890f897e60aSThomas Gleixner }
891f897e60aSThomas Gleixner
892f897e60aSThomas Gleixner /* Invoke the calibration routine */
893f897e60aSThomas Gleixner local_irq_disable();
894f897e60aSThomas Gleixner lapic_cal_handler(NULL);
895f897e60aSThomas Gleixner local_irq_enable();
896f897e60aSThomas Gleixner }
897f62bae50SIngo Molnar
898f62bae50SIngo Molnar local_irq_disable();
899f62bae50SIngo Molnar
900f62bae50SIngo Molnar /* Build delta t1-t2 as apic timer counts down */
901f62bae50SIngo Molnar delta = lapic_cal_t1 - lapic_cal_t2;
902f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
903f62bae50SIngo Molnar
904f62bae50SIngo Molnar deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
905f62bae50SIngo Molnar
906f62bae50SIngo Molnar /* we trust the PM based calibration if possible */
907f62bae50SIngo Molnar pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
908f62bae50SIngo Molnar &delta, &deltatsc);
909f62bae50SIngo Molnar
91052ae346bSDaniel Drake lapic_timer_period = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
9116eb4f082SJacob Pan lapic_init_clockevent();
912f62bae50SIngo Molnar
913f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
914411462f6SThomas Gleixner apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
915f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
91652ae346bSDaniel Drake lapic_timer_period);
917f62bae50SIngo Molnar
91859e21e3dSBorislav Petkov if (boot_cpu_has(X86_FEATURE_TSC)) {
919f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
920f62bae50SIngo Molnar "%ld.%04ld MHz.\n",
921f62bae50SIngo Molnar (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
922f62bae50SIngo Molnar (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
923f62bae50SIngo Molnar }
924f62bae50SIngo Molnar
925f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
926f62bae50SIngo Molnar "%u.%04u MHz.\n",
92752ae346bSDaniel Drake lapic_timer_period / (1000000 / HZ),
92852ae346bSDaniel Drake lapic_timer_period % (1000000 / HZ));
929f62bae50SIngo Molnar
930f62bae50SIngo Molnar /*
931f62bae50SIngo Molnar * Do a sanity check on the APIC calibration result
932f62bae50SIngo Molnar */
93352ae346bSDaniel Drake if (lapic_timer_period < (1000000 / HZ)) {
934f62bae50SIngo Molnar local_irq_enable();
9358d3bcc44SKefeng Wang pr_warn("APIC frequency too slow, disabling apic timer\n");
936f62bae50SIngo Molnar return -1;
937f62bae50SIngo Molnar }
938f62bae50SIngo Molnar
939f62bae50SIngo Molnar levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
940f62bae50SIngo Molnar
941f62bae50SIngo Molnar /*
942f897e60aSThomas Gleixner * PM timer calibration failed or not turned on so lets try APIC
943f897e60aSThomas Gleixner * timer based calibration, if a global clockevent device is
944f897e60aSThomas Gleixner * available.
945f62bae50SIngo Molnar */
946f897e60aSThomas Gleixner if (!pm_referenced && global_clock_event) {
947f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
948f62bae50SIngo Molnar
949f62bae50SIngo Molnar /*
950f62bae50SIngo Molnar * Setup the apic timer manually
951f62bae50SIngo Molnar */
952f62bae50SIngo Molnar levt->event_handler = lapic_cal_handler;
953b23d8e52SViresh Kumar lapic_timer_set_periodic(levt);
954f62bae50SIngo Molnar lapic_cal_loops = -1;
955f62bae50SIngo Molnar
956f62bae50SIngo Molnar /* Let the interrupts run */
957f62bae50SIngo Molnar local_irq_enable();
958f62bae50SIngo Molnar
959f62bae50SIngo Molnar while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
960f62bae50SIngo Molnar cpu_relax();
961f62bae50SIngo Molnar
962f62bae50SIngo Molnar /* Stop the lapic timer */
963c948c260SThomas Gleixner local_irq_disable();
964b23d8e52SViresh Kumar lapic_timer_shutdown(levt);
965f62bae50SIngo Molnar
966f62bae50SIngo Molnar /* Jiffies delta */
967f62bae50SIngo Molnar deltaj = lapic_cal_j2 - lapic_cal_j1;
968f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
969f62bae50SIngo Molnar
970f62bae50SIngo Molnar /* Check, if the jiffies result is consistent */
971f62bae50SIngo Molnar if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
972f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
973f62bae50SIngo Molnar else
974f62bae50SIngo Molnar levt->features |= CLOCK_EVT_FEAT_DUMMY;
975c948c260SThomas Gleixner }
976f62bae50SIngo Molnar local_irq_enable();
977f62bae50SIngo Molnar
978f62bae50SIngo Molnar if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
9798d3bcc44SKefeng Wang pr_warn("APIC timer disabled due to verification failure\n");
980f62bae50SIngo Molnar return -1;
981f62bae50SIngo Molnar }
982f62bae50SIngo Molnar
983f62bae50SIngo Molnar return 0;
984f62bae50SIngo Molnar }
985f62bae50SIngo Molnar
986f62bae50SIngo Molnar /*
987f62bae50SIngo Molnar * Setup the boot APIC
988f62bae50SIngo Molnar *
989f62bae50SIngo Molnar * Calibrate and verify the result.
990f62bae50SIngo Molnar */
setup_boot_APIC_clock(void)991f62bae50SIngo Molnar void __init setup_boot_APIC_clock(void)
992f62bae50SIngo Molnar {
993f62bae50SIngo Molnar /*
994f62bae50SIngo Molnar * The local apic timer can be disabled via the kernel
995f62bae50SIngo Molnar * commandline or from the CPU detection code. Register the lapic
996f62bae50SIngo Molnar * timer as a dummy clock event source on SMP systems, so the
997f62bae50SIngo Molnar * broadcast mechanism is used. On UP systems simply ignore it.
998f62bae50SIngo Molnar */
999f62bae50SIngo Molnar if (disable_apic_timer) {
1000f62bae50SIngo Molnar pr_info("Disabling APIC timer\n");
1001f62bae50SIngo Molnar /* No broadcast on UP ! */
1002f62bae50SIngo Molnar if (num_possible_cpus() > 1) {
1003f62bae50SIngo Molnar lapic_clockevent.mult = 1;
1004f62bae50SIngo Molnar setup_APIC_timer();
1005f62bae50SIngo Molnar }
1006f62bae50SIngo Molnar return;
1007f62bae50SIngo Molnar }
1008f62bae50SIngo Molnar
1009f62bae50SIngo Molnar if (calibrate_APIC_clock()) {
1010f62bae50SIngo Molnar /* No broadcast on UP ! */
1011f62bae50SIngo Molnar if (num_possible_cpus() > 1)
1012f62bae50SIngo Molnar setup_APIC_timer();
1013f62bae50SIngo Molnar return;
1014f62bae50SIngo Molnar }
1015f62bae50SIngo Molnar
1016f62bae50SIngo Molnar /*
1017f62bae50SIngo Molnar * If nmi_watchdog is set to IO_APIC, we need the
1018f62bae50SIngo Molnar * PIT/HPET going. Otherwise register lapic as a dummy
1019f62bae50SIngo Molnar * device.
1020f62bae50SIngo Molnar */
1021f62bae50SIngo Molnar lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
1022f62bae50SIngo Molnar
1023f62bae50SIngo Molnar /* Setup the lapic or request the broadcast */
1024f62bae50SIngo Molnar setup_APIC_timer();
102507c94a38SBorislav Petkov amd_e400_c1e_apic_setup();
1026f62bae50SIngo Molnar }
1027f62bae50SIngo Molnar
setup_secondary_APIC_clock(void)1028148f9bb8SPaul Gortmaker void setup_secondary_APIC_clock(void)
1029f62bae50SIngo Molnar {
1030f62bae50SIngo Molnar setup_APIC_timer();
103107c94a38SBorislav Petkov amd_e400_c1e_apic_setup();
1032f62bae50SIngo Molnar }
1033f62bae50SIngo Molnar
1034f62bae50SIngo Molnar /*
1035f62bae50SIngo Molnar * The guts of the apic timer interrupt
1036f62bae50SIngo Molnar */
local_apic_timer_interrupt(void)1037f62bae50SIngo Molnar static void local_apic_timer_interrupt(void)
1038f62bae50SIngo Molnar {
10393bec6defSThomas Gleixner struct clock_event_device *evt = this_cpu_ptr(&lapic_events);
1040f62bae50SIngo Molnar
1041f62bae50SIngo Molnar /*
1042f62bae50SIngo Molnar * Normally we should not be here till LAPIC has been initialized but
1043f62bae50SIngo Molnar * in some cases like kdump, its possible that there is a pending LAPIC
1044f62bae50SIngo Molnar * timer interrupt from previous kernel's context and is delivered in
1045f62bae50SIngo Molnar * new kernel the moment interrupts are enabled.
1046f62bae50SIngo Molnar *
1047f62bae50SIngo Molnar * Interrupts are enabled early and LAPIC is setup much later, hence
1048f62bae50SIngo Molnar * its possible that when we get here evt->event_handler is NULL.
1049f62bae50SIngo Molnar * Check for event_handler being NULL and discard the interrupt as
1050f62bae50SIngo Molnar * spurious.
1051f62bae50SIngo Molnar */
1052f62bae50SIngo Molnar if (!evt->event_handler) {
10538d3bcc44SKefeng Wang pr_warn("Spurious LAPIC timer interrupt on cpu %d\n",
10543bec6defSThomas Gleixner smp_processor_id());
1055f62bae50SIngo Molnar /* Switch it off */
1056b23d8e52SViresh Kumar lapic_timer_shutdown(evt);
1057f62bae50SIngo Molnar return;
1058f62bae50SIngo Molnar }
1059f62bae50SIngo Molnar
1060f62bae50SIngo Molnar /*
1061f62bae50SIngo Molnar * the NMI deadlock-detector uses this.
1062f62bae50SIngo Molnar */
1063f62bae50SIngo Molnar inc_irq_stat(apic_timer_irqs);
1064f62bae50SIngo Molnar
1065f62bae50SIngo Molnar evt->event_handler(evt);
1066f62bae50SIngo Molnar }
1067f62bae50SIngo Molnar
1068f62bae50SIngo Molnar /*
1069f62bae50SIngo Molnar * Local APIC timer interrupt. This is the most natural way for doing
1070f62bae50SIngo Molnar * local interrupts, but local timer interrupts can be emulated by
1071f62bae50SIngo Molnar * broadcast interrupts too. [in case the hw doesn't support APIC timers]
1072f62bae50SIngo Molnar *
1073f62bae50SIngo Molnar * [ if a single-CPU system runs an SMP kernel then we call the local
1074f62bae50SIngo Molnar * interrupt as well. Thus we cannot inline the local irq ... ]
1075f62bae50SIngo Molnar */
DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)1076db0338eeSThomas Gleixner DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)
1077f62bae50SIngo Molnar {
1078f62bae50SIngo Molnar struct pt_regs *old_regs = set_irq_regs(regs);
1079f62bae50SIngo Molnar
1080670c04adSDave Hansen apic_eoi();
1081cf910e83SSeiji Aguchi trace_local_timer_entry(LOCAL_TIMER_VECTOR);
1082cf910e83SSeiji Aguchi local_apic_timer_interrupt();
1083cf910e83SSeiji Aguchi trace_local_timer_exit(LOCAL_TIMER_VECTOR);
1084f62bae50SIngo Molnar
1085f62bae50SIngo Molnar set_irq_regs(old_regs);
1086f62bae50SIngo Molnar }
1087f62bae50SIngo Molnar
1088f62bae50SIngo Molnar /*
1089f62bae50SIngo Molnar * Local APIC start and shutdown
1090f62bae50SIngo Molnar */
1091f62bae50SIngo Molnar
1092f62bae50SIngo Molnar /**
1093f62bae50SIngo Molnar * clear_local_APIC - shutdown the local APIC
1094f62bae50SIngo Molnar *
1095f62bae50SIngo Molnar * This is called, when a CPU is disabled and before rebooting, so the state of
1096f62bae50SIngo Molnar * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
1097f62bae50SIngo Molnar * leftovers during boot.
1098f62bae50SIngo Molnar */
clear_local_APIC(void)1099f62bae50SIngo Molnar void clear_local_APIC(void)
1100f62bae50SIngo Molnar {
1101f62bae50SIngo Molnar int maxlvt;
1102f62bae50SIngo Molnar u32 v;
1103f62bae50SIngo Molnar
110478c32000SThomas Gleixner if (!apic_accessible())
1105f62bae50SIngo Molnar return;
1106f62bae50SIngo Molnar
1107f62bae50SIngo Molnar maxlvt = lapic_get_maxlvt();
1108f62bae50SIngo Molnar /*
1109f62bae50SIngo Molnar * Masking an LVT entry can trigger a local APIC error
1110f62bae50SIngo Molnar * if the vector is zero. Mask LVTERR first to prevent this.
1111f62bae50SIngo Molnar */
1112f62bae50SIngo Molnar if (maxlvt >= 3) {
1113f62bae50SIngo Molnar v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
1114f62bae50SIngo Molnar apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
1115f62bae50SIngo Molnar }
1116f62bae50SIngo Molnar /*
1117f62bae50SIngo Molnar * Careful: we have to set masks only first to deassert
1118f62bae50SIngo Molnar * any level-triggered sources.
1119f62bae50SIngo Molnar */
1120f62bae50SIngo Molnar v = apic_read(APIC_LVTT);
1121f62bae50SIngo Molnar apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
1122f62bae50SIngo Molnar v = apic_read(APIC_LVT0);
1123f62bae50SIngo Molnar apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
1124f62bae50SIngo Molnar v = apic_read(APIC_LVT1);
1125f62bae50SIngo Molnar apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
1126f62bae50SIngo Molnar if (maxlvt >= 4) {
1127f62bae50SIngo Molnar v = apic_read(APIC_LVTPC);
1128f62bae50SIngo Molnar apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
1129f62bae50SIngo Molnar }
1130f62bae50SIngo Molnar
1131f62bae50SIngo Molnar /* lets not touch this if we didn't frob it */
11324efc0670SAndi Kleen #ifdef CONFIG_X86_THERMAL_VECTOR
1133f62bae50SIngo Molnar if (maxlvt >= 5) {
1134f62bae50SIngo Molnar v = apic_read(APIC_LVTTHMR);
1135f62bae50SIngo Molnar apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
1136f62bae50SIngo Molnar }
1137f62bae50SIngo Molnar #endif
1138638bee71SH. Peter Anvin #ifdef CONFIG_X86_MCE_INTEL
1139638bee71SH. Peter Anvin if (maxlvt >= 6) {
1140638bee71SH. Peter Anvin v = apic_read(APIC_LVTCMCI);
1141638bee71SH. Peter Anvin if (!(v & APIC_LVT_MASKED))
1142638bee71SH. Peter Anvin apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
1143638bee71SH. Peter Anvin }
1144638bee71SH. Peter Anvin #endif
1145638bee71SH. Peter Anvin
1146f62bae50SIngo Molnar /*
1147f62bae50SIngo Molnar * Clean APIC state for other OSs:
1148f62bae50SIngo Molnar */
1149f62bae50SIngo Molnar apic_write(APIC_LVTT, APIC_LVT_MASKED);
1150f62bae50SIngo Molnar apic_write(APIC_LVT0, APIC_LVT_MASKED);
1151f62bae50SIngo Molnar apic_write(APIC_LVT1, APIC_LVT_MASKED);
1152f62bae50SIngo Molnar if (maxlvt >= 3)
1153f62bae50SIngo Molnar apic_write(APIC_LVTERR, APIC_LVT_MASKED);
1154f62bae50SIngo Molnar if (maxlvt >= 4)
1155f62bae50SIngo Molnar apic_write(APIC_LVTPC, APIC_LVT_MASKED);
1156f62bae50SIngo Molnar
1157f62bae50SIngo Molnar /* Integrated APIC (!82489DX) ? */
1158f62bae50SIngo Molnar if (lapic_is_integrated()) {
1159f62bae50SIngo Molnar if (maxlvt > 3)
1160f62bae50SIngo Molnar /* Clear ESR due to Pentium errata 3AP and 11AP */
1161f62bae50SIngo Molnar apic_write(APIC_ESR, 0);
1162f62bae50SIngo Molnar apic_read(APIC_ESR);
1163f62bae50SIngo Molnar }
1164f62bae50SIngo Molnar }
1165f62bae50SIngo Molnar
1166f62bae50SIngo Molnar /**
116760dcaad5SThomas Gleixner * apic_soft_disable - Clears and software disables the local APIC on hotplug
116860dcaad5SThomas Gleixner *
116960dcaad5SThomas Gleixner * Contrary to disable_local_APIC() this does not touch the enable bit in
117060dcaad5SThomas Gleixner * MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC
117160dcaad5SThomas Gleixner * bus would require a hardware reset as the APIC would lose track of bus
117260dcaad5SThomas Gleixner * arbitration. On systems with FSB delivery APICBASE could be disabled,
117360dcaad5SThomas Gleixner * but it has to be guaranteed that no interrupt is sent to the APIC while
117460dcaad5SThomas Gleixner * in that state and it's not clear from the SDM whether it still responds
117560dcaad5SThomas Gleixner * to INIT/SIPI messages. Stay on the safe side and use software disable.
117660dcaad5SThomas Gleixner */
apic_soft_disable(void)117760dcaad5SThomas Gleixner void apic_soft_disable(void)
117860dcaad5SThomas Gleixner {
117960dcaad5SThomas Gleixner u32 value;
118060dcaad5SThomas Gleixner
118160dcaad5SThomas Gleixner clear_local_APIC();
118260dcaad5SThomas Gleixner
118360dcaad5SThomas Gleixner /* Soft disable APIC (implies clearing of registers for 82489DX!). */
118460dcaad5SThomas Gleixner value = apic_read(APIC_SPIV);
118560dcaad5SThomas Gleixner value &= ~APIC_SPIV_APIC_ENABLED;
118660dcaad5SThomas Gleixner apic_write(APIC_SPIV, value);
118760dcaad5SThomas Gleixner }
118860dcaad5SThomas Gleixner
118960dcaad5SThomas Gleixner /**
1190f62bae50SIngo Molnar * disable_local_APIC - clear and disable the local APIC
1191f62bae50SIngo Molnar */
disable_local_APIC(void)1192f62bae50SIngo Molnar void disable_local_APIC(void)
1193f62bae50SIngo Molnar {
119478c32000SThomas Gleixner if (!apic_accessible())
1195f62bae50SIngo Molnar return;
1196f62bae50SIngo Molnar
119760dcaad5SThomas Gleixner apic_soft_disable();
1198f62bae50SIngo Molnar
1199f62bae50SIngo Molnar #ifdef CONFIG_X86_32
1200f62bae50SIngo Molnar /*
1201f62bae50SIngo Molnar * When LAPIC was disabled by the BIOS and enabled by the kernel,
1202f62bae50SIngo Molnar * restore the disabled state.
1203f62bae50SIngo Molnar */
1204f62bae50SIngo Molnar if (enabled_via_apicbase) {
1205f62bae50SIngo Molnar unsigned int l, h;
1206f62bae50SIngo Molnar
1207f62bae50SIngo Molnar rdmsr(MSR_IA32_APICBASE, l, h);
1208f62bae50SIngo Molnar l &= ~MSR_IA32_APICBASE_ENABLE;
1209f62bae50SIngo Molnar wrmsr(MSR_IA32_APICBASE, l, h);
1210f62bae50SIngo Molnar }
1211f62bae50SIngo Molnar #endif
1212f62bae50SIngo Molnar }
1213f62bae50SIngo Molnar
1214f62bae50SIngo Molnar /*
1215f62bae50SIngo Molnar * If Linux enabled the LAPIC against the BIOS default disable it down before
1216f62bae50SIngo Molnar * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
1217f62bae50SIngo Molnar * not power-off. Additionally clear all LVT entries before disable_local_APIC
1218f62bae50SIngo Molnar * for the case where Linux didn't enable the LAPIC.
1219f62bae50SIngo Molnar */
lapic_shutdown(void)1220f62bae50SIngo Molnar void lapic_shutdown(void)
1221f62bae50SIngo Molnar {
1222f62bae50SIngo Molnar unsigned long flags;
1223f62bae50SIngo Molnar
122493984fbdSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1225f62bae50SIngo Molnar return;
1226f62bae50SIngo Molnar
1227f62bae50SIngo Molnar local_irq_save(flags);
1228f62bae50SIngo Molnar
1229f62bae50SIngo Molnar #ifdef CONFIG_X86_32
1230f62bae50SIngo Molnar if (!enabled_via_apicbase)
1231f62bae50SIngo Molnar clear_local_APIC();
1232f62bae50SIngo Molnar else
1233f62bae50SIngo Molnar #endif
1234f62bae50SIngo Molnar disable_local_APIC();
1235f62bae50SIngo Molnar
1236f62bae50SIngo Molnar
1237f62bae50SIngo Molnar local_irq_restore(flags);
1238f62bae50SIngo Molnar }
1239f62bae50SIngo Molnar
1240f62bae50SIngo Molnar /**
1241f62bae50SIngo Molnar * sync_Arb_IDs - synchronize APIC bus arbitration IDs
1242f62bae50SIngo Molnar */
sync_Arb_IDs(void)1243f62bae50SIngo Molnar void __init sync_Arb_IDs(void)
1244f62bae50SIngo Molnar {
1245f62bae50SIngo Molnar /*
1246f62bae50SIngo Molnar * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1247f62bae50SIngo Molnar * needed on AMD.
1248f62bae50SIngo Molnar */
1249f62bae50SIngo Molnar if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1250f62bae50SIngo Molnar return;
1251f62bae50SIngo Molnar
1252f62bae50SIngo Molnar /*
1253f62bae50SIngo Molnar * Wait for idle.
1254f62bae50SIngo Molnar */
1255f62bae50SIngo Molnar apic_wait_icr_idle();
1256f62bae50SIngo Molnar
1257f62bae50SIngo Molnar apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
1258f62bae50SIngo Molnar apic_write(APIC_ICR, APIC_DEST_ALLINC |
1259f62bae50SIngo Molnar APIC_INT_LEVELTRIG | APIC_DM_INIT);
1260f62bae50SIngo Molnar }
1261f62bae50SIngo Molnar
12626444b40eSSean Christopherson enum apic_intr_mode_id apic_intr_mode __ro_after_init;
12630114a8e8SDou Liyang
__apic_intr_mode_select(void)126497992387SThomas Gleixner static int __init __apic_intr_mode_select(void)
1265f62bae50SIngo Molnar {
12660114a8e8SDou Liyang /* Check kernel option */
126749062454SThomas Gleixner if (apic_is_disabled) {
12680114a8e8SDou Liyang pr_info("APIC disabled via kernel command line\n");
12690114a8e8SDou Liyang return APIC_PIC;
12700114a8e8SDou Liyang }
1271f62bae50SIngo Molnar
12720114a8e8SDou Liyang /* Check BIOS */
12730114a8e8SDou Liyang #ifdef CONFIG_X86_64
12740114a8e8SDou Liyang /* On 64-bit, the APIC must be integrated, Check local APIC only */
12750114a8e8SDou Liyang if (!boot_cpu_has(X86_FEATURE_APIC)) {
127649062454SThomas Gleixner apic_is_disabled = true;
12770114a8e8SDou Liyang pr_info("APIC disabled by BIOS\n");
12780114a8e8SDou Liyang return APIC_PIC;
12790114a8e8SDou Liyang }
12800114a8e8SDou Liyang #else
12810114a8e8SDou Liyang /* On 32-bit, the APIC may be integrated APIC or 82489DX */
1282f62bae50SIngo Molnar
12830114a8e8SDou Liyang /* Neither 82489DX nor integrated APIC ? */
12840114a8e8SDou Liyang if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) {
128549062454SThomas Gleixner apic_is_disabled = true;
12860114a8e8SDou Liyang return APIC_PIC;
12870114a8e8SDou Liyang }
1288f62bae50SIngo Molnar
12890114a8e8SDou Liyang /* If the BIOS pretends there is an integrated APIC ? */
12900114a8e8SDou Liyang if (!boot_cpu_has(X86_FEATURE_APIC) &&
12910114a8e8SDou Liyang APIC_INTEGRATED(boot_cpu_apic_version)) {
129249062454SThomas Gleixner apic_is_disabled = true;
1293d10a9044SThomas Gleixner pr_err(FW_BUG "Local APIC not detected, force emulation\n");
12940114a8e8SDou Liyang return APIC_PIC;
12950114a8e8SDou Liyang }
1296f62bae50SIngo Molnar #endif
1297f62bae50SIngo Molnar
12980114a8e8SDou Liyang /* Check MP table or ACPI MADT configuration */
12990114a8e8SDou Liyang if (!smp_found_config) {
13000114a8e8SDou Liyang disable_ioapic_support();
13013e730dadSDou Liyang if (!acpi_lapic) {
13020114a8e8SDou Liyang pr_info("APIC: ACPI MADT or MP tables are not detected\n");
13033e730dadSDou Liyang return APIC_VIRTUAL_WIRE_NO_CONFIG;
13043e730dadSDou Liyang }
13050114a8e8SDou Liyang return APIC_VIRTUAL_WIRE;
13060114a8e8SDou Liyang }
13070114a8e8SDou Liyang
13083e730dadSDou Liyang #ifdef CONFIG_SMP
13093e730dadSDou Liyang /* If SMP should be disabled, then really disable it! */
13103e730dadSDou Liyang if (!setup_max_cpus) {
13113e730dadSDou Liyang pr_info("APIC: SMP mode deactivated\n");
13123e730dadSDou Liyang return APIC_SYMMETRIC_IO_NO_ROUTING;
13133e730dadSDou Liyang }
13143e730dadSDou Liyang #endif
13153e730dadSDou Liyang
13160114a8e8SDou Liyang return APIC_SYMMETRIC_IO;
13170114a8e8SDou Liyang }
13180114a8e8SDou Liyang
131997992387SThomas Gleixner /* Select the interrupt delivery mode for the BSP */
apic_intr_mode_select(void)132097992387SThomas Gleixner void __init apic_intr_mode_select(void)
132197992387SThomas Gleixner {
132297992387SThomas Gleixner apic_intr_mode = __apic_intr_mode_select();
132397992387SThomas Gleixner }
132497992387SThomas Gleixner
1325fc90ccfdSVille Syrjälä /*
1326fc90ccfdSVille Syrjälä * An initial setup of the virtual wire mode.
1327fc90ccfdSVille Syrjälä */
init_bsp_APIC(void)1328fc90ccfdSVille Syrjälä void __init init_bsp_APIC(void)
1329fc90ccfdSVille Syrjälä {
1330fc90ccfdSVille Syrjälä unsigned int value;
1331fc90ccfdSVille Syrjälä
1332fc90ccfdSVille Syrjälä /*
1333fc90ccfdSVille Syrjälä * Don't do the setup now if we have a SMP BIOS as the
1334fc90ccfdSVille Syrjälä * through-I/O-APIC virtual wire mode might be active.
1335fc90ccfdSVille Syrjälä */
1336fc90ccfdSVille Syrjälä if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC))
1337fc90ccfdSVille Syrjälä return;
1338fc90ccfdSVille Syrjälä
1339fc90ccfdSVille Syrjälä /*
1340fc90ccfdSVille Syrjälä * Do not trust the local APIC being empty at bootup.
1341fc90ccfdSVille Syrjälä */
1342fc90ccfdSVille Syrjälä clear_local_APIC();
1343fc90ccfdSVille Syrjälä
1344fc90ccfdSVille Syrjälä /*
1345fc90ccfdSVille Syrjälä * Enable APIC.
1346fc90ccfdSVille Syrjälä */
1347fc90ccfdSVille Syrjälä value = apic_read(APIC_SPIV);
1348fc90ccfdSVille Syrjälä value &= ~APIC_VECTOR_MASK;
1349fc90ccfdSVille Syrjälä value |= APIC_SPIV_APIC_ENABLED;
1350fc90ccfdSVille Syrjälä
1351fc90ccfdSVille Syrjälä #ifdef CONFIG_X86_32
1352fc90ccfdSVille Syrjälä /* This bit is reserved on P4/Xeon and should be cleared */
1353fc90ccfdSVille Syrjälä if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1354fc90ccfdSVille Syrjälä (boot_cpu_data.x86 == 15))
1355fc90ccfdSVille Syrjälä value &= ~APIC_SPIV_FOCUS_DISABLED;
1356fc90ccfdSVille Syrjälä else
1357fc90ccfdSVille Syrjälä #endif
1358fc90ccfdSVille Syrjälä value |= APIC_SPIV_FOCUS_DISABLED;
1359fc90ccfdSVille Syrjälä value |= SPURIOUS_APIC_VECTOR;
1360fc90ccfdSVille Syrjälä apic_write(APIC_SPIV, value);
1361fc90ccfdSVille Syrjälä
1362fc90ccfdSVille Syrjälä /*
1363fc90ccfdSVille Syrjälä * Set up the virtual wire mode.
1364fc90ccfdSVille Syrjälä */
1365fc90ccfdSVille Syrjälä apic_write(APIC_LVT0, APIC_DM_EXTINT);
1366fc90ccfdSVille Syrjälä value = APIC_DM_NMI;
1367fc90ccfdSVille Syrjälä if (!lapic_is_integrated()) /* 82489DX */
1368fc90ccfdSVille Syrjälä value |= APIC_LVT_LEVEL_TRIGGER;
1369fc90ccfdSVille Syrjälä if (apic_extnmi == APIC_EXTNMI_NONE)
1370fc90ccfdSVille Syrjälä value |= APIC_LVT_MASKED;
1371fc90ccfdSVille Syrjälä apic_write(APIC_LVT1, value);
1372fc90ccfdSVille Syrjälä }
1373fc90ccfdSVille Syrjälä
1374748b170cSThomas Gleixner static void __init apic_bsp_setup(bool upmode);
1375748b170cSThomas Gleixner
13764b1669e8SDou Liyang /* Init the interrupt delivery mode for the BSP */
apic_intr_mode_init(void)13774b1669e8SDou Liyang void __init apic_intr_mode_init(void)
13784b1669e8SDou Liyang {
13790c759131SDou Liyang bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT);
13803e730dadSDou Liyang
13814f45ed9fSDou Liyang switch (apic_intr_mode) {
13824b1669e8SDou Liyang case APIC_PIC:
13834b1669e8SDou Liyang pr_info("APIC: Keep in PIC mode(8259)\n");
13844b1669e8SDou Liyang return;
13854b1669e8SDou Liyang case APIC_VIRTUAL_WIRE:
13864b1669e8SDou Liyang pr_info("APIC: Switch to virtual wire mode setup\n");
13873e730dadSDou Liyang break;
13883e730dadSDou Liyang case APIC_VIRTUAL_WIRE_NO_CONFIG:
13893e730dadSDou Liyang pr_info("APIC: Switch to virtual wire mode setup with no configuration\n");
13903e730dadSDou Liyang upmode = true;
13913e730dadSDou Liyang break;
13924b1669e8SDou Liyang case APIC_SYMMETRIC_IO:
139379761ce8SColin Ian King pr_info("APIC: Switch to symmetric I/O mode setup\n");
13943e730dadSDou Liyang break;
13953e730dadSDou Liyang case APIC_SYMMETRIC_IO_NO_ROUTING:
139679761ce8SColin Ian King pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n");
13973e730dadSDou Liyang break;
13984b1669e8SDou Liyang }
13993e730dadSDou Liyang
14009d87f5b6SThomas Gleixner x86_64_probe_apic();
14019d87f5b6SThomas Gleixner
14029d87f5b6SThomas Gleixner x86_32_install_bigsmp();
14037a116a2dSKuppuswamy Sathyanarayanan
1404bb733e43SThomas Gleixner if (x86_platform.apic_post_init)
1405bb733e43SThomas Gleixner x86_platform.apic_post_init();
1406bb733e43SThomas Gleixner
14073e730dadSDou Liyang apic_bsp_setup(upmode);
1408f62bae50SIngo Molnar }
1409f62bae50SIngo Molnar
lapic_setup_esr(void)1410148f9bb8SPaul Gortmaker static void lapic_setup_esr(void)
1411f62bae50SIngo Molnar {
1412f62bae50SIngo Molnar unsigned int oldvalue, value, maxlvt;
1413f62bae50SIngo Molnar
1414f62bae50SIngo Molnar if (!lapic_is_integrated()) {
1415f62bae50SIngo Molnar pr_info("No ESR for 82489DX.\n");
1416f62bae50SIngo Molnar return;
1417f62bae50SIngo Molnar }
1418f62bae50SIngo Molnar
1419f62bae50SIngo Molnar if (apic->disable_esr) {
1420f62bae50SIngo Molnar /*
1421f62bae50SIngo Molnar * Something untraceable is creating bad interrupts on
1422f62bae50SIngo Molnar * secondary quads ... for the moment, just leave the
1423f62bae50SIngo Molnar * ESR disabled - we can't do anything useful with the
1424f62bae50SIngo Molnar * errors anyway - mbligh
1425f62bae50SIngo Molnar */
1426f62bae50SIngo Molnar pr_info("Leaving ESR disabled.\n");
1427f62bae50SIngo Molnar return;
1428f62bae50SIngo Molnar }
1429f62bae50SIngo Molnar
1430f62bae50SIngo Molnar maxlvt = lapic_get_maxlvt();
1431f62bae50SIngo Molnar if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1432f62bae50SIngo Molnar apic_write(APIC_ESR, 0);
1433f62bae50SIngo Molnar oldvalue = apic_read(APIC_ESR);
1434f62bae50SIngo Molnar
1435f62bae50SIngo Molnar /* enables sending errors */
1436f62bae50SIngo Molnar value = ERROR_APIC_VECTOR;
1437f62bae50SIngo Molnar apic_write(APIC_LVTERR, value);
1438f62bae50SIngo Molnar
1439f62bae50SIngo Molnar /*
1440f62bae50SIngo Molnar * spec says clear errors after enabling vector.
1441f62bae50SIngo Molnar */
1442f62bae50SIngo Molnar if (maxlvt > 3)
1443f62bae50SIngo Molnar apic_write(APIC_ESR, 0);
1444f62bae50SIngo Molnar value = apic_read(APIC_ESR);
1445f62bae50SIngo Molnar if (value != oldvalue)
1446f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "ESR value before enabling "
1447f62bae50SIngo Molnar "vector: 0x%08x after: 0x%08x\n",
1448f62bae50SIngo Molnar oldvalue, value);
1449f62bae50SIngo Molnar }
1450f62bae50SIngo Molnar
1451cc8bf191SThomas Gleixner #define APIC_IR_REGS APIC_ISR_NR
1452cc8bf191SThomas Gleixner #define APIC_IR_BITS (APIC_IR_REGS * 32)
1453cc8bf191SThomas Gleixner #define APIC_IR_MAPSIZE (APIC_IR_BITS / BITS_PER_LONG)
14549b217f33SDou Liyang
1455cc8bf191SThomas Gleixner union apic_ir {
1456cc8bf191SThomas Gleixner unsigned long map[APIC_IR_MAPSIZE];
1457cc8bf191SThomas Gleixner u32 regs[APIC_IR_REGS];
1458cc8bf191SThomas Gleixner };
1459cc8bf191SThomas Gleixner
apic_check_and_ack(union apic_ir * irr,union apic_ir * isr)1460cc8bf191SThomas Gleixner static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr)
1461cc8bf191SThomas Gleixner {
1462cc8bf191SThomas Gleixner int i, bit;
1463cc8bf191SThomas Gleixner
1464cc8bf191SThomas Gleixner /* Read the IRRs */
1465cc8bf191SThomas Gleixner for (i = 0; i < APIC_IR_REGS; i++)
1466cc8bf191SThomas Gleixner irr->regs[i] = apic_read(APIC_IRR + i * 0x10);
1467cc8bf191SThomas Gleixner
1468cc8bf191SThomas Gleixner /* Read the ISRs */
1469cc8bf191SThomas Gleixner for (i = 0; i < APIC_IR_REGS; i++)
1470cc8bf191SThomas Gleixner isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
1471cc8bf191SThomas Gleixner
1472cc8bf191SThomas Gleixner /*
1473cc8bf191SThomas Gleixner * If the ISR map is not empty. ACK the APIC and run another round
1474cc8bf191SThomas Gleixner * to verify whether a pending IRR has been unblocked and turned
1475cc8bf191SThomas Gleixner * into a ISR.
1476cc8bf191SThomas Gleixner */
1477cc8bf191SThomas Gleixner if (!bitmap_empty(isr->map, APIC_IR_BITS)) {
1478cc8bf191SThomas Gleixner /*
1479cc8bf191SThomas Gleixner * There can be multiple ISR bits set when a high priority
1480cc8bf191SThomas Gleixner * interrupt preempted a lower priority one. Issue an ACK
1481cc8bf191SThomas Gleixner * per set bit.
1482cc8bf191SThomas Gleixner */
1483cc8bf191SThomas Gleixner for_each_set_bit(bit, isr->map, APIC_IR_BITS)
1484670c04adSDave Hansen apic_eoi();
1485cc8bf191SThomas Gleixner return true;
1486cc8bf191SThomas Gleixner }
1487cc8bf191SThomas Gleixner
1488cc8bf191SThomas Gleixner return !bitmap_empty(irr->map, APIC_IR_BITS);
1489cc8bf191SThomas Gleixner }
1490cc8bf191SThomas Gleixner
14919b217f33SDou Liyang /*
14929b217f33SDou Liyang * After a crash, we no longer service the interrupts and a pending
14939b217f33SDou Liyang * interrupt from previous kernel might still have ISR bit set.
14949b217f33SDou Liyang *
1495cc8bf191SThomas Gleixner * Most probably by now the CPU has serviced that pending interrupt and it
1496670c04adSDave Hansen * might not have done the apic_eoi() because it thought, interrupt
1497cc8bf191SThomas Gleixner * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear
1498d9f6e12fSIngo Molnar * the ISR bit and cpu thinks it has already serviced the interrupt. Hence
1499cc8bf191SThomas Gleixner * a vector might get locked. It was noticed for timer irq (vector
1500cc8bf191SThomas Gleixner * 0x31). Issue an extra EOI to clear ISR.
1501cc8bf191SThomas Gleixner *
1502cc8bf191SThomas Gleixner * If there are pending IRR bits they turn into ISR bits after a higher
1503cc8bf191SThomas Gleixner * priority ISR bit has been acked.
15049b217f33SDou Liyang */
apic_pending_intr_clear(void)1505cc8bf191SThomas Gleixner static void apic_pending_intr_clear(void)
1506cc8bf191SThomas Gleixner {
1507cc8bf191SThomas Gleixner union apic_ir irr, isr;
1508cc8bf191SThomas Gleixner unsigned int i;
15099b217f33SDou Liyang
1510cc8bf191SThomas Gleixner /* 512 loops are way oversized and give the APIC a chance to obey. */
1511cc8bf191SThomas Gleixner for (i = 0; i < 512; i++) {
1512cc8bf191SThomas Gleixner if (!apic_check_and_ack(&irr, &isr))
1513cc8bf191SThomas Gleixner return;
15149b217f33SDou Liyang }
1515cc8bf191SThomas Gleixner /* Dump the IRR/ISR content if that failed */
1516cc8bf191SThomas Gleixner pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr.map, isr.map);
15179b217f33SDou Liyang }
15189b217f33SDou Liyang
1519f62bae50SIngo Molnar /**
1520f62bae50SIngo Molnar * setup_local_APIC - setup the local APIC
15210aa002feSTejun Heo *
1522543113d2SDou Liyang * Used to setup local APIC while initializing BSP or bringing up APs.
15230aa002feSTejun Heo * Always called with preemption disabled.
1524f62bae50SIngo Molnar */
setup_local_APIC(void)1525b753a2b7SDou Liyang static void setup_local_APIC(void)
1526f62bae50SIngo Molnar {
15270aa002feSTejun Heo int cpu = smp_processor_id();
15289b217f33SDou Liyang unsigned int value;
15298c3ba8d0SKerstin Jonsson
153049062454SThomas Gleixner if (apic_is_disabled) {
15317167d08eSHenrik Kretzschmar disable_ioapic_support();
1532f62bae50SIngo Molnar return;
1533f62bae50SIngo Molnar }
1534f62bae50SIngo Molnar
15352640da4cSThomas Gleixner /*
15362640da4cSThomas Gleixner * If this comes from kexec/kcrash the APIC might be enabled in
15372640da4cSThomas Gleixner * SPIV. Soft disable it before doing further initialization.
15382640da4cSThomas Gleixner */
15392640da4cSThomas Gleixner value = apic_read(APIC_SPIV);
15402640da4cSThomas Gleixner value &= ~APIC_SPIV_APIC_ENABLED;
15412640da4cSThomas Gleixner apic_write(APIC_SPIV, value);
15422640da4cSThomas Gleixner
1543f62bae50SIngo Molnar #ifdef CONFIG_X86_32
1544f62bae50SIngo Molnar /* Pound the ESR really hard over the head with a big hammer - mbligh */
1545f62bae50SIngo Molnar if (lapic_is_integrated() && apic->disable_esr) {
1546f62bae50SIngo Molnar apic_write(APIC_ESR, 0);
1547f62bae50SIngo Molnar apic_write(APIC_ESR, 0);
1548f62bae50SIngo Molnar apic_write(APIC_ESR, 0);
1549f62bae50SIngo Molnar apic_write(APIC_ESR, 0);
1550f62bae50SIngo Molnar }
1551f62bae50SIngo Molnar #endif
15525a3a46bdSThomas Gleixner /* Validate that the APIC is registered if required */
15535a3a46bdSThomas Gleixner BUG_ON(apic->apic_id_registered && !apic->apic_id_registered());
1554f62bae50SIngo Molnar
1555f62bae50SIngo Molnar /*
1556f62bae50SIngo Molnar * Intel recommends to set DFR, LDR and TPR before enabling
1557f62bae50SIngo Molnar * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
15582f6df03fSThomas Gleixner * document number 292116).
15592f6df03fSThomas Gleixner *
15602f6df03fSThomas Gleixner * Except for APICs which operate in physical destination mode.
1561f62bae50SIngo Molnar */
15622f6df03fSThomas Gleixner if (apic->init_apic_ldr)
1563f62bae50SIngo Molnar apic->init_apic_ldr();
1564f62bae50SIngo Molnar
1565f62bae50SIngo Molnar /*
1566229b969bSAndy Lutomirski * Set Task Priority to 'accept all except vectors 0-31'. An APIC
1567229b969bSAndy Lutomirski * vector in the 16-31 range could be delivered if TPR == 0, but we
1568229b969bSAndy Lutomirski * would think it's an exception and terrible things will happen. We
1569229b969bSAndy Lutomirski * never change this later on.
1570f62bae50SIngo Molnar */
1571f62bae50SIngo Molnar value = apic_read(APIC_TASKPRI);
1572f62bae50SIngo Molnar value &= ~APIC_TPRI_MASK;
1573229b969bSAndy Lutomirski value |= 0x10;
1574f62bae50SIngo Molnar apic_write(APIC_TASKPRI, value);
1575f62bae50SIngo Molnar
1576cc8bf191SThomas Gleixner /* Clear eventually stale ISR/IRR bits */
15779b217f33SDou Liyang apic_pending_intr_clear();
1578f62bae50SIngo Molnar
1579f62bae50SIngo Molnar /*
1580f62bae50SIngo Molnar * Now that we are all set up, enable the APIC
1581f62bae50SIngo Molnar */
1582f62bae50SIngo Molnar value = apic_read(APIC_SPIV);
1583f62bae50SIngo Molnar value &= ~APIC_VECTOR_MASK;
1584f62bae50SIngo Molnar /*
1585f62bae50SIngo Molnar * Enable APIC
1586f62bae50SIngo Molnar */
1587f62bae50SIngo Molnar value |= APIC_SPIV_APIC_ENABLED;
1588f62bae50SIngo Molnar
1589f62bae50SIngo Molnar #ifdef CONFIG_X86_32
1590f62bae50SIngo Molnar /*
1591f62bae50SIngo Molnar * Some unknown Intel IO/APIC (or APIC) errata is biting us with
1592f62bae50SIngo Molnar * certain networking cards. If high frequency interrupts are
1593f62bae50SIngo Molnar * happening on a particular IOAPIC pin, plus the IOAPIC routing
1594f62bae50SIngo Molnar * entry is masked/unmasked at a high rate as well then sooner or
1595f62bae50SIngo Molnar * later IOAPIC line gets 'stuck', no more interrupts are received
1596f62bae50SIngo Molnar * from the device. If focus CPU is disabled then the hang goes
1597f62bae50SIngo Molnar * away, oh well :-(
1598f62bae50SIngo Molnar *
1599f62bae50SIngo Molnar * [ This bug can be reproduced easily with a level-triggered
1600f62bae50SIngo Molnar * PCI Ne2000 networking cards and PII/PIII processors, dual
1601f62bae50SIngo Molnar * BX chipset. ]
1602f62bae50SIngo Molnar */
1603f62bae50SIngo Molnar /*
1604f62bae50SIngo Molnar * Actually disabling the focus CPU check just makes the hang less
1605d9f6e12fSIngo Molnar * frequent as it makes the interrupt distribution model be more
1606f62bae50SIngo Molnar * like LRU than MRU (the short-term load is more even across CPUs).
1607f62bae50SIngo Molnar */
1608f62bae50SIngo Molnar
1609f62bae50SIngo Molnar /*
1610f62bae50SIngo Molnar * - enable focus processor (bit==0)
1611f62bae50SIngo Molnar * - 64bit mode always use processor focus
1612f62bae50SIngo Molnar * so no need to set it
1613f62bae50SIngo Molnar */
1614f62bae50SIngo Molnar value &= ~APIC_SPIV_FOCUS_DISABLED;
1615f62bae50SIngo Molnar #endif
1616f62bae50SIngo Molnar
1617f62bae50SIngo Molnar /*
1618f62bae50SIngo Molnar * Set spurious IRQ vector
1619f62bae50SIngo Molnar */
1620f62bae50SIngo Molnar value |= SPURIOUS_APIC_VECTOR;
1621f62bae50SIngo Molnar apic_write(APIC_SPIV, value);
1622f62bae50SIngo Molnar
162339c89dffSThomas Gleixner perf_events_lapic_init();
162439c89dffSThomas Gleixner
1625f62bae50SIngo Molnar /*
1626f62bae50SIngo Molnar * Set up LVT0, LVT1:
1627f62bae50SIngo Molnar *
1628a1652bb8SJean Delvare * set up through-local-APIC on the boot CPU's LINT0. This is not
1629f62bae50SIngo Molnar * strictly necessary in pure symmetric-IO mode, but sometimes
1630f62bae50SIngo Molnar * we delegate interrupts to the 8259A.
1631f62bae50SIngo Molnar */
1632f62bae50SIngo Molnar /*
1633f62bae50SIngo Molnar * TODO: set up through-local-APIC from through-I/O-APIC? --macro
1634f62bae50SIngo Molnar */
1635f62bae50SIngo Molnar value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1636ecf600f8SThomas Gleixner if (!cpu && (pic_mode || !value || ioapic_is_disabled)) {
1637f62bae50SIngo Molnar value = APIC_DM_EXTINT;
16380aa002feSTejun Heo apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
1639f62bae50SIngo Molnar } else {
1640f62bae50SIngo Molnar value = APIC_DM_EXTINT | APIC_LVT_MASKED;
16410aa002feSTejun Heo apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
1642f62bae50SIngo Molnar }
1643f62bae50SIngo Molnar apic_write(APIC_LVT0, value);
1644f62bae50SIngo Molnar
1645f62bae50SIngo Molnar /*
1646b7c4948eSHidehiro Kawai * Only the BSP sees the LINT1 NMI signal by default. This can be
1647b7c4948eSHidehiro Kawai * modified by apic_extnmi= boot option.
1648f62bae50SIngo Molnar */
1649b7c4948eSHidehiro Kawai if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) ||
1650b7c4948eSHidehiro Kawai apic_extnmi == APIC_EXTNMI_ALL)
1651f62bae50SIngo Molnar value = APIC_DM_NMI;
1652f62bae50SIngo Molnar else
1653f62bae50SIngo Molnar value = APIC_DM_NMI | APIC_LVT_MASKED;
1654ae41a2a4SDou Liyang
1655ae41a2a4SDou Liyang /* Is 82489DX ? */
1656ae41a2a4SDou Liyang if (!lapic_is_integrated())
1657f62bae50SIngo Molnar value |= APIC_LVT_LEVEL_TRIGGER;
1658f62bae50SIngo Molnar apic_write(APIC_LVT1, value);
1659f62bae50SIngo Molnar
1660638bee71SH. Peter Anvin #ifdef CONFIG_X86_MCE_INTEL
1661638bee71SH. Peter Anvin /* Recheck CMCI information after local APIC is up on CPU #0 */
16620aa002feSTejun Heo if (!cpu)
1663638bee71SH. Peter Anvin cmci_recheck();
1664638bee71SH. Peter Anvin #endif
1665f62bae50SIngo Molnar }
1666f62bae50SIngo Molnar
end_local_APIC_setup(void)166705f7e46dSThomas Gleixner static void end_local_APIC_setup(void)
1668f62bae50SIngo Molnar {
1669f62bae50SIngo Molnar lapic_setup_esr();
1670f62bae50SIngo Molnar
1671f62bae50SIngo Molnar #ifdef CONFIG_X86_32
1672f62bae50SIngo Molnar {
1673f62bae50SIngo Molnar unsigned int value;
1674f62bae50SIngo Molnar /* Disable the local apic timer */
1675f62bae50SIngo Molnar value = apic_read(APIC_LVTT);
1676f62bae50SIngo Molnar value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1677f62bae50SIngo Molnar apic_write(APIC_LVTT, value);
1678f62bae50SIngo Molnar }
1679f62bae50SIngo Molnar #endif
1680f62bae50SIngo Molnar
1681f62bae50SIngo Molnar apic_pm_activate();
16822fb270f3SJan Beulich }
16832fb270f3SJan Beulich
16847f7fbf45SKenji Kaneshige /*
168505f7e46dSThomas Gleixner * APIC setup function for application processors. Called from smpboot.c
16867f7fbf45SKenji Kaneshige */
apic_ap_setup(void)168705f7e46dSThomas Gleixner void apic_ap_setup(void)
168805f7e46dSThomas Gleixner {
168905f7e46dSThomas Gleixner setup_local_APIC();
169005f7e46dSThomas Gleixner end_local_APIC_setup();
1691f62bae50SIngo Molnar }
1692f62bae50SIngo Molnar
1693d63107faSThomas Gleixner static __init void cpu_set_boot_apic(void);
1694d63107faSThomas Gleixner
apic_read_boot_cpu_id(bool x2apic)1695d10a9044SThomas Gleixner static __init void apic_read_boot_cpu_id(bool x2apic)
1696d10a9044SThomas Gleixner {
1697d10a9044SThomas Gleixner /*
1698d10a9044SThomas Gleixner * This can be invoked from check_x2apic() before the APIC has been
1699d10a9044SThomas Gleixner * selected. But that code knows for sure that the BIOS enabled
1700d10a9044SThomas Gleixner * X2APIC.
1701d10a9044SThomas Gleixner */
1702d10a9044SThomas Gleixner if (x2apic) {
1703d10a9044SThomas Gleixner boot_cpu_physical_apicid = native_apic_msr_read(APIC_ID);
1704d10a9044SThomas Gleixner boot_cpu_apic_version = GET_APIC_VERSION(native_apic_msr_read(APIC_LVR));
1705d10a9044SThomas Gleixner } else {
1706d10a9044SThomas Gleixner boot_cpu_physical_apicid = read_apic_id();
1707d10a9044SThomas Gleixner boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
1708d10a9044SThomas Gleixner }
1709d63107faSThomas Gleixner cpu_set_boot_apic();
1710d10a9044SThomas Gleixner }
1711d10a9044SThomas Gleixner
1712f62bae50SIngo Molnar #ifdef CONFIG_X86_X2APIC
1713bfb05070SThomas Gleixner int x2apic_mode;
1714db7d8e47SSean Christopherson EXPORT_SYMBOL_GPL(x2apic_mode);
171512e189d3SThomas Gleixner
171612e189d3SThomas Gleixner enum {
171712e189d3SThomas Gleixner X2APIC_OFF,
171812e189d3SThomas Gleixner X2APIC_DISABLED,
1719b8d1d163SDaniel Sneddon /* All states below here have X2APIC enabled */
1720b8d1d163SDaniel Sneddon X2APIC_ON,
1721b8d1d163SDaniel Sneddon X2APIC_ON_LOCKED
172212e189d3SThomas Gleixner };
172312e189d3SThomas Gleixner static int x2apic_state;
172412e189d3SThomas Gleixner
x2apic_hw_locked(void)1725b8d1d163SDaniel Sneddon static bool x2apic_hw_locked(void)
1726b8d1d163SDaniel Sneddon {
1727c83e35f4SIngo Molnar u64 x86_arch_cap_msr;
1728b8d1d163SDaniel Sneddon u64 msr;
1729b8d1d163SDaniel Sneddon
1730c83e35f4SIngo Molnar x86_arch_cap_msr = x86_read_arch_cap_msr();
1731c83e35f4SIngo Molnar if (x86_arch_cap_msr & ARCH_CAP_XAPIC_DISABLE) {
1732b8d1d163SDaniel Sneddon rdmsrl(MSR_IA32_XAPIC_DISABLE_STATUS, msr);
1733b8d1d163SDaniel Sneddon return (msr & LEGACY_XAPIC_DISABLED);
1734b8d1d163SDaniel Sneddon }
1735b8d1d163SDaniel Sneddon return false;
1736b8d1d163SDaniel Sneddon }
1737b8d1d163SDaniel Sneddon
__x2apic_disable(void)1738d786ad32SDenys Vlasenko static void __x2apic_disable(void)
173944e25ff9SThomas Gleixner {
174044e25ff9SThomas Gleixner u64 msr;
174144e25ff9SThomas Gleixner
174293984fbdSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_APIC))
1743659006bfSThomas Gleixner return;
1744659006bfSThomas Gleixner
174544e25ff9SThomas Gleixner rdmsrl(MSR_IA32_APICBASE, msr);
174644e25ff9SThomas Gleixner if (!(msr & X2APIC_ENABLE))
174744e25ff9SThomas Gleixner return;
174844e25ff9SThomas Gleixner /* Disable xapic and x2apic first and then reenable xapic mode */
174944e25ff9SThomas Gleixner wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
175044e25ff9SThomas Gleixner wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
175144e25ff9SThomas Gleixner printk_once(KERN_INFO "x2apic disabled\n");
175244e25ff9SThomas Gleixner }
175344e25ff9SThomas Gleixner
__x2apic_enable(void)1754d786ad32SDenys Vlasenko static void __x2apic_enable(void)
1755659006bfSThomas Gleixner {
1756659006bfSThomas Gleixner u64 msr;
1757659006bfSThomas Gleixner
1758659006bfSThomas Gleixner rdmsrl(MSR_IA32_APICBASE, msr);
1759659006bfSThomas Gleixner if (msr & X2APIC_ENABLE)
1760659006bfSThomas Gleixner return;
1761659006bfSThomas Gleixner wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
1762659006bfSThomas Gleixner printk_once(KERN_INFO "x2apic enabled\n");
1763659006bfSThomas Gleixner }
1764659006bfSThomas Gleixner
setup_nox2apic(char * str)1765bfb05070SThomas Gleixner static int __init setup_nox2apic(char *str)
1766bfb05070SThomas Gleixner {
1767bfb05070SThomas Gleixner if (x2apic_enabled()) {
1768bfb05070SThomas Gleixner int apicid = native_apic_msr_read(APIC_ID);
1769bfb05070SThomas Gleixner
1770bfb05070SThomas Gleixner if (apicid >= 255) {
17718d3bcc44SKefeng Wang pr_warn("Apicid: %08x, cannot enforce nox2apic\n",
1772bfb05070SThomas Gleixner apicid);
1773bfb05070SThomas Gleixner return 0;
1774bfb05070SThomas Gleixner }
1775b8d1d163SDaniel Sneddon if (x2apic_hw_locked()) {
1776b8d1d163SDaniel Sneddon pr_warn("APIC locked in x2apic mode, can't disable\n");
1777b8d1d163SDaniel Sneddon return 0;
1778b8d1d163SDaniel Sneddon }
17798d3bcc44SKefeng Wang pr_warn("x2apic already enabled.\n");
178044e25ff9SThomas Gleixner __x2apic_disable();
178144e25ff9SThomas Gleixner }
1782bfb05070SThomas Gleixner setup_clear_cpu_cap(X86_FEATURE_X2APIC);
178312e189d3SThomas Gleixner x2apic_state = X2APIC_DISABLED;
178444e25ff9SThomas Gleixner x2apic_mode = 0;
1785bfb05070SThomas Gleixner return 0;
1786bfb05070SThomas Gleixner }
1787bfb05070SThomas Gleixner early_param("nox2apic", setup_nox2apic);
1788bfb05070SThomas Gleixner
1789659006bfSThomas Gleixner /* Called from cpu_init() to enable x2apic on (secondary) cpus */
x2apic_setup(void)1790659006bfSThomas Gleixner void x2apic_setup(void)
1791659006bfSThomas Gleixner {
1792659006bfSThomas Gleixner /*
1793b8d1d163SDaniel Sneddon * Try to make the AP's APIC state match that of the BSP, but if the
1794b8d1d163SDaniel Sneddon * BSP is unlocked and the AP is locked then there is a state mismatch.
1795b8d1d163SDaniel Sneddon * Warn about the mismatch in case a GP fault occurs due to a locked AP
1796b8d1d163SDaniel Sneddon * trying to be turned off.
1797b8d1d163SDaniel Sneddon */
1798b8d1d163SDaniel Sneddon if (x2apic_state != X2APIC_ON_LOCKED && x2apic_hw_locked())
1799b8d1d163SDaniel Sneddon pr_warn("x2apic lock mismatch between BSP and AP.\n");
1800b8d1d163SDaniel Sneddon /*
1801b8d1d163SDaniel Sneddon * If x2apic is not in ON or LOCKED state, disable it if already enabled
1802659006bfSThomas Gleixner * from BIOS.
1803659006bfSThomas Gleixner */
1804b8d1d163SDaniel Sneddon if (x2apic_state < X2APIC_ON) {
1805659006bfSThomas Gleixner __x2apic_disable();
1806659006bfSThomas Gleixner return;
1807659006bfSThomas Gleixner }
1808659006bfSThomas Gleixner __x2apic_enable();
1809659006bfSThomas Gleixner }
1810659006bfSThomas Gleixner
1811513f8dc0SThomas Gleixner static __init void apic_set_fixmap(bool read_apic);
18125a88f354SThomas Gleixner
x2apic_disable(void)181344e25ff9SThomas Gleixner static __init void x2apic_disable(void)
1814fb209bd8SYinghai Lu {
1815a57e456aSThomas Gleixner u32 x2apic_id, state = x2apic_state;
1816fb209bd8SYinghai Lu
1817a57e456aSThomas Gleixner x2apic_mode = 0;
1818a57e456aSThomas Gleixner x2apic_state = X2APIC_DISABLED;
1819a57e456aSThomas Gleixner
1820a57e456aSThomas Gleixner if (state != X2APIC_ON)
1821a57e456aSThomas Gleixner return;
1822fb209bd8SYinghai Lu
18236d2d49d2SThomas Gleixner x2apic_id = read_apic_id();
1824fb209bd8SYinghai Lu if (x2apic_id >= 255)
1825fb209bd8SYinghai Lu panic("Cannot disable x2apic, id: %08x\n", x2apic_id);
1826fb209bd8SYinghai Lu
1827b8d1d163SDaniel Sneddon if (x2apic_hw_locked()) {
1828b8d1d163SDaniel Sneddon pr_warn("Cannot disable locked x2apic, id: %08x\n", x2apic_id);
1829b8d1d163SDaniel Sneddon return;
1830b8d1d163SDaniel Sneddon }
1831b8d1d163SDaniel Sneddon
183244e25ff9SThomas Gleixner __x2apic_disable();
1833513f8dc0SThomas Gleixner /*
1834513f8dc0SThomas Gleixner * Don't reread the APIC ID as it was already done from
1835513f8dc0SThomas Gleixner * check_x2apic() and the APIC driver still is a x2APIC variant,
1836513f8dc0SThomas Gleixner * which fails to do the read after x2APIC was disabled.
1837513f8dc0SThomas Gleixner */
1838513f8dc0SThomas Gleixner apic_set_fixmap(false);
1839fb209bd8SYinghai Lu }
1840fb209bd8SYinghai Lu
x2apic_enable(void)1841659006bfSThomas Gleixner static __init void x2apic_enable(void)
1842f62bae50SIngo Molnar {
1843659006bfSThomas Gleixner if (x2apic_state != X2APIC_OFF)
1844f62bae50SIngo Molnar return;
1845f62bae50SIngo Molnar
1846659006bfSThomas Gleixner x2apic_mode = 1;
184712e189d3SThomas Gleixner x2apic_state = X2APIC_ON;
1848659006bfSThomas Gleixner __x2apic_enable();
1849f62bae50SIngo Molnar }
1850d524165cSThomas Gleixner
try_to_enable_x2apic(int remap_mode)185162e61633SThomas Gleixner static __init void try_to_enable_x2apic(int remap_mode)
185207806c50SJiang Liu {
1853659006bfSThomas Gleixner if (x2apic_state == X2APIC_DISABLED)
185407806c50SJiang Liu return;
185507806c50SJiang Liu
185662e61633SThomas Gleixner if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
1857ab0f59c6SDavid Woodhouse u32 apic_limit = 255;
1858ab0f59c6SDavid Woodhouse
185926573a97SDavid Woodhouse /*
186026573a97SDavid Woodhouse * Using X2APIC without IR is not architecturally supported
186126573a97SDavid Woodhouse * on bare metal but may be supported in guests.
186207806c50SJiang Liu */
186326573a97SDavid Woodhouse if (!x86_init.hyper.x2apic_available()) {
186462e61633SThomas Gleixner pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
186544e25ff9SThomas Gleixner x2apic_disable();
186607806c50SJiang Liu return;
186707806c50SJiang Liu }
186807806c50SJiang Liu
186907806c50SJiang Liu /*
1870ab0f59c6SDavid Woodhouse * If the hypervisor supports extended destination ID in
1871ab0f59c6SDavid Woodhouse * MSI, that increases the maximum APIC ID that can be
1872ab0f59c6SDavid Woodhouse * used for non-remapped IRQ domains.
1873ab0f59c6SDavid Woodhouse */
1874ab0f59c6SDavid Woodhouse if (x86_init.hyper.msi_ext_dest_id()) {
1875ab0f59c6SDavid Woodhouse virt_ext_dest_id = 1;
1876ab0f59c6SDavid Woodhouse apic_limit = 32767;
1877ab0f59c6SDavid Woodhouse }
1878ab0f59c6SDavid Woodhouse
1879ab0f59c6SDavid Woodhouse /*
188026573a97SDavid Woodhouse * Without IR, all CPUs can be addressed by IOAPIC/MSI only
1881d9f6e12fSIngo Molnar * in physical mode, and CPUs with an APIC ID that cannot
188226573a97SDavid Woodhouse * be addressed must not be brought online.
188307806c50SJiang Liu */
1884ab0f59c6SDavid Woodhouse x2apic_set_max_apicid(apic_limit);
188555eae7deSThomas Gleixner x2apic_phys = 1;
188607806c50SJiang Liu }
1887659006bfSThomas Gleixner x2apic_enable();
188855eae7deSThomas Gleixner }
188955eae7deSThomas Gleixner
check_x2apic(void)189055eae7deSThomas Gleixner void __init check_x2apic(void)
189155eae7deSThomas Gleixner {
189255eae7deSThomas Gleixner if (x2apic_enabled()) {
189355eae7deSThomas Gleixner pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
189455eae7deSThomas Gleixner x2apic_mode = 1;
1895b8d1d163SDaniel Sneddon if (x2apic_hw_locked())
1896b8d1d163SDaniel Sneddon x2apic_state = X2APIC_ON_LOCKED;
1897b8d1d163SDaniel Sneddon else
189812e189d3SThomas Gleixner x2apic_state = X2APIC_ON;
1899d10a9044SThomas Gleixner apic_read_boot_cpu_id(true);
190062436a4dSBorislav Petkov } else if (!boot_cpu_has(X86_FEATURE_X2APIC)) {
190112e189d3SThomas Gleixner x2apic_state = X2APIC_DISABLED;
190255eae7deSThomas Gleixner }
190355eae7deSThomas Gleixner }
190455eae7deSThomas Gleixner #else /* CONFIG_X86_X2APIC */
check_x2apic(void)1905e3998434SMateusz Jończyk void __init check_x2apic(void)
190655eae7deSThomas Gleixner {
190755eae7deSThomas Gleixner if (!apic_is_x2apic_enabled())
1908e3998434SMateusz Jończyk return;
190955eae7deSThomas Gleixner /*
1910e3998434SMateusz Jończyk * Checkme: Can we simply turn off x2APIC here instead of disabling the APIC?
191155eae7deSThomas Gleixner */
1912e3998434SMateusz Jończyk pr_err("Kernel does not support x2APIC, please recompile with CONFIG_X86_X2APIC.\n");
1913e3998434SMateusz Jończyk pr_err("Disabling APIC, expect reduced performance and functionality.\n");
1914e3998434SMateusz Jończyk
191549062454SThomas Gleixner apic_is_disabled = true;
1916e3998434SMateusz Jończyk setup_clear_cpu_cap(X86_FEATURE_APIC);
191755eae7deSThomas Gleixner }
191855eae7deSThomas Gleixner
try_to_enable_x2apic(int remap_mode)191962e61633SThomas Gleixner static inline void try_to_enable_x2apic(int remap_mode) { }
__x2apic_enable(void)1920659006bfSThomas Gleixner static inline void __x2apic_enable(void) { }
192155eae7deSThomas Gleixner #endif /* !CONFIG_X86_X2APIC */
192255eae7deSThomas Gleixner
enable_IR_x2apic(void)1923ce69a784SGleb Natapov void __init enable_IR_x2apic(void)
1924ce69a784SGleb Natapov {
1925ce69a784SGleb Natapov unsigned long flags;
192607806c50SJiang Liu int ret, ir_stat;
1927b7f42ab2SYinghai Lu
1928ecf600f8SThomas Gleixner if (ioapic_is_disabled) {
192911277aabSDou Liyang pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
19302e63ad4bSWanpeng Li return;
193111277aabSDou Liyang }
19322e63ad4bSWanpeng Li
193307806c50SJiang Liu ir_stat = irq_remapping_prepare();
193407806c50SJiang Liu if (ir_stat < 0 && !x2apic_supported())
1935e670761fSYinghai Lu return;
1936ce69a784SGleb Natapov
193731dce14aSSuresh Siddha ret = save_ioapic_entries();
1938f62bae50SIngo Molnar if (ret) {
1939f62bae50SIngo Molnar pr_info("Saving IO-APIC state failed: %d\n", ret);
1940fb209bd8SYinghai Lu return;
1941f62bae50SIngo Molnar }
1942f62bae50SIngo Molnar
194305c3dc2cSSuresh Siddha local_irq_save(flags);
1944b81bb373SJacob Pan legacy_pic->mask_all();
194531dce14aSSuresh Siddha mask_ioapic_entries();
194605c3dc2cSSuresh Siddha
19476a6256f9SAdam Buchbinder /* If irq_remapping_prepare() succeeded, try to enable it */
194807806c50SJiang Liu if (ir_stat >= 0)
194911277aabSDou Liyang ir_stat = irq_remapping_enable();
195007806c50SJiang Liu /* ir_stat contains the remap mode or an error code */
195107806c50SJiang Liu try_to_enable_x2apic(ir_stat);
1952a31bc327SYinghai Lu
195307806c50SJiang Liu if (ir_stat < 0)
195431dce14aSSuresh Siddha restore_ioapic_entries();
1955b81bb373SJacob Pan legacy_pic->restore_mask();
1956f62bae50SIngo Molnar local_irq_restore(flags);
1957f62bae50SIngo Molnar }
195893758238SWeidong Han
1959f62bae50SIngo Molnar #ifdef CONFIG_X86_64
1960f62bae50SIngo Molnar /*
1961f62bae50SIngo Molnar * Detect and enable local APICs on non-SMP boards.
1962f62bae50SIngo Molnar * Original code written by Keir Fraser.
1963f62bae50SIngo Molnar * On AMD64 we trust the BIOS - if it says no APIC it is likely
1964f62bae50SIngo Molnar * not correctly set up (usually the APIC timer won't work etc.)
1965f62bae50SIngo Molnar */
detect_init_APIC(void)19661751adedSThomas Gleixner static bool __init detect_init_APIC(void)
1967f62bae50SIngo Molnar {
196893984fbdSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_APIC)) {
1969f62bae50SIngo Molnar pr_info("No local APIC present\n");
19701751adedSThomas Gleixner return false;
1971f62bae50SIngo Molnar }
1972f62bae50SIngo Molnar
197381287ad6SThomas Gleixner register_lapic_address(APIC_DEFAULT_PHYS_BASE);
19741751adedSThomas Gleixner return true;
1975f62bae50SIngo Molnar }
1976f62bae50SIngo Molnar #else
19775a7ae78fSThomas Gleixner
apic_verify(unsigned long addr)197881287ad6SThomas Gleixner static bool __init apic_verify(unsigned long addr)
19795a7ae78fSThomas Gleixner {
19805a7ae78fSThomas Gleixner u32 features, h, l;
19815a7ae78fSThomas Gleixner
19825a7ae78fSThomas Gleixner /*
19835a7ae78fSThomas Gleixner * The APIC feature bit should now be enabled
19845a7ae78fSThomas Gleixner * in `cpuid'
19855a7ae78fSThomas Gleixner */
19865a7ae78fSThomas Gleixner features = cpuid_edx(1);
19875a7ae78fSThomas Gleixner if (!(features & (1 << X86_FEATURE_APIC))) {
19888d3bcc44SKefeng Wang pr_warn("Could not enable APIC!\n");
19891751adedSThomas Gleixner return false;
19905a7ae78fSThomas Gleixner }
19915a7ae78fSThomas Gleixner set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
19925a7ae78fSThomas Gleixner
19935a7ae78fSThomas Gleixner /* The BIOS may have set up the APIC at some other address */
1994cbf2829bSBryan O'Donoghue if (boot_cpu_data.x86 >= 6) {
19955a7ae78fSThomas Gleixner rdmsr(MSR_IA32_APICBASE, l, h);
19965a7ae78fSThomas Gleixner if (l & MSR_IA32_APICBASE_ENABLE)
199781287ad6SThomas Gleixner addr = l & MSR_IA32_APICBASE_BASE;
1998cbf2829bSBryan O'Donoghue }
19995a7ae78fSThomas Gleixner
200081287ad6SThomas Gleixner register_lapic_address(addr);
20015a7ae78fSThomas Gleixner pr_info("Found and enabled local APIC!\n");
20021751adedSThomas Gleixner return true;
20035a7ae78fSThomas Gleixner }
20045a7ae78fSThomas Gleixner
apic_force_enable(unsigned long addr)20051751adedSThomas Gleixner bool __init apic_force_enable(unsigned long addr)
20065a7ae78fSThomas Gleixner {
20075a7ae78fSThomas Gleixner u32 h, l;
20085a7ae78fSThomas Gleixner
200949062454SThomas Gleixner if (apic_is_disabled)
20101751adedSThomas Gleixner return false;
20115a7ae78fSThomas Gleixner
20125a7ae78fSThomas Gleixner /*
20135a7ae78fSThomas Gleixner * Some BIOSes disable the local APIC in the APIC_BASE
20145a7ae78fSThomas Gleixner * MSR. This can only be done in software for Intel P6 or later
20155a7ae78fSThomas Gleixner * and AMD K7 (Model > 1) or later.
20165a7ae78fSThomas Gleixner */
2017cbf2829bSBryan O'Donoghue if (boot_cpu_data.x86 >= 6) {
20185a7ae78fSThomas Gleixner rdmsr(MSR_IA32_APICBASE, l, h);
20195a7ae78fSThomas Gleixner if (!(l & MSR_IA32_APICBASE_ENABLE)) {
20205a7ae78fSThomas Gleixner pr_info("Local APIC disabled by BIOS -- reenabling.\n");
20215a7ae78fSThomas Gleixner l &= ~MSR_IA32_APICBASE_BASE;
2022a906fdaaSThomas Gleixner l |= MSR_IA32_APICBASE_ENABLE | addr;
20235a7ae78fSThomas Gleixner wrmsr(MSR_IA32_APICBASE, l, h);
20245a7ae78fSThomas Gleixner enabled_via_apicbase = 1;
20255a7ae78fSThomas Gleixner }
2026cbf2829bSBryan O'Donoghue }
202781287ad6SThomas Gleixner return apic_verify(addr);
20285a7ae78fSThomas Gleixner }
20295a7ae78fSThomas Gleixner
2030f62bae50SIngo Molnar /*
2031f62bae50SIngo Molnar * Detect and initialize APIC
2032f62bae50SIngo Molnar */
detect_init_APIC(void)20331751adedSThomas Gleixner static bool __init detect_init_APIC(void)
2034f62bae50SIngo Molnar {
2035f62bae50SIngo Molnar /* Disabled by kernel option? */
203649062454SThomas Gleixner if (apic_is_disabled)
20371751adedSThomas Gleixner return false;
2038f62bae50SIngo Molnar
2039f62bae50SIngo Molnar switch (boot_cpu_data.x86_vendor) {
2040f62bae50SIngo Molnar case X86_VENDOR_AMD:
2041f62bae50SIngo Molnar if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
2042f62bae50SIngo Molnar (boot_cpu_data.x86 >= 15))
2043f62bae50SIngo Molnar break;
2044f62bae50SIngo Molnar goto no_apic;
2045da33dfefSPu Wen case X86_VENDOR_HYGON:
2046da33dfefSPu Wen break;
2047f62bae50SIngo Molnar case X86_VENDOR_INTEL:
2048f62bae50SIngo Molnar if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
204993984fbdSBorislav Petkov (boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC)))
2050f62bae50SIngo Molnar break;
2051f62bae50SIngo Molnar goto no_apic;
2052f62bae50SIngo Molnar default:
2053f62bae50SIngo Molnar goto no_apic;
2054f62bae50SIngo Molnar }
2055f62bae50SIngo Molnar
205693984fbdSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_APIC)) {
2057f62bae50SIngo Molnar /*
2058f62bae50SIngo Molnar * Over-ride BIOS and try to enable the local APIC only if
2059f62bae50SIngo Molnar * "lapic" specified.
2060f62bae50SIngo Molnar */
2061f62bae50SIngo Molnar if (!force_enable_local_apic) {
2062f62bae50SIngo Molnar pr_info("Local APIC disabled by BIOS -- "
2063f62bae50SIngo Molnar "you can enable it with \"lapic\"\n");
20641751adedSThomas Gleixner return false;
2065f62bae50SIngo Molnar }
20661751adedSThomas Gleixner if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE))
20671751adedSThomas Gleixner return false;
20685a7ae78fSThomas Gleixner } else {
206981287ad6SThomas Gleixner if (!apic_verify(APIC_DEFAULT_PHYS_BASE))
20701751adedSThomas Gleixner return false;
2071f62bae50SIngo Molnar }
2072f62bae50SIngo Molnar
2073f62bae50SIngo Molnar apic_pm_activate();
2074f62bae50SIngo Molnar
20751751adedSThomas Gleixner return true;
2076f62bae50SIngo Molnar
2077f62bae50SIngo Molnar no_apic:
2078f62bae50SIngo Molnar pr_info("No local APIC present or hardware disabled\n");
20791751adedSThomas Gleixner return false;
2080f62bae50SIngo Molnar }
2081f62bae50SIngo Molnar #endif
2082f62bae50SIngo Molnar
2083f62bae50SIngo Molnar /**
2084f62bae50SIngo Molnar * init_apic_mappings - initialize APIC mappings
2085f62bae50SIngo Molnar */
init_apic_mappings(void)2086f62bae50SIngo Molnar void __init init_apic_mappings(void)
2087f62bae50SIngo Molnar {
2088c84cb373SThomas Gleixner if (apic_validate_deadline_timer())
2089de308d18SBorislav Petkov pr_info("TSC deadline timer available\n");
2090bd9240a1SPeter Zijlstra
2091d10a9044SThomas Gleixner if (x2apic_mode)
2092f62bae50SIngo Molnar return;
2093f62bae50SIngo Molnar
2094e8122513SThomas Gleixner if (!smp_found_config) {
2095e8122513SThomas Gleixner if (!detect_init_APIC()) {
20964797f6b0SYinghai Lu pr_info("APIC: disable apic facility\n");
20974797f6b0SYinghai Lu apic_disable();
2098cec6be6dSCyrill Gorcunov }
2099e8122513SThomas Gleixner num_processors = 1;
2100e8122513SThomas Gleixner }
2101f62bae50SIngo Molnar }
2102f62bae50SIngo Molnar
apic_set_fixmap(bool read_apic)2103513f8dc0SThomas Gleixner static __init void apic_set_fixmap(bool read_apic)
21045a88f354SThomas Gleixner {
21055a88f354SThomas Gleixner set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
210678c32000SThomas Gleixner apic_mmio_base = APIC_BASE;
21075a88f354SThomas Gleixner apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
21085a88f354SThomas Gleixner apic_mmio_base, mp_lapic_addr);
2109513f8dc0SThomas Gleixner if (read_apic)
21105a88f354SThomas Gleixner apic_read_boot_cpu_id(false);
21115a88f354SThomas Gleixner }
21125a88f354SThomas Gleixner
register_lapic_address(unsigned long address)2113c0104d38SYinghai Lu void __init register_lapic_address(unsigned long address)
2114c0104d38SYinghai Lu {
211581287ad6SThomas Gleixner /* This should only happen once */
211681287ad6SThomas Gleixner WARN_ON_ONCE(mp_lapic_addr);
2117c0104d38SYinghai Lu mp_lapic_addr = address;
2118c0104d38SYinghai Lu
21195a88f354SThomas Gleixner if (!x2apic_mode)
2120513f8dc0SThomas Gleixner apic_set_fixmap(true);
2121c0104d38SYinghai Lu }
2122c0104d38SYinghai Lu
2123f62bae50SIngo Molnar /*
2124f62bae50SIngo Molnar * Local APIC interrupts
2125f62bae50SIngo Molnar */
2126f62bae50SIngo Molnar
21273c5e0267SThomas Gleixner /*
21283c5e0267SThomas Gleixner * Common handling code for spurious_interrupt and spurious_vector entry
21293c5e0267SThomas Gleixner * points below. No point in allowing the compiler to inline it twice.
2130f62bae50SIngo Molnar */
handle_spurious_interrupt(u8 vector)21313c5e0267SThomas Gleixner static noinline void handle_spurious_interrupt(u8 vector)
2132f62bae50SIngo Molnar {
2133f62bae50SIngo Molnar u32 v;
2134f62bae50SIngo Molnar
213561069de7SThomas Gleixner trace_spurious_apic_entry(vector);
213661069de7SThomas Gleixner
2137f62bae50SIngo Molnar inc_irq_stat(irq_spurious_count);
2138f62bae50SIngo Molnar
2139f8a8fe61SThomas Gleixner /*
2140f8a8fe61SThomas Gleixner * If this is a spurious interrupt then do not acknowledge
2141f8a8fe61SThomas Gleixner */
2142f8a8fe61SThomas Gleixner if (vector == SPURIOUS_APIC_VECTOR) {
2143f8a8fe61SThomas Gleixner /* See SDM vol 3 */
2144f8a8fe61SThomas Gleixner pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n",
2145f8a8fe61SThomas Gleixner smp_processor_id());
2146f8a8fe61SThomas Gleixner goto out;
2147f8a8fe61SThomas Gleixner }
2148eddc0e92SSeiji Aguchi
2149f8a8fe61SThomas Gleixner /*
2150f8a8fe61SThomas Gleixner * If it is a vectored one, verify it's set in the ISR. If set,
2151f8a8fe61SThomas Gleixner * acknowledge it.
2152f8a8fe61SThomas Gleixner */
2153f8a8fe61SThomas Gleixner v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
2154f8a8fe61SThomas Gleixner if (v & (1 << (vector & 0x1f))) {
2155f8a8fe61SThomas Gleixner pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
2156f8a8fe61SThomas Gleixner vector, smp_processor_id());
2157670c04adSDave Hansen apic_eoi();
2158f8a8fe61SThomas Gleixner } else {
2159f8a8fe61SThomas Gleixner pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
2160f8a8fe61SThomas Gleixner vector, smp_processor_id());
2161f8a8fe61SThomas Gleixner }
2162f8a8fe61SThomas Gleixner out:
21632414e021SJan Beulich trace_spurious_apic_exit(vector);
2164f62bae50SIngo Molnar }
2165f62bae50SIngo Molnar
21663c5e0267SThomas Gleixner /**
21673c5e0267SThomas Gleixner * spurious_interrupt - Catch all for interrupts raised on unused vectors
21683c5e0267SThomas Gleixner * @regs: Pointer to pt_regs on stack
21693c5e0267SThomas Gleixner * @vector: The vector number
21703c5e0267SThomas Gleixner *
21713c5e0267SThomas Gleixner * This is invoked from ASM entry code to catch all interrupts which
21723c5e0267SThomas Gleixner * trigger on an entry which is routed to the common_spurious idtentry
21733c5e0267SThomas Gleixner * point.
21743c5e0267SThomas Gleixner */
DEFINE_IDTENTRY_IRQ(spurious_interrupt)21753c5e0267SThomas Gleixner DEFINE_IDTENTRY_IRQ(spurious_interrupt)
21763c5e0267SThomas Gleixner {
21773c5e0267SThomas Gleixner handle_spurious_interrupt(vector);
21783c5e0267SThomas Gleixner }
21793c5e0267SThomas Gleixner
DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)2180db0338eeSThomas Gleixner DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)
2181633260faSThomas Gleixner {
21823c5e0267SThomas Gleixner handle_spurious_interrupt(SPURIOUS_APIC_VECTOR);
2183f62bae50SIngo Molnar }
2184f62bae50SIngo Molnar
2185f62bae50SIngo Molnar /*
2186f62bae50SIngo Molnar * This interrupt should never happen with our APIC/SMP architecture
2187f62bae50SIngo Molnar */
DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)2188db0338eeSThomas Gleixner DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
2189f62bae50SIngo Molnar {
21902b398bd9SYouquan Song static const char * const error_interrupt_reason[] = {
21912b398bd9SYouquan Song "Send CS error", /* APIC Error Bit 0 */
21922b398bd9SYouquan Song "Receive CS error", /* APIC Error Bit 1 */
21932b398bd9SYouquan Song "Send accept error", /* APIC Error Bit 2 */
21942b398bd9SYouquan Song "Receive accept error", /* APIC Error Bit 3 */
21952b398bd9SYouquan Song "Redirectable IPI", /* APIC Error Bit 4 */
21962b398bd9SYouquan Song "Send illegal vector", /* APIC Error Bit 5 */
21972b398bd9SYouquan Song "Received illegal vector", /* APIC Error Bit 6 */
21982b398bd9SYouquan Song "Illegal register address", /* APIC Error Bit 7 */
21992b398bd9SYouquan Song };
220061069de7SThomas Gleixner u32 v, i = 0;
220161069de7SThomas Gleixner
220261069de7SThomas Gleixner trace_error_apic_entry(ERROR_APIC_VECTOR);
2203f62bae50SIngo Molnar
2204f62bae50SIngo Molnar /* First tickle the hardware, only then report what went on. -- REW */
2205023de4a0SMaciej W. Rozycki if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
2206f62bae50SIngo Molnar apic_write(APIC_ESR, 0);
220760283df7SRichard Weinberger v = apic_read(APIC_ESR);
2208670c04adSDave Hansen apic_eoi();
2209f62bae50SIngo Molnar atomic_inc(&irq_err_count);
2210f62bae50SIngo Molnar
221160283df7SRichard Weinberger apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x",
221260283df7SRichard Weinberger smp_processor_id(), v);
22132b398bd9SYouquan Song
221460283df7SRichard Weinberger v &= 0xff;
221560283df7SRichard Weinberger while (v) {
221660283df7SRichard Weinberger if (v & 0x1)
22172b398bd9SYouquan Song apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
22182b398bd9SYouquan Song i++;
221960283df7SRichard Weinberger v >>= 1;
22204b8073e4SPeter Senna Tschudin }
22212b398bd9SYouquan Song
22222b398bd9SYouquan Song apic_printk(APIC_DEBUG, KERN_CONT "\n");
22232b398bd9SYouquan Song
2224cf910e83SSeiji Aguchi trace_error_apic_exit(ERROR_APIC_VECTOR);
2225f62bae50SIngo Molnar }
2226f62bae50SIngo Molnar
2227f62bae50SIngo Molnar /**
2228f62bae50SIngo Molnar * connect_bsp_APIC - attach the APIC to the interrupt system
2229f62bae50SIngo Molnar */
connect_bsp_APIC(void)223005f7e46dSThomas Gleixner static void __init connect_bsp_APIC(void)
2231f62bae50SIngo Molnar {
2232f62bae50SIngo Molnar #ifdef CONFIG_X86_32
2233f62bae50SIngo Molnar if (pic_mode) {
2234f62bae50SIngo Molnar /*
2235f62bae50SIngo Molnar * Do not trust the local APIC being empty at bootup.
2236f62bae50SIngo Molnar */
2237f62bae50SIngo Molnar clear_local_APIC();
2238f62bae50SIngo Molnar /*
2239f62bae50SIngo Molnar * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
2240f62bae50SIngo Molnar * local APIC to INT and NMI lines.
2241f62bae50SIngo Molnar */
2242f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "leaving PIC mode, "
2243f62bae50SIngo Molnar "enabling APIC mode.\n");
2244c0eaa453SCyrill Gorcunov imcr_pic_to_apic();
2245f62bae50SIngo Molnar }
2246f62bae50SIngo Molnar #endif
2247f62bae50SIngo Molnar }
2248f62bae50SIngo Molnar
2249f62bae50SIngo Molnar /**
2250f62bae50SIngo Molnar * disconnect_bsp_APIC - detach the APIC from the interrupt system
2251f62bae50SIngo Molnar * @virt_wire_setup: indicates, whether virtual wire mode is selected
2252f62bae50SIngo Molnar *
2253f62bae50SIngo Molnar * Virtual wire mode is necessary to deliver legacy interrupts even when the
2254f62bae50SIngo Molnar * APIC is disabled.
2255f62bae50SIngo Molnar */
disconnect_bsp_APIC(int virt_wire_setup)2256f62bae50SIngo Molnar void disconnect_bsp_APIC(int virt_wire_setup)
2257f62bae50SIngo Molnar {
2258f62bae50SIngo Molnar unsigned int value;
2259f62bae50SIngo Molnar
2260f62bae50SIngo Molnar #ifdef CONFIG_X86_32
2261f62bae50SIngo Molnar if (pic_mode) {
2262f62bae50SIngo Molnar /*
2263f62bae50SIngo Molnar * Put the board back into PIC mode (has an effect only on
2264f62bae50SIngo Molnar * certain older boards). Note that APIC interrupts, including
2265f62bae50SIngo Molnar * IPIs, won't work beyond this point! The only exception are
2266f62bae50SIngo Molnar * INIT IPIs.
2267f62bae50SIngo Molnar */
2268f62bae50SIngo Molnar apic_printk(APIC_VERBOSE, "disabling APIC mode, "
2269f62bae50SIngo Molnar "entering PIC mode.\n");
2270c0eaa453SCyrill Gorcunov imcr_apic_to_pic();
2271f62bae50SIngo Molnar return;
2272f62bae50SIngo Molnar }
2273f62bae50SIngo Molnar #endif
2274f62bae50SIngo Molnar
2275f62bae50SIngo Molnar /* Go back to Virtual Wire compatibility mode */
2276f62bae50SIngo Molnar
2277f62bae50SIngo Molnar /* For the spurious interrupt use vector F, and enable it */
2278f62bae50SIngo Molnar value = apic_read(APIC_SPIV);
2279f62bae50SIngo Molnar value &= ~APIC_VECTOR_MASK;
2280f62bae50SIngo Molnar value |= APIC_SPIV_APIC_ENABLED;
2281f62bae50SIngo Molnar value |= 0xf;
2282f62bae50SIngo Molnar apic_write(APIC_SPIV, value);
2283f62bae50SIngo Molnar
2284f62bae50SIngo Molnar if (!virt_wire_setup) {
2285f62bae50SIngo Molnar /*
2286f62bae50SIngo Molnar * For LVT0 make it edge triggered, active high,
2287f62bae50SIngo Molnar * external and enabled
2288f62bae50SIngo Molnar */
2289f62bae50SIngo Molnar value = apic_read(APIC_LVT0);
2290f62bae50SIngo Molnar value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2291f62bae50SIngo Molnar APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2292f62bae50SIngo Molnar APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2293f62bae50SIngo Molnar value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2294f62bae50SIngo Molnar value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
2295f62bae50SIngo Molnar apic_write(APIC_LVT0, value);
2296f62bae50SIngo Molnar } else {
2297f62bae50SIngo Molnar /* Disable LVT0 */
2298f62bae50SIngo Molnar apic_write(APIC_LVT0, APIC_LVT_MASKED);
2299f62bae50SIngo Molnar }
2300f62bae50SIngo Molnar
2301f62bae50SIngo Molnar /*
2302f62bae50SIngo Molnar * For LVT1 make it edge triggered, active high,
2303f62bae50SIngo Molnar * nmi and enabled
2304f62bae50SIngo Molnar */
2305f62bae50SIngo Molnar value = apic_read(APIC_LVT1);
2306f62bae50SIngo Molnar value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2307f62bae50SIngo Molnar APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2308f62bae50SIngo Molnar APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2309f62bae50SIngo Molnar value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2310f62bae50SIngo Molnar value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
2311f62bae50SIngo Molnar apic_write(APIC_LVT1, value);
2312f62bae50SIngo Molnar }
2313f62bae50SIngo Molnar
23148f54969dSGu Zheng /*
23158f54969dSGu Zheng * The number of allocated logical CPU IDs. Since logical CPU IDs are allocated
23168f54969dSGu Zheng * contiguously, it equals to current allocated max logical CPU ID plus 1.
231712bf98b9SDou Liyang * All allocated CPU IDs should be in the [0, nr_logical_cpuids) range,
231812bf98b9SDou Liyang * so the maximum of nr_logical_cpuids is nr_cpu_ids.
23198f54969dSGu Zheng *
23208f54969dSGu Zheng * NOTE: Reserve 0 for BSP.
23218f54969dSGu Zheng */
23228f54969dSGu Zheng static int nr_logical_cpuids = 1;
23238f54969dSGu Zheng
23248f54969dSGu Zheng /*
23258f54969dSGu Zheng * Used to store mapping between logical CPU IDs and APIC IDs.
23268f54969dSGu Zheng */
23277e75178aSDavid Woodhouse int cpuid_to_apicid[] = {
23288f54969dSGu Zheng [0 ... NR_CPUS - 1] = -1,
23298f54969dSGu Zheng };
23308f54969dSGu Zheng
arch_match_cpu_phys_id(int cpu,u64 phys_id)2331dd926880SJohan Hovold bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
2332dd926880SJohan Hovold {
2333dd926880SJohan Hovold return phys_id == cpuid_to_apicid[cpu];
2334dd926880SJohan Hovold }
2335dd926880SJohan Hovold
2336d0055f35SVlastimil Babka #ifdef CONFIG_SMP
cpu_mark_primary_thread(unsigned int cpu,unsigned int apicid)2337f54d4434SThomas Gleixner static void cpu_mark_primary_thread(unsigned int cpu, unsigned int apicid)
23386a4d2657SThomas Gleixner {
23396a4d2657SThomas Gleixner /* Isolate the SMT bit(s) in the APICID and check for 0 */
2340f54d4434SThomas Gleixner u32 mask = (1U << (fls(smp_num_siblings) - 1)) - 1;
2341f54d4434SThomas Gleixner
2342f54d4434SThomas Gleixner if (smp_num_siblings == 1 || !(apicid & mask))
2343f54d4434SThomas Gleixner cpumask_set_cpu(cpu, &__cpu_primary_thread_mask);
23446a4d2657SThomas Gleixner }
23455da80b28SThomas Gleixner
23465da80b28SThomas Gleixner /*
23475da80b28SThomas Gleixner * Due to the utter mess of CPUID evaluation smp_num_siblings is not valid
23485da80b28SThomas Gleixner * during early boot. Initialize the primary thread mask before SMP
23495da80b28SThomas Gleixner * bringup.
23505da80b28SThomas Gleixner */
smp_init_primary_thread_mask(void)23515da80b28SThomas Gleixner static int __init smp_init_primary_thread_mask(void)
23525da80b28SThomas Gleixner {
23535da80b28SThomas Gleixner unsigned int cpu;
23545da80b28SThomas Gleixner
2355e0621d5dSThomas Gleixner /*
2356e0621d5dSThomas Gleixner * XEN/PV provides either none or useless topology information.
2357e0621d5dSThomas Gleixner * Pretend that all vCPUs are primary threads.
2358e0621d5dSThomas Gleixner */
2359e0621d5dSThomas Gleixner if (xen_pv_domain()) {
2360e0621d5dSThomas Gleixner cpumask_copy(&__cpu_primary_thread_mask, cpu_possible_mask);
2361e0621d5dSThomas Gleixner return 0;
2362e0621d5dSThomas Gleixner }
2363e0621d5dSThomas Gleixner
23645da80b28SThomas Gleixner for (cpu = 0; cpu < nr_logical_cpuids; cpu++)
23655da80b28SThomas Gleixner cpu_mark_primary_thread(cpu, cpuid_to_apicid[cpu]);
23665da80b28SThomas Gleixner return 0;
23675da80b28SThomas Gleixner }
23685da80b28SThomas Gleixner early_initcall(smp_init_primary_thread_mask);
2369f54d4434SThomas Gleixner #else
cpu_mark_primary_thread(unsigned int cpu,unsigned int apicid)2370f54d4434SThomas Gleixner static inline void cpu_mark_primary_thread(unsigned int cpu, unsigned int apicid) { }
2371d0055f35SVlastimil Babka #endif
23726a4d2657SThomas Gleixner
23738f54969dSGu Zheng /*
23748f54969dSGu Zheng * Should use this API to allocate logical CPU IDs to keep nr_logical_cpuids
23758f54969dSGu Zheng * and cpuid_to_apicid[] synchronized.
23768f54969dSGu Zheng */
allocate_logical_cpuid(int apicid)23778f54969dSGu Zheng static int allocate_logical_cpuid(int apicid)
23788f54969dSGu Zheng {
23798f54969dSGu Zheng int i;
23808f54969dSGu Zheng
23818f54969dSGu Zheng /*
23828f54969dSGu Zheng * cpuid <-> apicid mapping is persistent, so when a cpu is up,
23838f54969dSGu Zheng * check if the kernel has allocated a cpuid for it.
23848f54969dSGu Zheng */
23858f54969dSGu Zheng for (i = 0; i < nr_logical_cpuids; i++) {
23868f54969dSGu Zheng if (cpuid_to_apicid[i] == apicid)
23878f54969dSGu Zheng return i;
23888f54969dSGu Zheng }
23898f54969dSGu Zheng
23908f54969dSGu Zheng /* Allocate a new cpuid. */
23918f54969dSGu Zheng if (nr_logical_cpuids >= nr_cpu_ids) {
23929b130ad5SAlexey Dobriyan WARN_ONCE(1, "APIC: NR_CPUS/possible_cpus limit of %u reached. "
23938f54969dSGu Zheng "Processor %d/0x%x and the rest are ignored.\n",
2394bb3f0a52SDou Liyang nr_cpu_ids, nr_logical_cpuids, apicid);
2395bb3f0a52SDou Liyang return -EINVAL;
23968f54969dSGu Zheng }
23978f54969dSGu Zheng
23988f54969dSGu Zheng cpuid_to_apicid[nr_logical_cpuids] = apicid;
23998f54969dSGu Zheng return nr_logical_cpuids++;
24008f54969dSGu Zheng }
24018f54969dSGu Zheng
cpu_update_apic(int cpu,int apicid)2402249ada2cSThomas Gleixner static void cpu_update_apic(int cpu, int apicid)
2403f62bae50SIngo Molnar {
2404f62bae50SIngo Molnar #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
2405f62bae50SIngo Molnar early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
2406f62bae50SIngo Molnar #endif
2407f62bae50SIngo Molnar set_cpu_possible(cpu, true);
2408f7c28833SGu Zheng physid_set(apicid, phys_cpu_present_map);
2409f62bae50SIngo Molnar set_cpu_present(cpu, true);
24102b85b3d2SDou Liyang num_processors++;
24117e1f85f9SJiang Liu
24125da80b28SThomas Gleixner if (system_state != SYSTEM_BOOTING)
2413f54d4434SThomas Gleixner cpu_mark_primary_thread(cpu, apicid);
2414d63107faSThomas Gleixner }
2415f54d4434SThomas Gleixner
cpu_set_boot_apic(void)2416d63107faSThomas Gleixner static __init void cpu_set_boot_apic(void)
2417d63107faSThomas Gleixner {
2418d63107faSThomas Gleixner cpuid_to_apicid[0] = boot_cpu_physical_apicid;
2419249ada2cSThomas Gleixner cpu_update_apic(0, boot_cpu_physical_apicid);
242079c9a17cSThomas Gleixner x86_32_probe_bigsmp_early();
2421d63107faSThomas Gleixner }
2422d63107faSThomas Gleixner
generic_processor_info(int apicid)2423249ada2cSThomas Gleixner int generic_processor_info(int apicid)
2424d63107faSThomas Gleixner {
2425d63107faSThomas Gleixner int cpu, max = nr_cpu_ids;
2426d63107faSThomas Gleixner
2427d63107faSThomas Gleixner /* The boot CPU must be set before MADT/MPTABLE parsing happens */
2428d63107faSThomas Gleixner if (cpuid_to_apicid[0] == BAD_APICID)
2429d63107faSThomas Gleixner panic("Boot CPU APIC not registered yet\n");
2430d63107faSThomas Gleixner
2431d63107faSThomas Gleixner if (apicid == boot_cpu_physical_apicid)
2432d63107faSThomas Gleixner return 0;
2433d63107faSThomas Gleixner
2434d63107faSThomas Gleixner if (disabled_cpu_apicid == apicid) {
2435d63107faSThomas Gleixner int thiscpu = num_processors + disabled_cpus;
2436d63107faSThomas Gleixner
2437d63107faSThomas Gleixner pr_warn("APIC: Disabling requested cpu. Processor %d/0x%x ignored.\n",
2438d63107faSThomas Gleixner thiscpu, apicid);
2439d63107faSThomas Gleixner
2440d63107faSThomas Gleixner disabled_cpus++;
2441d63107faSThomas Gleixner return -ENODEV;
2442d63107faSThomas Gleixner }
2443d63107faSThomas Gleixner
2444d63107faSThomas Gleixner if (num_processors >= nr_cpu_ids) {
2445d63107faSThomas Gleixner int thiscpu = max + disabled_cpus;
2446d63107faSThomas Gleixner
2447d63107faSThomas Gleixner pr_warn("APIC: NR_CPUS/possible_cpus limit of %i reached. "
2448d63107faSThomas Gleixner "Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2449d63107faSThomas Gleixner
2450d63107faSThomas Gleixner disabled_cpus++;
2451d63107faSThomas Gleixner return -EINVAL;
2452d63107faSThomas Gleixner }
2453d63107faSThomas Gleixner
2454d63107faSThomas Gleixner cpu = allocate_logical_cpuid(apicid);
2455d63107faSThomas Gleixner if (cpu < 0) {
2456d63107faSThomas Gleixner disabled_cpus++;
2457d63107faSThomas Gleixner return -EINVAL;
2458d63107faSThomas Gleixner }
2459d63107faSThomas Gleixner
2460249ada2cSThomas Gleixner cpu_update_apic(cpu, apicid);
24617e1f85f9SJiang Liu return cpu;
2462f62bae50SIngo Molnar }
2463f62bae50SIngo Molnar
2464d63107faSThomas Gleixner
__irq_msi_compose_msg(struct irq_cfg * cfg,struct msi_msg * msg,bool dmar)2465f598181aSDavid Woodhouse void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg,
2466f598181aSDavid Woodhouse bool dmar)
2467f598181aSDavid Woodhouse {
24686285aa50SThomas Gleixner memset(msg, 0, sizeof(*msg));
2469f598181aSDavid Woodhouse
24706285aa50SThomas Gleixner msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
24716285aa50SThomas Gleixner msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical;
24726285aa50SThomas Gleixner msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF;
2473f598181aSDavid Woodhouse
24746285aa50SThomas Gleixner msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED;
24756285aa50SThomas Gleixner msg->arch_data.vector = cfg->vector;
2476f598181aSDavid Woodhouse
24776285aa50SThomas Gleixner msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
2478f598181aSDavid Woodhouse /*
2479f598181aSDavid Woodhouse * Only the IOMMU itself can use the trick of putting destination
2480f598181aSDavid Woodhouse * APIC ID into the high bits of the address. Anything else would
2481f598181aSDavid Woodhouse * just be writing to memory if it tried that, and needs IR to
2482ab0f59c6SDavid Woodhouse * address APICs which can't be addressed in the normal 32-bit
2483ab0f59c6SDavid Woodhouse * address range at 0xFFExxxxx. That is typically just 8 bits, but
2484ab0f59c6SDavid Woodhouse * some hypervisors allow the extended destination ID field in bits
2485ab0f59c6SDavid Woodhouse * 5-11 to be used, giving support for 15 bits of APIC IDs in total.
2486f598181aSDavid Woodhouse */
2487f598181aSDavid Woodhouse if (dmar)
24886285aa50SThomas Gleixner msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8;
2489ab0f59c6SDavid Woodhouse else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000)
2490ab0f59c6SDavid Woodhouse msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8;
2491f598181aSDavid Woodhouse else
24926285aa50SThomas Gleixner WARN_ON_ONCE(cfg->dest_apicid > 0xFF);
2493f598181aSDavid Woodhouse }
2494f598181aSDavid Woodhouse
x86_msi_msg_get_destid(struct msi_msg * msg,bool extid)24956285aa50SThomas Gleixner u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid)
24966285aa50SThomas Gleixner {
24976285aa50SThomas Gleixner u32 dest = msg->arch_addr_lo.destid_0_7;
24986285aa50SThomas Gleixner
24996285aa50SThomas Gleixner if (extid)
25006285aa50SThomas Gleixner dest |= msg->arch_addr_hi.destid_8_31 << 8;
25016285aa50SThomas Gleixner return dest;
25026285aa50SThomas Gleixner }
25036285aa50SThomas Gleixner EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid);
25046285aa50SThomas Gleixner
apic_bsp_up_setup(void)2505374aab33SThomas Gleixner static void __init apic_bsp_up_setup(void)
250605f7e46dSThomas Gleixner {
2507374aab33SThomas Gleixner #ifdef CONFIG_X86_64
25085d64d209SDou Liyang apic_write(APIC_ID, apic->set_apic_id(boot_cpu_physical_apicid));
2509374aab33SThomas Gleixner #endif
2510374aab33SThomas Gleixner physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
251105f7e46dSThomas Gleixner }
251205f7e46dSThomas Gleixner
251305f7e46dSThomas Gleixner /**
251405f7e46dSThomas Gleixner * apic_bsp_setup - Setup function for local apic and io-apic
2515374aab33SThomas Gleixner * @upmode: Force UP mode (for APIC_init_uniprocessor)
251605f7e46dSThomas Gleixner */
apic_bsp_setup(bool upmode)2517748b170cSThomas Gleixner static void __init apic_bsp_setup(bool upmode)
251805f7e46dSThomas Gleixner {
251905f7e46dSThomas Gleixner connect_bsp_APIC();
2520374aab33SThomas Gleixner if (upmode)
2521374aab33SThomas Gleixner apic_bsp_up_setup();
252205f7e46dSThomas Gleixner setup_local_APIC();
252305f7e46dSThomas Gleixner
252405f7e46dSThomas Gleixner enable_IO_APIC();
2525374aab33SThomas Gleixner end_local_APIC_setup();
2526374aab33SThomas Gleixner irq_remap_enable_fault_handling();
252705f7e46dSThomas Gleixner setup_IO_APIC();
25287d65f9e8SThomas Gleixner lapic_update_legacy_vectors();
2529e714a91fSThomas Gleixner }
2530e714a91fSThomas Gleixner
253130b8b006SThomas Gleixner #ifdef CONFIG_UP_LATE_INIT
up_late_init(void)253230b8b006SThomas Gleixner void __init up_late_init(void)
253330b8b006SThomas Gleixner {
25340c759131SDou Liyang if (apic_intr_mode == APIC_PIC)
25350c759131SDou Liyang return;
25360c759131SDou Liyang
25370c759131SDou Liyang /* Setup local timer */
25380c759131SDou Liyang x86_init.timers.setup_percpu_clockev();
253930b8b006SThomas Gleixner }
254030b8b006SThomas Gleixner #endif
254130b8b006SThomas Gleixner
2542e714a91fSThomas Gleixner /*
2543f62bae50SIngo Molnar * Power management
2544f62bae50SIngo Molnar */
2545f62bae50SIngo Molnar #ifdef CONFIG_PM
2546f62bae50SIngo Molnar
2547f62bae50SIngo Molnar static struct {
2548f62bae50SIngo Molnar /*
2549f62bae50SIngo Molnar * 'active' is true if the local APIC was enabled by us and
2550f62bae50SIngo Molnar * not the BIOS; this signifies that we are also responsible
2551f62bae50SIngo Molnar * for disabling it before entering apm/acpi suspend
2552f62bae50SIngo Molnar */
2553f62bae50SIngo Molnar int active;
2554f62bae50SIngo Molnar /* r/w apic fields */
2555f62bae50SIngo Molnar unsigned int apic_id;
2556f62bae50SIngo Molnar unsigned int apic_taskpri;
2557f62bae50SIngo Molnar unsigned int apic_ldr;
2558f62bae50SIngo Molnar unsigned int apic_dfr;
2559f62bae50SIngo Molnar unsigned int apic_spiv;
2560f62bae50SIngo Molnar unsigned int apic_lvtt;
2561f62bae50SIngo Molnar unsigned int apic_lvtpc;
2562f62bae50SIngo Molnar unsigned int apic_lvt0;
2563f62bae50SIngo Molnar unsigned int apic_lvt1;
2564f62bae50SIngo Molnar unsigned int apic_lvterr;
2565f62bae50SIngo Molnar unsigned int apic_tmict;
2566f62bae50SIngo Molnar unsigned int apic_tdcr;
2567f62bae50SIngo Molnar unsigned int apic_thmr;
256842baa258SJuergen Gross unsigned int apic_cmci;
2569f62bae50SIngo Molnar } apic_pm_state;
2570f62bae50SIngo Molnar
lapic_suspend(void)2571f3c6ea1bSRafael J. Wysocki static int lapic_suspend(void)
2572f62bae50SIngo Molnar {
2573f62bae50SIngo Molnar unsigned long flags;
2574f62bae50SIngo Molnar int maxlvt;
2575f62bae50SIngo Molnar
2576f62bae50SIngo Molnar if (!apic_pm_state.active)
2577f62bae50SIngo Molnar return 0;
2578f62bae50SIngo Molnar
2579f62bae50SIngo Molnar maxlvt = lapic_get_maxlvt();
2580f62bae50SIngo Molnar
2581f62bae50SIngo Molnar apic_pm_state.apic_id = apic_read(APIC_ID);
2582f62bae50SIngo Molnar apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2583f62bae50SIngo Molnar apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2584f62bae50SIngo Molnar apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2585f62bae50SIngo Molnar apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2586f62bae50SIngo Molnar apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
2587f62bae50SIngo Molnar if (maxlvt >= 4)
2588f62bae50SIngo Molnar apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
2589f62bae50SIngo Molnar apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2590f62bae50SIngo Molnar apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2591f62bae50SIngo Molnar apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2592f62bae50SIngo Molnar apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2593f62bae50SIngo Molnar apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
25944efc0670SAndi Kleen #ifdef CONFIG_X86_THERMAL_VECTOR
2595f62bae50SIngo Molnar if (maxlvt >= 5)
2596f62bae50SIngo Molnar apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2597f62bae50SIngo Molnar #endif
259842baa258SJuergen Gross #ifdef CONFIG_X86_MCE_INTEL
259942baa258SJuergen Gross if (maxlvt >= 6)
260042baa258SJuergen Gross apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI);
260142baa258SJuergen Gross #endif
2602f62bae50SIngo Molnar
2603f62bae50SIngo Molnar local_irq_save(flags);
26040f378d73STony W Wang-oc
26050f378d73STony W Wang-oc /*
26060f378d73STony W Wang-oc * Mask IOAPIC before disabling the local APIC to prevent stale IRR
26070f378d73STony W Wang-oc * entries on some implementations.
26080f378d73STony W Wang-oc */
26090f378d73STony W Wang-oc mask_ioapic_entries();
26100f378d73STony W Wang-oc
2611f62bae50SIngo Molnar disable_local_APIC();
2612fc1edaf9SSuresh Siddha
261395a02e97SSuresh Siddha irq_remapping_disable();
2614fc1edaf9SSuresh Siddha
2615f62bae50SIngo Molnar local_irq_restore(flags);
2616f62bae50SIngo Molnar return 0;
2617f62bae50SIngo Molnar }
2618f62bae50SIngo Molnar
lapic_resume(void)2619f3c6ea1bSRafael J. Wysocki static void lapic_resume(void)
2620f62bae50SIngo Molnar {
2621f62bae50SIngo Molnar unsigned int l, h;
2622f62bae50SIngo Molnar unsigned long flags;
262331dce14aSSuresh Siddha int maxlvt;
2624b24696bcSFenghua Yu
2625f62bae50SIngo Molnar if (!apic_pm_state.active)
2626f3c6ea1bSRafael J. Wysocki return;
2627f62bae50SIngo Molnar
2628b24696bcSFenghua Yu local_irq_save(flags);
2629336224baSJoerg Roedel
263031dce14aSSuresh Siddha /*
263131dce14aSSuresh Siddha * IO-APIC and PIC have their own resume routines.
263231dce14aSSuresh Siddha * We just mask them here to make sure the interrupt
263331dce14aSSuresh Siddha * subsystem is completely quiet while we enable x2apic
263431dce14aSSuresh Siddha * and interrupt-remapping.
263531dce14aSSuresh Siddha */
263631dce14aSSuresh Siddha mask_ioapic_entries();
2637b81bb373SJacob Pan legacy_pic->mask_all();
2638f62bae50SIngo Molnar
2639659006bfSThomas Gleixner if (x2apic_mode) {
2640659006bfSThomas Gleixner __x2apic_enable();
2641659006bfSThomas Gleixner } else {
2642f62bae50SIngo Molnar /*
2643f62bae50SIngo Molnar * Make sure the APICBASE points to the right address
2644f62bae50SIngo Molnar *
2645f62bae50SIngo Molnar * FIXME! This will be wrong if we ever support suspend on
2646f62bae50SIngo Molnar * SMP! We'll need to do this as part of the CPU restore!
2647f62bae50SIngo Molnar */
2648cbf2829bSBryan O'Donoghue if (boot_cpu_data.x86 >= 6) {
2649f62bae50SIngo Molnar rdmsr(MSR_IA32_APICBASE, l, h);
2650f62bae50SIngo Molnar l &= ~MSR_IA32_APICBASE_BASE;
2651f62bae50SIngo Molnar l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2652f62bae50SIngo Molnar wrmsr(MSR_IA32_APICBASE, l, h);
2653f62bae50SIngo Molnar }
2654cbf2829bSBryan O'Donoghue }
2655f62bae50SIngo Molnar
2656b24696bcSFenghua Yu maxlvt = lapic_get_maxlvt();
2657f62bae50SIngo Molnar apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2658f62bae50SIngo Molnar apic_write(APIC_ID, apic_pm_state.apic_id);
2659f62bae50SIngo Molnar apic_write(APIC_DFR, apic_pm_state.apic_dfr);
2660f62bae50SIngo Molnar apic_write(APIC_LDR, apic_pm_state.apic_ldr);
2661f62bae50SIngo Molnar apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
2662f62bae50SIngo Molnar apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
2663f62bae50SIngo Molnar apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
2664f62bae50SIngo Molnar apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
266542baa258SJuergen Gross #ifdef CONFIG_X86_THERMAL_VECTOR
2666f62bae50SIngo Molnar if (maxlvt >= 5)
2667f62bae50SIngo Molnar apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
2668f62bae50SIngo Molnar #endif
266942baa258SJuergen Gross #ifdef CONFIG_X86_MCE_INTEL
267042baa258SJuergen Gross if (maxlvt >= 6)
267142baa258SJuergen Gross apic_write(APIC_LVTCMCI, apic_pm_state.apic_cmci);
267242baa258SJuergen Gross #endif
2673f62bae50SIngo Molnar if (maxlvt >= 4)
2674f62bae50SIngo Molnar apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
2675f62bae50SIngo Molnar apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
2676f62bae50SIngo Molnar apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
2677f62bae50SIngo Molnar apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
2678f62bae50SIngo Molnar apic_write(APIC_ESR, 0);
2679f62bae50SIngo Molnar apic_read(APIC_ESR);
2680f62bae50SIngo Molnar apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
2681f62bae50SIngo Molnar apic_write(APIC_ESR, 0);
2682f62bae50SIngo Molnar apic_read(APIC_ESR);
2683f62bae50SIngo Molnar
268495a02e97SSuresh Siddha irq_remapping_reenable(x2apic_mode);
268531dce14aSSuresh Siddha
2686f62bae50SIngo Molnar local_irq_restore(flags);
2687f62bae50SIngo Molnar }
2688f62bae50SIngo Molnar
2689f62bae50SIngo Molnar /*
2690f62bae50SIngo Molnar * This device has no shutdown method - fully functioning local APICs
2691f62bae50SIngo Molnar * are needed on every CPU up until machine_halt/restart/poweroff.
2692f62bae50SIngo Molnar */
2693f62bae50SIngo Molnar
2694f3c6ea1bSRafael J. Wysocki static struct syscore_ops lapic_syscore_ops = {
2695f62bae50SIngo Molnar .resume = lapic_resume,
2696f62bae50SIngo Molnar .suspend = lapic_suspend,
2697f62bae50SIngo Molnar };
2698f62bae50SIngo Molnar
apic_pm_activate(void)2699148f9bb8SPaul Gortmaker static void apic_pm_activate(void)
2700f62bae50SIngo Molnar {
2701f62bae50SIngo Molnar apic_pm_state.active = 1;
2702f62bae50SIngo Molnar }
2703f62bae50SIngo Molnar
init_lapic_sysfs(void)2704f62bae50SIngo Molnar static int __init init_lapic_sysfs(void)
2705f62bae50SIngo Molnar {
2706f62bae50SIngo Molnar /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
270793984fbdSBorislav Petkov if (boot_cpu_has(X86_FEATURE_APIC))
2708f3c6ea1bSRafael J. Wysocki register_syscore_ops(&lapic_syscore_ops);
2709f62bae50SIngo Molnar
2710f3c6ea1bSRafael J. Wysocki return 0;
2711f62bae50SIngo Molnar }
2712b24696bcSFenghua Yu
2713b24696bcSFenghua Yu /* local apic needs to resume before other devices access its registers. */
2714b24696bcSFenghua Yu core_initcall(init_lapic_sysfs);
2715f62bae50SIngo Molnar
2716f62bae50SIngo Molnar #else /* CONFIG_PM */
2717f62bae50SIngo Molnar
apic_pm_activate(void)2718f62bae50SIngo Molnar static void apic_pm_activate(void) { }
2719f62bae50SIngo Molnar
2720f62bae50SIngo Molnar #endif /* CONFIG_PM */
2721f62bae50SIngo Molnar
2722f62bae50SIngo Molnar #ifdef CONFIG_X86_64
2723e0e42142SYinghai Lu
2724148f9bb8SPaul Gortmaker static int multi_checked;
2725148f9bb8SPaul Gortmaker static int multi;
2726e0e42142SYinghai Lu
set_multi(const struct dmi_system_id * d)2727148f9bb8SPaul Gortmaker static int set_multi(const struct dmi_system_id *d)
2728e0e42142SYinghai Lu {
2729e0e42142SYinghai Lu if (multi)
2730e0e42142SYinghai Lu return 0;
27316f0aced6SCyrill Gorcunov pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
2732e0e42142SYinghai Lu multi = 1;
2733e0e42142SYinghai Lu return 0;
2734e0e42142SYinghai Lu }
2735e0e42142SYinghai Lu
2736148f9bb8SPaul Gortmaker static const struct dmi_system_id multi_dmi_table[] = {
2737e0e42142SYinghai Lu {
2738e0e42142SYinghai Lu .callback = set_multi,
2739e0e42142SYinghai Lu .ident = "IBM System Summit2",
2740e0e42142SYinghai Lu .matches = {
2741e0e42142SYinghai Lu DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2742e0e42142SYinghai Lu DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2743e0e42142SYinghai Lu },
2744e0e42142SYinghai Lu },
2745e0e42142SYinghai Lu {}
2746e0e42142SYinghai Lu };
2747e0e42142SYinghai Lu
dmi_check_multi(void)2748148f9bb8SPaul Gortmaker static void dmi_check_multi(void)
2749e0e42142SYinghai Lu {
2750e0e42142SYinghai Lu if (multi_checked)
2751e0e42142SYinghai Lu return;
2752e0e42142SYinghai Lu
2753e0e42142SYinghai Lu dmi_check_system(multi_dmi_table);
2754e0e42142SYinghai Lu multi_checked = 1;
2755e0e42142SYinghai Lu }
2756f62bae50SIngo Molnar
2757f62bae50SIngo Molnar /*
2758e0e42142SYinghai Lu * apic_is_clustered_box() -- Check if we can expect good TSC
2759e0e42142SYinghai Lu *
2760e0e42142SYinghai Lu * Thus far, the major user of this is IBM's Summit2 series:
2761e0e42142SYinghai Lu * Clustered boxes may have unsynced TSC problems if they are
2762e0e42142SYinghai Lu * multi-chassis.
2763e0e42142SYinghai Lu * Use DMI to check them
2764f62bae50SIngo Molnar */
apic_is_clustered_box(void)2765148f9bb8SPaul Gortmaker int apic_is_clustered_box(void)
2766e0e42142SYinghai Lu {
2767e0e42142SYinghai Lu dmi_check_multi();
2768411cf9eeSOren Twaig return multi;
2769f62bae50SIngo Molnar }
2770f62bae50SIngo Molnar #endif
2771f62bae50SIngo Molnar
2772f62bae50SIngo Molnar /*
2773f62bae50SIngo Molnar * APIC command line parameters
2774f62bae50SIngo Molnar */
setup_disableapic(char * arg)2775f62bae50SIngo Molnar static int __init setup_disableapic(char *arg)
2776f62bae50SIngo Molnar {
277749062454SThomas Gleixner apic_is_disabled = true;
2778f62bae50SIngo Molnar setup_clear_cpu_cap(X86_FEATURE_APIC);
2779f62bae50SIngo Molnar return 0;
2780f62bae50SIngo Molnar }
2781f62bae50SIngo Molnar early_param("disableapic", setup_disableapic);
2782f62bae50SIngo Molnar
2783f62bae50SIngo Molnar /* same as disableapic, for compatibility */
setup_nolapic(char * arg)2784f62bae50SIngo Molnar static int __init setup_nolapic(char *arg)
2785f62bae50SIngo Molnar {
2786f62bae50SIngo Molnar return setup_disableapic(arg);
2787f62bae50SIngo Molnar }
2788f62bae50SIngo Molnar early_param("nolapic", setup_nolapic);
2789f62bae50SIngo Molnar
parse_lapic_timer_c2_ok(char * arg)2790f62bae50SIngo Molnar static int __init parse_lapic_timer_c2_ok(char *arg)
2791f62bae50SIngo Molnar {
2792f62bae50SIngo Molnar local_apic_timer_c2_ok = 1;
2793f62bae50SIngo Molnar return 0;
2794f62bae50SIngo Molnar }
2795f62bae50SIngo Molnar early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2796f62bae50SIngo Molnar
parse_disable_apic_timer(char * arg)2797f62bae50SIngo Molnar static int __init parse_disable_apic_timer(char *arg)
2798f62bae50SIngo Molnar {
2799f62bae50SIngo Molnar disable_apic_timer = 1;
2800f62bae50SIngo Molnar return 0;
2801f62bae50SIngo Molnar }
2802f62bae50SIngo Molnar early_param("noapictimer", parse_disable_apic_timer);
2803f62bae50SIngo Molnar
parse_nolapic_timer(char * arg)2804f62bae50SIngo Molnar static int __init parse_nolapic_timer(char *arg)
2805f62bae50SIngo Molnar {
2806f62bae50SIngo Molnar disable_apic_timer = 1;
2807f62bae50SIngo Molnar return 0;
2808f62bae50SIngo Molnar }
2809f62bae50SIngo Molnar early_param("nolapic_timer", parse_nolapic_timer);
2810f62bae50SIngo Molnar
apic_set_verbosity(char * arg)2811f62bae50SIngo Molnar static int __init apic_set_verbosity(char *arg)
2812f62bae50SIngo Molnar {
2813f62bae50SIngo Molnar if (!arg) {
2814ecf600f8SThomas Gleixner if (IS_ENABLED(CONFIG_X86_32))
2815f62bae50SIngo Molnar return -EINVAL;
2816ecf600f8SThomas Gleixner
2817ecf600f8SThomas Gleixner ioapic_is_disabled = false;
2818ecf600f8SThomas Gleixner return 0;
2819f62bae50SIngo Molnar }
2820f62bae50SIngo Molnar
2821f62bae50SIngo Molnar if (strcmp("debug", arg) == 0)
2822f62bae50SIngo Molnar apic_verbosity = APIC_DEBUG;
2823f62bae50SIngo Molnar else if (strcmp("verbose", arg) == 0)
2824f62bae50SIngo Molnar apic_verbosity = APIC_VERBOSE;
28254fcab669SDou Liyang #ifdef CONFIG_X86_64
2826f62bae50SIngo Molnar else {
28278d3bcc44SKefeng Wang pr_warn("APIC Verbosity level %s not recognised"
2828f62bae50SIngo Molnar " use apic=verbose or apic=debug\n", arg);
2829f62bae50SIngo Molnar return -EINVAL;
2830f62bae50SIngo Molnar }
28314fcab669SDou Liyang #endif
2832f62bae50SIngo Molnar
2833f62bae50SIngo Molnar return 0;
2834f62bae50SIngo Molnar }
2835f62bae50SIngo Molnar early_param("apic", apic_set_verbosity);
2836f62bae50SIngo Molnar
lapic_insert_resource(void)2837f62bae50SIngo Molnar static int __init lapic_insert_resource(void)
2838f62bae50SIngo Molnar {
283978c32000SThomas Gleixner if (!apic_mmio_base)
2840f62bae50SIngo Molnar return -1;
2841f62bae50SIngo Molnar
2842f62bae50SIngo Molnar /* Put local APIC into the resource map. */
284378c32000SThomas Gleixner lapic_resource.start = apic_mmio_base;
2844f62bae50SIngo Molnar lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2845f62bae50SIngo Molnar insert_resource(&iomem_resource, &lapic_resource);
2846f62bae50SIngo Molnar
2847f62bae50SIngo Molnar return 0;
2848f62bae50SIngo Molnar }
2849f62bae50SIngo Molnar
2850f62bae50SIngo Molnar /*
28511506c8dcSIngo Molnar * need call insert after e820__reserve_resources()
2852f62bae50SIngo Molnar * that is using request_resource
2853f62bae50SIngo Molnar */
2854f62bae50SIngo Molnar late_initcall(lapic_insert_resource);
2855151e0c7dSHATAYAMA Daisuke
apic_set_disabled_cpu_apicid(char * arg)2856151e0c7dSHATAYAMA Daisuke static int __init apic_set_disabled_cpu_apicid(char *arg)
2857151e0c7dSHATAYAMA Daisuke {
2858151e0c7dSHATAYAMA Daisuke if (!arg || !get_option(&arg, &disabled_cpu_apicid))
2859151e0c7dSHATAYAMA Daisuke return -EINVAL;
2860151e0c7dSHATAYAMA Daisuke
2861151e0c7dSHATAYAMA Daisuke return 0;
2862151e0c7dSHATAYAMA Daisuke }
2863151e0c7dSHATAYAMA Daisuke early_param("disable_cpu_apicid", apic_set_disabled_cpu_apicid);
2864b7c4948eSHidehiro Kawai
apic_set_extnmi(char * arg)2865b7c4948eSHidehiro Kawai static int __init apic_set_extnmi(char *arg)
2866b7c4948eSHidehiro Kawai {
2867b7c4948eSHidehiro Kawai if (!arg)
2868b7c4948eSHidehiro Kawai return -EINVAL;
2869b7c4948eSHidehiro Kawai
2870b7c4948eSHidehiro Kawai if (!strncmp("all", arg, 3))
2871b7c4948eSHidehiro Kawai apic_extnmi = APIC_EXTNMI_ALL;
2872b7c4948eSHidehiro Kawai else if (!strncmp("none", arg, 4))
2873b7c4948eSHidehiro Kawai apic_extnmi = APIC_EXTNMI_NONE;
2874b7c4948eSHidehiro Kawai else if (!strncmp("bsp", arg, 3))
2875b7c4948eSHidehiro Kawai apic_extnmi = APIC_EXTNMI_BSP;
2876b7c4948eSHidehiro Kawai else {
2877b7c4948eSHidehiro Kawai pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg);
2878b7c4948eSHidehiro Kawai return -EINVAL;
2879b7c4948eSHidehiro Kawai }
2880b7c4948eSHidehiro Kawai
2881b7c4948eSHidehiro Kawai return 0;
2882b7c4948eSHidehiro Kawai }
2883b7c4948eSHidehiro Kawai early_param("apic_extnmi", apic_set_extnmi);
2884