xref: /openbmc/linux/arch/x86/include/asm/idtentry.h (revision 32e93cae)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_IDTENTRY_H
3 #define _ASM_X86_IDTENTRY_H
4 
5 /* Interrupts/Exceptions */
6 #include <asm/trapnr.h>
7 
8 #define IDT_ALIGN	(8 * (1 + HAS_KERNEL_IBT))
9 
10 #ifndef __ASSEMBLY__
11 #include <linux/entry-common.h>
12 #include <linux/hardirq.h>
13 
14 #include <asm/irq_stack.h>
15 
16 typedef void (*idtentry_t)(struct pt_regs *regs);
17 
18 /**
19  * DECLARE_IDTENTRY - Declare functions for simple IDT entry points
20  *		      No error code pushed by hardware
21  * @vector:	Vector number (ignored for C)
22  * @func:	Function name of the entry point
23  *
24  * Declares four functions:
25  * - The ASM entry point: asm_##func
26  * - The XEN PV trap entry point: xen_##func (maybe unused)
27  * - The C handler called from the FRED event dispatcher (maybe unused)
28  * - The C handler called from the ASM entry point
29  *
30  * Note: This is the C variant of DECLARE_IDTENTRY(). As the name says it
31  * declares the entry points for usage in C code. There is an ASM variant
32  * as well which is used to emit the entry stubs in entry_32/64.S.
33  */
34 #define DECLARE_IDTENTRY(vector, func)					\
35 	asmlinkage void asm_##func(void);				\
36 	asmlinkage void xen_asm_##func(void);				\
37 	void fred_##func(struct pt_regs *regs);				\
38 	__visible void func(struct pt_regs *regs)
39 
40 /**
41  * DEFINE_IDTENTRY - Emit code for simple IDT entry points
42  * @func:	Function name of the entry point
43  *
44  * @func is called from ASM entry code with interrupts disabled.
45  *
46  * The macro is written so it acts as function definition. Append the
47  * body with a pair of curly brackets.
48  *
49  * irqentry_enter() contains common code which has to be invoked before
50  * arbitrary code in the body. irqentry_exit() contains common code
51  * which has to run before returning to the low level assembly code.
52  */
53 #define DEFINE_IDTENTRY(func)						\
54 static __always_inline void __##func(struct pt_regs *regs);		\
55 									\
56 __visible noinstr void func(struct pt_regs *regs)			\
57 {									\
58 	irqentry_state_t state = irqentry_enter(regs);			\
59 									\
60 	instrumentation_begin();					\
61 	__##func (regs);						\
62 	instrumentation_end();						\
63 	irqentry_exit(regs, state);					\
64 }									\
65 									\
66 static __always_inline void __##func(struct pt_regs *regs)
67 
68 /* Special case for 32bit IRET 'trap' */
69 #define DECLARE_IDTENTRY_SW	DECLARE_IDTENTRY
70 #define DEFINE_IDTENTRY_SW	DEFINE_IDTENTRY
71 
72 /**
73  * DECLARE_IDTENTRY_ERRORCODE - Declare functions for simple IDT entry points
74  *				Error code pushed by hardware
75  * @vector:	Vector number (ignored for C)
76  * @func:	Function name of the entry point
77  *
78  * Declares three functions:
79  * - The ASM entry point: asm_##func
80  * - The XEN PV trap entry point: xen_##func (maybe unused)
81  * - The C handler called from the ASM entry point
82  *
83  * Same as DECLARE_IDTENTRY, but has an extra error_code argument for the
84  * C-handler.
85  */
86 #define DECLARE_IDTENTRY_ERRORCODE(vector, func)			\
87 	asmlinkage void asm_##func(void);				\
88 	asmlinkage void xen_asm_##func(void);				\
89 	__visible void func(struct pt_regs *regs, unsigned long error_code)
90 
91 /**
92  * DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points
93  *			       Error code pushed by hardware
94  * @func:	Function name of the entry point
95  *
96  * Same as DEFINE_IDTENTRY, but has an extra error_code argument
97  */
98 #define DEFINE_IDTENTRY_ERRORCODE(func)					\
99 static __always_inline void __##func(struct pt_regs *regs,		\
100 				     unsigned long error_code);		\
101 									\
102 __visible noinstr void func(struct pt_regs *regs,			\
103 			    unsigned long error_code)			\
104 {									\
105 	irqentry_state_t state = irqentry_enter(regs);			\
106 									\
107 	instrumentation_begin();					\
108 	__##func (regs, error_code);					\
109 	instrumentation_end();						\
110 	irqentry_exit(regs, state);					\
111 }									\
112 									\
113 static __always_inline void __##func(struct pt_regs *regs,		\
114 				     unsigned long error_code)
115 
116 /**
117  * DECLARE_IDTENTRY_RAW - Declare functions for raw IDT entry points
118  *		      No error code pushed by hardware
119  * @vector:	Vector number (ignored for C)
120  * @func:	Function name of the entry point
121  *
122  * Maps to DECLARE_IDTENTRY().
123  */
124 #define DECLARE_IDTENTRY_RAW(vector, func)				\
125 	DECLARE_IDTENTRY(vector, func)
126 
127 /**
128  * DEFINE_IDTENTRY_RAW - Emit code for raw IDT entry points
129  * @func:	Function name of the entry point
130  *
131  * @func is called from ASM entry code with interrupts disabled.
132  *
133  * The macro is written so it acts as function definition. Append the
134  * body with a pair of curly brackets.
135  *
136  * Contrary to DEFINE_IDTENTRY() this does not invoke the
137  * idtentry_enter/exit() helpers before and after the body invocation. This
138  * needs to be done in the body itself if applicable. Use if extra work
139  * is required before the enter/exit() helpers are invoked.
140  */
141 #define DEFINE_IDTENTRY_RAW(func)					\
142 __visible noinstr void func(struct pt_regs *regs)
143 
144 /**
145  * DEFINE_FREDENTRY_RAW - Emit code for raw FRED entry points
146  * @func:	Function name of the entry point
147  *
148  * @func is called from the FRED event dispatcher with interrupts disabled.
149  *
150  * See @DEFINE_IDTENTRY_RAW for further details.
151  */
152 #define DEFINE_FREDENTRY_RAW(func)					\
153 noinstr void fred_##func(struct pt_regs *regs)
154 
155 /**
156  * DECLARE_IDTENTRY_RAW_ERRORCODE - Declare functions for raw IDT entry points
157  *				    Error code pushed by hardware
158  * @vector:	Vector number (ignored for C)
159  * @func:	Function name of the entry point
160  *
161  * Maps to DECLARE_IDTENTRY_ERRORCODE()
162  */
163 #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func)			\
164 	DECLARE_IDTENTRY_ERRORCODE(vector, func)
165 
166 /**
167  * DEFINE_IDTENTRY_RAW_ERRORCODE - Emit code for raw IDT entry points
168  * @func:	Function name of the entry point
169  *
170  * @func is called from ASM entry code with interrupts disabled.
171  *
172  * The macro is written so it acts as function definition. Append the
173  * body with a pair of curly brackets.
174  *
175  * Contrary to DEFINE_IDTENTRY_ERRORCODE() this does not invoke the
176  * irqentry_enter/exit() helpers before and after the body invocation. This
177  * needs to be done in the body itself if applicable. Use if extra work
178  * is required before the enter/exit() helpers are invoked.
179  */
180 #define DEFINE_IDTENTRY_RAW_ERRORCODE(func)				\
181 __visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
182 
183 /**
184  * DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry
185  *			  points (common/spurious)
186  * @vector:	Vector number (ignored for C)
187  * @func:	Function name of the entry point
188  *
189  * Maps to DECLARE_IDTENTRY_ERRORCODE()
190  */
191 #define DECLARE_IDTENTRY_IRQ(vector, func)				\
192 	DECLARE_IDTENTRY_ERRORCODE(vector, func)
193 
194 /**
195  * DEFINE_IDTENTRY_IRQ - Emit code for device interrupt IDT entry points
196  * @func:	Function name of the entry point
197  *
198  * The vector number is pushed by the low level entry stub and handed
199  * to the function as error_code argument which needs to be truncated
200  * to an u8 because the push is sign extending.
201  *
202  * irq_enter/exit_rcu() are invoked before the function body and the
203  * KVM L1D flush request is set. Stack switching to the interrupt stack
204  * has to be done in the function body if necessary.
205  */
206 #define DEFINE_IDTENTRY_IRQ(func)					\
207 static void __##func(struct pt_regs *regs, u32 vector);			\
208 									\
209 __visible noinstr void func(struct pt_regs *regs,			\
210 			    unsigned long error_code)			\
211 {									\
212 	irqentry_state_t state = irqentry_enter(regs);			\
213 	u32 vector = (u32)(u8)error_code;				\
214 									\
215 	kvm_set_cpu_l1tf_flush_l1d();                                   \
216 	instrumentation_begin();					\
217 	run_irq_on_irqstack_cond(__##func, regs, vector);		\
218 	instrumentation_end();						\
219 	irqentry_exit(regs, state);					\
220 }									\
221 									\
222 static noinline void __##func(struct pt_regs *regs, u32 vector)
223 
224 /**
225  * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points
226  * @vector:	Vector number (ignored for C)
227  * @func:	Function name of the entry point
228  *
229  * Declares three functions:
230  * - The ASM entry point: asm_##func
231  * - The XEN PV trap entry point: xen_##func (maybe unused)
232  * - The C handler called from the ASM entry point
233  *
234  * Maps to DECLARE_IDTENTRY().
235  */
236 #define DECLARE_IDTENTRY_SYSVEC(vector, func)				\
237 	DECLARE_IDTENTRY(vector, func)
238 
239 /**
240  * DEFINE_IDTENTRY_SYSVEC - Emit code for system vector IDT entry points
241  * @func:	Function name of the entry point
242  *
243  * irqentry_enter/exit() and irq_enter/exit_rcu() are invoked before the
244  * function body. KVM L1D flush request is set.
245  *
246  * Runs the function on the interrupt stack if the entry hit kernel mode
247  */
248 #define DEFINE_IDTENTRY_SYSVEC(func)					\
249 static void __##func(struct pt_regs *regs);				\
250 									\
251 static __always_inline void instr_##func(struct pt_regs *regs)		\
252 {									\
253 	run_sysvec_on_irqstack_cond(__##func, regs);			\
254 }									\
255 									\
256 __visible noinstr void func(struct pt_regs *regs)			\
257 {									\
258 	irqentry_state_t state = irqentry_enter(regs);			\
259 									\
260 	kvm_set_cpu_l1tf_flush_l1d();                                   \
261 	instrumentation_begin();					\
262 	instr_##func (regs);						\
263 	instrumentation_end();						\
264 	irqentry_exit(regs, state);					\
265 }									\
266 									\
267 void fred_##func(struct pt_regs *regs)					\
268 {									\
269 	instr_##func (regs);						\
270 }									\
271 									\
272 static noinline void __##func(struct pt_regs *regs)
273 
274 /**
275  * DEFINE_IDTENTRY_SYSVEC_SIMPLE - Emit code for simple system vector IDT
276  *				   entry points
277  * @func:	Function name of the entry point
278  *
279  * Runs the function on the interrupted stack. No switch to IRQ stack and
280  * only the minimal __irq_enter/exit() handling.
281  *
282  * Only use for 'empty' vectors like reschedule IPI and KVM posted
283  * interrupt vectors.
284  */
285 #define DEFINE_IDTENTRY_SYSVEC_SIMPLE(func)				\
286 static __always_inline void __##func(struct pt_regs *regs);		\
287 									\
288 static __always_inline void instr_##func(struct pt_regs *regs)		\
289 {									\
290 	__irq_enter_raw();						\
291 	__##func (regs);						\
292 	__irq_exit_raw();						\
293 }									\
294 									\
295 __visible noinstr void func(struct pt_regs *regs)			\
296 {									\
297 	irqentry_state_t state = irqentry_enter(regs);			\
298 									\
299 	kvm_set_cpu_l1tf_flush_l1d();                                   \
300 	instrumentation_begin();					\
301 	instr_##func (regs);						\
302 	instrumentation_end();						\
303 	irqentry_exit(regs, state);					\
304 }									\
305 									\
306 void fred_##func(struct pt_regs *regs)					\
307 {									\
308 	instr_##func (regs);						\
309 }									\
310 									\
311 static __always_inline void __##func(struct pt_regs *regs)
312 
313 /**
314  * DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point
315  * @vector:	Vector number (ignored for C)
316  * @func:	Function name of the entry point
317  *
318  * Declares three functions:
319  * - The ASM entry point: asm_##func
320  * - The XEN PV trap entry point: xen_##func (maybe unused)
321  * - The C handler called from the ASM entry point
322  *
323  * Maps to DECLARE_IDTENTRY(). Distinct entry point to handle the 32/64-bit
324  * difference
325  */
326 #define DECLARE_IDTENTRY_XENCB(vector, func)				\
327 	DECLARE_IDTENTRY(vector, func)
328 
329 #ifdef CONFIG_X86_64
330 /**
331  * DECLARE_IDTENTRY_IST - Declare functions for IST handling IDT entry points
332  * @vector:	Vector number (ignored for C)
333  * @func:	Function name of the entry point
334  *
335  * Maps to DECLARE_IDTENTRY_RAW, but declares also the NOIST C handler
336  * which is called from the ASM entry point on user mode entry
337  */
338 #define DECLARE_IDTENTRY_IST(vector, func)				\
339 	DECLARE_IDTENTRY_RAW(vector, func);				\
340 	__visible void noist_##func(struct pt_regs *regs)
341 
342 /**
343  * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point
344  * @vector:	Vector number (ignored for C)
345  * @func:	Function name of the entry point
346  *
347  * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the
348  * safe_stack C handler.
349  */
350 #define DECLARE_IDTENTRY_VC(vector, func)				\
351 	DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func);			\
352 	__visible noinstr void kernel_##func(struct pt_regs *regs, unsigned long error_code);	\
353 	__visible noinstr void   user_##func(struct pt_regs *regs, unsigned long error_code)
354 
355 /**
356  * DEFINE_IDTENTRY_IST - Emit code for IST entry points
357  * @func:	Function name of the entry point
358  *
359  * Maps to DEFINE_IDTENTRY_RAW
360  */
361 #define DEFINE_IDTENTRY_IST(func)					\
362 	DEFINE_IDTENTRY_RAW(func)
363 
364 /**
365  * DEFINE_IDTENTRY_NOIST - Emit code for NOIST entry points which
366  *			   belong to a IST entry point (MCE, DB)
367  * @func:	Function name of the entry point. Must be the same as
368  *		the function name of the corresponding IST variant
369  *
370  * Maps to DEFINE_IDTENTRY_RAW().
371  */
372 #define DEFINE_IDTENTRY_NOIST(func)					\
373 	DEFINE_IDTENTRY_RAW(noist_##func)
374 
375 /**
376  * DECLARE_IDTENTRY_DF - Declare functions for double fault
377  * @vector:	Vector number (ignored for C)
378  * @func:	Function name of the entry point
379  *
380  * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE
381  */
382 #define DECLARE_IDTENTRY_DF(vector, func)				\
383 	DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func)
384 
385 /**
386  * DEFINE_IDTENTRY_DF - Emit code for double fault
387  * @func:	Function name of the entry point
388  *
389  * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
390  */
391 #define DEFINE_IDTENTRY_DF(func)					\
392 	DEFINE_IDTENTRY_RAW_ERRORCODE(func)
393 
394 /**
395  * DEFINE_IDTENTRY_VC_KERNEL - Emit code for VMM communication handler
396 			       when raised from kernel mode
397  * @func:	Function name of the entry point
398  *
399  * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
400  */
401 #define DEFINE_IDTENTRY_VC_KERNEL(func)				\
402 	DEFINE_IDTENTRY_RAW_ERRORCODE(kernel_##func)
403 
404 /**
405  * DEFINE_IDTENTRY_VC_USER - Emit code for VMM communication handler
406 			     when raised from user mode
407  * @func:	Function name of the entry point
408  *
409  * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
410  */
411 #define DEFINE_IDTENTRY_VC_USER(func)				\
412 	DEFINE_IDTENTRY_RAW_ERRORCODE(user_##func)
413 
414 #else	/* CONFIG_X86_64 */
415 
416 /**
417  * DECLARE_IDTENTRY_DF - Declare functions for double fault 32bit variant
418  * @vector:	Vector number (ignored for C)
419  * @func:	Function name of the entry point
420  *
421  * Declares two functions:
422  * - The ASM entry point: asm_##func
423  * - The C handler called from the C shim
424  */
425 #define DECLARE_IDTENTRY_DF(vector, func)				\
426 	asmlinkage void asm_##func(void);				\
427 	__visible void func(struct pt_regs *regs,			\
428 			    unsigned long error_code,			\
429 			    unsigned long address)
430 
431 /**
432  * DEFINE_IDTENTRY_DF - Emit code for double fault on 32bit
433  * @func:	Function name of the entry point
434  *
435  * This is called through the doublefault shim which already provides
436  * cr2 in the address argument.
437  */
438 #define DEFINE_IDTENTRY_DF(func)					\
439 __visible noinstr void func(struct pt_regs *regs,			\
440 			    unsigned long error_code,			\
441 			    unsigned long address)
442 
443 #endif	/* !CONFIG_X86_64 */
444 
445 /* C-Code mapping */
446 #define DECLARE_IDTENTRY_NMI		DECLARE_IDTENTRY_RAW
447 #define DEFINE_IDTENTRY_NMI		DEFINE_IDTENTRY_RAW
448 #define DEFINE_FREDENTRY_NMI		DEFINE_FREDENTRY_RAW
449 
450 #ifdef CONFIG_X86_64
451 #define DECLARE_IDTENTRY_MCE		DECLARE_IDTENTRY_IST
452 #define DEFINE_IDTENTRY_MCE		DEFINE_IDTENTRY_IST
453 #define DEFINE_IDTENTRY_MCE_USER	DEFINE_IDTENTRY_NOIST
454 #define DEFINE_FREDENTRY_MCE		DEFINE_FREDENTRY_RAW
455 
456 #define DECLARE_IDTENTRY_DEBUG		DECLARE_IDTENTRY_IST
457 #define DEFINE_IDTENTRY_DEBUG		DEFINE_IDTENTRY_IST
458 #define DEFINE_IDTENTRY_DEBUG_USER	DEFINE_IDTENTRY_NOIST
459 #define DEFINE_FREDENTRY_DEBUG		DEFINE_FREDENTRY_RAW
460 #endif
461 
462 #else /* !__ASSEMBLY__ */
463 
464 /*
465  * The ASM variants for DECLARE_IDTENTRY*() which emit the ASM entry stubs.
466  */
467 #define DECLARE_IDTENTRY(vector, func)					\
468 	idtentry vector asm_##func func has_error_code=0
469 
470 #define DECLARE_IDTENTRY_ERRORCODE(vector, func)			\
471 	idtentry vector asm_##func func has_error_code=1
472 
473 /* Special case for 32bit IRET 'trap'. Do not emit ASM code */
474 #define DECLARE_IDTENTRY_SW(vector, func)
475 
476 #define DECLARE_IDTENTRY_RAW(vector, func)				\
477 	DECLARE_IDTENTRY(vector, func)
478 
479 #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func)			\
480 	DECLARE_IDTENTRY_ERRORCODE(vector, func)
481 
482 /* Entries for common/spurious (device) interrupts */
483 #define DECLARE_IDTENTRY_IRQ(vector, func)				\
484 	idtentry_irq vector func
485 
486 /* System vector entries */
487 #define DECLARE_IDTENTRY_SYSVEC(vector, func)				\
488 	idtentry_sysvec vector func
489 
490 #ifdef CONFIG_X86_64
491 # define DECLARE_IDTENTRY_MCE(vector, func)				\
492 	idtentry_mce_db vector asm_##func func
493 
494 # define DECLARE_IDTENTRY_DEBUG(vector, func)				\
495 	idtentry_mce_db vector asm_##func func
496 
497 # define DECLARE_IDTENTRY_DF(vector, func)				\
498 	idtentry_df vector asm_##func func
499 
500 # define DECLARE_IDTENTRY_XENCB(vector, func)				\
501 	DECLARE_IDTENTRY(vector, func)
502 
503 # define DECLARE_IDTENTRY_VC(vector, func)				\
504 	idtentry_vc vector asm_##func func
505 
506 #else
507 # define DECLARE_IDTENTRY_MCE(vector, func)				\
508 	DECLARE_IDTENTRY(vector, func)
509 
510 /* No ASM emitted for DF as this goes through a C shim */
511 # define DECLARE_IDTENTRY_DF(vector, func)
512 
513 /* No ASM emitted for XEN hypervisor callback */
514 # define DECLARE_IDTENTRY_XENCB(vector, func)
515 
516 #endif
517 
518 /* No ASM code emitted for NMI */
519 #define DECLARE_IDTENTRY_NMI(vector, func)
520 
521 /*
522  * ASM code to emit the common vector entry stubs where each stub is
523  * packed into IDT_ALIGN bytes.
524  *
525  * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because
526  * GCC treats the local vector variable as unsigned int and would expand
527  * all vectors above 0x7F to a 5 byte push. The original code did an
528  * adjustment of the vector number to be in the signed byte range to avoid
529  * this. While clever it's mindboggling counterintuitive and requires the
530  * odd conversion back to a real vector number in the C entry points. Using
531  * .byte achieves the same thing and the only fixup needed in the C entry
532  * point is to mask off the bits above bit 7 because the push is sign
533  * extending.
534  */
535 	.align IDT_ALIGN
536 SYM_CODE_START(irq_entries_start)
537     vector=FIRST_EXTERNAL_VECTOR
538     .rept NR_EXTERNAL_VECTORS
539 	UNWIND_HINT_IRET_REGS
540 0 :
541 	ENDBR
542 	.byte	0x6a, vector
543 	jmp	asm_common_interrupt
544 	/* Ensure that the above is IDT_ALIGN bytes max */
545 	.fill 0b + IDT_ALIGN - ., 1, 0xcc
546 	vector = vector+1
547     .endr
548 SYM_CODE_END(irq_entries_start)
549 
550 #ifdef CONFIG_X86_LOCAL_APIC
551 	.align IDT_ALIGN
552 SYM_CODE_START(spurious_entries_start)
553     vector=FIRST_SYSTEM_VECTOR
554     .rept NR_SYSTEM_VECTORS
555 	UNWIND_HINT_IRET_REGS
556 0 :
557 	ENDBR
558 	.byte	0x6a, vector
559 	jmp	asm_spurious_interrupt
560 	/* Ensure that the above is IDT_ALIGN bytes max */
561 	.fill 0b + IDT_ALIGN - ., 1, 0xcc
562 	vector = vector+1
563     .endr
564 SYM_CODE_END(spurious_entries_start)
565 #endif
566 
567 #endif /* __ASSEMBLY__ */
568 
569 /*
570  * The actual entry points. Note that DECLARE_IDTENTRY*() serves two
571  * purposes:
572  *  - provide the function declarations when included from C-Code
573  *  - emit the ASM stubs when included from entry_32/64.S
574  *
575  * This avoids duplicate defines and ensures that everything is consistent.
576  */
577 
578 /*
579  * Dummy trap number so the low level ASM macro vector number checks do not
580  * match which results in emitting plain IDTENTRY stubs without bells and
581  * whistles.
582  */
583 #define X86_TRAP_OTHER		0xFFFF
584 
585 /* Simple exception entry points. No hardware error code */
586 DECLARE_IDTENTRY(X86_TRAP_DE,		exc_divide_error);
587 DECLARE_IDTENTRY(X86_TRAP_OF,		exc_overflow);
588 DECLARE_IDTENTRY(X86_TRAP_BR,		exc_bounds);
589 DECLARE_IDTENTRY(X86_TRAP_NM,		exc_device_not_available);
590 DECLARE_IDTENTRY(X86_TRAP_OLD_MF,	exc_coproc_segment_overrun);
591 DECLARE_IDTENTRY(X86_TRAP_SPURIOUS,	exc_spurious_interrupt_bug);
592 DECLARE_IDTENTRY(X86_TRAP_MF,		exc_coprocessor_error);
593 DECLARE_IDTENTRY(X86_TRAP_XF,		exc_simd_coprocessor_error);
594 
595 /* 32bit software IRET trap. Do not emit ASM code */
596 DECLARE_IDTENTRY_SW(X86_TRAP_IRET,	iret_error);
597 
598 /* Simple exception entries with error code pushed by hardware */
599 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_TS,	exc_invalid_tss);
600 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_NP,	exc_segment_not_present);
601 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS,	exc_stack_segment);
602 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_GP,	exc_general_protection);
603 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_AC,	exc_alignment_check);
604 
605 /* Raw exception entries which need extra work */
606 DECLARE_IDTENTRY_RAW(X86_TRAP_UD,		exc_invalid_op);
607 DECLARE_IDTENTRY_RAW(X86_TRAP_BP,		exc_int3);
608 DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_PF,	exc_page_fault);
609 
610 #if defined(CONFIG_IA32_EMULATION)
611 DECLARE_IDTENTRY_RAW(IA32_SYSCALL_VECTOR,	int80_emulation);
612 #endif
613 
614 #ifdef CONFIG_X86_MCE
615 #ifdef CONFIG_X86_64
616 DECLARE_IDTENTRY_MCE(X86_TRAP_MC,	exc_machine_check);
617 #else
618 DECLARE_IDTENTRY_RAW(X86_TRAP_MC,	exc_machine_check);
619 #endif
620 #ifdef CONFIG_XEN_PV
621 DECLARE_IDTENTRY_RAW(X86_TRAP_MC,	xenpv_exc_machine_check);
622 #endif
623 #endif
624 
625 /* NMI */
626 
627 #if IS_ENABLED(CONFIG_KVM_INTEL)
628 /*
629  * Special entry point for VMX which invokes this on the kernel stack, even for
630  * 64-bit, i.e. without using an IST.  asm_exc_nmi() requires an IST to work
631  * correctly vs. the NMI 'executing' marker.  Used for 32-bit kernels as well
632  * to avoid more ifdeffery.
633  */
634 DECLARE_IDTENTRY(X86_TRAP_NMI,		exc_nmi_kvm_vmx);
635 #endif
636 
637 DECLARE_IDTENTRY_NMI(X86_TRAP_NMI,	exc_nmi);
638 #ifdef CONFIG_XEN_PV
639 DECLARE_IDTENTRY_RAW(X86_TRAP_NMI,	xenpv_exc_nmi);
640 #endif
641 
642 /* #DB */
643 #ifdef CONFIG_X86_64
644 DECLARE_IDTENTRY_DEBUG(X86_TRAP_DB,	exc_debug);
645 #else
646 DECLARE_IDTENTRY_RAW(X86_TRAP_DB,	exc_debug);
647 #endif
648 #ifdef CONFIG_XEN_PV
649 DECLARE_IDTENTRY_RAW(X86_TRAP_DB,	xenpv_exc_debug);
650 #endif
651 
652 /* #DF */
653 DECLARE_IDTENTRY_DF(X86_TRAP_DF,	exc_double_fault);
654 #ifdef CONFIG_XEN_PV
655 DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF,	xenpv_exc_double_fault);
656 #endif
657 
658 /* #CP */
659 #ifdef CONFIG_X86_CET
660 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP,	exc_control_protection);
661 #endif
662 
663 /* #VC */
664 #ifdef CONFIG_AMD_MEM_ENCRYPT
665 DECLARE_IDTENTRY_VC(X86_TRAP_VC,	exc_vmm_communication);
666 #endif
667 
668 #ifdef CONFIG_XEN_PV
669 DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER,	exc_xen_hypervisor_callback);
670 DECLARE_IDTENTRY_RAW(X86_TRAP_OTHER,	exc_xen_unknown_trap);
671 #endif
672 
673 #ifdef CONFIG_INTEL_TDX_GUEST
674 DECLARE_IDTENTRY(X86_TRAP_VE,		exc_virtualization_exception);
675 #endif
676 
677 /* Device interrupts common/spurious */
678 DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER,	common_interrupt);
679 #ifdef CONFIG_X86_LOCAL_APIC
680 DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER,	spurious_interrupt);
681 #endif
682 
683 /* System vector entry points */
684 #ifdef CONFIG_X86_LOCAL_APIC
685 DECLARE_IDTENTRY_SYSVEC(ERROR_APIC_VECTOR,		sysvec_error_interrupt);
686 DECLARE_IDTENTRY_SYSVEC(SPURIOUS_APIC_VECTOR,		sysvec_spurious_apic_interrupt);
687 DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR,		sysvec_apic_timer_interrupt);
688 DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR,	sysvec_x86_platform_ipi);
689 #endif
690 
691 #ifdef CONFIG_SMP
692 DECLARE_IDTENTRY(RESCHEDULE_VECTOR,			sysvec_reschedule_ipi);
693 DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR,			sysvec_reboot);
694 DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR,	sysvec_call_function_single);
695 DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR,		sysvec_call_function);
696 #else
697 # define fred_sysvec_reschedule_ipi			NULL
698 # define fred_sysvec_reboot				NULL
699 # define fred_sysvec_call_function_single		NULL
700 # define fred_sysvec_call_function			NULL
701 #endif
702 
703 #ifdef CONFIG_X86_LOCAL_APIC
704 # ifdef CONFIG_X86_MCE_THRESHOLD
705 DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR,		sysvec_threshold);
706 # else
707 # define fred_sysvec_threshold				NULL
708 # endif
709 
710 # ifdef CONFIG_X86_MCE_AMD
711 DECLARE_IDTENTRY_SYSVEC(DEFERRED_ERROR_VECTOR,		sysvec_deferred_error);
712 # else
713 # define fred_sysvec_deferred_error			NULL
714 # endif
715 
716 # ifdef CONFIG_X86_THERMAL_VECTOR
717 DECLARE_IDTENTRY_SYSVEC(THERMAL_APIC_VECTOR,		sysvec_thermal);
718 # else
719 # define fred_sysvec_thermal				NULL
720 # endif
721 
722 # ifdef CONFIG_IRQ_WORK
723 DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR,		sysvec_irq_work);
724 # else
725 # define fred_sysvec_irq_work				NULL
726 # endif
727 #endif
728 
729 #ifdef CONFIG_HAVE_KVM
730 DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR,		sysvec_kvm_posted_intr_ipi);
731 DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR,	sysvec_kvm_posted_intr_wakeup_ipi);
732 DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR,	sysvec_kvm_posted_intr_nested_ipi);
733 #else
734 # define fred_sysvec_kvm_posted_intr_ipi		NULL
735 # define fred_sysvec_kvm_posted_intr_wakeup_ipi		NULL
736 # define fred_sysvec_kvm_posted_intr_nested_ipi		NULL
737 #endif
738 
739 #if IS_ENABLED(CONFIG_HYPERV)
740 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_hyperv_callback);
741 DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR,	sysvec_hyperv_reenlightenment);
742 DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR,		sysvec_hyperv_stimer0);
743 #endif
744 
745 #if IS_ENABLED(CONFIG_ACRN_GUEST)
746 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_acrn_hv_callback);
747 #endif
748 
749 #ifdef CONFIG_XEN_PVHVM
750 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_xen_hvm_callback);
751 #endif
752 
753 #ifdef CONFIG_KVM_GUEST
754 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_kvm_asyncpf_interrupt);
755 #endif
756 
757 #undef X86_TRAP_OTHER
758 
759 #endif
760