1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Interrupt descriptor table related code 4 */ 5 #include <linux/interrupt.h> 6 7 #include <asm/traps.h> 8 #include <asm/proto.h> 9 #include <asm/desc.h> 10 #include <asm/hw_irq.h> 11 12 struct idt_data { 13 unsigned int vector; 14 unsigned int segment; 15 struct idt_bits bits; 16 const void *addr; 17 }; 18 19 #define DPL0 0x0 20 #define DPL3 0x3 21 22 #define DEFAULT_STACK 0 23 24 #define G(_vector, _addr, _ist, _type, _dpl, _segment) \ 25 { \ 26 .vector = _vector, \ 27 .bits.ist = _ist, \ 28 .bits.type = _type, \ 29 .bits.dpl = _dpl, \ 30 .bits.p = 1, \ 31 .addr = _addr, \ 32 .segment = _segment, \ 33 } 34 35 /* Interrupt gate */ 36 #define INTG(_vector, _addr) \ 37 G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL0, __KERNEL_CS) 38 39 /* System interrupt gate */ 40 #define SYSG(_vector, _addr) \ 41 G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS) 42 43 /* 44 * Interrupt gate with interrupt stack. The _ist index is the index in 45 * the tss.ist[] array, but for the descriptor it needs to start at 1. 46 */ 47 #define ISTG(_vector, _addr, _ist) \ 48 G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS) 49 50 /* Task gate */ 51 #define TSKG(_vector, _gdt) \ 52 G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3) 53 54 /* 55 * Early traps running on the DEFAULT_STACK because the other interrupt 56 * stacks work only after cpu_init(). 57 */ 58 static const __initconst struct idt_data early_idts[] = { 59 INTG(X86_TRAP_DB, debug), 60 SYSG(X86_TRAP_BP, int3), 61 #ifdef CONFIG_X86_32 62 INTG(X86_TRAP_PF, page_fault), 63 #endif 64 }; 65 66 /* 67 * The default IDT entries which are set up in trap_init() before 68 * cpu_init() is invoked. Interrupt stacks cannot be used at that point and 69 * the traps which use them are reinitialized with IST after cpu_init() has 70 * set up TSS. 71 */ 72 static const __initconst struct idt_data def_idts[] = { 73 INTG(X86_TRAP_DE, divide_error), 74 INTG(X86_TRAP_NMI, nmi), 75 INTG(X86_TRAP_BR, bounds), 76 INTG(X86_TRAP_UD, invalid_op), 77 INTG(X86_TRAP_NM, device_not_available), 78 INTG(X86_TRAP_OLD_MF, coprocessor_segment_overrun), 79 INTG(X86_TRAP_TS, invalid_TSS), 80 INTG(X86_TRAP_NP, segment_not_present), 81 INTG(X86_TRAP_SS, stack_segment), 82 INTG(X86_TRAP_GP, general_protection), 83 INTG(X86_TRAP_SPURIOUS, spurious_interrupt_bug), 84 INTG(X86_TRAP_MF, coprocessor_error), 85 INTG(X86_TRAP_AC, alignment_check), 86 INTG(X86_TRAP_XF, simd_coprocessor_error), 87 88 #ifdef CONFIG_X86_32 89 TSKG(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS), 90 #else 91 INTG(X86_TRAP_DF, double_fault), 92 #endif 93 INTG(X86_TRAP_DB, debug), 94 95 #ifdef CONFIG_X86_MCE 96 INTG(X86_TRAP_MC, &machine_check), 97 #endif 98 99 SYSG(X86_TRAP_OF, overflow), 100 #if defined(CONFIG_IA32_EMULATION) 101 SYSG(IA32_SYSCALL_VECTOR, entry_INT80_compat), 102 #elif defined(CONFIG_X86_32) 103 SYSG(IA32_SYSCALL_VECTOR, entry_INT80_32), 104 #endif 105 }; 106 107 /* 108 * The APIC and SMP idt entries 109 */ 110 static const __initconst struct idt_data apic_idts[] = { 111 #ifdef CONFIG_SMP 112 INTG(RESCHEDULE_VECTOR, reschedule_interrupt), 113 INTG(CALL_FUNCTION_VECTOR, call_function_interrupt), 114 INTG(CALL_FUNCTION_SINGLE_VECTOR, call_function_single_interrupt), 115 INTG(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt), 116 INTG(REBOOT_VECTOR, reboot_interrupt), 117 #endif 118 119 #ifdef CONFIG_X86_THERMAL_VECTOR 120 INTG(THERMAL_APIC_VECTOR, thermal_interrupt), 121 #endif 122 123 #ifdef CONFIG_X86_MCE_THRESHOLD 124 INTG(THRESHOLD_APIC_VECTOR, threshold_interrupt), 125 #endif 126 127 #ifdef CONFIG_X86_MCE_AMD 128 INTG(DEFERRED_ERROR_VECTOR, deferred_error_interrupt), 129 #endif 130 131 #ifdef CONFIG_X86_LOCAL_APIC 132 INTG(LOCAL_TIMER_VECTOR, apic_timer_interrupt), 133 INTG(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi), 134 # ifdef CONFIG_HAVE_KVM 135 INTG(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), 136 INTG(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), 137 INTG(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), 138 # endif 139 # ifdef CONFIG_IRQ_WORK 140 INTG(IRQ_WORK_VECTOR, irq_work_interrupt), 141 # endif 142 #ifdef CONFIG_X86_UV 143 INTG(UV_BAU_MESSAGE, uv_bau_message_intr1), 144 #endif 145 INTG(SPURIOUS_APIC_VECTOR, spurious_interrupt), 146 INTG(ERROR_APIC_VECTOR, error_interrupt), 147 #endif 148 }; 149 150 #ifdef CONFIG_X86_64 151 /* 152 * Early traps running on the DEFAULT_STACK because the other interrupt 153 * stacks work only after cpu_init(). 154 */ 155 static const __initconst struct idt_data early_pf_idts[] = { 156 INTG(X86_TRAP_PF, page_fault), 157 }; 158 159 /* 160 * Override for the debug_idt. Same as the default, but with interrupt 161 * stack set to DEFAULT_STACK (0). Required for NMI trap handling. 162 */ 163 static const __initconst struct idt_data dbg_idts[] = { 164 INTG(X86_TRAP_DB, debug), 165 }; 166 #endif 167 168 /* Must be page-aligned because the real IDT is used in a fixmap. */ 169 gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss; 170 171 struct desc_ptr idt_descr __ro_after_init = { 172 .size = (IDT_ENTRIES * 2 * sizeof(unsigned long)) - 1, 173 .address = (unsigned long) idt_table, 174 }; 175 176 #ifdef CONFIG_X86_64 177 /* No need to be aligned, but done to keep all IDTs defined the same way. */ 178 gate_desc debug_idt_table[IDT_ENTRIES] __page_aligned_bss; 179 180 /* 181 * The exceptions which use Interrupt stacks. They are setup after 182 * cpu_init() when the TSS has been initialized. 183 */ 184 static const __initconst struct idt_data ist_idts[] = { 185 ISTG(X86_TRAP_DB, debug, IST_INDEX_DB), 186 ISTG(X86_TRAP_NMI, nmi, IST_INDEX_NMI), 187 ISTG(X86_TRAP_DF, double_fault, IST_INDEX_DF), 188 #ifdef CONFIG_X86_MCE 189 ISTG(X86_TRAP_MC, &machine_check, IST_INDEX_MCE), 190 #endif 191 }; 192 193 /* 194 * Override for the debug_idt. Same as the default, but with interrupt 195 * stack set to DEFAULT_STACK (0). Required for NMI trap handling. 196 */ 197 const struct desc_ptr debug_idt_descr = { 198 .size = IDT_ENTRIES * 16 - 1, 199 .address = (unsigned long) debug_idt_table, 200 }; 201 #endif 202 203 static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d) 204 { 205 unsigned long addr = (unsigned long) d->addr; 206 207 gate->offset_low = (u16) addr; 208 gate->segment = (u16) d->segment; 209 gate->bits = d->bits; 210 gate->offset_middle = (u16) (addr >> 16); 211 #ifdef CONFIG_X86_64 212 gate->offset_high = (u32) (addr >> 32); 213 gate->reserved = 0; 214 #endif 215 } 216 217 static void 218 idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys) 219 { 220 gate_desc desc; 221 222 for (; size > 0; t++, size--) { 223 idt_init_desc(&desc, t); 224 write_idt_entry(idt, t->vector, &desc); 225 if (sys) 226 set_bit(t->vector, system_vectors); 227 } 228 } 229 230 static void set_intr_gate(unsigned int n, const void *addr) 231 { 232 struct idt_data data; 233 234 BUG_ON(n > 0xFF); 235 236 memset(&data, 0, sizeof(data)); 237 data.vector = n; 238 data.addr = addr; 239 data.segment = __KERNEL_CS; 240 data.bits.type = GATE_INTERRUPT; 241 data.bits.p = 1; 242 243 idt_setup_from_table(idt_table, &data, 1, false); 244 } 245 246 /** 247 * idt_setup_early_traps - Initialize the idt table with early traps 248 * 249 * On X8664 these traps do not use interrupt stacks as they can't work 250 * before cpu_init() is invoked and sets up TSS. The IST variants are 251 * installed after that. 252 */ 253 void __init idt_setup_early_traps(void) 254 { 255 idt_setup_from_table(idt_table, early_idts, ARRAY_SIZE(early_idts), 256 true); 257 load_idt(&idt_descr); 258 } 259 260 /** 261 * idt_setup_traps - Initialize the idt table with default traps 262 */ 263 void __init idt_setup_traps(void) 264 { 265 idt_setup_from_table(idt_table, def_idts, ARRAY_SIZE(def_idts), true); 266 } 267 268 #ifdef CONFIG_X86_64 269 /** 270 * idt_setup_early_pf - Initialize the idt table with early pagefault handler 271 * 272 * On X8664 this does not use interrupt stacks as they can't work before 273 * cpu_init() is invoked and sets up TSS. The IST variant is installed 274 * after that. 275 * 276 * FIXME: Why is 32bit and 64bit installing the PF handler at different 277 * places in the early setup code? 278 */ 279 void __init idt_setup_early_pf(void) 280 { 281 idt_setup_from_table(idt_table, early_pf_idts, 282 ARRAY_SIZE(early_pf_idts), true); 283 } 284 285 /** 286 * idt_setup_ist_traps - Initialize the idt table with traps using IST 287 */ 288 void __init idt_setup_ist_traps(void) 289 { 290 idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true); 291 } 292 293 /** 294 * idt_setup_debugidt_traps - Initialize the debug idt table with debug traps 295 */ 296 void __init idt_setup_debugidt_traps(void) 297 { 298 memcpy(&debug_idt_table, &idt_table, IDT_ENTRIES * 16); 299 300 idt_setup_from_table(debug_idt_table, dbg_idts, ARRAY_SIZE(dbg_idts), false); 301 } 302 #endif 303 304 /** 305 * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates 306 */ 307 void __init idt_setup_apic_and_irq_gates(void) 308 { 309 int i = FIRST_EXTERNAL_VECTOR; 310 void *entry; 311 312 idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true); 313 314 for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) { 315 entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR); 316 set_intr_gate(i, entry); 317 } 318 319 #ifdef CONFIG_X86_LOCAL_APIC 320 for_each_clear_bit_from(i, system_vectors, NR_VECTORS) { 321 set_bit(i, system_vectors); 322 entry = spurious_entries_start + 8 * (i - FIRST_SYSTEM_VECTOR); 323 set_intr_gate(i, entry); 324 } 325 #endif 326 } 327 328 /** 329 * idt_setup_early_handler - Initializes the idt table with early handlers 330 */ 331 void __init idt_setup_early_handler(void) 332 { 333 int i; 334 335 for (i = 0; i < NUM_EXCEPTION_VECTORS; i++) 336 set_intr_gate(i, early_idt_handler_array[i]); 337 #ifdef CONFIG_X86_32 338 for ( ; i < NR_VECTORS; i++) 339 set_intr_gate(i, early_ignore_irq); 340 #endif 341 load_idt(&idt_descr); 342 } 343 344 /** 345 * idt_invalidate - Invalidate interrupt descriptor table 346 * @addr: The virtual address of the 'invalid' IDT 347 */ 348 void idt_invalidate(void *addr) 349 { 350 struct desc_ptr idt = { .address = (unsigned long) addr, .size = 0 }; 351 352 load_idt(&idt); 353 } 354 355 void __init update_intr_gate(unsigned int n, const void *addr) 356 { 357 if (WARN_ON_ONCE(!test_bit(n, system_vectors))) 358 return; 359 set_intr_gate(n, addr); 360 } 361 362 void alloc_intr_gate(unsigned int n, const void *addr) 363 { 364 BUG_ON(n < FIRST_SYSTEM_VECTOR); 365 if (!test_and_set_bit(n, system_vectors)) 366 set_intr_gate(n, addr); 367 } 368