1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 29a163ed8SThomas Gleixner /* 39a163ed8SThomas Gleixner * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar 49a163ed8SThomas Gleixner * 59a163ed8SThomas Gleixner * This file contains the lowest level x86-specific interrupt 69a163ed8SThomas Gleixner * entry, irq-stacks and irq statistics code. All the remaining 79a163ed8SThomas Gleixner * irq logic is done by the generic kernel/irq/ code and 89a163ed8SThomas Gleixner * by the x86-specific irq controller code. (e.g. i8259.c and 99a163ed8SThomas Gleixner * io_apic.c.) 109a163ed8SThomas Gleixner */ 119a163ed8SThomas Gleixner 129a163ed8SThomas Gleixner #include <linux/seq_file.h> 139a163ed8SThomas Gleixner #include <linux/interrupt.h> 14447ae316SNicolai Stange #include <linux/irq.h> 159a163ed8SThomas Gleixner #include <linux/kernel_stat.h> 169a163ed8SThomas Gleixner #include <linux/notifier.h> 179a163ed8SThomas Gleixner #include <linux/cpu.h> 189a163ed8SThomas Gleixner #include <linux/delay.h> 1972ade5f9SJaswinder Singh Rajput #include <linux/uaccess.h> 2042f8faecSLai Jiangshan #include <linux/percpu.h> 215c1eb089SEric Dumazet #include <linux/mm.h> 229a163ed8SThomas Gleixner 239a163ed8SThomas Gleixner #include <asm/apic.h> 247614e913SAndi Kleen #include <asm/nospec-branch.h> 259a163ed8SThomas Gleixner 26de9b10afSThomas Gleixner #ifdef CONFIG_DEBUG_STACKOVERFLOW 2753b56502SIngo Molnar 2853b56502SIngo Molnar int sysctl_panic_on_stackoverflow __read_mostly; 2953b56502SIngo Molnar 30de9b10afSThomas Gleixner /* Debugging check for stack overflow: is there less than 1KB free? */ 31de9b10afSThomas Gleixner static int check_stack_overflow(void) 32de9b10afSThomas Gleixner { 33de9b10afSThomas Gleixner long sp; 34de9b10afSThomas Gleixner 35de9b10afSThomas Gleixner __asm__ __volatile__("andl %%esp,%0" : 36de9b10afSThomas Gleixner "=r" (sp) : "0" (THREAD_SIZE - 1)); 37de9b10afSThomas Gleixner 38de9b10afSThomas Gleixner return sp < (sizeof(struct thread_info) + STACK_WARN); 39de9b10afSThomas Gleixner } 40de9b10afSThomas Gleixner 41de9b10afSThomas Gleixner static void print_stack_overflow(void) 42de9b10afSThomas Gleixner { 43de9b10afSThomas Gleixner printk(KERN_WARNING "low stack detected by irq handler\n"); 44de9b10afSThomas Gleixner dump_stack(); 4555af7796SMitsuo Hayasaka if (sysctl_panic_on_stackoverflow) 4655af7796SMitsuo Hayasaka panic("low stack detected by irq handler - check messages\n"); 47de9b10afSThomas Gleixner } 48de9b10afSThomas Gleixner 49de9b10afSThomas Gleixner #else 50de9b10afSThomas Gleixner static inline int check_stack_overflow(void) { return 0; } 51de9b10afSThomas Gleixner static inline void print_stack_overflow(void) { } 52de9b10afSThomas Gleixner #endif 53de9b10afSThomas Gleixner 54198d208dSSteven Rostedt DEFINE_PER_CPU(struct irq_stack *, hardirq_stack); 55198d208dSSteven Rostedt DEFINE_PER_CPU(struct irq_stack *, softirq_stack); 569a163ed8SThomas Gleixner 57403d8efcSThomas Gleixner static void call_on_stack(void *func, void *stack) 589a163ed8SThomas Gleixner { 59403d8efcSThomas Gleixner asm volatile("xchgl %%ebx,%%esp \n" 607614e913SAndi Kleen CALL_NOSPEC 6104b361abSAndi Kleen "movl %%ebx,%%esp \n" 62403d8efcSThomas Gleixner : "=b" (stack) 63403d8efcSThomas Gleixner : "0" (stack), 647614e913SAndi Kleen [thunk_target] "D"(func) 65403d8efcSThomas Gleixner : "memory", "cc", "edx", "ecx", "eax"); 6604b361abSAndi Kleen } 6704b361abSAndi Kleen 68198d208dSSteven Rostedt static inline void *current_stack(void) 69198d208dSSteven Rostedt { 70196bd485SAndrey Ryabinin return (void *)(current_stack_pointer & ~(THREAD_SIZE - 1)); 71198d208dSSteven Rostedt } 72198d208dSSteven Rostedt 73bd0b9ac4SThomas Gleixner static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc) 749a163ed8SThomas Gleixner { 75198d208dSSteven Rostedt struct irq_stack *curstk, *irqstk; 76bd0b9ac4SThomas Gleixner u32 *isp, *prev_esp, arg1; 779a163ed8SThomas Gleixner 78198d208dSSteven Rostedt curstk = (struct irq_stack *) current_stack(); 79198d208dSSteven Rostedt irqstk = __this_cpu_read(hardirq_stack); 809a163ed8SThomas Gleixner 819a163ed8SThomas Gleixner /* 829a163ed8SThomas Gleixner * this is where we switch to the IRQ stack. However, if we are 839a163ed8SThomas Gleixner * already using the IRQ stack (because we interrupted a hardirq 849a163ed8SThomas Gleixner * handler) we can't do that and just have to keep using the 859a163ed8SThomas Gleixner * current stack (which is the irq stack already after all) 869a163ed8SThomas Gleixner */ 87198d208dSSteven Rostedt if (unlikely(curstk == irqstk)) 88de9b10afSThomas Gleixner return 0; 899a163ed8SThomas Gleixner 90198d208dSSteven Rostedt isp = (u32 *) ((char *)irqstk + sizeof(*irqstk)); 91198d208dSSteven Rostedt 92198d208dSSteven Rostedt /* Save the next esp at the bottom of the stack */ 93198d208dSSteven Rostedt prev_esp = (u32 *)irqstk; 94196bd485SAndrey Ryabinin *prev_esp = current_stack_pointer; 959a163ed8SThomas Gleixner 9604b361abSAndi Kleen if (unlikely(overflow)) 97403d8efcSThomas Gleixner call_on_stack(print_stack_overflow, isp); 9804b361abSAndi Kleen 99403d8efcSThomas Gleixner asm volatile("xchgl %%ebx,%%esp \n" 1007614e913SAndi Kleen CALL_NOSPEC 1019a163ed8SThomas Gleixner "movl %%ebx,%%esp \n" 102bd0b9ac4SThomas Gleixner : "=a" (arg1), "=b" (isp) 103bd0b9ac4SThomas Gleixner : "0" (desc), "1" (isp), 1047614e913SAndi Kleen [thunk_target] "D" (desc->handle_irq) 105403d8efcSThomas Gleixner : "memory", "cc", "ecx"); 1069a163ed8SThomas Gleixner return 1; 1079a163ed8SThomas Gleixner } 1089a163ed8SThomas Gleixner 1099a163ed8SThomas Gleixner /* 1109a163ed8SThomas Gleixner * allocate per-cpu stacks for hardirq and for softirq processing 1119a163ed8SThomas Gleixner */ 112148f9bb8SPaul Gortmaker void irq_ctx_init(int cpu) 1139a163ed8SThomas Gleixner { 114198d208dSSteven Rostedt struct irq_stack *irqstk; 1159a163ed8SThomas Gleixner 116198d208dSSteven Rostedt if (per_cpu(hardirq_stack, cpu)) 1179a163ed8SThomas Gleixner return; 1189a163ed8SThomas Gleixner 119198d208dSSteven Rostedt irqstk = page_address(alloc_pages_node(cpu_to_node(cpu), 12038e7c572SThomas Gleixner THREADINFO_GFP, 12138e7c572SThomas Gleixner THREAD_SIZE_ORDER)); 122198d208dSSteven Rostedt per_cpu(hardirq_stack, cpu) = irqstk; 1239a163ed8SThomas Gleixner 124198d208dSSteven Rostedt irqstk = page_address(alloc_pages_node(cpu_to_node(cpu), 12538e7c572SThomas Gleixner THREADINFO_GFP, 12638e7c572SThomas Gleixner THREAD_SIZE_ORDER)); 127198d208dSSteven Rostedt per_cpu(softirq_stack, cpu) = irqstk; 1289a163ed8SThomas Gleixner 129403d8efcSThomas Gleixner printk(KERN_DEBUG "CPU %u irqstacks, hard=%p soft=%p\n", 130198d208dSSteven Rostedt cpu, per_cpu(hardirq_stack, cpu), per_cpu(softirq_stack, cpu)); 1319a163ed8SThomas Gleixner } 1329a163ed8SThomas Gleixner 1337d65f4a6SFrederic Weisbecker void do_softirq_own_stack(void) 1349a163ed8SThomas Gleixner { 135198d208dSSteven Rostedt struct irq_stack *irqstk; 1360788aa6aSSteven Rostedt u32 *isp, *prev_esp; 1379a163ed8SThomas Gleixner 138198d208dSSteven Rostedt irqstk = __this_cpu_read(softirq_stack); 1399a163ed8SThomas Gleixner 1409a163ed8SThomas Gleixner /* build the stack frame on the softirq stack */ 141198d208dSSteven Rostedt isp = (u32 *) ((char *)irqstk + sizeof(*irqstk)); 1429a163ed8SThomas Gleixner 1430788aa6aSSteven Rostedt /* Push the previous esp onto the stack */ 144198d208dSSteven Rostedt prev_esp = (u32 *)irqstk; 145196bd485SAndrey Ryabinin *prev_esp = current_stack_pointer; 1460788aa6aSSteven Rostedt 147403d8efcSThomas Gleixner call_on_stack(__do_softirq, isp); 1489a163ed8SThomas Gleixner } 149403d8efcSThomas Gleixner 150a782a7e4SThomas Gleixner bool handle_irq(struct irq_desc *desc, struct pt_regs *regs) 1519b2b76a3SJeremy Fitzhardinge { 152bd0b9ac4SThomas Gleixner int overflow = check_stack_overflow(); 1539b2b76a3SJeremy Fitzhardinge 154a782a7e4SThomas Gleixner if (IS_ERR_OR_NULL(desc)) 1559b2b76a3SJeremy Fitzhardinge return false; 1569b2b76a3SJeremy Fitzhardinge 157bd0b9ac4SThomas Gleixner if (user_mode(regs) || !execute_on_irq_stack(overflow, desc)) { 1589b2b76a3SJeremy Fitzhardinge if (unlikely(overflow)) 1599b2b76a3SJeremy Fitzhardinge print_stack_overflow(); 160bd0b9ac4SThomas Gleixner generic_handle_irq_desc(desc); 1619b2b76a3SJeremy Fitzhardinge } 1629b2b76a3SJeremy Fitzhardinge 1639b2b76a3SJeremy Fitzhardinge return true; 1649b2b76a3SJeremy Fitzhardinge } 165