1e99b6f46SAndy Lutomirski // SPDX-License-Identifier: GPL-2.0 2e99b6f46SAndy Lutomirski #include <linux/mm.h> 3e99b6f46SAndy Lutomirski #include <linux/sched.h> 4e99b6f46SAndy Lutomirski #include <linux/sched/debug.h> 5e99b6f46SAndy Lutomirski #include <linux/init_task.h> 6e99b6f46SAndy Lutomirski #include <linux/fs.h> 7e99b6f46SAndy Lutomirski 8e99b6f46SAndy Lutomirski #include <linux/uaccess.h> 9e99b6f46SAndy Lutomirski #include <asm/pgtable.h> 10e99b6f46SAndy Lutomirski #include <asm/processor.h> 11e99b6f46SAndy Lutomirski #include <asm/desc.h> 12*7d8d8cfdSAndy Lutomirski #include <asm/traps.h> 13e99b6f46SAndy Lutomirski 14*7d8d8cfdSAndy Lutomirski extern void double_fault(void); 15e99b6f46SAndy Lutomirski #define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM) 16e99b6f46SAndy Lutomirski 17*7d8d8cfdSAndy Lutomirski #define TSS(x) this_cpu_read(cpu_tss_rw.x86_tss.x) 18*7d8d8cfdSAndy Lutomirski 19*7d8d8cfdSAndy Lutomirski static void set_df_gdt_entry(unsigned int cpu); 20*7d8d8cfdSAndy Lutomirski 21*7d8d8cfdSAndy Lutomirski /* 22*7d8d8cfdSAndy Lutomirski * Called by double_fault with CR0.TS and EFLAGS.NT cleared. The CPU thinks 23*7d8d8cfdSAndy Lutomirski * we're running the doublefault task. Cannot return. 24*7d8d8cfdSAndy Lutomirski */ 25*7d8d8cfdSAndy Lutomirski asmlinkage notrace void __noreturn doublefault_shim(void) 26e99b6f46SAndy Lutomirski { 27*7d8d8cfdSAndy Lutomirski unsigned long cr2; 28*7d8d8cfdSAndy Lutomirski struct pt_regs regs; 29e99b6f46SAndy Lutomirski 30dc4e0021SAndy Lutomirski BUILD_BUG_ON(sizeof(struct doublefault_stack) != PAGE_SIZE); 31dc4e0021SAndy Lutomirski 32*7d8d8cfdSAndy Lutomirski cr2 = native_read_cr2(); 33e99b6f46SAndy Lutomirski 34*7d8d8cfdSAndy Lutomirski /* Reset back to the normal kernel task. */ 35*7d8d8cfdSAndy Lutomirski force_reload_TR(); 36*7d8d8cfdSAndy Lutomirski set_df_gdt_entry(smp_processor_id()); 37e99b6f46SAndy Lutomirski 38*7d8d8cfdSAndy Lutomirski trace_hardirqs_off(); 39e99b6f46SAndy Lutomirski 40*7d8d8cfdSAndy Lutomirski /* 41*7d8d8cfdSAndy Lutomirski * Fill in pt_regs. A downside of doing this in C is that the unwinder 42*7d8d8cfdSAndy Lutomirski * won't see it (no ENCODE_FRAME_POINTER), so a nested stack dump 43*7d8d8cfdSAndy Lutomirski * won't successfully unwind to the source of the double fault. 44*7d8d8cfdSAndy Lutomirski * The main dump from do_double_fault() is fine, though, since it 45*7d8d8cfdSAndy Lutomirski * uses these regs directly. 46*7d8d8cfdSAndy Lutomirski * 47*7d8d8cfdSAndy Lutomirski * If anyone ever cares, this could be moved to asm. 48*7d8d8cfdSAndy Lutomirski */ 49*7d8d8cfdSAndy Lutomirski regs.ss = TSS(ss); 50*7d8d8cfdSAndy Lutomirski regs.__ssh = 0; 51*7d8d8cfdSAndy Lutomirski regs.sp = TSS(sp); 52*7d8d8cfdSAndy Lutomirski regs.flags = TSS(flags); 53*7d8d8cfdSAndy Lutomirski regs.cs = TSS(cs); 54*7d8d8cfdSAndy Lutomirski /* We won't go through the entry asm, so we can leave __csh as 0. */ 55*7d8d8cfdSAndy Lutomirski regs.__csh = 0; 56*7d8d8cfdSAndy Lutomirski regs.ip = TSS(ip); 57*7d8d8cfdSAndy Lutomirski regs.orig_ax = 0; 58*7d8d8cfdSAndy Lutomirski regs.gs = TSS(gs); 59*7d8d8cfdSAndy Lutomirski regs.__gsh = 0; 60*7d8d8cfdSAndy Lutomirski regs.fs = TSS(fs); 61*7d8d8cfdSAndy Lutomirski regs.__fsh = 0; 62*7d8d8cfdSAndy Lutomirski regs.es = TSS(es); 63*7d8d8cfdSAndy Lutomirski regs.__esh = 0; 64*7d8d8cfdSAndy Lutomirski regs.ds = TSS(ds); 65*7d8d8cfdSAndy Lutomirski regs.__dsh = 0; 66*7d8d8cfdSAndy Lutomirski regs.ax = TSS(ax); 67*7d8d8cfdSAndy Lutomirski regs.bp = TSS(bp); 68*7d8d8cfdSAndy Lutomirski regs.di = TSS(di); 69*7d8d8cfdSAndy Lutomirski regs.si = TSS(si); 70*7d8d8cfdSAndy Lutomirski regs.dx = TSS(dx); 71*7d8d8cfdSAndy Lutomirski regs.cx = TSS(cx); 72*7d8d8cfdSAndy Lutomirski regs.bx = TSS(bx); 73e99b6f46SAndy Lutomirski 74*7d8d8cfdSAndy Lutomirski do_double_fault(®s, 0, cr2); 75e99b6f46SAndy Lutomirski 76*7d8d8cfdSAndy Lutomirski /* 77*7d8d8cfdSAndy Lutomirski * x86_32 does not save the original CR3 anywhere on a task switch. 78*7d8d8cfdSAndy Lutomirski * This means that, even if we wanted to return, we would need to find 79*7d8d8cfdSAndy Lutomirski * some way to reconstruct CR3. We could make a credible guess based 80*7d8d8cfdSAndy Lutomirski * on cpu_tlbstate, but that would be racy and would not account for 81*7d8d8cfdSAndy Lutomirski * PTI. 82*7d8d8cfdSAndy Lutomirski * 83*7d8d8cfdSAndy Lutomirski * Instead, don't bother. We can return through 84*7d8d8cfdSAndy Lutomirski * rewind_stack_do_exit() instead. 85*7d8d8cfdSAndy Lutomirski */ 86*7d8d8cfdSAndy Lutomirski panic("cannot return from double fault\n"); 87e99b6f46SAndy Lutomirski } 88*7d8d8cfdSAndy Lutomirski NOKPROBE_SYMBOL(doublefault_shim); 89e99b6f46SAndy Lutomirski 90dc4e0021SAndy Lutomirski DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = { 91dc4e0021SAndy Lutomirski .tss = { 92dc4e0021SAndy Lutomirski /* 93dc4e0021SAndy Lutomirski * No sp0 or ss0 -- we never run CPL != 0 with this TSS 94dc4e0021SAndy Lutomirski * active. sp is filled in later. 95dc4e0021SAndy Lutomirski */ 96e99b6f46SAndy Lutomirski .ldt = 0, 97e99b6f46SAndy Lutomirski .io_bitmap_base = IO_BITMAP_OFFSET_INVALID, 98e99b6f46SAndy Lutomirski 99*7d8d8cfdSAndy Lutomirski .ip = (unsigned long) double_fault, 100*7d8d8cfdSAndy Lutomirski .flags = X86_EFLAGS_FIXED, 101e99b6f46SAndy Lutomirski .es = __USER_DS, 102e99b6f46SAndy Lutomirski .cs = __KERNEL_CS, 103e99b6f46SAndy Lutomirski .ss = __KERNEL_DS, 104e99b6f46SAndy Lutomirski .ds = __USER_DS, 105e99b6f46SAndy Lutomirski .fs = __KERNEL_PERCPU, 106e99b6f46SAndy Lutomirski #ifndef CONFIG_X86_32_LAZY_GS 107e99b6f46SAndy Lutomirski .gs = __KERNEL_STACK_CANARY, 108e99b6f46SAndy Lutomirski #endif 109e99b6f46SAndy Lutomirski 110e99b6f46SAndy Lutomirski .__cr3 = __pa_nodebug(swapper_pg_dir), 111dc4e0021SAndy Lutomirski }, 112e99b6f46SAndy Lutomirski }; 113dc4e0021SAndy Lutomirski 114*7d8d8cfdSAndy Lutomirski static void set_df_gdt_entry(unsigned int cpu) 115*7d8d8cfdSAndy Lutomirski { 116*7d8d8cfdSAndy Lutomirski /* Set up doublefault TSS pointer in the GDT */ 117*7d8d8cfdSAndy Lutomirski __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, 118*7d8d8cfdSAndy Lutomirski &get_cpu_entry_area(cpu)->doublefault_stack.tss); 119*7d8d8cfdSAndy Lutomirski 120*7d8d8cfdSAndy Lutomirski } 121*7d8d8cfdSAndy Lutomirski 122dc4e0021SAndy Lutomirski void doublefault_init_cpu_tss(void) 123dc4e0021SAndy Lutomirski { 124dc4e0021SAndy Lutomirski unsigned int cpu = smp_processor_id(); 125dc4e0021SAndy Lutomirski struct cpu_entry_area *cea = get_cpu_entry_area(cpu); 126dc4e0021SAndy Lutomirski 127dc4e0021SAndy Lutomirski /* 128dc4e0021SAndy Lutomirski * The linker isn't smart enough to initialize percpu variables that 129dc4e0021SAndy Lutomirski * point to other places in percpu space. 130dc4e0021SAndy Lutomirski */ 131dc4e0021SAndy Lutomirski this_cpu_write(doublefault_stack.tss.sp, 132dc4e0021SAndy Lutomirski (unsigned long)&cea->doublefault_stack.stack + 133dc4e0021SAndy Lutomirski sizeof(doublefault_stack.stack)); 134dc4e0021SAndy Lutomirski 135*7d8d8cfdSAndy Lutomirski set_df_gdt_entry(cpu); 136dc4e0021SAndy Lutomirski } 137