10603839bSHuacai Chen // SPDX-License-Identifier: GPL-2.0 20603839bSHuacai Chen /* 30603839bSHuacai Chen * Author: Huacai Chen <chenhuacai@loongson.cn> 40603839bSHuacai Chen * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 50603839bSHuacai Chen */ 6efada2afSWANG Xuerui #include <linux/bitfield.h> 70603839bSHuacai Chen #include <linux/bitops.h> 80603839bSHuacai Chen #include <linux/bug.h> 90603839bSHuacai Chen #include <linux/compiler.h> 100603839bSHuacai Chen #include <linux/context_tracking.h> 110603839bSHuacai Chen #include <linux/entry-common.h> 120603839bSHuacai Chen #include <linux/init.h> 130603839bSHuacai Chen #include <linux/kernel.h> 144e62d1d8SYouling Tang #include <linux/kexec.h> 150603839bSHuacai Chen #include <linux/module.h> 160603839bSHuacai Chen #include <linux/extable.h> 170603839bSHuacai Chen #include <linux/mm.h> 180603839bSHuacai Chen #include <linux/sched/mm.h> 190603839bSHuacai Chen #include <linux/sched/debug.h> 200603839bSHuacai Chen #include <linux/smp.h> 210603839bSHuacai Chen #include <linux/spinlock.h> 220603839bSHuacai Chen #include <linux/kallsyms.h> 230603839bSHuacai Chen #include <linux/memblock.h> 240603839bSHuacai Chen #include <linux/interrupt.h> 250603839bSHuacai Chen #include <linux/ptrace.h> 260603839bSHuacai Chen #include <linux/kgdb.h> 270603839bSHuacai Chen #include <linux/kdebug.h> 280603839bSHuacai Chen #include <linux/kprobes.h> 290603839bSHuacai Chen #include <linux/notifier.h> 300603839bSHuacai Chen #include <linux/irq.h> 310603839bSHuacai Chen #include <linux/perf_event.h> 320603839bSHuacai Chen 330603839bSHuacai Chen #include <asm/addrspace.h> 340603839bSHuacai Chen #include <asm/bootinfo.h> 350603839bSHuacai Chen #include <asm/branch.h> 360603839bSHuacai Chen #include <asm/break.h> 370603839bSHuacai Chen #include <asm/cpu.h> 380603839bSHuacai Chen #include <asm/fpu.h> 390603839bSHuacai Chen #include <asm/loongarch.h> 400603839bSHuacai Chen #include <asm/mmu_context.h> 410603839bSHuacai Chen #include <asm/pgtable.h> 420603839bSHuacai Chen #include <asm/ptrace.h> 430603839bSHuacai Chen #include <asm/sections.h> 440603839bSHuacai Chen #include <asm/siginfo.h> 450603839bSHuacai Chen #include <asm/stacktrace.h> 460603839bSHuacai Chen #include <asm/tlb.h> 470603839bSHuacai Chen #include <asm/types.h> 4849232773SQing Zhang #include <asm/unwind.h> 490603839bSHuacai Chen 500603839bSHuacai Chen #include "access-helper.h" 510603839bSHuacai Chen 520603839bSHuacai Chen extern asmlinkage void handle_ade(void); 530603839bSHuacai Chen extern asmlinkage void handle_ale(void); 540603839bSHuacai Chen extern asmlinkage void handle_sys(void); 550603839bSHuacai Chen extern asmlinkage void handle_bp(void); 560603839bSHuacai Chen extern asmlinkage void handle_ri(void); 570603839bSHuacai Chen extern asmlinkage void handle_fpu(void); 580603839bSHuacai Chen extern asmlinkage void handle_fpe(void); 590603839bSHuacai Chen extern asmlinkage void handle_lbt(void); 600603839bSHuacai Chen extern asmlinkage void handle_lsx(void); 610603839bSHuacai Chen extern asmlinkage void handle_lasx(void); 620603839bSHuacai Chen extern asmlinkage void handle_reserved(void); 630603839bSHuacai Chen extern asmlinkage void handle_watch(void); 640603839bSHuacai Chen extern asmlinkage void handle_vint(void); 650603839bSHuacai Chen 660603839bSHuacai Chen static void show_backtrace(struct task_struct *task, const struct pt_regs *regs, 670603839bSHuacai Chen const char *loglvl, bool user) 680603839bSHuacai Chen { 690603839bSHuacai Chen unsigned long addr; 7049232773SQing Zhang struct unwind_state state; 7149232773SQing Zhang struct pt_regs *pregs = (struct pt_regs *)regs; 7249232773SQing Zhang 7349232773SQing Zhang if (!task) 7449232773SQing Zhang task = current; 750603839bSHuacai Chen 760603839bSHuacai Chen printk("%sCall Trace:", loglvl); 7749232773SQing Zhang for (unwind_start(&state, task, pregs); 7849232773SQing Zhang !unwind_done(&state); unwind_next_frame(&state)) { 7949232773SQing Zhang addr = unwind_get_return_address(&state); 800603839bSHuacai Chen print_ip_sym(loglvl, addr); 810603839bSHuacai Chen } 820603839bSHuacai Chen printk("%s\n", loglvl); 830603839bSHuacai Chen } 840603839bSHuacai Chen 850603839bSHuacai Chen static void show_stacktrace(struct task_struct *task, 860603839bSHuacai Chen const struct pt_regs *regs, const char *loglvl, bool user) 870603839bSHuacai Chen { 880603839bSHuacai Chen int i; 890603839bSHuacai Chen const int field = 2 * sizeof(unsigned long); 900603839bSHuacai Chen unsigned long stackdata; 910603839bSHuacai Chen unsigned long *sp = (unsigned long *)regs->regs[3]; 920603839bSHuacai Chen 930603839bSHuacai Chen printk("%sStack :", loglvl); 940603839bSHuacai Chen i = 0; 950603839bSHuacai Chen while ((unsigned long) sp & (PAGE_SIZE - 1)) { 960603839bSHuacai Chen if (i && ((i % (64 / field)) == 0)) { 970603839bSHuacai Chen pr_cont("\n"); 980603839bSHuacai Chen printk("%s ", loglvl); 990603839bSHuacai Chen } 1000603839bSHuacai Chen if (i > 39) { 1010603839bSHuacai Chen pr_cont(" ..."); 1020603839bSHuacai Chen break; 1030603839bSHuacai Chen } 1040603839bSHuacai Chen 1050603839bSHuacai Chen if (__get_addr(&stackdata, sp++, user)) { 1060603839bSHuacai Chen pr_cont(" (Bad stack address)"); 1070603839bSHuacai Chen break; 1080603839bSHuacai Chen } 1090603839bSHuacai Chen 1100603839bSHuacai Chen pr_cont(" %0*lx", field, stackdata); 1110603839bSHuacai Chen i++; 1120603839bSHuacai Chen } 1130603839bSHuacai Chen pr_cont("\n"); 1140603839bSHuacai Chen show_backtrace(task, regs, loglvl, user); 1150603839bSHuacai Chen } 1160603839bSHuacai Chen 1170603839bSHuacai Chen void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) 1180603839bSHuacai Chen { 1190603839bSHuacai Chen struct pt_regs regs; 1200603839bSHuacai Chen 1210603839bSHuacai Chen regs.csr_crmd = 0; 1220603839bSHuacai Chen if (sp) { 1230603839bSHuacai Chen regs.csr_era = 0; 1240603839bSHuacai Chen regs.regs[1] = 0; 1250603839bSHuacai Chen regs.regs[3] = (unsigned long)sp; 1260603839bSHuacai Chen } else { 1270603839bSHuacai Chen if (!task || task == current) 1280603839bSHuacai Chen prepare_frametrace(®s); 1290603839bSHuacai Chen else { 1300603839bSHuacai Chen regs.csr_era = task->thread.reg01; 1310603839bSHuacai Chen regs.regs[1] = 0; 1320603839bSHuacai Chen regs.regs[3] = task->thread.reg03; 1330603839bSHuacai Chen regs.regs[22] = task->thread.reg22; 1340603839bSHuacai Chen } 1350603839bSHuacai Chen } 1360603839bSHuacai Chen 1370603839bSHuacai Chen show_stacktrace(task, ®s, loglvl, false); 1380603839bSHuacai Chen } 1390603839bSHuacai Chen 1400603839bSHuacai Chen static void show_code(unsigned int *pc, bool user) 1410603839bSHuacai Chen { 1420603839bSHuacai Chen long i; 1430603839bSHuacai Chen unsigned int insn; 1440603839bSHuacai Chen 1450603839bSHuacai Chen printk("Code:"); 1460603839bSHuacai Chen 1470603839bSHuacai Chen for(i = -3 ; i < 6 ; i++) { 1480603839bSHuacai Chen if (__get_inst(&insn, pc + i, user)) { 1490603839bSHuacai Chen pr_cont(" (Bad address in era)\n"); 1500603839bSHuacai Chen break; 1510603839bSHuacai Chen } 1520603839bSHuacai Chen pr_cont("%c%08x%c", (i?' ':'<'), insn, (i?' ':'>')); 1530603839bSHuacai Chen } 1540603839bSHuacai Chen pr_cont("\n"); 1550603839bSHuacai Chen } 1560603839bSHuacai Chen 157efada2afSWANG Xuerui static void print_bool_fragment(const char *key, unsigned long val, bool first) 158efada2afSWANG Xuerui { 159efada2afSWANG Xuerui /* e.g. "+PG", "-DA" */ 160efada2afSWANG Xuerui pr_cont("%s%c%s", first ? "" : " ", val ? '+' : '-', key); 161efada2afSWANG Xuerui } 162efada2afSWANG Xuerui 163efada2afSWANG Xuerui static void print_plv_fragment(const char *key, int val) 164efada2afSWANG Xuerui { 165efada2afSWANG Xuerui /* e.g. "PLV0", "PPLV3" */ 166efada2afSWANG Xuerui pr_cont("%s%d", key, val); 167efada2afSWANG Xuerui } 168efada2afSWANG Xuerui 169efada2afSWANG Xuerui static void print_memory_type_fragment(const char *key, unsigned long val) 170efada2afSWANG Xuerui { 171efada2afSWANG Xuerui const char *humanized_type; 172efada2afSWANG Xuerui 173efada2afSWANG Xuerui switch (val) { 174efada2afSWANG Xuerui case 0: 175efada2afSWANG Xuerui humanized_type = "SUC"; 176efada2afSWANG Xuerui break; 177efada2afSWANG Xuerui case 1: 178efada2afSWANG Xuerui humanized_type = "CC"; 179efada2afSWANG Xuerui break; 180efada2afSWANG Xuerui case 2: 181efada2afSWANG Xuerui humanized_type = "WUC"; 182efada2afSWANG Xuerui break; 183efada2afSWANG Xuerui default: 184efada2afSWANG Xuerui pr_cont(" %s=Reserved(%lu)", key, val); 185efada2afSWANG Xuerui return; 186efada2afSWANG Xuerui } 187efada2afSWANG Xuerui 188efada2afSWANG Xuerui /* e.g. " DATM=WUC" */ 189efada2afSWANG Xuerui pr_cont(" %s=%s", key, humanized_type); 190efada2afSWANG Xuerui } 191efada2afSWANG Xuerui 192efada2afSWANG Xuerui static void print_crmd(unsigned long x) 193efada2afSWANG Xuerui { 194efada2afSWANG Xuerui printk(" CRMD: %08lx (", x); 195efada2afSWANG Xuerui print_plv_fragment("PLV", (int) FIELD_GET(CSR_CRMD_PLV, x)); 196efada2afSWANG Xuerui print_bool_fragment("IE", FIELD_GET(CSR_CRMD_IE, x), false); 197efada2afSWANG Xuerui print_bool_fragment("DA", FIELD_GET(CSR_CRMD_DA, x), false); 198efada2afSWANG Xuerui print_bool_fragment("PG", FIELD_GET(CSR_CRMD_PG, x), false); 199efada2afSWANG Xuerui print_memory_type_fragment("DACF", FIELD_GET(CSR_CRMD_DACF, x)); 200efada2afSWANG Xuerui print_memory_type_fragment("DACM", FIELD_GET(CSR_CRMD_DACM, x)); 201efada2afSWANG Xuerui print_bool_fragment("WE", FIELD_GET(CSR_CRMD_WE, x), false); 202efada2afSWANG Xuerui pr_cont(")\n"); 203efada2afSWANG Xuerui } 204efada2afSWANG Xuerui 205ce7f0b18SWANG Xuerui static void print_prmd(unsigned long x) 206ce7f0b18SWANG Xuerui { 207ce7f0b18SWANG Xuerui printk(" PRMD: %08lx (", x); 208ce7f0b18SWANG Xuerui print_plv_fragment("PPLV", (int) FIELD_GET(CSR_PRMD_PPLV, x)); 209ce7f0b18SWANG Xuerui print_bool_fragment("PIE", FIELD_GET(CSR_PRMD_PIE, x), false); 210ce7f0b18SWANG Xuerui print_bool_fragment("PWE", FIELD_GET(CSR_PRMD_PWE, x), false); 211ce7f0b18SWANG Xuerui pr_cont(")\n"); 212ce7f0b18SWANG Xuerui } 213ce7f0b18SWANG Xuerui 214*9718d96cSWANG Xuerui static void print_euen(unsigned long x) 215*9718d96cSWANG Xuerui { 216*9718d96cSWANG Xuerui printk(" EUEN: %08lx (", x); 217*9718d96cSWANG Xuerui print_bool_fragment("FPE", FIELD_GET(CSR_EUEN_FPEN, x), true); 218*9718d96cSWANG Xuerui print_bool_fragment("SXE", FIELD_GET(CSR_EUEN_LSXEN, x), false); 219*9718d96cSWANG Xuerui print_bool_fragment("ASXE", FIELD_GET(CSR_EUEN_LASXEN, x), false); 220*9718d96cSWANG Xuerui print_bool_fragment("BTE", FIELD_GET(CSR_EUEN_LBTEN, x), false); 221*9718d96cSWANG Xuerui pr_cont(")\n"); 222*9718d96cSWANG Xuerui } 223*9718d96cSWANG Xuerui 2240603839bSHuacai Chen static void __show_regs(const struct pt_regs *regs) 2250603839bSHuacai Chen { 2260603839bSHuacai Chen const int field = 2 * sizeof(unsigned long); 2270603839bSHuacai Chen unsigned int excsubcode; 2280603839bSHuacai Chen unsigned int exccode; 2290603839bSHuacai Chen 2300603839bSHuacai Chen show_regs_print_info(KERN_DEFAULT); 2310603839bSHuacai Chen 232f6a79b60SWANG Xuerui /* Print saved GPRs except $zero (substituting with PC/ERA) */ 233f6a79b60SWANG Xuerui #define GPR_FIELD(x) field, regs->regs[x] 234f6a79b60SWANG Xuerui printk("pc %0*lx ra %0*lx tp %0*lx sp %0*lx\n", 235f6a79b60SWANG Xuerui field, regs->csr_era, GPR_FIELD(1), GPR_FIELD(2), GPR_FIELD(3)); 236f6a79b60SWANG Xuerui printk("a0 %0*lx a1 %0*lx a2 %0*lx a3 %0*lx\n", 237f6a79b60SWANG Xuerui GPR_FIELD(4), GPR_FIELD(5), GPR_FIELD(6), GPR_FIELD(7)); 238f6a79b60SWANG Xuerui printk("a4 %0*lx a5 %0*lx a6 %0*lx a7 %0*lx\n", 239f6a79b60SWANG Xuerui GPR_FIELD(8), GPR_FIELD(9), GPR_FIELD(10), GPR_FIELD(11)); 240f6a79b60SWANG Xuerui printk("t0 %0*lx t1 %0*lx t2 %0*lx t3 %0*lx\n", 241f6a79b60SWANG Xuerui GPR_FIELD(12), GPR_FIELD(13), GPR_FIELD(14), GPR_FIELD(15)); 242f6a79b60SWANG Xuerui printk("t4 %0*lx t5 %0*lx t6 %0*lx t7 %0*lx\n", 243f6a79b60SWANG Xuerui GPR_FIELD(16), GPR_FIELD(17), GPR_FIELD(18), GPR_FIELD(19)); 244f6a79b60SWANG Xuerui printk("t8 %0*lx u0 %0*lx s9 %0*lx s0 %0*lx\n", 245f6a79b60SWANG Xuerui GPR_FIELD(20), GPR_FIELD(21), GPR_FIELD(22), GPR_FIELD(23)); 246f6a79b60SWANG Xuerui printk("s1 %0*lx s2 %0*lx s3 %0*lx s4 %0*lx\n", 247f6a79b60SWANG Xuerui GPR_FIELD(24), GPR_FIELD(25), GPR_FIELD(26), GPR_FIELD(27)); 248f6a79b60SWANG Xuerui printk("s5 %0*lx s6 %0*lx s7 %0*lx s8 %0*lx\n", 249f6a79b60SWANG Xuerui GPR_FIELD(28), GPR_FIELD(29), GPR_FIELD(30), GPR_FIELD(31)); 2500603839bSHuacai Chen 251f6a79b60SWANG Xuerui /* The slot for $zero is reused as the syscall restart flag */ 252f6a79b60SWANG Xuerui if (regs->regs[0]) 253f6a79b60SWANG Xuerui printk("syscall restart flag: %0*lx\n", GPR_FIELD(0)); 254863b3795SWANG Xuerui 255863b3795SWANG Xuerui if (user_mode(regs)) { 256863b3795SWANG Xuerui printk(" ra: %0*lx\n", GPR_FIELD(1)); 257863b3795SWANG Xuerui printk(" ERA: %0*lx\n", field, regs->csr_era); 258863b3795SWANG Xuerui } else { 259863b3795SWANG Xuerui printk(" ra: %0*lx %pS\n", GPR_FIELD(1), (void *) regs->regs[1]); 260863b3795SWANG Xuerui printk(" ERA: %0*lx %pS\n", field, regs->csr_era, (void *) regs->csr_era); 261863b3795SWANG Xuerui } 262f6a79b60SWANG Xuerui #undef GPR_FIELD 2630603839bSHuacai Chen 26405fa8d49SWANG Xuerui /* Print saved important CSRs */ 265efada2afSWANG Xuerui print_crmd(regs->csr_crmd); 266ce7f0b18SWANG Xuerui print_prmd(regs->csr_prmd); 267*9718d96cSWANG Xuerui print_euen(regs->csr_euen); 26805fa8d49SWANG Xuerui printk(" ECFG: %08lx\n", regs->csr_ecfg); 26905fa8d49SWANG Xuerui printk("ESTAT: %08lx\n", regs->csr_estat); 2700603839bSHuacai Chen 2710603839bSHuacai Chen exccode = ((regs->csr_estat) & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT; 2720603839bSHuacai Chen excsubcode = ((regs->csr_estat) & CSR_ESTAT_ESUBCODE) >> CSR_ESTAT_ESUBCODE_SHIFT; 2730603839bSHuacai Chen printk("ExcCode : %x (SubCode %x)\n", exccode, excsubcode); 2740603839bSHuacai Chen 2750603839bSHuacai Chen if (exccode >= EXCCODE_TLBL && exccode <= EXCCODE_ALE) 2760603839bSHuacai Chen printk("BadVA : %0*lx\n", field, regs->csr_badvaddr); 2770603839bSHuacai Chen 2780603839bSHuacai Chen printk("PrId : %08x (%s)\n", read_cpucfg(LOONGARCH_CPUCFG0), 2790603839bSHuacai Chen cpu_family_string()); 2800603839bSHuacai Chen } 2810603839bSHuacai Chen 2820603839bSHuacai Chen void show_regs(struct pt_regs *regs) 2830603839bSHuacai Chen { 2840603839bSHuacai Chen __show_regs((struct pt_regs *)regs); 2850603839bSHuacai Chen dump_stack(); 2860603839bSHuacai Chen } 2870603839bSHuacai Chen 2880603839bSHuacai Chen void show_registers(struct pt_regs *regs) 2890603839bSHuacai Chen { 2900603839bSHuacai Chen __show_regs(regs); 2910603839bSHuacai Chen print_modules(); 2920603839bSHuacai Chen printk("Process %s (pid: %d, threadinfo=%p, task=%p)\n", 2930603839bSHuacai Chen current->comm, current->pid, current_thread_info(), current); 2940603839bSHuacai Chen 2950603839bSHuacai Chen show_stacktrace(current, regs, KERN_DEFAULT, user_mode(regs)); 2960603839bSHuacai Chen show_code((void *)regs->csr_era, user_mode(regs)); 2970603839bSHuacai Chen printk("\n"); 2980603839bSHuacai Chen } 2990603839bSHuacai Chen 3000603839bSHuacai Chen static DEFINE_RAW_SPINLOCK(die_lock); 3010603839bSHuacai Chen 3020603839bSHuacai Chen void __noreturn die(const char *str, struct pt_regs *regs) 3030603839bSHuacai Chen { 3040603839bSHuacai Chen static int die_counter; 3050603839bSHuacai Chen int sig = SIGSEGV; 3060603839bSHuacai Chen 3070603839bSHuacai Chen oops_enter(); 3080603839bSHuacai Chen 3090603839bSHuacai Chen if (notify_die(DIE_OOPS, str, regs, 0, current->thread.trap_nr, 3100603839bSHuacai Chen SIGSEGV) == NOTIFY_STOP) 3110603839bSHuacai Chen sig = 0; 3120603839bSHuacai Chen 3130603839bSHuacai Chen console_verbose(); 3140603839bSHuacai Chen raw_spin_lock_irq(&die_lock); 3150603839bSHuacai Chen bust_spinlocks(1); 3160603839bSHuacai Chen 3170603839bSHuacai Chen printk("%s[#%d]:\n", str, ++die_counter); 3180603839bSHuacai Chen show_registers(regs); 3190603839bSHuacai Chen add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 3200603839bSHuacai Chen raw_spin_unlock_irq(&die_lock); 3210603839bSHuacai Chen 3220603839bSHuacai Chen oops_exit(); 3230603839bSHuacai Chen 3244e62d1d8SYouling Tang if (regs && kexec_should_crash(current)) 3254e62d1d8SYouling Tang crash_kexec(regs); 3264e62d1d8SYouling Tang 3270603839bSHuacai Chen if (in_interrupt()) 3280603839bSHuacai Chen panic("Fatal exception in interrupt"); 3290603839bSHuacai Chen 3300603839bSHuacai Chen if (panic_on_oops) 3310603839bSHuacai Chen panic("Fatal exception"); 3320603839bSHuacai Chen 3330603839bSHuacai Chen make_task_dead(sig); 3340603839bSHuacai Chen } 3350603839bSHuacai Chen 3360603839bSHuacai Chen static inline void setup_vint_size(unsigned int size) 3370603839bSHuacai Chen { 3380603839bSHuacai Chen unsigned int vs; 3390603839bSHuacai Chen 3400603839bSHuacai Chen vs = ilog2(size/4); 3410603839bSHuacai Chen 3420603839bSHuacai Chen if (vs == 0 || vs > 7) 3430603839bSHuacai Chen panic("vint_size %d Not support yet", vs); 3440603839bSHuacai Chen 3450603839bSHuacai Chen csr_xchg32(vs<<CSR_ECFG_VS_SHIFT, CSR_ECFG_VS, LOONGARCH_CSR_ECFG); 3460603839bSHuacai Chen } 3470603839bSHuacai Chen 3480603839bSHuacai Chen /* 3490603839bSHuacai Chen * Send SIGFPE according to FCSR Cause bits, which must have already 3500603839bSHuacai Chen * been masked against Enable bits. This is impotant as Inexact can 3510603839bSHuacai Chen * happen together with Overflow or Underflow, and `ptrace' can set 3520603839bSHuacai Chen * any bits. 3530603839bSHuacai Chen */ 3540603839bSHuacai Chen void force_fcsr_sig(unsigned long fcsr, void __user *fault_addr, 3550603839bSHuacai Chen struct task_struct *tsk) 3560603839bSHuacai Chen { 3570603839bSHuacai Chen int si_code = FPE_FLTUNK; 3580603839bSHuacai Chen 3590603839bSHuacai Chen if (fcsr & FPU_CSR_INV_X) 3600603839bSHuacai Chen si_code = FPE_FLTINV; 3610603839bSHuacai Chen else if (fcsr & FPU_CSR_DIV_X) 3620603839bSHuacai Chen si_code = FPE_FLTDIV; 3630603839bSHuacai Chen else if (fcsr & FPU_CSR_OVF_X) 3640603839bSHuacai Chen si_code = FPE_FLTOVF; 3650603839bSHuacai Chen else if (fcsr & FPU_CSR_UDF_X) 3660603839bSHuacai Chen si_code = FPE_FLTUND; 3670603839bSHuacai Chen else if (fcsr & FPU_CSR_INE_X) 3680603839bSHuacai Chen si_code = FPE_FLTRES; 3690603839bSHuacai Chen 3700603839bSHuacai Chen force_sig_fault(SIGFPE, si_code, fault_addr); 3710603839bSHuacai Chen } 3720603839bSHuacai Chen 3730603839bSHuacai Chen int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcsr) 3740603839bSHuacai Chen { 3750603839bSHuacai Chen int si_code; 3760603839bSHuacai Chen 3770603839bSHuacai Chen switch (sig) { 3780603839bSHuacai Chen case 0: 3790603839bSHuacai Chen return 0; 3800603839bSHuacai Chen 3810603839bSHuacai Chen case SIGFPE: 3820603839bSHuacai Chen force_fcsr_sig(fcsr, fault_addr, current); 3830603839bSHuacai Chen return 1; 3840603839bSHuacai Chen 3850603839bSHuacai Chen case SIGBUS: 3860603839bSHuacai Chen force_sig_fault(SIGBUS, BUS_ADRERR, fault_addr); 3870603839bSHuacai Chen return 1; 3880603839bSHuacai Chen 3890603839bSHuacai Chen case SIGSEGV: 3900603839bSHuacai Chen mmap_read_lock(current->mm); 3910603839bSHuacai Chen if (vma_lookup(current->mm, (unsigned long)fault_addr)) 3920603839bSHuacai Chen si_code = SEGV_ACCERR; 3930603839bSHuacai Chen else 3940603839bSHuacai Chen si_code = SEGV_MAPERR; 3950603839bSHuacai Chen mmap_read_unlock(current->mm); 3960603839bSHuacai Chen force_sig_fault(SIGSEGV, si_code, fault_addr); 3970603839bSHuacai Chen return 1; 3980603839bSHuacai Chen 3990603839bSHuacai Chen default: 4000603839bSHuacai Chen force_sig(sig); 4010603839bSHuacai Chen return 1; 4020603839bSHuacai Chen } 4030603839bSHuacai Chen } 4040603839bSHuacai Chen 4050603839bSHuacai Chen /* 4060603839bSHuacai Chen * Delayed fp exceptions when doing a lazy ctx switch 4070603839bSHuacai Chen */ 4080603839bSHuacai Chen asmlinkage void noinstr do_fpe(struct pt_regs *regs, unsigned long fcsr) 4090603839bSHuacai Chen { 4100603839bSHuacai Chen int sig; 4110603839bSHuacai Chen void __user *fault_addr; 4120603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 4130603839bSHuacai Chen 4140603839bSHuacai Chen if (notify_die(DIE_FP, "FP exception", regs, 0, current->thread.trap_nr, 4150603839bSHuacai Chen SIGFPE) == NOTIFY_STOP) 4160603839bSHuacai Chen goto out; 4170603839bSHuacai Chen 4180603839bSHuacai Chen /* Clear FCSR.Cause before enabling interrupts */ 4190603839bSHuacai Chen write_fcsr(LOONGARCH_FCSR0, fcsr & ~mask_fcsr_x(fcsr)); 4200603839bSHuacai Chen local_irq_enable(); 4210603839bSHuacai Chen 4220603839bSHuacai Chen die_if_kernel("FP exception in kernel code", regs); 4230603839bSHuacai Chen 4240603839bSHuacai Chen sig = SIGFPE; 4250603839bSHuacai Chen fault_addr = (void __user *) regs->csr_era; 4260603839bSHuacai Chen 4270603839bSHuacai Chen /* Send a signal if required. */ 4280603839bSHuacai Chen process_fpemu_return(sig, fault_addr, fcsr); 4290603839bSHuacai Chen 4300603839bSHuacai Chen out: 4310603839bSHuacai Chen local_irq_disable(); 4320603839bSHuacai Chen irqentry_exit(regs, state); 4330603839bSHuacai Chen } 4340603839bSHuacai Chen 4350603839bSHuacai Chen asmlinkage void noinstr do_ade(struct pt_regs *regs) 4360603839bSHuacai Chen { 4370603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 4380603839bSHuacai Chen 4390603839bSHuacai Chen die_if_kernel("Kernel ade access", regs); 4400603839bSHuacai Chen force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)regs->csr_badvaddr); 4410603839bSHuacai Chen 4420603839bSHuacai Chen irqentry_exit(regs, state); 4430603839bSHuacai Chen } 4440603839bSHuacai Chen 44561a6fcccSHuacai Chen /* sysctl hooks */ 44661a6fcccSHuacai Chen int unaligned_enabled __read_mostly = 1; /* Enabled by default */ 44761a6fcccSHuacai Chen int no_unaligned_warning __read_mostly = 1; /* Only 1 warning by default */ 44861a6fcccSHuacai Chen 4490603839bSHuacai Chen asmlinkage void noinstr do_ale(struct pt_regs *regs) 4500603839bSHuacai Chen { 4510603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 4520603839bSHuacai Chen 45341596803SHuacai Chen #ifndef CONFIG_ARCH_STRICT_ALIGN 45441596803SHuacai Chen die_if_kernel("Kernel ale access", regs); 45541596803SHuacai Chen force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)regs->csr_badvaddr); 45641596803SHuacai Chen #else 45741596803SHuacai Chen unsigned int *pc; 45841596803SHuacai Chen 45961a6fcccSHuacai Chen perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->csr_badvaddr); 46061a6fcccSHuacai Chen 46161a6fcccSHuacai Chen /* 46261a6fcccSHuacai Chen * Did we catch a fault trying to load an instruction? 46361a6fcccSHuacai Chen */ 46461a6fcccSHuacai Chen if (regs->csr_badvaddr == regs->csr_era) 46561a6fcccSHuacai Chen goto sigbus; 46661a6fcccSHuacai Chen if (user_mode(regs) && !test_thread_flag(TIF_FIXADE)) 46761a6fcccSHuacai Chen goto sigbus; 46861a6fcccSHuacai Chen if (!unaligned_enabled) 46961a6fcccSHuacai Chen goto sigbus; 47061a6fcccSHuacai Chen if (!no_unaligned_warning) 47161a6fcccSHuacai Chen show_registers(regs); 47261a6fcccSHuacai Chen 47361a6fcccSHuacai Chen pc = (unsigned int *)exception_era(regs); 47461a6fcccSHuacai Chen 47561a6fcccSHuacai Chen emulate_load_store_insn(regs, (void __user *)regs->csr_badvaddr, pc); 47661a6fcccSHuacai Chen 47761a6fcccSHuacai Chen goto out; 47861a6fcccSHuacai Chen 47961a6fcccSHuacai Chen sigbus: 4800603839bSHuacai Chen die_if_kernel("Kernel ale access", regs); 4810603839bSHuacai Chen force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)regs->csr_badvaddr); 48261a6fcccSHuacai Chen out: 48341596803SHuacai Chen #endif 4840603839bSHuacai Chen irqentry_exit(regs, state); 4850603839bSHuacai Chen } 4860603839bSHuacai Chen 4872d2c3952SYouling Tang #ifdef CONFIG_GENERIC_BUG 4882d2c3952SYouling Tang int is_valid_bugaddr(unsigned long addr) 4892d2c3952SYouling Tang { 4902d2c3952SYouling Tang return 1; 4912d2c3952SYouling Tang } 4922d2c3952SYouling Tang #endif /* CONFIG_GENERIC_BUG */ 4932d2c3952SYouling Tang 4942d2c3952SYouling Tang static void bug_handler(struct pt_regs *regs) 4952d2c3952SYouling Tang { 4962d2c3952SYouling Tang switch (report_bug(regs->csr_era, regs)) { 4972d2c3952SYouling Tang case BUG_TRAP_TYPE_BUG: 4982d2c3952SYouling Tang case BUG_TRAP_TYPE_NONE: 4992d2c3952SYouling Tang die_if_kernel("Oops - BUG", regs); 5002d2c3952SYouling Tang force_sig(SIGTRAP); 5012d2c3952SYouling Tang break; 5022d2c3952SYouling Tang 5032d2c3952SYouling Tang case BUG_TRAP_TYPE_WARN: 5042d2c3952SYouling Tang /* Skip the BUG instruction and continue */ 5052d2c3952SYouling Tang regs->csr_era += LOONGARCH_INSN_SIZE; 5062d2c3952SYouling Tang break; 5072d2c3952SYouling Tang } 5082d2c3952SYouling Tang } 5092d2c3952SYouling Tang 5100603839bSHuacai Chen asmlinkage void noinstr do_bp(struct pt_regs *regs) 5110603839bSHuacai Chen { 5120603839bSHuacai Chen bool user = user_mode(regs); 5130603839bSHuacai Chen unsigned int opcode, bcode; 5140603839bSHuacai Chen unsigned long era = exception_era(regs); 5150603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 5160603839bSHuacai Chen 5176d4cc40fSTiezhu Yang if (regs->csr_prmd & CSR_PRMD_PIE) 5180603839bSHuacai Chen local_irq_enable(); 5196d4cc40fSTiezhu Yang 5200603839bSHuacai Chen current->thread.trap_nr = read_csr_excode(); 5210603839bSHuacai Chen if (__get_inst(&opcode, (u32 *)era, user)) 5220603839bSHuacai Chen goto out_sigsegv; 5230603839bSHuacai Chen 5240603839bSHuacai Chen bcode = (opcode & 0x7fff); 5250603839bSHuacai Chen 5260603839bSHuacai Chen /* 5270603839bSHuacai Chen * notify the kprobe handlers, if instruction is likely to 5280603839bSHuacai Chen * pertain to them. 5290603839bSHuacai Chen */ 5300603839bSHuacai Chen switch (bcode) { 5310603839bSHuacai Chen case BRK_KPROBE_BP: 5326d4cc40fSTiezhu Yang if (kprobe_breakpoint_handler(regs)) 5330603839bSHuacai Chen goto out; 5340603839bSHuacai Chen else 5350603839bSHuacai Chen break; 5360603839bSHuacai Chen case BRK_KPROBE_SSTEPBP: 5376d4cc40fSTiezhu Yang if (kprobe_singlestep_handler(regs)) 5380603839bSHuacai Chen goto out; 5390603839bSHuacai Chen else 5400603839bSHuacai Chen break; 5410603839bSHuacai Chen case BRK_UPROBE_BP: 5420603839bSHuacai Chen if (notify_die(DIE_UPROBE, "Uprobe", regs, bcode, 5430603839bSHuacai Chen current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) 5440603839bSHuacai Chen goto out; 5450603839bSHuacai Chen else 5460603839bSHuacai Chen break; 5470603839bSHuacai Chen case BRK_UPROBE_XOLBP: 5480603839bSHuacai Chen if (notify_die(DIE_UPROBE_XOL, "Uprobe_XOL", regs, bcode, 5490603839bSHuacai Chen current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) 5500603839bSHuacai Chen goto out; 5510603839bSHuacai Chen else 5520603839bSHuacai Chen break; 5530603839bSHuacai Chen default: 5540603839bSHuacai Chen if (notify_die(DIE_TRAP, "Break", regs, bcode, 5550603839bSHuacai Chen current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) 5560603839bSHuacai Chen goto out; 5570603839bSHuacai Chen else 5580603839bSHuacai Chen break; 5590603839bSHuacai Chen } 5600603839bSHuacai Chen 5610603839bSHuacai Chen switch (bcode) { 5620603839bSHuacai Chen case BRK_BUG: 5632d2c3952SYouling Tang bug_handler(regs); 5640603839bSHuacai Chen break; 5650603839bSHuacai Chen case BRK_DIVZERO: 5660603839bSHuacai Chen die_if_kernel("Break instruction in kernel code", regs); 5670603839bSHuacai Chen force_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)regs->csr_era); 5680603839bSHuacai Chen break; 5690603839bSHuacai Chen case BRK_OVERFLOW: 5700603839bSHuacai Chen die_if_kernel("Break instruction in kernel code", regs); 5710603839bSHuacai Chen force_sig_fault(SIGFPE, FPE_INTOVF, (void __user *)regs->csr_era); 5720603839bSHuacai Chen break; 5730603839bSHuacai Chen default: 5740603839bSHuacai Chen die_if_kernel("Break instruction in kernel code", regs); 5750603839bSHuacai Chen force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->csr_era); 5760603839bSHuacai Chen break; 5770603839bSHuacai Chen } 5780603839bSHuacai Chen 5790603839bSHuacai Chen out: 5806d4cc40fSTiezhu Yang if (regs->csr_prmd & CSR_PRMD_PIE) 5810603839bSHuacai Chen local_irq_disable(); 5826d4cc40fSTiezhu Yang 5830603839bSHuacai Chen irqentry_exit(regs, state); 5840603839bSHuacai Chen return; 5850603839bSHuacai Chen 5860603839bSHuacai Chen out_sigsegv: 5870603839bSHuacai Chen force_sig(SIGSEGV); 5880603839bSHuacai Chen goto out; 5890603839bSHuacai Chen } 5900603839bSHuacai Chen 5910603839bSHuacai Chen asmlinkage void noinstr do_watch(struct pt_regs *regs) 5920603839bSHuacai Chen { 593edffa33cSQing Zhang irqentry_state_t state = irqentry_enter(regs); 594edffa33cSQing Zhang 595424421a7SQing Zhang #ifndef CONFIG_HAVE_HW_BREAKPOINT 596424421a7SQing Zhang pr_warn("Hardware watch point handler not implemented!\n"); 597424421a7SQing Zhang #else 598424421a7SQing Zhang if (test_tsk_thread_flag(current, TIF_SINGLESTEP)) { 599424421a7SQing Zhang int llbit = (csr_read32(LOONGARCH_CSR_LLBCTL) & 0x1); 600424421a7SQing Zhang unsigned long pc = instruction_pointer(regs); 601424421a7SQing Zhang union loongarch_instruction *ip = (union loongarch_instruction *)pc; 602424421a7SQing Zhang 603424421a7SQing Zhang if (llbit) { 604424421a7SQing Zhang /* 605424421a7SQing Zhang * When the ll-sc combo is encountered, it is regarded as an single 606424421a7SQing Zhang * instruction. So don't clear llbit and reset CSR.FWPS.Skip until 607424421a7SQing Zhang * the llsc execution is completed. 608424421a7SQing Zhang */ 609424421a7SQing Zhang csr_write32(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS); 610424421a7SQing Zhang csr_write32(CSR_LLBCTL_KLO, LOONGARCH_CSR_LLBCTL); 611424421a7SQing Zhang goto out; 612424421a7SQing Zhang } 613424421a7SQing Zhang 614424421a7SQing Zhang if (pc == current->thread.single_step) { 615424421a7SQing Zhang /* 616424421a7SQing Zhang * Certain insns are occasionally not skipped when CSR.FWPS.Skip is 617424421a7SQing Zhang * set, such as fld.d/fst.d. So singlestep needs to compare whether 618424421a7SQing Zhang * the csr_era is equal to the value of singlestep which last time set. 619424421a7SQing Zhang */ 620424421a7SQing Zhang if (!is_self_loop_ins(ip, regs)) { 621424421a7SQing Zhang /* 622424421a7SQing Zhang * Check if the given instruction the target pc is equal to the 623424421a7SQing Zhang * current pc, If yes, then we should not set the CSR.FWPS.SKIP 624424421a7SQing Zhang * bit to break the original instruction stream. 625424421a7SQing Zhang */ 626424421a7SQing Zhang csr_write32(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS); 627424421a7SQing Zhang goto out; 628424421a7SQing Zhang } 629424421a7SQing Zhang } 630424421a7SQing Zhang } else { 631edffa33cSQing Zhang breakpoint_handler(regs); 632edffa33cSQing Zhang watchpoint_handler(regs); 633424421a7SQing Zhang } 634edffa33cSQing Zhang 635424421a7SQing Zhang force_sig(SIGTRAP); 636424421a7SQing Zhang out: 637424421a7SQing Zhang #endif 638edffa33cSQing Zhang irqentry_exit(regs, state); 6390603839bSHuacai Chen } 6400603839bSHuacai Chen 6410603839bSHuacai Chen asmlinkage void noinstr do_ri(struct pt_regs *regs) 6420603839bSHuacai Chen { 64306e76aceSHuacai Chen int status = SIGILL; 6440603839bSHuacai Chen unsigned int opcode = 0; 6450603839bSHuacai Chen unsigned int __user *era = (unsigned int __user *)exception_era(regs); 6460603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 6470603839bSHuacai Chen 6480603839bSHuacai Chen local_irq_enable(); 6490603839bSHuacai Chen current->thread.trap_nr = read_csr_excode(); 6500603839bSHuacai Chen 6510603839bSHuacai Chen if (notify_die(DIE_RI, "RI Fault", regs, 0, current->thread.trap_nr, 6520603839bSHuacai Chen SIGILL) == NOTIFY_STOP) 6530603839bSHuacai Chen goto out; 6540603839bSHuacai Chen 6550603839bSHuacai Chen die_if_kernel("Reserved instruction in kernel code", regs); 6560603839bSHuacai Chen 6570603839bSHuacai Chen if (unlikely(get_user(opcode, era) < 0)) { 6580603839bSHuacai Chen status = SIGSEGV; 6590603839bSHuacai Chen current->thread.error_code = 1; 6600603839bSHuacai Chen } 6610603839bSHuacai Chen 6620603839bSHuacai Chen force_sig(status); 6630603839bSHuacai Chen 6640603839bSHuacai Chen out: 6650603839bSHuacai Chen local_irq_disable(); 6660603839bSHuacai Chen irqentry_exit(regs, state); 6670603839bSHuacai Chen } 6680603839bSHuacai Chen 6690603839bSHuacai Chen static void init_restore_fp(void) 6700603839bSHuacai Chen { 6710603839bSHuacai Chen if (!used_math()) { 6720603839bSHuacai Chen /* First time FP context user. */ 6730603839bSHuacai Chen init_fpu(); 6740603839bSHuacai Chen } else { 6750603839bSHuacai Chen /* This task has formerly used the FP context */ 6760603839bSHuacai Chen if (!is_fpu_owner()) 6770603839bSHuacai Chen own_fpu_inatomic(1); 6780603839bSHuacai Chen } 6790603839bSHuacai Chen 6800603839bSHuacai Chen BUG_ON(!is_fp_enabled()); 6810603839bSHuacai Chen } 6820603839bSHuacai Chen 6830603839bSHuacai Chen asmlinkage void noinstr do_fpu(struct pt_regs *regs) 6840603839bSHuacai Chen { 6850603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 6860603839bSHuacai Chen 6870603839bSHuacai Chen local_irq_enable(); 6880603839bSHuacai Chen die_if_kernel("do_fpu invoked from kernel context!", regs); 6890603839bSHuacai Chen 6900603839bSHuacai Chen preempt_disable(); 6910603839bSHuacai Chen init_restore_fp(); 6920603839bSHuacai Chen preempt_enable(); 6930603839bSHuacai Chen 6940603839bSHuacai Chen local_irq_disable(); 6950603839bSHuacai Chen irqentry_exit(regs, state); 6960603839bSHuacai Chen } 6970603839bSHuacai Chen 6980603839bSHuacai Chen asmlinkage void noinstr do_lsx(struct pt_regs *regs) 6990603839bSHuacai Chen { 7000603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 7010603839bSHuacai Chen 7020603839bSHuacai Chen local_irq_enable(); 7030603839bSHuacai Chen force_sig(SIGILL); 7040603839bSHuacai Chen local_irq_disable(); 7050603839bSHuacai Chen 7060603839bSHuacai Chen irqentry_exit(regs, state); 7070603839bSHuacai Chen } 7080603839bSHuacai Chen 7090603839bSHuacai Chen asmlinkage void noinstr do_lasx(struct pt_regs *regs) 7100603839bSHuacai Chen { 7110603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 7120603839bSHuacai Chen 7130603839bSHuacai Chen local_irq_enable(); 7140603839bSHuacai Chen force_sig(SIGILL); 7150603839bSHuacai Chen local_irq_disable(); 7160603839bSHuacai Chen 7170603839bSHuacai Chen irqentry_exit(regs, state); 7180603839bSHuacai Chen } 7190603839bSHuacai Chen 7200603839bSHuacai Chen asmlinkage void noinstr do_lbt(struct pt_regs *regs) 7210603839bSHuacai Chen { 7220603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 7230603839bSHuacai Chen 7240603839bSHuacai Chen local_irq_enable(); 7250603839bSHuacai Chen force_sig(SIGILL); 7260603839bSHuacai Chen local_irq_disable(); 7270603839bSHuacai Chen 7280603839bSHuacai Chen irqentry_exit(regs, state); 7290603839bSHuacai Chen } 7300603839bSHuacai Chen 7310603839bSHuacai Chen asmlinkage void noinstr do_reserved(struct pt_regs *regs) 7320603839bSHuacai Chen { 7330603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 7340603839bSHuacai Chen 7350603839bSHuacai Chen local_irq_enable(); 7360603839bSHuacai Chen /* 7370603839bSHuacai Chen * Game over - no way to handle this if it ever occurs. Most probably 7380603839bSHuacai Chen * caused by a fatal error after another hardware/software error. 7390603839bSHuacai Chen */ 7400603839bSHuacai Chen pr_err("Caught reserved exception %u on pid:%d [%s] - should not happen\n", 7410603839bSHuacai Chen read_csr_excode(), current->pid, current->comm); 7420603839bSHuacai Chen die_if_kernel("do_reserved exception", regs); 7430603839bSHuacai Chen force_sig(SIGUNUSED); 7440603839bSHuacai Chen 7450603839bSHuacai Chen local_irq_disable(); 7460603839bSHuacai Chen 7470603839bSHuacai Chen irqentry_exit(regs, state); 7480603839bSHuacai Chen } 7490603839bSHuacai Chen 7500603839bSHuacai Chen asmlinkage void cache_parity_error(void) 7510603839bSHuacai Chen { 7520603839bSHuacai Chen /* For the moment, report the problem and hang. */ 7530603839bSHuacai Chen pr_err("Cache error exception:\n"); 7540603839bSHuacai Chen pr_err("csr_merrctl == %08x\n", csr_read32(LOONGARCH_CSR_MERRCTL)); 7550603839bSHuacai Chen pr_err("csr_merrera == %016llx\n", csr_read64(LOONGARCH_CSR_MERRERA)); 7560603839bSHuacai Chen panic("Can't handle the cache error!"); 7570603839bSHuacai Chen } 7580603839bSHuacai Chen 7590603839bSHuacai Chen asmlinkage void noinstr handle_loongarch_irq(struct pt_regs *regs) 7600603839bSHuacai Chen { 7610603839bSHuacai Chen struct pt_regs *old_regs; 7620603839bSHuacai Chen 7630603839bSHuacai Chen irq_enter_rcu(); 7640603839bSHuacai Chen old_regs = set_irq_regs(regs); 7650603839bSHuacai Chen handle_arch_irq(regs); 7660603839bSHuacai Chen set_irq_regs(old_regs); 7670603839bSHuacai Chen irq_exit_rcu(); 7680603839bSHuacai Chen } 7690603839bSHuacai Chen 7700603839bSHuacai Chen asmlinkage void noinstr do_vint(struct pt_regs *regs, unsigned long sp) 7710603839bSHuacai Chen { 7720603839bSHuacai Chen register int cpu; 7730603839bSHuacai Chen register unsigned long stack; 7740603839bSHuacai Chen irqentry_state_t state = irqentry_enter(regs); 7750603839bSHuacai Chen 7760603839bSHuacai Chen cpu = smp_processor_id(); 7770603839bSHuacai Chen 7780603839bSHuacai Chen if (on_irq_stack(cpu, sp)) 7790603839bSHuacai Chen handle_loongarch_irq(regs); 7800603839bSHuacai Chen else { 7810603839bSHuacai Chen stack = per_cpu(irq_stack, cpu) + IRQ_STACK_START; 7820603839bSHuacai Chen 7830603839bSHuacai Chen /* Save task's sp on IRQ stack for unwinding */ 7840603839bSHuacai Chen *(unsigned long *)stack = sp; 7850603839bSHuacai Chen 7860603839bSHuacai Chen __asm__ __volatile__( 7870603839bSHuacai Chen "move $s0, $sp \n" /* Preserve sp */ 7880603839bSHuacai Chen "move $sp, %[stk] \n" /* Switch stack */ 7890603839bSHuacai Chen "move $a0, %[regs] \n" 7900603839bSHuacai Chen "bl handle_loongarch_irq \n" 7910603839bSHuacai Chen "move $sp, $s0 \n" /* Restore sp */ 7920603839bSHuacai Chen : /* No outputs */ 7930603839bSHuacai Chen : [stk] "r" (stack), [regs] "r" (regs) 7940603839bSHuacai Chen : "$a0", "$a1", "$a2", "$a3", "$a4", "$a5", "$a6", "$a7", "$s0", 7950603839bSHuacai Chen "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8", 7960603839bSHuacai Chen "memory"); 7970603839bSHuacai Chen } 7980603839bSHuacai Chen 7990603839bSHuacai Chen irqentry_exit(regs, state); 8000603839bSHuacai Chen } 8010603839bSHuacai Chen 8020603839bSHuacai Chen unsigned long eentry; 8030603839bSHuacai Chen unsigned long tlbrentry; 8040603839bSHuacai Chen 8050603839bSHuacai Chen long exception_handlers[VECSIZE * 128 / sizeof(long)] __aligned(SZ_64K); 8060603839bSHuacai Chen 8070603839bSHuacai Chen static void configure_exception_vector(void) 8080603839bSHuacai Chen { 8090603839bSHuacai Chen eentry = (unsigned long)exception_handlers; 8100603839bSHuacai Chen tlbrentry = (unsigned long)exception_handlers + 80*VECSIZE; 8110603839bSHuacai Chen 8120603839bSHuacai Chen csr_write64(eentry, LOONGARCH_CSR_EENTRY); 8130603839bSHuacai Chen csr_write64(eentry, LOONGARCH_CSR_MERRENTRY); 8140603839bSHuacai Chen csr_write64(tlbrentry, LOONGARCH_CSR_TLBRENTRY); 8150603839bSHuacai Chen } 8160603839bSHuacai Chen 8170603839bSHuacai Chen void per_cpu_trap_init(int cpu) 8180603839bSHuacai Chen { 8190603839bSHuacai Chen unsigned int i; 8200603839bSHuacai Chen 8210603839bSHuacai Chen setup_vint_size(VECSIZE); 8220603839bSHuacai Chen 8230603839bSHuacai Chen configure_exception_vector(); 8240603839bSHuacai Chen 8250603839bSHuacai Chen if (!cpu_data[cpu].asid_cache) 8260603839bSHuacai Chen cpu_data[cpu].asid_cache = asid_first_version(cpu); 8270603839bSHuacai Chen 8280603839bSHuacai Chen mmgrab(&init_mm); 8290603839bSHuacai Chen current->active_mm = &init_mm; 8300603839bSHuacai Chen BUG_ON(current->mm); 8310603839bSHuacai Chen enter_lazy_tlb(&init_mm, current); 8320603839bSHuacai Chen 8330603839bSHuacai Chen /* Initialise exception handlers */ 8340603839bSHuacai Chen if (cpu == 0) 8350603839bSHuacai Chen for (i = 0; i < 64; i++) 8360603839bSHuacai Chen set_handler(i * VECSIZE, handle_reserved, VECSIZE); 8370603839bSHuacai Chen 838d4b6f156SHuacai Chen tlb_init(cpu); 8390603839bSHuacai Chen cpu_cache_init(); 8400603839bSHuacai Chen } 8410603839bSHuacai Chen 8420603839bSHuacai Chen /* Install CPU exception handler */ 8430603839bSHuacai Chen void set_handler(unsigned long offset, void *addr, unsigned long size) 8440603839bSHuacai Chen { 8450603839bSHuacai Chen memcpy((void *)(eentry + offset), addr, size); 8460603839bSHuacai Chen local_flush_icache_range(eentry + offset, eentry + offset + size); 8470603839bSHuacai Chen } 8480603839bSHuacai Chen 8490603839bSHuacai Chen static const char panic_null_cerr[] = 8500603839bSHuacai Chen "Trying to set NULL cache error exception handler\n"; 8510603839bSHuacai Chen 8520603839bSHuacai Chen /* 8530603839bSHuacai Chen * Install uncached CPU exception handler. 8540603839bSHuacai Chen * This is suitable only for the cache error exception which is the only 8550603839bSHuacai Chen * exception handler that is being run uncached. 8560603839bSHuacai Chen */ 8570603839bSHuacai Chen void set_merr_handler(unsigned long offset, void *addr, unsigned long size) 8580603839bSHuacai Chen { 8590603839bSHuacai Chen unsigned long uncached_eentry = TO_UNCACHE(__pa(eentry)); 8600603839bSHuacai Chen 8610603839bSHuacai Chen if (!addr) 8620603839bSHuacai Chen panic(panic_null_cerr); 8630603839bSHuacai Chen 8640603839bSHuacai Chen memcpy((void *)(uncached_eentry + offset), addr, size); 8650603839bSHuacai Chen } 8660603839bSHuacai Chen 8670603839bSHuacai Chen void __init trap_init(void) 8680603839bSHuacai Chen { 8690603839bSHuacai Chen long i; 8700603839bSHuacai Chen 8710603839bSHuacai Chen /* Set interrupt vector handler */ 8729e36fa42SWANG Xuerui for (i = EXCCODE_INT_START; i <= EXCCODE_INT_END; i++) 8730603839bSHuacai Chen set_handler(i * VECSIZE, handle_vint, VECSIZE); 8740603839bSHuacai Chen 8750603839bSHuacai Chen set_handler(EXCCODE_ADE * VECSIZE, handle_ade, VECSIZE); 8760603839bSHuacai Chen set_handler(EXCCODE_ALE * VECSIZE, handle_ale, VECSIZE); 8770603839bSHuacai Chen set_handler(EXCCODE_SYS * VECSIZE, handle_sys, VECSIZE); 8780603839bSHuacai Chen set_handler(EXCCODE_BP * VECSIZE, handle_bp, VECSIZE); 8790603839bSHuacai Chen set_handler(EXCCODE_INE * VECSIZE, handle_ri, VECSIZE); 8800603839bSHuacai Chen set_handler(EXCCODE_IPE * VECSIZE, handle_ri, VECSIZE); 8810603839bSHuacai Chen set_handler(EXCCODE_FPDIS * VECSIZE, handle_fpu, VECSIZE); 8820603839bSHuacai Chen set_handler(EXCCODE_LSXDIS * VECSIZE, handle_lsx, VECSIZE); 8830603839bSHuacai Chen set_handler(EXCCODE_LASXDIS * VECSIZE, handle_lasx, VECSIZE); 8840603839bSHuacai Chen set_handler(EXCCODE_FPE * VECSIZE, handle_fpe, VECSIZE); 8850603839bSHuacai Chen set_handler(EXCCODE_BTDIS * VECSIZE, handle_lbt, VECSIZE); 8860603839bSHuacai Chen set_handler(EXCCODE_WATCH * VECSIZE, handle_watch, VECSIZE); 8870603839bSHuacai Chen 8880603839bSHuacai Chen cache_error_setup(); 8890603839bSHuacai Chen 8900603839bSHuacai Chen local_flush_icache_range(eentry, eentry + 0x400); 8910603839bSHuacai Chen } 892