1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012 Regents of the University of California 4 * Copyright (C) 2017 SiFive 5 */ 6 7#include <linux/init.h> 8#include <linux/linkage.h> 9 10#include <asm/asm.h> 11#include <asm/csr.h> 12#include <asm/unistd.h> 13#include <asm/thread_info.h> 14#include <asm/asm-offsets.h> 15#include <asm/errata_list.h> 16 17#if !IS_ENABLED(CONFIG_PREEMPTION) 18.set resume_kernel, restore_all 19#endif 20 21ENTRY(handle_exception) 22 /* 23 * If coming from userspace, preserve the user thread pointer and load 24 * the kernel thread pointer. If we came from the kernel, the scratch 25 * register will contain 0, and we should continue on the current TP. 26 */ 27 csrrw tp, CSR_SCRATCH, tp 28 bnez tp, _save_context 29 30_restore_kernel_tpsp: 31 csrr tp, CSR_SCRATCH 32 REG_S sp, TASK_TI_KERNEL_SP(tp) 33 34#ifdef CONFIG_VMAP_STACK 35 addi sp, sp, -(PT_SIZE_ON_STACK) 36 srli sp, sp, THREAD_SHIFT 37 andi sp, sp, 0x1 38 bnez sp, handle_kernel_stack_overflow 39 REG_L sp, TASK_TI_KERNEL_SP(tp) 40#endif 41 42_save_context: 43 REG_S sp, TASK_TI_USER_SP(tp) 44 REG_L sp, TASK_TI_KERNEL_SP(tp) 45 addi sp, sp, -(PT_SIZE_ON_STACK) 46 REG_S x1, PT_RA(sp) 47 REG_S x3, PT_GP(sp) 48 REG_S x5, PT_T0(sp) 49 REG_S x6, PT_T1(sp) 50 REG_S x7, PT_T2(sp) 51 REG_S x8, PT_S0(sp) 52 REG_S x9, PT_S1(sp) 53 REG_S x10, PT_A0(sp) 54 REG_S x11, PT_A1(sp) 55 REG_S x12, PT_A2(sp) 56 REG_S x13, PT_A3(sp) 57 REG_S x14, PT_A4(sp) 58 REG_S x15, PT_A5(sp) 59 REG_S x16, PT_A6(sp) 60 REG_S x17, PT_A7(sp) 61 REG_S x18, PT_S2(sp) 62 REG_S x19, PT_S3(sp) 63 REG_S x20, PT_S4(sp) 64 REG_S x21, PT_S5(sp) 65 REG_S x22, PT_S6(sp) 66 REG_S x23, PT_S7(sp) 67 REG_S x24, PT_S8(sp) 68 REG_S x25, PT_S9(sp) 69 REG_S x26, PT_S10(sp) 70 REG_S x27, PT_S11(sp) 71 REG_S x28, PT_T3(sp) 72 REG_S x29, PT_T4(sp) 73 REG_S x30, PT_T5(sp) 74 REG_S x31, PT_T6(sp) 75 76 /* 77 * Disable user-mode memory access as it should only be set in the 78 * actual user copy routines. 79 * 80 * Disable the FPU to detect illegal usage of floating point in kernel 81 * space. 82 */ 83 li t0, SR_SUM | SR_FS 84 85 REG_L s0, TASK_TI_USER_SP(tp) 86 csrrc s1, CSR_STATUS, t0 87 csrr s2, CSR_EPC 88 csrr s3, CSR_TVAL 89 csrr s4, CSR_CAUSE 90 csrr s5, CSR_SCRATCH 91 REG_S s0, PT_SP(sp) 92 REG_S s1, PT_STATUS(sp) 93 REG_S s2, PT_EPC(sp) 94 REG_S s3, PT_BADADDR(sp) 95 REG_S s4, PT_CAUSE(sp) 96 REG_S s5, PT_TP(sp) 97 98 /* 99 * Set the scratch register to 0, so that if a recursive exception 100 * occurs, the exception vector knows it came from the kernel 101 */ 102 csrw CSR_SCRATCH, x0 103 104 /* Load the global pointer */ 105.option push 106.option norelax 107 la gp, __global_pointer$ 108.option pop 109 110#ifdef CONFIG_TRACE_IRQFLAGS 111 call trace_hardirqs_off 112#endif 113 114#ifdef CONFIG_CONTEXT_TRACKING 115 /* If previous state is in user mode, call context_tracking_user_exit. */ 116 li a0, SR_PP 117 and a0, s1, a0 118 bnez a0, skip_context_tracking 119 call context_tracking_user_exit 120skip_context_tracking: 121#endif 122 123 /* 124 * MSB of cause differentiates between 125 * interrupts and exceptions 126 */ 127 bge s4, zero, 1f 128 129 la ra, ret_from_exception 130 131 /* Handle interrupts */ 132 move a0, sp /* pt_regs */ 133 la a1, generic_handle_arch_irq 134 jr a1 1351: 136 /* 137 * Exceptions run with interrupts enabled or disabled depending on the 138 * state of SR_PIE in m/sstatus. 139 */ 140 andi t0, s1, SR_PIE 141 beqz t0, 1f 142 /* kprobes, entered via ebreak, must have interrupts disabled. */ 143 li t0, EXC_BREAKPOINT 144 beq s4, t0, 1f 145#ifdef CONFIG_TRACE_IRQFLAGS 146 call trace_hardirqs_on 147#endif 148 csrs CSR_STATUS, SR_IE 149 1501: 151 la ra, ret_from_exception 152 /* Handle syscalls */ 153 li t0, EXC_SYSCALL 154 beq s4, t0, handle_syscall 155 156 /* Handle other exceptions */ 157 slli t0, s4, RISCV_LGPTR 158 la t1, excp_vect_table 159 la t2, excp_vect_table_end 160 move a0, sp /* pt_regs */ 161 add t0, t1, t0 162 /* Check if exception code lies within bounds */ 163 bgeu t0, t2, 1f 164 REG_L t0, 0(t0) 165 jr t0 1661: 167 tail do_trap_unknown 168 169handle_syscall: 170#ifdef CONFIG_RISCV_M_MODE 171 /* 172 * When running is M-Mode (no MMU config), MPIE does not get set. 173 * As a result, we need to force enable interrupts here because 174 * handle_exception did not do set SR_IE as it always sees SR_PIE 175 * being cleared. 176 */ 177 csrs CSR_STATUS, SR_IE 178#endif 179#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING) 180 /* Recover a0 - a7 for system calls */ 181 REG_L a0, PT_A0(sp) 182 REG_L a1, PT_A1(sp) 183 REG_L a2, PT_A2(sp) 184 REG_L a3, PT_A3(sp) 185 REG_L a4, PT_A4(sp) 186 REG_L a5, PT_A5(sp) 187 REG_L a6, PT_A6(sp) 188 REG_L a7, PT_A7(sp) 189#endif 190 /* save the initial A0 value (needed in signal handlers) */ 191 REG_S a0, PT_ORIG_A0(sp) 192 /* 193 * Advance SEPC to avoid executing the original 194 * scall instruction on sret 195 */ 196 addi s2, s2, 0x4 197 REG_S s2, PT_EPC(sp) 198 /* Trace syscalls, but only if requested by the user. */ 199 REG_L t0, TASK_TI_FLAGS(tp) 200 andi t0, t0, _TIF_SYSCALL_WORK 201 bnez t0, handle_syscall_trace_enter 202check_syscall_nr: 203 /* Check to make sure we don't jump to a bogus syscall number. */ 204 li t0, __NR_syscalls 205 la s0, sys_ni_syscall 206 /* 207 * Syscall number held in a7. 208 * If syscall number is above allowed value, redirect to ni_syscall. 209 */ 210 bgeu a7, t0, 1f 211 /* Call syscall */ 212 la s0, sys_call_table 213 slli t0, a7, RISCV_LGPTR 214 add s0, s0, t0 215 REG_L s0, 0(s0) 2161: 217 jalr s0 218 219ret_from_syscall: 220 /* Set user a0 to kernel a0 */ 221 REG_S a0, PT_A0(sp) 222 /* 223 * We didn't execute the actual syscall. 224 * Seccomp already set return value for the current task pt_regs. 225 * (If it was configured with SECCOMP_RET_ERRNO/TRACE) 226 */ 227ret_from_syscall_rejected: 228 /* Trace syscalls, but only if requested by the user. */ 229 REG_L t0, TASK_TI_FLAGS(tp) 230 andi t0, t0, _TIF_SYSCALL_WORK 231 bnez t0, handle_syscall_trace_exit 232 233ret_from_exception: 234 REG_L s0, PT_STATUS(sp) 235 csrc CSR_STATUS, SR_IE 236#ifdef CONFIG_TRACE_IRQFLAGS 237 call trace_hardirqs_off 238#endif 239#ifdef CONFIG_RISCV_M_MODE 240 /* the MPP value is too large to be used as an immediate arg for addi */ 241 li t0, SR_MPP 242 and s0, s0, t0 243#else 244 andi s0, s0, SR_SPP 245#endif 246 bnez s0, resume_kernel 247 248resume_userspace: 249 /* Interrupts must be disabled here so flags are checked atomically */ 250 REG_L s0, TASK_TI_FLAGS(tp) /* current_thread_info->flags */ 251 andi s1, s0, _TIF_WORK_MASK 252 bnez s1, work_pending 253 254#ifdef CONFIG_CONTEXT_TRACKING 255 call context_tracking_user_enter 256#endif 257 258 /* Save unwound kernel stack pointer in thread_info */ 259 addi s0, sp, PT_SIZE_ON_STACK 260 REG_S s0, TASK_TI_KERNEL_SP(tp) 261 262 /* 263 * Save TP into the scratch register , so we can find the kernel data 264 * structures again. 265 */ 266 csrw CSR_SCRATCH, tp 267 268restore_all: 269#ifdef CONFIG_TRACE_IRQFLAGS 270 REG_L s1, PT_STATUS(sp) 271 andi t0, s1, SR_PIE 272 beqz t0, 1f 273 call trace_hardirqs_on 274 j 2f 2751: 276 call trace_hardirqs_off 2772: 278#endif 279 REG_L a0, PT_STATUS(sp) 280 /* 281 * The current load reservation is effectively part of the processor's 282 * state, in the sense that load reservations cannot be shared between 283 * different hart contexts. We can't actually save and restore a load 284 * reservation, so instead here we clear any existing reservation -- 285 * it's always legal for implementations to clear load reservations at 286 * any point (as long as the forward progress guarantee is kept, but 287 * we'll ignore that here). 288 * 289 * Dangling load reservations can be the result of taking a trap in the 290 * middle of an LR/SC sequence, but can also be the result of a taken 291 * forward branch around an SC -- which is how we implement CAS. As a 292 * result we need to clear reservations between the last CAS and the 293 * jump back to the new context. While it is unlikely the store 294 * completes, implementations are allowed to expand reservations to be 295 * arbitrarily large. 296 */ 297 REG_L a2, PT_EPC(sp) 298 REG_SC x0, a2, PT_EPC(sp) 299 300 csrw CSR_STATUS, a0 301 csrw CSR_EPC, a2 302 303 REG_L x1, PT_RA(sp) 304 REG_L x3, PT_GP(sp) 305 REG_L x4, PT_TP(sp) 306 REG_L x5, PT_T0(sp) 307 REG_L x6, PT_T1(sp) 308 REG_L x7, PT_T2(sp) 309 REG_L x8, PT_S0(sp) 310 REG_L x9, PT_S1(sp) 311 REG_L x10, PT_A0(sp) 312 REG_L x11, PT_A1(sp) 313 REG_L x12, PT_A2(sp) 314 REG_L x13, PT_A3(sp) 315 REG_L x14, PT_A4(sp) 316 REG_L x15, PT_A5(sp) 317 REG_L x16, PT_A6(sp) 318 REG_L x17, PT_A7(sp) 319 REG_L x18, PT_S2(sp) 320 REG_L x19, PT_S3(sp) 321 REG_L x20, PT_S4(sp) 322 REG_L x21, PT_S5(sp) 323 REG_L x22, PT_S6(sp) 324 REG_L x23, PT_S7(sp) 325 REG_L x24, PT_S8(sp) 326 REG_L x25, PT_S9(sp) 327 REG_L x26, PT_S10(sp) 328 REG_L x27, PT_S11(sp) 329 REG_L x28, PT_T3(sp) 330 REG_L x29, PT_T4(sp) 331 REG_L x30, PT_T5(sp) 332 REG_L x31, PT_T6(sp) 333 334 REG_L x2, PT_SP(sp) 335 336#ifdef CONFIG_RISCV_M_MODE 337 mret 338#else 339 sret 340#endif 341 342#if IS_ENABLED(CONFIG_PREEMPTION) 343resume_kernel: 344 REG_L s0, TASK_TI_PREEMPT_COUNT(tp) 345 bnez s0, restore_all 346 REG_L s0, TASK_TI_FLAGS(tp) 347 andi s0, s0, _TIF_NEED_RESCHED 348 beqz s0, restore_all 349 call preempt_schedule_irq 350 j restore_all 351#endif 352 353work_pending: 354 /* Enter slow path for supplementary processing */ 355 la ra, ret_from_exception 356 andi s1, s0, _TIF_NEED_RESCHED 357 bnez s1, work_resched 358work_notifysig: 359 /* Handle pending signals and notify-resume requests */ 360 csrs CSR_STATUS, SR_IE /* Enable interrupts for do_notify_resume() */ 361 move a0, sp /* pt_regs */ 362 move a1, s0 /* current_thread_info->flags */ 363 tail do_notify_resume 364work_resched: 365 tail schedule 366 367/* Slow paths for ptrace. */ 368handle_syscall_trace_enter: 369 move a0, sp 370 call do_syscall_trace_enter 371 move t0, a0 372 REG_L a0, PT_A0(sp) 373 REG_L a1, PT_A1(sp) 374 REG_L a2, PT_A2(sp) 375 REG_L a3, PT_A3(sp) 376 REG_L a4, PT_A4(sp) 377 REG_L a5, PT_A5(sp) 378 REG_L a6, PT_A6(sp) 379 REG_L a7, PT_A7(sp) 380 bnez t0, ret_from_syscall_rejected 381 j check_syscall_nr 382handle_syscall_trace_exit: 383 move a0, sp 384 call do_syscall_trace_exit 385 j ret_from_exception 386 387#ifdef CONFIG_VMAP_STACK 388handle_kernel_stack_overflow: 389 la sp, shadow_stack 390 addi sp, sp, SHADOW_OVERFLOW_STACK_SIZE 391 392 //save caller register to shadow stack 393 addi sp, sp, -(PT_SIZE_ON_STACK) 394 REG_S x1, PT_RA(sp) 395 REG_S x5, PT_T0(sp) 396 REG_S x6, PT_T1(sp) 397 REG_S x7, PT_T2(sp) 398 REG_S x10, PT_A0(sp) 399 REG_S x11, PT_A1(sp) 400 REG_S x12, PT_A2(sp) 401 REG_S x13, PT_A3(sp) 402 REG_S x14, PT_A4(sp) 403 REG_S x15, PT_A5(sp) 404 REG_S x16, PT_A6(sp) 405 REG_S x17, PT_A7(sp) 406 REG_S x28, PT_T3(sp) 407 REG_S x29, PT_T4(sp) 408 REG_S x30, PT_T5(sp) 409 REG_S x31, PT_T6(sp) 410 411 la ra, restore_caller_reg 412 tail get_overflow_stack 413 414restore_caller_reg: 415 //save per-cpu overflow stack 416 REG_S a0, -8(sp) 417 //restore caller register from shadow_stack 418 REG_L x1, PT_RA(sp) 419 REG_L x5, PT_T0(sp) 420 REG_L x6, PT_T1(sp) 421 REG_L x7, PT_T2(sp) 422 REG_L x10, PT_A0(sp) 423 REG_L x11, PT_A1(sp) 424 REG_L x12, PT_A2(sp) 425 REG_L x13, PT_A3(sp) 426 REG_L x14, PT_A4(sp) 427 REG_L x15, PT_A5(sp) 428 REG_L x16, PT_A6(sp) 429 REG_L x17, PT_A7(sp) 430 REG_L x28, PT_T3(sp) 431 REG_L x29, PT_T4(sp) 432 REG_L x30, PT_T5(sp) 433 REG_L x31, PT_T6(sp) 434 435 //load per-cpu overflow stack 436 REG_L sp, -8(sp) 437 addi sp, sp, -(PT_SIZE_ON_STACK) 438 439 //save context to overflow stack 440 REG_S x1, PT_RA(sp) 441 REG_S x3, PT_GP(sp) 442 REG_S x5, PT_T0(sp) 443 REG_S x6, PT_T1(sp) 444 REG_S x7, PT_T2(sp) 445 REG_S x8, PT_S0(sp) 446 REG_S x9, PT_S1(sp) 447 REG_S x10, PT_A0(sp) 448 REG_S x11, PT_A1(sp) 449 REG_S x12, PT_A2(sp) 450 REG_S x13, PT_A3(sp) 451 REG_S x14, PT_A4(sp) 452 REG_S x15, PT_A5(sp) 453 REG_S x16, PT_A6(sp) 454 REG_S x17, PT_A7(sp) 455 REG_S x18, PT_S2(sp) 456 REG_S x19, PT_S3(sp) 457 REG_S x20, PT_S4(sp) 458 REG_S x21, PT_S5(sp) 459 REG_S x22, PT_S6(sp) 460 REG_S x23, PT_S7(sp) 461 REG_S x24, PT_S8(sp) 462 REG_S x25, PT_S9(sp) 463 REG_S x26, PT_S10(sp) 464 REG_S x27, PT_S11(sp) 465 REG_S x28, PT_T3(sp) 466 REG_S x29, PT_T4(sp) 467 REG_S x30, PT_T5(sp) 468 REG_S x31, PT_T6(sp) 469 470 REG_L s0, TASK_TI_KERNEL_SP(tp) 471 csrr s1, CSR_STATUS 472 csrr s2, CSR_EPC 473 csrr s3, CSR_TVAL 474 csrr s4, CSR_CAUSE 475 csrr s5, CSR_SCRATCH 476 REG_S s0, PT_SP(sp) 477 REG_S s1, PT_STATUS(sp) 478 REG_S s2, PT_EPC(sp) 479 REG_S s3, PT_BADADDR(sp) 480 REG_S s4, PT_CAUSE(sp) 481 REG_S s5, PT_TP(sp) 482 move a0, sp 483 tail handle_bad_stack 484#endif 485 486END(handle_exception) 487 488ENTRY(ret_from_fork) 489 la ra, ret_from_exception 490 tail schedule_tail 491ENDPROC(ret_from_fork) 492 493ENTRY(ret_from_kernel_thread) 494 call schedule_tail 495 /* Call fn(arg) */ 496 la ra, ret_from_exception 497 move a0, s1 498 jr s0 499ENDPROC(ret_from_kernel_thread) 500 501 502/* 503 * Integer register context switch 504 * The callee-saved registers must be saved and restored. 505 * 506 * a0: previous task_struct (must be preserved across the switch) 507 * a1: next task_struct 508 * 509 * The value of a0 and a1 must be preserved by this function, as that's how 510 * arguments are passed to schedule_tail. 511 */ 512ENTRY(__switch_to) 513 /* Save context into prev->thread */ 514 li a4, TASK_THREAD_RA 515 add a3, a0, a4 516 add a4, a1, a4 517 REG_S ra, TASK_THREAD_RA_RA(a3) 518 REG_S sp, TASK_THREAD_SP_RA(a3) 519 REG_S s0, TASK_THREAD_S0_RA(a3) 520 REG_S s1, TASK_THREAD_S1_RA(a3) 521 REG_S s2, TASK_THREAD_S2_RA(a3) 522 REG_S s3, TASK_THREAD_S3_RA(a3) 523 REG_S s4, TASK_THREAD_S4_RA(a3) 524 REG_S s5, TASK_THREAD_S5_RA(a3) 525 REG_S s6, TASK_THREAD_S6_RA(a3) 526 REG_S s7, TASK_THREAD_S7_RA(a3) 527 REG_S s8, TASK_THREAD_S8_RA(a3) 528 REG_S s9, TASK_THREAD_S9_RA(a3) 529 REG_S s10, TASK_THREAD_S10_RA(a3) 530 REG_S s11, TASK_THREAD_S11_RA(a3) 531 /* Restore context from next->thread */ 532 REG_L ra, TASK_THREAD_RA_RA(a4) 533 REG_L sp, TASK_THREAD_SP_RA(a4) 534 REG_L s0, TASK_THREAD_S0_RA(a4) 535 REG_L s1, TASK_THREAD_S1_RA(a4) 536 REG_L s2, TASK_THREAD_S2_RA(a4) 537 REG_L s3, TASK_THREAD_S3_RA(a4) 538 REG_L s4, TASK_THREAD_S4_RA(a4) 539 REG_L s5, TASK_THREAD_S5_RA(a4) 540 REG_L s6, TASK_THREAD_S6_RA(a4) 541 REG_L s7, TASK_THREAD_S7_RA(a4) 542 REG_L s8, TASK_THREAD_S8_RA(a4) 543 REG_L s9, TASK_THREAD_S9_RA(a4) 544 REG_L s10, TASK_THREAD_S10_RA(a4) 545 REG_L s11, TASK_THREAD_S11_RA(a4) 546 /* The offset of thread_info in task_struct is zero. */ 547 move tp, a1 548 ret 549ENDPROC(__switch_to) 550 551#ifndef CONFIG_MMU 552#define do_page_fault do_trap_unknown 553#endif 554 555 .section ".rodata" 556 .align LGREG 557 /* Exception vector table */ 558ENTRY(excp_vect_table) 559 RISCV_PTR do_trap_insn_misaligned 560 ALT_INSN_FAULT(RISCV_PTR do_trap_insn_fault) 561 RISCV_PTR do_trap_insn_illegal 562 RISCV_PTR do_trap_break 563 RISCV_PTR do_trap_load_misaligned 564 RISCV_PTR do_trap_load_fault 565 RISCV_PTR do_trap_store_misaligned 566 RISCV_PTR do_trap_store_fault 567 RISCV_PTR do_trap_ecall_u /* system call, gets intercepted */ 568 RISCV_PTR do_trap_ecall_s 569 RISCV_PTR do_trap_unknown 570 RISCV_PTR do_trap_ecall_m 571 /* instruciton page fault */ 572 ALT_PAGE_FAULT(RISCV_PTR do_page_fault) 573 RISCV_PTR do_page_fault /* load page fault */ 574 RISCV_PTR do_trap_unknown 575 RISCV_PTR do_page_fault /* store page fault */ 576excp_vect_table_end: 577END(excp_vect_table) 578 579#ifndef CONFIG_MMU 580ENTRY(__user_rt_sigreturn) 581 li a7, __NR_rt_sigreturn 582 scall 583END(__user_rt_sigreturn) 584#endif 585