1/* 2 * Low-level exception handling code 3 * 4 * Copyright (C) 2012 ARM Ltd. 5 * Authors: Catalin Marinas <catalin.marinas@arm.com> 6 * Will Deacon <will.deacon@arm.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21#include <linux/init.h> 22#include <linux/linkage.h> 23 24#include <asm/assembler.h> 25#include <asm/asm-offsets.h> 26#include <asm/errno.h> 27#include <asm/esr.h> 28#include <asm/thread_info.h> 29#include <asm/unistd.h> 30#include <asm/unistd32.h> 31 32/* 33 * Bad Abort numbers 34 *----------------- 35 */ 36#define BAD_SYNC 0 37#define BAD_IRQ 1 38#define BAD_FIQ 2 39#define BAD_ERROR 3 40 41 .macro kernel_entry, el, regsize = 64 42 sub sp, sp, #S_FRAME_SIZE - S_LR // room for LR, SP, SPSR, ELR 43 .if \regsize == 32 44 mov w0, w0 // zero upper 32 bits of x0 45 .endif 46 push x28, x29 47 push x26, x27 48 push x24, x25 49 push x22, x23 50 push x20, x21 51 push x18, x19 52 push x16, x17 53 push x14, x15 54 push x12, x13 55 push x10, x11 56 push x8, x9 57 push x6, x7 58 push x4, x5 59 push x2, x3 60 push x0, x1 61 .if \el == 0 62 mrs x21, sp_el0 63 .else 64 add x21, sp, #S_FRAME_SIZE 65 .endif 66 mrs x22, elr_el1 67 mrs x23, spsr_el1 68 stp lr, x21, [sp, #S_LR] 69 stp x22, x23, [sp, #S_PC] 70 71 /* 72 * Set syscallno to -1 by default (overridden later if real syscall). 73 */ 74 .if \el == 0 75 mvn x21, xzr 76 str x21, [sp, #S_SYSCALLNO] 77 .endif 78 79 /* 80 * Registers that may be useful after this macro is invoked: 81 * 82 * x21 - aborted SP 83 * x22 - aborted PC 84 * x23 - aborted PSTATE 85 */ 86 .endm 87 88 .macro kernel_exit, el, ret = 0 89 ldp x21, x22, [sp, #S_PC] // load ELR, SPSR 90 .if \el == 0 91 ldr x23, [sp, #S_SP] // load return stack pointer 92 .endif 93 .if \ret 94 ldr x1, [sp, #S_X1] // preserve x0 (syscall return) 95 add sp, sp, S_X2 96 .else 97 pop x0, x1 98 .endif 99 pop x2, x3 // load the rest of the registers 100 pop x4, x5 101 pop x6, x7 102 pop x8, x9 103 msr elr_el1, x21 // set up the return data 104 msr spsr_el1, x22 105 .if \el == 0 106 msr sp_el0, x23 107 .endif 108 pop x10, x11 109 pop x12, x13 110 pop x14, x15 111 pop x16, x17 112 pop x18, x19 113 pop x20, x21 114 pop x22, x23 115 pop x24, x25 116 pop x26, x27 117 pop x28, x29 118 ldr lr, [sp], #S_FRAME_SIZE - S_LR // load LR and restore SP 119 eret // return to kernel 120 .endm 121 122 .macro get_thread_info, rd 123 mov \rd, sp 124 and \rd, \rd, #~((1 << 13) - 1) // top of 8K stack 125 .endm 126 127/* 128 * These are the registers used in the syscall handler, and allow us to 129 * have in theory up to 7 arguments to a function - x0 to x6. 130 * 131 * x7 is reserved for the system call number in 32-bit mode. 132 */ 133sc_nr .req x25 // number of system calls 134scno .req x26 // syscall number 135stbl .req x27 // syscall table pointer 136tsk .req x28 // current thread_info 137 138/* 139 * Interrupt handling. 140 */ 141 .macro irq_handler 142 ldr x1, handle_arch_irq 143 mov x0, sp 144 blr x1 145 .endm 146 147 .text 148 149/* 150 * Exception vectors. 151 */ 152 153 .align 11 154ENTRY(vectors) 155 ventry el1_sync_invalid // Synchronous EL1t 156 ventry el1_irq_invalid // IRQ EL1t 157 ventry el1_fiq_invalid // FIQ EL1t 158 ventry el1_error_invalid // Error EL1t 159 160 ventry el1_sync // Synchronous EL1h 161 ventry el1_irq // IRQ EL1h 162 ventry el1_fiq_invalid // FIQ EL1h 163 ventry el1_error_invalid // Error EL1h 164 165 ventry el0_sync // Synchronous 64-bit EL0 166 ventry el0_irq // IRQ 64-bit EL0 167 ventry el0_fiq_invalid // FIQ 64-bit EL0 168 ventry el0_error_invalid // Error 64-bit EL0 169 170#ifdef CONFIG_COMPAT 171 ventry el0_sync_compat // Synchronous 32-bit EL0 172 ventry el0_irq_compat // IRQ 32-bit EL0 173 ventry el0_fiq_invalid_compat // FIQ 32-bit EL0 174 ventry el0_error_invalid_compat // Error 32-bit EL0 175#else 176 ventry el0_sync_invalid // Synchronous 32-bit EL0 177 ventry el0_irq_invalid // IRQ 32-bit EL0 178 ventry el0_fiq_invalid // FIQ 32-bit EL0 179 ventry el0_error_invalid // Error 32-bit EL0 180#endif 181END(vectors) 182 183/* 184 * Invalid mode handlers 185 */ 186 .macro inv_entry, el, reason, regsize = 64 187 kernel_entry el, \regsize 188 mov x0, sp 189 mov x1, #\reason 190 mrs x2, esr_el1 191 b bad_mode 192 .endm 193 194el0_sync_invalid: 195 inv_entry 0, BAD_SYNC 196ENDPROC(el0_sync_invalid) 197 198el0_irq_invalid: 199 inv_entry 0, BAD_IRQ 200ENDPROC(el0_irq_invalid) 201 202el0_fiq_invalid: 203 inv_entry 0, BAD_FIQ 204ENDPROC(el0_fiq_invalid) 205 206el0_error_invalid: 207 inv_entry 0, BAD_ERROR 208ENDPROC(el0_error_invalid) 209 210#ifdef CONFIG_COMPAT 211el0_fiq_invalid_compat: 212 inv_entry 0, BAD_FIQ, 32 213ENDPROC(el0_fiq_invalid_compat) 214 215el0_error_invalid_compat: 216 inv_entry 0, BAD_ERROR, 32 217ENDPROC(el0_error_invalid_compat) 218#endif 219 220el1_sync_invalid: 221 inv_entry 1, BAD_SYNC 222ENDPROC(el1_sync_invalid) 223 224el1_irq_invalid: 225 inv_entry 1, BAD_IRQ 226ENDPROC(el1_irq_invalid) 227 228el1_fiq_invalid: 229 inv_entry 1, BAD_FIQ 230ENDPROC(el1_fiq_invalid) 231 232el1_error_invalid: 233 inv_entry 1, BAD_ERROR 234ENDPROC(el1_error_invalid) 235 236/* 237 * EL1 mode handlers. 238 */ 239 .align 6 240el1_sync: 241 kernel_entry 1 242 mrs x1, esr_el1 // read the syndrome register 243 lsr x24, x1, #ESR_EL1_EC_SHIFT // exception class 244 cmp x24, #ESR_EL1_EC_DABT_EL1 // data abort in EL1 245 b.eq el1_da 246 cmp x24, #ESR_EL1_EC_SYS64 // configurable trap 247 b.eq el1_undef 248 cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception 249 b.eq el1_sp_pc 250 cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception 251 b.eq el1_sp_pc 252 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL1 253 b.eq el1_undef 254 cmp x24, #ESR_EL1_EC_BREAKPT_EL1 // debug exception in EL1 255 b.ge el1_dbg 256 b el1_inv 257el1_da: 258 /* 259 * Data abort handling 260 */ 261 mrs x0, far_el1 262 enable_dbg_if_not_stepping x2 263 // re-enable interrupts if they were enabled in the aborted context 264 tbnz x23, #7, 1f // PSR_I_BIT 265 enable_irq 2661: 267 mov x2, sp // struct pt_regs 268 bl do_mem_abort 269 270 // disable interrupts before pulling preserved data off the stack 271 disable_irq 272 kernel_exit 1 273el1_sp_pc: 274 /* 275 * Stack or PC alignment exception handling 276 */ 277 mrs x0, far_el1 278 mov x1, x25 279 mov x2, sp 280 b do_sp_pc_abort 281el1_undef: 282 /* 283 * Undefined instruction 284 */ 285 mov x0, sp 286 b do_undefinstr 287el1_dbg: 288 /* 289 * Debug exception handling 290 */ 291 tbz x24, #0, el1_inv // EL1 only 292 mrs x0, far_el1 293 mov x2, sp // struct pt_regs 294 bl do_debug_exception 295 296 kernel_exit 1 297el1_inv: 298 // TODO: add support for undefined instructions in kernel mode 299 mov x0, sp 300 mov x1, #BAD_SYNC 301 mrs x2, esr_el1 302 b bad_mode 303ENDPROC(el1_sync) 304 305 .align 6 306el1_irq: 307 kernel_entry 1 308 enable_dbg_if_not_stepping x0 309#ifdef CONFIG_TRACE_IRQFLAGS 310 bl trace_hardirqs_off 311#endif 312#ifdef CONFIG_PREEMPT 313 get_thread_info tsk 314 ldr x24, [tsk, #TI_PREEMPT] // get preempt count 315 add x0, x24, #1 // increment it 316 str x0, [tsk, #TI_PREEMPT] 317#endif 318 irq_handler 319#ifdef CONFIG_PREEMPT 320 str x24, [tsk, #TI_PREEMPT] // restore preempt count 321 cbnz x24, 1f // preempt count != 0 322 ldr x0, [tsk, #TI_FLAGS] // get flags 323 tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling? 324 bl el1_preempt 3251: 326#endif 327#ifdef CONFIG_TRACE_IRQFLAGS 328 bl trace_hardirqs_on 329#endif 330 kernel_exit 1 331ENDPROC(el1_irq) 332 333#ifdef CONFIG_PREEMPT 334el1_preempt: 335 mov x24, lr 3361: enable_dbg 337 bl preempt_schedule_irq // irq en/disable is done inside 338 ldr x0, [tsk, #TI_FLAGS] // get new tasks TI_FLAGS 339 tbnz x0, #TIF_NEED_RESCHED, 1b // needs rescheduling? 340 ret x24 341#endif 342 343/* 344 * EL0 mode handlers. 345 */ 346 .align 6 347el0_sync: 348 kernel_entry 0 349 mrs x25, esr_el1 // read the syndrome register 350 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class 351 cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state 352 b.eq el0_svc 353 adr lr, ret_from_exception 354 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 355 b.eq el0_da 356 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 357 b.eq el0_ia 358 cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access 359 b.eq el0_fpsimd_acc 360 cmp x24, #ESR_EL1_EC_FP_EXC64 // FP/ASIMD exception 361 b.eq el0_fpsimd_exc 362 cmp x24, #ESR_EL1_EC_SYS64 // configurable trap 363 b.eq el0_undef 364 cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception 365 b.eq el0_sp_pc 366 cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception 367 b.eq el0_sp_pc 368 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0 369 b.eq el0_undef 370 cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0 371 b.ge el0_dbg 372 b el0_inv 373 374#ifdef CONFIG_COMPAT 375 .align 6 376el0_sync_compat: 377 kernel_entry 0, 32 378 mrs x25, esr_el1 // read the syndrome register 379 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class 380 cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state 381 b.eq el0_svc_compat 382 adr lr, ret_from_exception 383 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 384 b.eq el0_da 385 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 386 b.eq el0_ia 387 cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access 388 b.eq el0_fpsimd_acc 389 cmp x24, #ESR_EL1_EC_FP_EXC32 // FP/ASIMD exception 390 b.eq el0_fpsimd_exc 391 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0 392 b.eq el0_undef 393 cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0 394 b.ge el0_dbg 395 b el0_inv 396el0_svc_compat: 397 /* 398 * AArch32 syscall handling 399 */ 400 adr stbl, compat_sys_call_table // load compat syscall table pointer 401 uxtw scno, w7 // syscall number in w7 (r7) 402 mov sc_nr, #__NR_compat_syscalls 403 b el0_svc_naked 404 405 .align 6 406el0_irq_compat: 407 kernel_entry 0, 32 408 b el0_irq_naked 409#endif 410 411el0_da: 412 /* 413 * Data abort handling 414 */ 415 mrs x0, far_el1 416 disable_step x1 417 isb 418 enable_dbg 419 // enable interrupts before calling the main handler 420 enable_irq 421 mov x1, x25 422 mov x2, sp 423 b do_mem_abort 424el0_ia: 425 /* 426 * Instruction abort handling 427 */ 428 mrs x0, far_el1 429 disable_step x1 430 isb 431 enable_dbg 432 // enable interrupts before calling the main handler 433 enable_irq 434 orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts 435 mov x2, sp 436 b do_mem_abort 437el0_fpsimd_acc: 438 /* 439 * Floating Point or Advanced SIMD access 440 */ 441 mov x0, x25 442 mov x1, sp 443 b do_fpsimd_acc 444el0_fpsimd_exc: 445 /* 446 * Floating Point or Advanced SIMD exception 447 */ 448 mov x0, x25 449 mov x1, sp 450 b do_fpsimd_exc 451el0_sp_pc: 452 /* 453 * Stack or PC alignment exception handling 454 */ 455 mrs x0, far_el1 456 disable_step x1 457 isb 458 enable_dbg 459 // enable interrupts before calling the main handler 460 enable_irq 461 mov x1, x25 462 mov x2, sp 463 b do_sp_pc_abort 464el0_undef: 465 /* 466 * Undefined instruction 467 */ 468 mov x0, sp 469 b do_undefinstr 470el0_dbg: 471 /* 472 * Debug exception handling 473 */ 474 tbnz x24, #0, el0_inv // EL0 only 475 mrs x0, far_el1 476 disable_step x1 477 mov x1, x25 478 mov x2, sp 479 b do_debug_exception 480el0_inv: 481 mov x0, sp 482 mov x1, #BAD_SYNC 483 mrs x2, esr_el1 484 b bad_mode 485ENDPROC(el0_sync) 486 487 .align 6 488el0_irq: 489 kernel_entry 0 490el0_irq_naked: 491 disable_step x1 492 isb 493 enable_dbg 494#ifdef CONFIG_TRACE_IRQFLAGS 495 bl trace_hardirqs_off 496#endif 497 get_thread_info tsk 498#ifdef CONFIG_PREEMPT 499 ldr x24, [tsk, #TI_PREEMPT] // get preempt count 500 add x23, x24, #1 // increment it 501 str x23, [tsk, #TI_PREEMPT] 502#endif 503 irq_handler 504#ifdef CONFIG_PREEMPT 505 ldr x0, [tsk, #TI_PREEMPT] 506 str x24, [tsk, #TI_PREEMPT] 507 cmp x0, x23 508 b.eq 1f 509 mov x1, #0 510 str x1, [x1] // BUG 5111: 512#endif 513#ifdef CONFIG_TRACE_IRQFLAGS 514 bl trace_hardirqs_on 515#endif 516 b ret_to_user 517ENDPROC(el0_irq) 518 519/* 520 * This is the return code to user mode for abort handlers 521 */ 522ret_from_exception: 523 get_thread_info tsk 524 b ret_to_user 525ENDPROC(ret_from_exception) 526 527/* 528 * Register switch for AArch64. The callee-saved registers need to be saved 529 * and restored. On entry: 530 * x0 = previous task_struct (must be preserved across the switch) 531 * x1 = next task_struct 532 * Previous and next are guaranteed not to be the same. 533 * 534 */ 535ENTRY(cpu_switch_to) 536 add x8, x0, #THREAD_CPU_CONTEXT 537 mov x9, sp 538 stp x19, x20, [x8], #16 // store callee-saved registers 539 stp x21, x22, [x8], #16 540 stp x23, x24, [x8], #16 541 stp x25, x26, [x8], #16 542 stp x27, x28, [x8], #16 543 stp x29, x9, [x8], #16 544 str lr, [x8] 545 add x8, x1, #THREAD_CPU_CONTEXT 546 ldp x19, x20, [x8], #16 // restore callee-saved registers 547 ldp x21, x22, [x8], #16 548 ldp x23, x24, [x8], #16 549 ldp x25, x26, [x8], #16 550 ldp x27, x28, [x8], #16 551 ldp x29, x9, [x8], #16 552 ldr lr, [x8] 553 mov sp, x9 554 ret 555ENDPROC(cpu_switch_to) 556 557/* 558 * This is the fast syscall return path. We do as little as possible here, 559 * and this includes saving x0 back into the kernel stack. 560 */ 561ret_fast_syscall: 562 disable_irq // disable interrupts 563 ldr x1, [tsk, #TI_FLAGS] 564 and x2, x1, #_TIF_WORK_MASK 565 cbnz x2, fast_work_pending 566 tbz x1, #TIF_SINGLESTEP, fast_exit 567 disable_dbg 568 enable_step x2 569fast_exit: 570 kernel_exit 0, ret = 1 571 572/* 573 * Ok, we need to do extra processing, enter the slow path. 574 */ 575fast_work_pending: 576 str x0, [sp, #S_X0] // returned x0 577work_pending: 578 tbnz x1, #TIF_NEED_RESCHED, work_resched 579 /* TIF_SIGPENDING or TIF_NOTIFY_RESUME case */ 580 ldr x2, [sp, #S_PSTATE] 581 mov x0, sp // 'regs' 582 tst x2, #PSR_MODE_MASK // user mode regs? 583 b.ne no_work_pending // returning to kernel 584 enable_irq // enable interrupts for do_notify_resume() 585 bl do_notify_resume 586 b ret_to_user 587work_resched: 588 enable_dbg 589 bl schedule 590 591/* 592 * "slow" syscall return path. 593 */ 594ret_to_user: 595 disable_irq // disable interrupts 596 ldr x1, [tsk, #TI_FLAGS] 597 and x2, x1, #_TIF_WORK_MASK 598 cbnz x2, work_pending 599 tbz x1, #TIF_SINGLESTEP, no_work_pending 600 disable_dbg 601 enable_step x2 602no_work_pending: 603 kernel_exit 0, ret = 0 604ENDPROC(ret_to_user) 605 606/* 607 * This is how we return from a fork. 608 */ 609ENTRY(ret_from_fork) 610 bl schedule_tail 611 cbz x19, 1f // not a kernel thread 612 mov x0, x20 613 blr x19 6141: get_thread_info tsk 615 b ret_to_user 616ENDPROC(ret_from_fork) 617 618/* 619 * SVC handler. 620 */ 621 .align 6 622el0_svc: 623 adrp stbl, sys_call_table // load syscall table pointer 624 uxtw scno, w8 // syscall number in w8 625 mov sc_nr, #__NR_syscalls 626el0_svc_naked: // compat entry point 627 stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number 628 disable_step x16 629 isb 630 enable_dbg 631 enable_irq 632 633 get_thread_info tsk 634 ldr x16, [tsk, #TI_FLAGS] // check for syscall tracing 635 tbnz x16, #TIF_SYSCALL_TRACE, __sys_trace // are we tracing syscalls? 636 adr lr, ret_fast_syscall // return address 637 cmp scno, sc_nr // check upper syscall limit 638 b.hs ni_sys 639 ldr x16, [stbl, scno, lsl #3] // address in the syscall table 640 br x16 // call sys_* routine 641ni_sys: 642 mov x0, sp 643 b do_ni_syscall 644ENDPROC(el0_svc) 645 646 /* 647 * This is the really slow path. We're going to be doing context 648 * switches, and waiting for our parent to respond. 649 */ 650__sys_trace: 651 mov x1, sp 652 mov w0, #0 // trace entry 653 bl syscall_trace 654 adr lr, __sys_trace_return // return address 655 uxtw scno, w0 // syscall number (possibly new) 656 mov x1, sp // pointer to regs 657 cmp scno, sc_nr // check upper syscall limit 658 b.hs ni_sys 659 ldp x0, x1, [sp] // restore the syscall args 660 ldp x2, x3, [sp, #S_X2] 661 ldp x4, x5, [sp, #S_X4] 662 ldp x6, x7, [sp, #S_X6] 663 ldr x16, [stbl, scno, lsl #3] // address in the syscall table 664 br x16 // call sys_* routine 665 666__sys_trace_return: 667 str x0, [sp] // save returned x0 668 mov x1, sp 669 mov w0, #1 // trace exit 670 bl syscall_trace 671 b ret_to_user 672 673/* 674 * Special system call wrappers. 675 */ 676ENTRY(sys_rt_sigreturn_wrapper) 677 mov x0, sp 678 b sys_rt_sigreturn 679ENDPROC(sys_rt_sigreturn_wrapper) 680 681ENTRY(handle_arch_irq) 682 .quad 0 683